ChuNa7a/ChuSQL
1
0 Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

修bug，完善功能
build / build (push) Blocked by required conditions
Details
tests / build (push) Waiting to run
Details
tests / tests (push) Waiting to run
Details

Browse Source
This commit is contained in:
ChuNa7a committed 2026-10-16 23:52:12 +08:00
1 parent 30364b1c4b
commit 6996e19694
125 files changed
+15343 -3746
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

chusql-core/engine/chusql-core-engine.cabal
+2 -1
View File
@@ -10,7 +10,8 @@ build-type: Simple
library
hs-source-dirs: src
exposed-modules: ChuSQL.Core.Engine.Algebra.Cost
exposed-modules: ChuSQL.Core.Engine.Domain
ChuSQL.Core.Engine.Algebra.Cost
ChuSQL.Core.Engine.Algebra.Eval
ChuSQL.Core.Engine.Algebra.Expr
ChuSQL.Core.Engine.Algebra.JoinOrder
chusql-core/engine/src/ChuSQL/Core/Engine.hs
+94 -46
View File
@@ -1,18 +1,19 @@
module ChuSQL.Core.Engine (runStatement, runStatementM, rowsOf, BoundWrite, bindWrite, executeWriteM) where
module ChuSQL.Core.Engine (runStatement, runStatementM, rowsOf, BoundWrite, bindWrite, executeWriteM, validateTableRows) where
import ChuSQL.Core.Engine.Algebra.Eval (evalCondForRowM, evalExprM)
import ChuSQL.Core.Engine.Algebra.Expr (evalCondForRow, evalExpr)
import ChuSQL.Core.Engine.Domain
import ChuSQL.Core.Engine.Runtime.Dependencies (domainDependencies, queryTables, typeDependencies, validateDependencies)
import ChuSQL.Core.Engine.Runtime.Query (bindQuery, executeQueryM)
import ChuSQL.Core.Engine.Runtime.Relation (bindWriteColumns, orderRow)
import ChuSQL.Core.Engine.Runtime.Dependencies (queryTables, domainDependencies, typeDependencies, validateDependencies)
import ChuSQL.Core.Engine.Semantic (prepare, prepareCheckExpression)
import ChuSQL.Core.Engine.Storage (MonadStorage (..))
import ChuSQL.Core.Engine.Storage.Memory (MemoryStorage (runMemoryStorage))
import ChuSQL.Core.Engine.Syntax.AST
import ChuSQL.Core.Engine.Syntax.Parser (parseExpression)
import ChuSQL.Core.Model
import Data.HashSet qualified as HS
import Data.HashMap.Strict qualified as HM
import Data.HashSet qualified as HS
import Data.Maybe (mapMaybe)
-- 引擎入口：检查语义、按语句分发执行，另有内存实现与泛型版本。
@@ -69,26 +70,27 @@ bindWrite :: Database -> Statement -> Either String BoundWrite
bindWrite db statement = case statement of
Select{} -> unsupported
SelectDistinct{} -> unsupported
_ | null (queryTables statement) -> unsupported
| otherwise -> do
resolved <- prepare catalog statement >>= bindWriteColumns catalog
let sources = queryTables resolved
mapM_ (lookupTable catalog) sources
let names = sources ++ domainDependencies catalog sources ++ typeDependencies catalog (declaredTypes resolved)
dependencies = [(name, table) | (name, table) <- catalog, name `elem` names]
action <- bindWriteAction catalog resolved
Right (BoundWrite catalog dependencies action)
_
| null (queryTables statement) -> unsupported
| otherwise -> do
resolved <- prepare catalog statement >>= bindWriteColumns catalog
let sources = queryTables resolved
mapM_ (lookupTable catalog) sources
let names = sources ++ domainDependencies catalog sources ++ typeDependencies catalog (declaredTypes resolved)
dependencies = [(name, table) | (name, table) <- catalog, name `elem` names]
action <- bindWriteAction catalog resolved
Right (BoundWrite catalog dependencies action)
where
catalog = [(name, table{tableRows = []}) | (name, table) <- db]
unsupported = Left "bound writes require an existing table target"
-- | 新声明类型也须保留领域依赖
-- \| 新声明类型也须保留领域依赖
declaredTypes resolved = case resolved of
AddColumn _ (_, column) -> [columnType column]
AlterColumnType _ _ ty -> [ty]
_ -> []
-- | 复查结构后执行可复用的绑定写计划
executeWriteM :: MonadStorage m => BoundWrite -> m (Either String [Row])
executeWriteM :: (MonadStorage m) => BoundWrite -> m (Either String [Row])
executeWriteM (BoundWrite catalog dependencies action) = do
current <- schema
case current >>= validateDependencies dependencies of
@@ -99,15 +101,24 @@ executeWriteM (BoundWrite catalog dependencies action) = do
bindWriteAction :: Database -> Statement -> Either String WriteAction
bindWriteAction catalog statement = case statement of
Insert target columns rows -> do
table <- lookupTable catalog target
checks <- checkPlan table
let plan = InsertPlan (bindInsertTargets (tableCols table) columns) (length columns)
(autoColumn table) checks (constraintPlan (tableCols table))
declared <- lookupTable catalog target
columnsEffective <- effectiveColumns catalog (tableCols declared)
let table = declared{tableCols = columnsEffective}
checks <- checkPlan catalog table
let plan =
InsertPlan
(bindInsertTargets (tableCols table) columns)
(length columns)
(autoColumn table)
checks
(constraintPlan (tableCols table))
Right (InsertAction target plan rows)
Update target assignments condition -> do
table <- lookupTable catalog target
declared <- lookupTable catalog target
columnsEffective <- effectiveColumns catalog (tableCols declared)
let table = declared{tableCols = columnsEffective}
bound <- bindUpdateTargets (tableCols table) assignments
checks <- checkPlan table
checks <- checkPlan catalog table
let plan = UpdatePlan (map fst (tableCols table)) bound condition checks (constraintPlan (tableCols table))
Right (UpdateAction target plan)
Delete target condition -> do
@@ -116,7 +127,7 @@ bindWriteAction catalog statement = case statement of
_ -> Right (StructureAction statement)
-- | 直接执行已绑定的数据修改或结构动作
executeWriteActionM :: MonadStorage m => Database -> WriteAction -> m (Either String [Row])
executeWriteActionM :: (MonadStorage m) => Database -> WriteAction -> m (Either String [Row])
executeWriteActionM _ (InsertAction target plan rows) = do
current <- existingRows target plan
case current >>= \current' -> buildRows plan current' rows of
@@ -151,14 +162,14 @@ executeWriteActionM catalog (DeleteAction target columns condition) = do
result <- replaceAll target kept
pure (result >> Right [])
where
-- | 取整数主键，否则使用原有整表删除路径
-- \| 取整数主键，否则使用原有整表删除路径
rowId row = case lookup "id" row of
Just (VInt key) -> Right key
_ -> Left "row has no integer id"
executeWriteActionM catalog (StructureAction statement) = runStatementUncheckedM catalog statement
-- | 绑定并执行查询语句
runQueryM :: MonadStorage m => Database -> Statement -> m (Either String [Row])
runQueryM :: (MonadStorage m) => Database -> Statement -> m (Either String [Row])
runQueryM db q = case bindQuery db q of
Left e -> pure (Left e)
Right bound -> executeQueryM bound
@@ -170,11 +181,19 @@ runStatementUncheckedM db q@Select{} = runQueryM db q
runStatementUncheckedM _ Insert{} = pure (Left "INSERT requires a bound write plan")
runStatementUncheckedM _ Update{} = pure (Left "UPDATE requires a bound write plan")
runStatementUncheckedM _ Delete{} = pure (Left "DELETE requires a bound write plan")
runStatementUncheckedM _ (CreateTable name cols) = do
result <- createTable name cols
pure (result >> Right [])
runStatementUncheckedM _ (CreateDomain name base) = do
result <- createTable (domainTableName name) [("base", plainColumn base)]
runStatementUncheckedM db (CreateTable name cols) = case validateDomainDefaults db cols of
Left err -> pure (Left err)
Right () -> do
result <- guardDomainCreation db (map (columnType . snd) cols) (createTable name cols)
pure (result >> Right [])
runStatementUncheckedM db (CreateDomain name definition) = case canonicalDomain db name definition of
Left err -> pure (Left err)
Right normalized -> do
let DomainDefinition column _ = normalized
result <- guardDomainCreation db [columnType column] (createTable (domainTableName name) (domainColumns normalized))
pure (result >> Right [])
runStatementUncheckedM _ (AlterDomain name action) = do
result <- alterDomain (domainTableName name) (prepareDomainAlter name action)
pure (result >> Right [])
runStatementUncheckedM _ (DropDomain name) = do
result <- dropTable (domainTableName name)
@@ -201,7 +220,7 @@ runStatementUncheckedM _ (DropColumn tbl col) = do
result <- dropColumn tbl col
pure (result >> Right [])
runStatementUncheckedM db (AddColumn tbl def) =
alterTable db tbl (\table -> addColumnTo def (tableCols table) (tableRows table))
alterTable db tbl (\table -> addColumnTo db def (tableCols table) (tableRows table))
runStatementUncheckedM db (RenameColumn tbl old new) =
alterTable db tbl (\table -> renameColumnIn old new (tableCols table) (tableRows table))
runStatementUncheckedM db (AlterColumnType tbl col ty) =
@@ -228,12 +247,22 @@ runStatementUncheckedM _ q@DropDatabase{} = runStatementM q
runStatementUncheckedM _ q@UseDatabase{} = runStatementM q
runStatementUncheckedM _ ShowDatabases = runStatementM ShowDatabases
runStatementUncheckedM _ BeginTransaction = pure (Left "BEGIN is executed by the session")
runStatementUncheckedM _ BeginSerializableTransaction = pure (Left "BEGIN is executed by the session")
runStatementUncheckedM _ CommitTransaction = pure (Left "COMMIT is executed by the session")
runStatementUncheckedM _ RollbackTransaction = pure (Left "ROLLBACK is executed by the session")
runStatementUncheckedM _ Savepoint{} = pure (Left "SAVEPOINT is executed by the session")
runStatementUncheckedM _ RollbackToSavepoint{} = pure (Left "ROLLBACK TO is executed by the session")
runStatementUncheckedM _ ReleaseSavepoint{} = pure (Left "RELEASE is executed by the session")
-- | 建表时在实际发布点复查领域定义
guardDomainCreation :: (MonadStorage m) => Database -> [ColumnType] -> m (Either String a) -> m (Either String a)
guardDomainCreation db types action
| null dependencies = action
| otherwise = withWriteSchemaGuard dependencies action
where
names = typeDependencies db types
dependencies = [(name, table) | (name, table) <- db, name `elem` names]
-- | 只有需要读全表才能判断约束时才扫表
existingRows :: (MonadStorage m) => String -> InsertPlan -> m (Either String [Row])
existingRows tbl (InsertPlan _ _ auto _ (ConstraintPlan _ _ unique))
@@ -317,11 +346,14 @@ autoStart (Just (name, _)) rows = case [n | r <- rows, Just (VInt n) <- [lookup
ns -> maximum ns + 1
-- | 一次解析出来的 CHECK 条件
data CheckPlan = CheckPlan [String] [(String, Expr)]
data CheckPlan = CheckPlan [String] [(String, Expr)] [(String, Int, Expr)]
-- | 按 schema 解析并绑定表上全部 CHECK
checkPlan :: Table -> Either String CheckPlan
checkPlan table = CheckPlan (map fst (tableCols table)) <$> mapM parse (mapMaybe checked (tableCols table))
checkPlan :: Database -> Table -> Either String CheckPlan
checkPlan db table = do
local <- mapM parse (mapMaybe checked (tableCols table))
domains <- domainCheckPlan db (tableCols table)
Right (CheckPlan (map fst (tableCols table)) local domains)
where
-- \| 只挑出带 CHECK 文本的列
checked (name, col) = (,) name <$> columnCheck col
@@ -334,9 +366,10 @@ checkPlan table = CheckPlan (map fst (tableCols table)) <$> mapM parse (mapMaybe
-- | 执行解析好的 CHECK 条件
runChecks :: CheckPlan -> Row -> Either String ()
runChecks (CheckPlan columns plan) row = do
runChecks (CheckPlan columns plan domains) row = do
ordered <- orderRow columns row
mapM_ (check ordered) plan
runDomainChecks domains ordered
where
-- \| 跑一条 CHECK
check ordered (name, e) = case evalCondForRow e ordered of
@@ -344,6 +377,16 @@ runChecks (CheckPlan columns plan) row = do
Right True -> Right ()
Right False -> Left ("CHECK on " ++ name ++ " failed")
-- 校验最终行集的领域、检查与唯一约束。
validateTableRows :: Database -> Table -> Either String ()
validateTableRows db table = do
effective <- effectiveColumns db (tableCols table)
plan <- checkPlan db table
let rows = tableRows table
mapM_ (runChecks plan) rows
ordered <- mapM (orderRow (map fst effective)) rows
enforceConstraints (constraintPlan effective) [] ordered
-- | 只取第一行（报错提示用）
firstLine :: String -> String
firstLine = takeWhile (/= '\n')
@@ -370,7 +413,7 @@ enforceConstraints (ConstraintPlan columns required unique) current rows = do
enforceNotNull :: [(String, Expr)] -> [Row] -> Either String ()
enforceNotNull targets rows = mapM_ check targets
where
-- | 按 schema 槽位检查 NOT NULL
-- \| 按 schema 槽位检查 NOT NULL
check (name, reference) = do
values <- mapM (evalExpr reference) rows
if VNull `elem` values
@@ -411,7 +454,7 @@ data BoundAssignment = BoundAssignment Int String Column Expr
bindUpdateTargets :: [(String, Column)] -> [(String, Expr)] -> Either String [BoundAssignment]
bindUpdateTargets columns = mapM bindTarget
where
-- | 只接受唯一目标并保留列定义
-- \| 只接受唯一目标并保留列定义
bindTarget (name, expression) = case [(slot, column) | (slot, (key, column)) <- zip [0 ..] columns, key == name] of
[(slot, column)] -> Right (BoundAssignment slot name column expression)
[] -> Left ("UPDATE: unknown target column: " ++ name)
@@ -429,9 +472,11 @@ applyUpdatesM db (assignment@(BoundAssignment _ _ _ e) : rest) row = do
-- | 校验目标位置并只覆盖该槽位的值
writeAssignment :: BoundAssignment -> Row -> Value -> Either String Row
writeAssignment (BoundAssignment slot name column _) row value = case splitAt slot row of
(before, (key, _) : after) | slot >= 0, key == name -> do
coerced <- coerceValue (columnType column) value
Right (before ++ ((name, coerced) : after))
(before, (key, _) : after)
| slot >= 0
, key == name -> do
coerced <- coerceValue (columnType column) value
Right (before ++ ((name, coerced) : after))
(_, []) -> Left ("missing bound column: " ++ name)
_ -> Left ("bound column position mismatch: " ++ name)
@@ -471,20 +516,23 @@ alterTable db tbl change = case lookupTable db tbl of
-- \| 改完再统一查一遍约束
checkAltered (cols, rows) = do
let table = Table tbl cols rows Nothing Nothing Nothing
plan <- checkPlan table
mapM_ (runChecks plan) rows
enforceConstraints (constraintPlan cols) [] rows
validateDomainDefaults db cols
validateTableRows db table
Right (cols, rows)
-- | ADD COLUMN：老行补默认值或 NULL
addColumnTo :: (String, Column) -> [(String, Column)] -> [Row] -> Either String ([(String, Column)], [Row])
addColumnTo (name, col) cols rows
addColumnTo :: Database -> (String, Column) -> [(String, Column)] -> [Row] -> Either String ([(String, Column)], [Row])
addColumnTo db (name, declared) cols rows
| name `elem` map fst cols = Left ("ADD COLUMN: column already exists: " ++ name)
| otherwise = do
values <- fill
Right (cols ++ [(name, col)], [r ++ [(name, v)] | (r, v) <- zip rows values])
effective <- effectiveColumns db [(name, declared)]
values <- case effective of
[(_, column)] -> fill column
_ -> Left "invalid added column layout"
Right (cols ++ [(name, declared)], [r ++ [(name, v)] | (r, v) <- zip rows values])
where
fill
-- \| 使用领域继承属性填充已有行
fill col
| columnAutoIncrement col = mapM (coerceValue (columnType col) . VInt) [1 .. length rows]
| Just d <- columnDefault col = mapM (\_ -> coerceValue (columnType col) d) rows
| columnNullable col = mapM (\_ -> coerceValue (columnType col) VNull) rows
chusql-core/engine/src/ChuSQL/Core/Engine/Algebra/Cost.hs
+3 -3
View File
@@ -1,7 +1,7 @@
module ChuSQL.Core.Engine.Algebra.Cost (tableMetaOf, probeCost, pointRows, rangeRows, selectivityOf, indexCheaper) where
import Data.Maybe (fromMaybe)
import ChuSQL.Core.Model
import Data.Maybe (fromMaybe)
-- 统计与成本估算：行数、不同值数、列直方图与索引探路页数。
@@ -33,9 +33,9 @@ estimateRows h lo hi = max 0 (min total (round (sum (zipWith overlap [0 ..] (his
where
total = sum (histBuckets h)
width = if histHigh h > histLow h then (histHigh h - histLow h) / fromIntegral (length (histBuckets h)) else 0
lower = fromMaybe (- (1 / 0)) (boundOf lo)
lower = fromMaybe (-(1 / 0)) (boundOf lo)
upper = fromMaybe (1 / 0) (boundOf hi)
-- | 第 i 个桶与区间重叠的行数
-- \| 第 i 个桶与区间重叠的行数
overlap :: Int -> Int -> Double
overlap i count
| width <= 0 = if lower <= histLow h && histLow h <= upper then fromIntegral count else 0
chusql-core/engine/src/ChuSQL/Core/Engine/Algebra/Eval.hs
+94 -65
View File
@@ -1,24 +1,26 @@
{-# LANGUAGE TupleSections #-}
module ChuSQL.Core.Engine.Algebra.Eval (evalRelOp, evalRelOpM, evalExprM, evalCondForRowM) where
import ChuSQL.Core.Engine.Algebra.Cost (tableMetaOf)
import ChuSQL.Core.Engine.Algebra.Expr (evalCondForRow, evalExpr, hasSubquery, quantifiedCompare, subquerySlotsIn, subqueryColumns)
import ChuSQL.Core.Engine.Algebra.Optimize (optimize)
import ChuSQL.Core.Engine.Algebra.Expr (evalCondForRow, evalExpr, hasSubquery, quantifiedCompare, subqueryColumns, subquerySlotsIn)
import ChuSQL.Core.Engine.Algebra.Op
import ChuSQL.Core.Engine.Algebra.Optimize (optimize)
import ChuSQL.Core.Engine.Algebra.Planner (translate)
import ChuSQL.Core.Engine.Algebra.Sort (sortRows, sortBoundRows)
import ChuSQL.Core.Engine.Algebra.Sort (sortBoundRows, sortRows)
import ChuSQL.Core.Engine.Builtin (aggregateArg, builtinOfExpr, inValues, invokeBuiltin, unsupportedAggregate)
import ChuSQL.Core.Engine.Runtime.Relation (bindRelOpColumns, orderRow)
import ChuSQL.Core.Engine.Builtin (aggregateArg, builtinOfExpr, invokeBuiltin, unsupportedAggregate, inValues)
import ChuSQL.Core.Model
import ChuSQL.Core.Engine.Storage (IndexResult (..), MonadStorage (..))
import ChuSQL.Core.Engine.Storage.Memory (runMemoryStorage)
import ChuSQL.Core.Engine.Syntax.AST (Expr (..), JoinKind (..), Subquery (..))
import ChuSQL.Core.Model
import Control.Monad (filterM)
import Control.Monad.Trans.Class (lift)
import Control.Monad.Trans.State.Strict (StateT, evalStateT, gets, modify')
import Data.Bifunctor (first)
import Data.HashMap.Strict qualified as HM
import Data.HashSet qualified as HS
import Data.List (nub)
import qualified Data.HashMap.Strict as HM
import qualified Data.HashSet as HS
-- 执行：按算子树算出结果行，外层行供相关子查询引用。
@@ -26,7 +28,7 @@ import qualified Data.HashSet as HS
type Execution m a = StateT (HM.HashMap String [Row]) m a
-- | 首次使用时执行，成功结果供后续引用复用
materialize :: Monad m => String -> Execution m (Either String [Row]) -> Execution m (Either String [Row])
materialize :: (Monad m) => String -> Execution m (Either String [Row]) -> Execution m (Either String [Row])
materialize key action = do
cached <- gets (HM.lookup key)
case cached of
@@ -46,7 +48,7 @@ project cs row = [(c, v) | c <- cs, Just v <- [lookup c row]]
-- | 给列名加别名前缀
addPrefix :: Maybe String -> Row -> Row
addPrefix mAlias = map (\(k, v) -> (qualify mAlias k, v))
addPrefix mAlias = map (first (qualify mAlias))
-- | 给一批行加别名前缀
prefixRows :: Maybe String -> [Row] -> [Row]
@@ -74,7 +76,7 @@ rangeCondition mAlias col lo hi = case (lo, hi) of
(Nothing, Nothing) -> LitBool True
where
c = Col (qualify mAlias col)
-- | 一个端点的比较，闭端再加上等号
-- \| 一个端点的比较，闭端再加上等号
bound rel (v, inclusive)
| inclusive = Or (rel c (valueLiteral v)) (Eq c (valueLiteral v))
| otherwise = rel c (valueLiteral v)
@@ -102,10 +104,18 @@ data EquiKeys = EquiKeys
-- | 认出 a = b 这种两列等值条件，并且能分清谁在左谁在右
equiKeys :: [String] -> [String] -> Expr -> Maybe EquiKeys
equiKeys lcols rcols (Eq left right)
| Just a <- columnName left, Just b <- columnName right
, a /= b, a `elem` lcols, b `elem` rcols = Just (EquiKeys a b)
| Just a <- columnName left, Just b <- columnName right
, a /= b, b `elem` lcols, a `elem` rcols = Just (EquiKeys b a)
| Just a <- columnName left
, Just b <- columnName right
, a /= b
, a `elem` lcols
, b `elem` rcols =
Just (EquiKeys a b)
| Just a <- columnName left
, Just b <- columnName right
, a /= b
, b `elem` lcols
, a `elem` rcols =
Just (EquiKeys b a)
equiKeys _ _ _ = Nothing
-- | 提取执行列的诊断名称
@@ -144,26 +154,32 @@ semiRows kind left right outer leftRows rightRows = do
let keys = HS.fromList [canonicalValue value | value <- values, value /= VNull]
hasNull = VNull `elem` values
hashSafe = all safeNumber values
-- | NULL 与数值边界沿用表达式比较规则
-- \| NULL 与数值边界沿用表达式比较规则
keep row = do
value <- evalExpr left (row ++ outer)
if value == VNull
then Right (kind == NonMatching || kind == NotIn && null values)
else if hashSafe && safeNumber value
then let found = HS.member (canonicalValue value) keys
in Right (case kind of
Matching -> found
NonMatching -> not found
NotIn -> not found && not hasNull)
else do
result <- inValues value values
Right (case kind of
Matching -> result == VBool True
NonMatching -> result /= VBool True
NotIn -> result == VBool False)
else
if hashSafe && safeNumber value
then
let found = HS.member (canonicalValue value) keys
in Right
( case kind of
Matching -> found
NonMatching -> not found
NotIn -> not found && not hasNull
)
else do
result <- inValues value values
Right
( case kind of
Matching -> result == VBool True
NonMatching -> result /= VBool True
NotIn -> result == VBool False
)
filterM keep leftRows
where
-- | 大数混合比较退回既有逐值语义
-- \| 大数混合比较退回既有逐值语义
safeNumber (VInt value) = abs (toInteger value) <= 9007199254740992
safeNumber (VFloat value) = abs value <= 9007199254740992
safeNumber _ = True
@@ -178,7 +194,7 @@ hashJoin :: JoinKind -> EquiKeys -> [String] -> [Row] -> [Row] -> [Row]
hashJoin kind ks ncols lrows rrows = concatMap probe lrows
where
buckets = HM.fromListWith (flip (++)) [(v, [r]) | r <- rrows, Just v <- [joinKeyOf (ekRight ks) r]]
-- | 拿左行的键去桶里找
-- \| 拿左行的键去桶里找
probe l =
let miss = case kind of
InnerJoin -> []
@@ -204,7 +220,7 @@ filterPairsM ::
filterPairsM db outer kind ncols cond lrows rrows = go lrows [] HM.empty
where
refs = subquerySlotsIn cond
-- | 逐左行配对，左连接没配上就补 NULL
-- \| 逐左行配对，左连接没配上就补 NULL
go [] acc _ = pure (Right (reverse acc))
go (l : ls) acc cache = do
hits <- keep l rrows [] cache
@@ -215,7 +231,7 @@ filterPairsM db outer kind ncols cond lrows rrows = go lrows [] HM.empty
(LeftJoin, []) -> [l ++ nullRow]
_ -> hs
go ls (reverse hs' ++ acc) nextCache
-- | 拿一行右行去比条件
-- \| 拿一行右行去比条件
keep _ [] acc cache = pure (Right (reverse acc, cache))
keep l (r : rs) acc cache = do
let env = l ++ r ++ outer
@@ -259,13 +275,18 @@ probeIndexJoin db outer kind mAlias t ks rcols cond l = case lookup (ekLeft ks)
Left err -> pure (Left err)
Right ordered -> fmap (fmap Just) (joinRowsIn db outer kind rcols cond [l] ordered)
-- | 列与键都不是浮点时才敢用索引
-- | 仅用已支持且无浮点歧义的索引键
indexSafe :: Database -> String -> String -> Value -> Bool
indexSafe db t col k = case lookupTable db t of
Left _ -> False
Right tbl -> case lookup col (tableCols tbl) of
Nothing -> False
Just c -> not (floatType (columnType c)) && not (floatValue k)
Just c -> scalarKey k && not (floatType (columnType c)) && not (floatValue k)
where
-- \| 只有已实现编码的键可以点查
scalarKey VInt{} = True
scalarKey VStr{} = True
scalarKey _ = False
-- | 是不是浮点列类型
floatType :: ColumnType -> Bool
@@ -288,7 +309,7 @@ groupKey outer keys row = mapM (\key -> evalExpr key (row ++ outer)) keys
groupRows :: Row -> [Expr] -> [Row] -> Either String [([Value], [Row])]
groupRows outer keys rows = go rows (HM.empty, [])
where
-- | 逐行塞进对应分组，并记住分组首次出现的顺序
-- \| 逐行塞进对应分组，并记住分组首次出现的顺序
go [] (acc, order) = Right [(k, reverse (HM.lookupDefault [] k acc)) | k <- reverse order]
go (r : rs) (acc, order) = do
k <- groupKey outer keys r
@@ -342,7 +363,7 @@ bindingValue column env = case lookup column env of
bindingSamples :: [Expr] -> Row -> [Row] -> Either String [(String, Row)]
bindingSamples refs outer rows = go rows HM.empty []
where
-- | 逐行按绑定键去重
-- \| 逐行按绑定键去重
go [] _ acc = Right (reverse acc)
go (r : rs) seen acc = do
let env = r ++ outer
@@ -366,7 +387,7 @@ bindingsTableM db outer exprs rows = case bindingSamples refs outer rows of
pure (fmap HM.fromList (sequence built))
where
refs = nub (concatMap subquerySlotsIn exprs)
-- | 一个绑定样本上把子查询全换成字面量
-- \| 一个绑定样本上把子查询全换成字面量
swap (key, env) = do
es <- mapM (\e -> substSubqueries db e env) exprs
pure (fmap (key,) (sequence es))
@@ -391,7 +412,7 @@ boundExpr refs env table = do
filterRowsM :: (Monad m) => (Row -> m (Either String Bool)) -> [Row] -> m (Either String [Row])
filterRowsM p = go []
where
-- | 逐行留下通过条件的
-- \| 逐行留下通过条件的
go acc [] = pure (Right (reverse acc))
go acc (r : rs) = do
keep <- p r
@@ -404,7 +425,7 @@ filterRowsM p = go []
computeM :: (MonadStorage m) => Database -> Row -> [(String, Expr)] -> Row -> Execution m (Either String Row)
computeM db outer items row = go items []
where
-- | 逐项求值并攒成一行
-- \| 逐项求值并攒成一行
go [] acc = pure (Right (reverse acc))
go ((label, e) : rest) acc = do
v <- evalExprIn db e (row ++ outer)
@@ -417,7 +438,7 @@ evalExprM :: (MonadStorage m) => Database -> Expr -> Row -> m (Either String Val
evalExprM db e env = evalStateT (evalExprIn db e env) HM.empty
-- | 在当前执行中求值并共享子查询物化
evalExprIn :: MonadStorage m => Database -> Expr -> Row -> Execution m (Either String Value)
evalExprIn :: (MonadStorage m) => Database -> Expr -> Row -> Execution m (Either String Value)
evalExprIn db e env = do
e' <- substSubqueries db e env
pure (e' >>= \x -> evalExpr x env)
@@ -427,7 +448,7 @@ evalCondForRowM :: (MonadStorage m) => Database -> Expr -> Row -> m (Either Stri
evalCondForRowM db e env = evalStateT (evalCondIn db e env) HM.empty
-- | 在当前执行中判定条件并共享物化
evalCondIn :: MonadStorage m => Database -> Expr -> Row -> Execution m (Either String Bool)
evalCondIn :: (MonadStorage m) => Database -> Expr -> Row -> Execution m (Either String Bool)
evalCondIn db e env = do
e' <- substSubqueries db e env
pure (e' >>= \x -> evalCondForRow x env)
@@ -465,29 +486,35 @@ substSubqueries db e env = case e of
InSub a sq negated -> do
lhs <- substSubqueries db a env
rows <- runSubqueryRows db sq env
pure (do
a' <- lhs
rs <- rows
vs <- mapM rowValue rs
Right (InList a' (map valueLiteral vs) negated))
pure
( do
a' <- lhs
rs <- rows
vs <- mapM rowValue rs
Right (InList a' (map valueLiteral vs) negated)
)
ExistsSub sq negated -> do
rows <- runSubqueryRows db sq env
pure (fmap (\rs -> LitBool (if negated then null rs else not (null rs))) rows)
QuantCmp op a sq q -> do
lhs <- substSubqueries db a env
rows <- runSubqueryRows db sq env
pure (do
a' <- lhs
rs <- rows
vs <- mapM rowValue rs
Right (quantifiedCompare op a' (map valueLiteral vs) q))
pure
( do
a' <- lhs
rs <- rows
vs <- mapM rowValue rs
Right (quantifiedCompare op a' (map valueLiteral vs) q)
)
InList a es negated -> do
lhs <- substSubqueries db a env
es' <- mapM (\x -> substSubqueries db x env) es
pure (do
a' <- lhs
items <- sequence es'
Right (InList a' items negated))
pure
( do
a' <- lhs
items <- sequence es'
Right (InList a' items negated)
)
Add a b -> bin Add a b
Sub a b -> bin Sub a b
Mul a b -> bin Mul a b
@@ -510,7 +537,7 @@ substSubqueries db e env = case e of
MaxOf a -> un MaxOf a
_ -> pure (Right e)
where
-- | 递归处理两个子表达式
-- \| 递归处理两个子表达式
bin ctor a b = do
ra <- substSubqueries db a env
case ra of
@@ -518,7 +545,7 @@ substSubqueries db e env = case e of
Right a' -> do
rb <- substSubqueries db b env
pure (ctor a' <$> rb)
-- | 递归处理一个子表达式
-- \| 递归处理一个子表达式
un ctor a = do
ra <- substSubqueries db a env
pure (ctor <$> ra)
@@ -531,14 +558,16 @@ runSubqueryRows db sq outer = case translate (subqueryStatement sq) of
Left err -> pure (Left err)
Right bindings -> case bindRelOpColumns db (map fst (bindings ++ outer)) (optimize db op) of
Left err -> pure (Left err)
Right bound -> materialize (show ("subquery", subqueryStatement sq, bindings, fallback))
(evalRelOpIn db (bindings ++ outer) bound)
Right bound ->
materialize
(show ("subquery", subqueryStatement sq, bindings, fallback))
(evalRelOpIn db (bindings ++ outer) bound)
where
-- | 原始调用未证明依赖完整，保留整个外层行
-- \| 原始调用未证明依赖完整，保留整个外层行
fallback = case sq of
Subquery{} -> Just outer
BoundSubquery{} -> Nothing
-- | 给外层物理列补回已解析的限定名
-- \| 给外层物理列补回已解析的限定名
bind (column, expression) = (,) column <$> bindingReferenceValue expression outer
-- | 单子求值：Scan/点查/范围查都问存储
@@ -567,7 +596,7 @@ evalRelOpIn db outer (Compute items op) = do
pure (table >>= \t -> traverse (computeBound t) rs)
where
refs = nub (concatMap (subquerySlotsIn . snd) items)
-- | 用这一行绑定换好的表达式算出一行
-- \| 用这一行绑定换好的表达式算出一行
computeBound table r = do
es <- boundExprs refs (r ++ outer) table
mapM (\(label, e) -> (,) label <$> evalExpr e (r ++ outer)) (zip (map fst items) es)
@@ -603,7 +632,7 @@ evalRelOpIn db outer (Filter e op) = do
pure (table >>= \t -> filterM (keepBound t) rs)
where
refs = nub (subquerySlotsIn e)
-- | 用这一行绑定换好的条件判定
-- \| 用这一行绑定换好的条件判定
keepBound table r = do
e' <- boundExpr refs (r ++ outer) table
evalCondForRow e' (r ++ outer)
@@ -631,7 +660,7 @@ evalRelOpIn db outer (SemiJoin kind left right l r query) = do
Left err -> pure (Left err)
Right [] -> pure (Right [])
Right leftRows -> do
-- | 独立 IN 沿用子查询身份，跨算子复用结果
-- \| 独立 IN 沿用子查询身份，跨算子复用结果
let key = case query of
Just statement -> show ("subquery", statement, [] :: Row, Nothing :: Maybe Row)
Nothing -> show ("derived", r, outer)
@@ -684,14 +713,14 @@ scanTable db tbl cols = do
then lift (scanShards width tbl cols)
else lift (maybe (scan tbl) (scanColumns tbl) cols)
where
-- | 统计里的行数，没有统计算 0
-- \| 统计里的行数，没有统计算 0
rows = maybe 0 metaRowCount (tableMetaOf db tbl)
-- | 按输出值去重并保留首次行序
distinctRows :: [Row] -> [Row]
distinctRows = go HS.empty
where
-- | 只输出尚未遇到的值组合
-- \| 只输出尚未遇到的值组合
go _ [] = []
go seen (row : rows)
| HS.member key seen = go seen rows
chusql-core/engine/src/ChuSQL/Core/Engine/Algebra/Expr.hs
+13 -13
View File
@@ -1,9 +1,9 @@
module ChuSQL.Core.Engine.Algebra.Expr (evalExpr, evalCondForRow, colsInExpr, subqueryRefsIn, subquerySlotsIn, subqueryColumns, aggregatesIn, hasAggregate, bareColumns, hasSubquery, hasDivision, inValues, threeValuedNot, quantifiedCompare) where
import ChuSQL.Core.Model
import ChuSQL.Core.Engine.Syntax.AST
import ChuSQL.Core.Engine.Builtin (Operator (..), compareNode, executeOperator, inValues, threeValuedNot)
import ChuSQL.Core.Engine.Runtime.SQL (invokeSQLFunction, constructSQLValue)
import ChuSQL.Core.Engine.Runtime.SQL (constructSQLValue, invokeSQLFunction)
import ChuSQL.Core.Engine.Syntax.AST
import ChuSQL.Core.Model
-- 表达式求值：三值逻辑，NULL 参与运算结果仍是 NULL。
@@ -59,9 +59,9 @@ evalExpr (InList a es negated) row = do
r <- inValues v vs
Right (if negated then threeValuedNot r else r)
evalExpr (ScalarSub _) _ = Left subqueryNeedsDatabase
evalExpr (InSub {}) _ = Left subqueryNeedsDatabase
evalExpr (InSub{}) _ = Left subqueryNeedsDatabase
evalExpr (ExistsSub _ _) _ = Left subqueryNeedsDatabase
evalExpr (QuantCmp {}) _ = Left subqueryNeedsDatabase
evalExpr (QuantCmp{}) _ = Left subqueryNeedsDatabase
evalExpr agg _ = Left ("aggregate functions are not allowed here: " ++ show agg)
-- | 子查询要连上存储才能跑
@@ -157,9 +157,9 @@ subqueryReferences references e = case e of
MaxOf a -> collect a
_ -> []
where
-- | 递归收集当前提取器的引用
-- \| 递归收集当前提取器的引用
collect = subqueryReferences references
-- | 两个子表达式的外层列拼起来
-- \| 两个子表达式的外层列拼起来
both a b = collect a ++ collect b
-- | 表达式里有没有除法（除法可能报错，不能提前或延后求值）
@@ -185,7 +185,7 @@ hasDivision e = case e of
QuantCmp _ a _ _ -> hasDivision a
_ -> False
where
-- | 两个子表达式里任意一个有除法
-- \| 两个子表达式里任意一个有除法
both a b = hasDivision a || hasDivision b
-- | 表达式里有没有子查询
@@ -196,10 +196,10 @@ hasSubquery e = case e of
FunctionCall _ arguments -> any hasSubquery arguments
BoundFunction _ _ arguments -> any hasSubquery arguments
ScalarSub _ -> True
InSub {} -> True
InSub{} -> True
InList a es _ -> any hasSubquery (a : es)
ExistsSub _ _ -> True
QuantCmp {} -> True
QuantCmp{} -> True
Add a b -> both a b
Sub a b -> both a b
Mul a b -> both a b
@@ -222,7 +222,7 @@ hasSubquery e = case e of
MaxOf a -> hasSubquery a
_ -> False
where
-- | 两个子表达式里任意一个有
-- \| 两个子表达式里任意一个有
both a b = hasSubquery a || hasSubquery b
-- | 表达式里出现的聚合调用（按出现顺序）
@@ -258,7 +258,7 @@ aggregatesIn e = case e of
IsNotNull a -> aggregatesIn a
_ -> []
where
-- | 两个子表达式的聚合拼起来
-- \| 两个子表达式的聚合拼起来
both a b = aggregatesIn a ++ aggregatesIn b
-- | 表达式里有没有聚合调用
@@ -297,7 +297,7 @@ bareColumns e = case e of
QuantCmp _ a sq _ -> bareColumns a ++ subqueryRefs sq
_ -> []
where
-- | 两个子表达式的裸列拼起来
-- \| 两个子表达式的裸列拼起来
both a b = bareColumns a ++ bareColumns b
-- | 判断条件真假：只有 TRUE 通过，NULL 当作不通过
chusql-core/engine/src/ChuSQL/Core/Engine/Algebra/JoinOrder.hs
+20 -18
View File
@@ -1,12 +1,12 @@
module ChuSQL.Core.Engine.Algebra.JoinOrder (reorderJoins) where
import Data.Maybe (fromMaybe)
import ChuSQL.Core.Engine.Algebra.Cost (pointRows, rangeRows, selectivityOf, tableMetaOf)
import ChuSQL.Core.Engine.Algebra.Expr (colsInExpr, hasDivision, hasSubquery)
import ChuSQL.Core.Engine.Algebra.Op (RelOp (..), relOpCols)
import ChuSQL.Core.Engine.Syntax.AST (Expr (..), JoinKind (..))
import ChuSQL.Core.Model
import Data.List (partition)
import Data.Maybe (fromMaybe)
-- 内连接链顺序优化：按统计估算中间结果，只在行序不可观察处重排。
@@ -40,7 +40,7 @@ innerChain (Join InnerJoin l r c) = do
(ls, lcs) <- innerChain l
(rs, rcs) <- innerChain r
pure (ls ++ rs, lcs ++ rcs ++ [c])
innerChain (Join {}) = Nothing
innerChain (Join{}) = Nothing
innerChain op = Just ([op], [])
-- | 叶子算子的估算行数，拿不到统计就是 Nothing
@@ -81,7 +81,7 @@ scaleRows db p x rows = case p of
GtE _ _ -> quarter
LtE _ _ -> quarter
NotEq _ _ -> quarter
QuantCmp {} -> quarter
QuantCmp{} -> quarter
And a b -> max 1 (min (scaleRows db a x rows) (scaleRows db b x rows))
_ -> quarter
where
@@ -89,7 +89,8 @@ scaleRows db p x rows = case p of
pick k lo hi = case leafSource x of
Just (alias, t)
| Just m <- tableMetaOf db t
, Just s <- selectivityOf m (unqualify alias k) lo hi -> max 1 (round (fromIntegral rows * s))
, Just s <- selectivityOf m (unqualify alias k) lo hi ->
max 1 (round (fromIntegral rows * s))
_ -> quarter
-- | 叶子对应的别名与表名
@@ -103,9 +104,10 @@ leafSource _ = Nothing
-- | 某列在叶子统计里的不同值数
distinctOf :: Database -> RelOp -> String -> Maybe Int
distinctOf db leaf col = do (alias, t) <- leafSource leaf
m <- tableMetaOf db t
lookup (unqualify alias col) [(c, n) | (c, n, _) <- metaDistinct m]
distinctOf db leaf col = do
(alias, t) <- leafSource leaf
m <- tableMetaOf db t
lookup (unqualify alias col) [(c, n) | (c, n, _) <- metaDistinct m]
-- | 若干叶子里某列最大的不同值数
distinctOn :: Database -> [RelOp] -> String -> Maybe Int
@@ -121,12 +123,12 @@ keyDistinct db have x conds = case [v | Eq (Col a) (Col b) <- conds, Just v <- [
where
haveCols = concatMap (relOpCols db) have
xCols = relOpCols db x
-- | 这一对列是不是两侧之间的等值键
-- \| 这一对列是不是两侧之间的等值键
pair a b
| a `elem` haveCols, b `elem` xCols = bigger (distinctOn db have a) (distinctOf db x b)
| b `elem` haveCols, a `elem` xCols = bigger (distinctOn db have b) (distinctOf db x a)
| otherwise = Nothing
-- | 两个可选统计取大的那个
-- \| 两个可选统计取大的那个
bigger (Just a) (Just b) = Just (max a b)
bigger (Just a) Nothing = Just a
bigger Nothing (Just b) = Just b
@@ -144,7 +146,7 @@ stepRows db have acc x conds = case keyDistinct db have x conds of
chainCost :: Database -> [RelOp] -> [Expr] -> Int
chainCost db (l : ls) conds = go [l] (fromMaybe 1 (leafRows db l)) ls
where
-- | 逐步接上后面的叶子
-- \| 逐步接上后面的叶子
go _ _ [] = 0
go have acc (x : xs) =
let card = stepRows db have acc x conds
@@ -159,11 +161,11 @@ growOrder db leaves conds have rest = case rest of
let next = pick rest
in growOrder db leaves conds (have ++ [next]) (filter (/= next) rest)
where
-- | 候选里成本最低的那个，同价取序号小的
-- \| 候选里成本最低的那个，同价取序号小的
pick (i : is) = foldl cheaper i is
pick [] = 0
cheaper i j = if costOf j < costOf i then j else i
-- | 这个前缀的估算成本
-- \| 这个前缀的估算成本
costOf i = chainCost db [leaves !! k | k <- have ++ [i]] conds
-- | 每个起点贪心一遍，取最便宜的顺序
@@ -171,13 +173,13 @@ bestOrder :: Database -> [RelOp] -> [Expr] -> [RelOp]
bestOrder db leaves conds = pick (map start [0 .. n - 1])
where
n = length leaves
-- | 从第 i 张表开始的贪心顺序
-- \| 从第 i 张表开始的贪心顺序
start i = [leaves !! k | k <- growOrder db leaves conds [i] (filter (/= i) [0 .. n - 1])]
-- | 成本最低的那个
-- \| 成本最低的那个
pick (o : os) = foldl cheaper o os
pick [] = []
cheaper o o' = if cost o' < cost o then o' else o
-- | 一个顺序的估算成本
-- \| 一个顺序的估算成本
cost o = chainCost db o conds
-- | 统计齐全、没有除法且确实更省时给出新顺序
@@ -188,7 +190,7 @@ cheaperOrder db leaves conds
| bestCost < textCost = Just best
| otherwise = Nothing
where
-- | 这个叶子是不是拿不到统计
-- \| 这个叶子是不是拿不到统计
unknown x = case leafRows db x of
Just _ -> False
Nothing -> True
@@ -200,13 +202,13 @@ cheaperOrder db leaves conds
rebuildChain :: Database -> [RelOp] -> [Expr] -> RelOp
rebuildChain db (l : ls) conds = go (relOpCols db l) l ls conds
where
-- | 逐个接上后面的叶子
-- \| 逐个接上后面的叶子
go _ acc [] rest = applyConds rest acc
go have acc (x : xs) rest =
let have' = have ++ relOpCols db x
(now, later) = partition (\c -> colsInExpr c `subsetOf` have') rest
in go have' (Join InnerJoin acc x (conjOf now)) xs later
-- | 若干条件用 AND 串起来
-- \| 若干条件用 AND 串起来
conjOf [] = LitBool True
conjOf (c : cs) = foldl And c cs
rebuildChain _ [] _ = Unit
chusql-core/engine/src/ChuSQL/Core/Engine/Algebra/Op.hs
+22 -8
View File
@@ -1,7 +1,7 @@
module ChuSQL.Core.Engine.Algebra.Op (RelOp (..), SemiKind (..), relOpCols, renderPlan) where
import ChuSQL.Core.Engine.Syntax.AST (Expr, JoinKind (..), SortDir, Statement)
import ChuSQL.Core.Model
import ChuSQL.Core.Engine.Syntax.AST (Expr, Statement, JoinKind (..), SortDir)
-- 关系代数算子定义、输出列推导与算子树文本渲染。
@@ -23,8 +23,8 @@ data RelOp
| BoundSort [(Expr, SortDir)] RelOp
| Limit Int RelOp
| Join JoinKind RelOp RelOp Expr
-- | 末项仅保存独立 IN 的物化身份
| SemiJoin SemiKind Expr Expr RelOp RelOp (Maybe Statement)
| -- | 末项仅保存独立 IN 的物化身份
SemiJoin SemiKind Expr Expr RelOp RelOp (Maybe Statement)
| Derived (Maybe String) RelOp
deriving (Show, Eq)
@@ -63,7 +63,7 @@ kindLabel LeftJoin = "LeftJoin"
renderPlan :: RelOp -> String
renderPlan = go 0
where
-- | 按缩进渲染一个算子
-- \| 按缩进渲染一个算子
go ind op = case op of
Scan a t cols -> pad ++ "Scan " ++ show a ++ " " ++ show t ++ maybe "" ((" columns=" ++) . show) cols
Lookup _ t c k -> pad ++ "Lookup " ++ show t ++ "." ++ show c ++ " = " ++ show k
@@ -80,15 +80,29 @@ renderPlan = go 0
++ "\n"
++ go (ind + 1) x
Distinct x -> pad ++ "Distinct\n" ++ go (ind + 1) x
BoundAggregate keys aggs x -> pad ++ "BoundAggregate keys=" ++ show keys
++ " aggs=" ++ show aggs ++ "\n" ++ go (ind + 1) x
BoundAggregate keys aggs x ->
pad
++ "BoundAggregate keys="
++ show keys
++ " aggs="
++ show aggs
++ "\n"
++ go (ind + 1) x
Unit -> pad ++ "Unit"
Sort s x -> pad ++ "Sort " ++ show s ++ "\n" ++ go (ind + 1) x
BoundSort s x -> pad ++ "BoundSort " ++ show s ++ "\n" ++ go (ind + 1) x
Limit n x -> pad ++ "Limit " ++ show n ++ "\n" ++ go (ind + 1) x
Join k l r c -> pad ++ kindLabel k ++ " on " ++ show c ++ "\n" ++ go (ind + 1) l ++ "\n" ++ go (ind + 1) r
SemiJoin k left right l r _ -> pad ++ "SemiJoin " ++ show k ++ " keys=" ++ show (left, right)
++ "\n" ++ go (ind + 1) l ++ "\n" ++ go (ind + 1) r
SemiJoin k left right l r _ ->
pad
++ "SemiJoin "
++ show k
++ " keys="
++ show (left, right)
++ "\n"
++ go (ind + 1) l
++ "\n"
++ go (ind + 1) r
Derived a x -> pad ++ "Derived " ++ show a ++ "\n" ++ go (ind + 1) x
where
pad = replicate (ind * 2) ' '
chusql-core/engine/src/ChuSQL/Core/Engine/Algebra/Optimize.hs
+26 -12
View File
@@ -5,8 +5,8 @@ import ChuSQL.Core.Engine.Algebra.Expr (colsInExpr, evalExpr, hasDivision)
import ChuSQL.Core.Engine.Algebra.JoinOrder (reorderJoins)
import ChuSQL.Core.Engine.Algebra.Op (RelOp (..), SemiKind (..), relOpCols)
import ChuSQL.Core.Engine.Algebra.Planner (translate)
import ChuSQL.Core.Engine.Syntax.AST (Expr (..), FromClause (..), JoinKind (..), Statement (..), Subquery (..))
import ChuSQL.Core.Model
import ChuSQL.Core.Engine.Syntax.AST (Expr (..), JoinKind (..), Statement (..), FromClause (..), Subquery (..))
import Data.List (nub)
-- 查询优化：谓词下推、投影下推、常量折叠、扫描方式与连接顺序选择。
@@ -30,7 +30,7 @@ combineConj (e : es) = Just (foldl And e es)
partitionPred :: [String] -> [Expr] -> ([Expr], [Expr])
partitionPred have = foldr pick ([], [])
where
-- | 按列归属收到左边或右边
-- \| 按列归属收到左边或右边
pick e (yes, no)
| colsInExpr e `isSubsetOf` have = (e : yes, no)
| otherwise = (yes, e : no)
@@ -111,8 +111,13 @@ pushProject db _ (Distinct x) = Distinct (pushProject db (relOpCols db x) x)
pushProject db need (Sort spec x) = Sort spec (pushProject db (need ++ map fst spec) x)
pushProject db need (Limit n x) = Limit n (pushProject db need x)
pushProject db need (SemiJoin kind left right l r query) =
SemiJoin kind left right (pushProject db (need ++ colsInExpr left) l)
(pushProject db (colsInExpr right) r) query
SemiJoin
kind
left
right
(pushProject db (need ++ colsInExpr left) l)
(pushProject db (colsInExpr right) r)
query
pushProject db need (Join kind l r c)
| needsAll need = Join kind (pushProject db need l) (pushProject db need r) c
| otherwise =
@@ -128,15 +133,15 @@ pushProject db need leaf@(Scan a t _)
| otherwise = Scan a t (Just (nub [column | requested <- need, let column = physicalColumn requested, column `elem` available]))
where
available = map (unqualify a) (relOpCols db (Scan a t Nothing))
-- | 把扫描限定列换成存储物理列名
-- \| 把扫描限定列换成存储物理列名
physicalColumn column = unqualify (Just (deriveQualifier a t)) (unqualify a column)
pushProject db need (Derived a x)
| needsAll need || all (`elem` wanted) (relOpCols db x) = Derived a (pushProject db ["*"] x)
| otherwise = Derived a (Project wanted (pushProject db wanted x))
where
wanted = nub (map (unqualify a) need)
pushProject db need leaf@(Lookup {}) = pushProjectLeaf db need leaf
pushProject db need leaf@(Range {}) = pushProjectLeaf db need leaf
pushProject db need leaf@(Lookup{}) = pushProjectLeaf db need leaf
pushProject db need leaf@(Range{}) = pushProjectLeaf db need leaf
-- | 折一个表达式节点
foldNode :: Expr -> Expr
@@ -186,9 +191,11 @@ worthRange db t col lo hi = case tableMetaOf db t of
-- | 将独立 IN 或简单等值 EXISTS 变为半连接
semiJoin :: Database -> Expr -> RelOp -> Maybe RelOp
semiJoin db (InSub left query negated) source
| null (subqueryRefs query), safeKey left
| null (subqueryRefs query)
, safeKey left
, Right right <- translate (subqueryStatement query)
, [column] <- relOpCols db right, column /= "*" =
, [column] <- relOpCols db right
, column /= "*" =
Just (SemiJoin (if negated then NotIn else Matching) left (Col column) source right (Just (subqueryStatement query)))
semiJoin db (ExistsSub query negated) source
| [reference] <- subqueryRefs query
@@ -199,8 +206,15 @@ semiJoin db (ExistsSub query negated) source
, Just column <- localKey columns reference a b
, let physical = if reference `elem` relOpCols db source then reference else reverse (takeWhile (/= '.') (reverse reference))
, physical `elem` relOpCols db source =
Just (SemiJoin (if negated then NonMatching else Matching) (Col physical) (Col "$semikey") source
(Compute (items ++ [("$semikey", Col column)]) scan) Nothing)
Just
( SemiJoin
(if negated then NonMatching else Matching)
(Col physical)
(Col "$semikey")
source
(Compute (items ++ [("$semikey", Col column)]) scan)
Nothing
)
semiJoin _ _ _ = Nothing
-- | 只允许无求值错误的左键
@@ -297,7 +311,7 @@ rewriteNode db op = pushOne db op
optimizeRelOp :: Database -> RelOp -> RelOp
optimizeRelOp db = rewriteNode db . descend
where
-- | 先递归处理子算子
-- \| 先递归处理子算子
descend (Filter p x) = Filter p (optimizeRelOp db x)
descend (Project cols x) = Project cols (optimizeRelOp db x)
descend (Compute items x) = Compute items (optimizeRelOp db x)
chusql-core/engine/src/ChuSQL/Core/Engine/Algebra/Planner.hs
+19 -13
View File
@@ -27,7 +27,7 @@ source fromC mWhere = do
nameAggs :: [(String, Expr)] -> [(String, Expr)]
nameAggs items = go (1 :: Int) [] (concatMap (aggregatesIn . snd) items)
where
-- | 逐个聚合起名，重复的不再起名
-- \| 逐个聚合起名，重复的不再起名
go :: Int -> [(String, Expr)] -> [Expr] -> [(String, Expr)]
go _ acc [] = reverse acc
go n acc (e : es)
@@ -63,7 +63,7 @@ rewriteAggs aggs e = case [label | (label, agg) <- aggs, agg == e] of
IsNotNull a -> IsNotNull (go a)
other -> other
where
-- | 递归替换当前查询层的聚合引用
-- \| 递归替换当前查询层的聚合引用
go = rewriteAggs aggs
-- | 排序 + 截断
@@ -82,10 +82,10 @@ translate q = case q of
SelectDistinct inner -> do
ordering <- distinctOrdering inner
plan <- case inner of
s@Select{} -> translate s { selectOrderBy = [], selectLimit = Nothing }
s@Select{} -> translate s{selectOrderBy = [], selectLimit = Nothing}
_ -> Left "DISTINCT requires SELECT"
let labels = case inner of
Select { selectItems = items } -> map fst items
Select{selectItems = items} -> map fst items
_ -> []
keys <- mapM (distinctKey labels) ordering
namedOrdering <- mapM (columnKey labels) ordering
@@ -97,32 +97,38 @@ translate q = case q of
Select items fromC mWhere groupBy having orderBy mLimit -> do
plan0 <- source fromC mWhere
expressions <- mapM (orderExpression items . fst) orderBy
let aggs = nameAggs (items ++ [("having", expression) | expression <- maybe [] pure having]
++ map ("order",) expressions)
let aggs =
nameAggs
( items
++ [("having", expression) | expression <- maybe [] pure having]
++ map ("order",) expressions
)
aggregate = not (null groupBy && null aggs && null having)
grouped = if aggregate then Aggregate groupBy aggs plan0 else plan0
plan = maybe grouped (\condition -> Filter (rewriteAggs aggs condition) grouped) having
items' = [(label, rewriteAggs aggs expression) | (label, expression) <- items]
ordering = zip (map (rewriteAggs aggs) expressions) (map snd orderBy)
-- | 简单列排序保留可优化的名称算子
-- \| 简单列排序保留可优化的名称算子
named (Col column, direction) = Just (column, direction)
named _ = Nothing
-- | 保持普通投影在 LIMIT 之后求值
-- \| 保持普通投影在 LIMIT 之后求值
projected child
| all (\(label, expression) -> expression == Col label) items' = Project (map fst items') child
| otherwise = Compute items' child
Right (case mapM named ordering of
Just spec -> projected (limited spec mLimit plan)
Nothing -> orderedProjection items' ordering mLimit plan)
Right
( case mapM named ordering of
Just spec -> projected (limited spec mLimit plan)
Nothing -> orderedProjection items' ordering mLimit plan
)
_ -> Left "only SELECT is supported by translate"
where
-- | 简单投影的序号映射到来源列
-- \| 简单投影的序号映射到来源列
columnKey columns (column, direction) = do
position <- orderPosition (length columns) column
case position of
Just slot -> Right (columns !! slot, direction)
Nothing -> Left "DISTINCT: sorting requires an output position"
-- | 去重后按指定输出位置或唯一名称排序
-- \| 去重后按指定输出位置或唯一名称排序
distinctKey columns (column, direction) = do
position <- orderPosition (length columns) column
case position of
chusql-core/engine/src/ChuSQL/Core/Engine/Algebra/Sort.hs
+6 -6
View File
@@ -1,8 +1,8 @@
module ChuSQL.Core.Engine.Algebra.Sort (sortRows, sortBoundRows, compareValue) where
import ChuSQL.Core.Model
import ChuSQL.Core.Engine.Syntax.AST
import ChuSQL.Core.Engine.Algebra.Expr (evalExpr)
import ChuSQL.Core.Engine.Syntax.AST
import ChuSQL.Core.Model
import Data.List (sortBy)
-- 排序：按多列规格排出结果行，每列可升可降。
@@ -34,9 +34,9 @@ sortRows :: [(String, SortDir)] -> [Row] -> [Row]
sortRows [] rows = rows
sortRows spec rows = map snd (sortBy cmp (map decorate rows))
where
-- | 取键并带上原行
-- \| 取键并带上原行
decorate row = (sortKey spec row, row)
-- | 比较两个带键的行
-- \| 比较两个带键的行
cmp (k1, _) (k2, _) = compareKeys (map snd spec) k1 k2
-- | 用绑定槽位取键并稳定排序
@@ -47,9 +47,9 @@ sortBoundRows spec outer rows = do
Right (map snd (sortBy compareRows decorated))
where
directions = map snd spec
-- | 每行只计算一次绑定排序键
-- \| 每行只计算一次绑定排序键
decorate row = do
keys <- mapM (\(expression, _) -> evalExpr expression (row ++ outer)) spec
Right (map Just keys, row)
-- | 比较排序键并保留同键行序
-- \| 比较排序键并保留同键行序
compareRows (left, _) (right, _) = compareKeys directions left right
chusql-core/engine/src/ChuSQL/Core/Engine/Builtin.hs
+24 -18
View File
@@ -19,7 +19,8 @@ module ChuSQL.Core.Engine.Builtin (
) where
import ChuSQL.Core.Engine.Syntax.AST (CompareOp (..), Expr (..))
import ChuSQL.Core.Model (Value (..), compareValue)
import ChuSQL.Core.Model (ColumnType (..), Value (..), compareValue, hasTypeCapability)
import ChuSQL.Core.Runtime (TypeCapability (..), builtinTypes, validateValue)
import Control.Monad (foldM)
import Data.Char (toLower)
@@ -201,12 +202,12 @@ compareNode CmpLtE = LtE
executeOperator :: Operator -> [Value] -> Either String Value
executeOperator op vs = case (op, vs) of
(OpNeg, [x]) -> evalNeg x
(OpGt, [x, y]) -> compareValues (== GT) x y
(OpLt, [x, y]) -> compareValues (< EQ) x y
(OpEq, [x, y]) -> compareValues (== EQ) x y
(OpGtE, [x, y]) -> compareValues (>= EQ) x y
(OpLtE, [x, y]) -> compareValues (<= EQ) x y
(OpNe, [x, y]) -> compareValues (/= EQ) x y
(OpGt, [x, y]) -> compareValues OrdCapability (== GT) x y
(OpLt, [x, y]) -> compareValues OrdCapability (< EQ) x y
(OpEq, [x, y]) -> compareValues EqCapability (== EQ) x y
(OpGtE, [x, y]) -> compareValues OrdCapability (>= EQ) x y
(OpLtE, [x, y]) -> compareValues OrdCapability (<= EQ) x y
(OpNe, [x, y]) -> compareValues EqCapability (/= EQ) x y
(OpAnd, [x, y]) -> andValues x y
(OpOr, [x, y]) -> orValues x y
(_, [x, y]) | Just a <- arithOf op -> evalArith a x y
@@ -262,21 +263,26 @@ asDouble (VFloat d) = Just d
asDouble _ = Nothing
-- | 比较两个值：NULL 给 NULL，其余按全序比
compareValues :: (Ordering -> Bool) -> Value -> Value -> Either String Value
compareValues rel x y
compareValues :: TypeCapability -> (Ordering -> Bool) -> Value -> Value -> Either String Value
compareValues capability rel x y
| x == VNull || y == VNull = Right VNull
| comparableValue x y = Right (VBool (rel (compareValue x y)))
| comparableValue capability x y = Right (VBool (rel (compareValue x y)))
| otherwise = Left "type error: cannot compare these values"
-- | 两个值是否落在可以用全序比较的同一族里
comparableValue :: Value -> Value -> Bool
comparableValue (VInt _) (VFloat _) = True
comparableValue (VFloat _) (VInt _) = True
comparableValue (VInt _) (VInt _) = True
comparableValue (VFloat _) (VFloat _) = True
comparableValue (VStr _) (VStr _) = True
comparableValue (VBool _) (VBool _) = True
comparableValue _ _ = False
comparableValue :: TypeCapability -> Value -> Value -> Bool
comparableValue _ (VInt _) (VFloat _) = True
comparableValue _ (VFloat _) (VInt _) = True
comparableValue _ (VInt _) (VInt _) = True
comparableValue _ (VFloat _) (VFloat _) = True
comparableValue _ (VStr _) (VStr _) = True
comparableValue _ (VBool _) (VBool _) = True
comparableValue capability (VRuntime left x) (VRuntime right y) =
left == right
&& hasTypeCapability capability (CRuntime left)
&& validateValue builtinTypes left x == Right ()
&& validateValue builtinTypes right y == Right ()
comparableValue _ _ _ = False
-- | 逻辑与的三值真值表
andValues :: Value -> Value -> Either String Value
chusql-core/engine/src/ChuSQL/Core/Engine/Domain.hs
+235
View File
@@ -0,0 +1,235 @@
module ChuSQL.Core.Engine.Domain (
domainColumns,
domainDefinition,
canonicalDomain,
effectiveColumns,
domainCheckPlan,
runDomainChecks,
validateDomainDefaults,
prepareDomainAlter,
) where
import ChuSQL.Core.Engine.Algebra.Expr (evalExpr)
import ChuSQL.Core.Engine.Runtime.Relation (orderRow)
import ChuSQL.Core.Engine.Semantic (prepareDomainCheckExpression)
import ChuSQL.Core.Engine.Syntax.AST
import ChuSQL.Core.Engine.Syntax.Parser (parseExpression)
import ChuSQL.Core.Model
import Control.Applicative ((<|>))
import Control.Monad (when)
import Data.Char (isAlphaNum, isAscii, toLower)
import Data.List (isPrefixOf, nub)
import Data.Maybe (fromMaybe)
-- 领域元数据、约束继承、默认值及结构变更的纯校验。
-- | 将领域定义编码为目录列
domainColumns :: DomainDefinition -> [(String, Column)]
domainColumns (DomainDefinition base checks) =
("base", base{columnCheck = Nothing})
: [("check_" ++ name, (plainColumn CBool){columnCheck = Just text}) | (name, text) <- checks]
-- | 从新旧目录恢复领域定义
domainDefinition :: Table -> Either String DomainDefinition
domainDefinition table = do
base <- maybe (Left "invalid domain catalog: missing base") Right (lookup "base" (tableCols table))
checks <- mapM readCheck [(name, col) | (name, col) <- tableCols table, name /= "base"]
let legacy = maybe [] (\text -> [("domain_check_0", text)]) (columnCheck base)
Right (DomainDefinition base{columnCheck = Nothing} (legacy ++ checks))
where
-- \| 恢复一条具名检查条件
readCheck (name, col)
| "check_" `isPrefixOf` name, Just text <- columnCheck col = Right (drop 6 name, text)
| otherwise = Left "invalid domain constraint catalog"
-- | 为匿名约束命名并验证完整定义
canonicalDomain :: Database -> String -> DomainDefinition -> Either String DomainDefinition
canonicalDomain db name definition = do
normalized <- normalize definition
let candidate = install name normalized db
validateOwn candidate name
Right normalized
where
-- \| 规范化约束名称并拒绝重复声明
normalize (DomainDefinition base checks) = do
when
(columnAutoIncrement base || columnUnique base || columnPrimaryKey base)
(Left "DOMAIN does not support row-set constraints")
let named = [map toLower key | (key, _) <- checks, not (null key)]
when (length named /= length (nub named)) (Left "duplicate domain constraint name")
mapM_ validName named
assigned <- assign named 1 checks
Right (DomainDefinition base assigned)
-- \| 验证约束标识符
validName key
| null key || length key > 48 || not (all (\c -> isAscii c && (isAlphaNum c || c == '_')) key) =
Left "invalid domain constraint name"
| otherwise = Right ()
-- \| 为未命名约束分配稳定名称
assign _ _ [] = Right []
assign used index ((key, text) : rest)
| null key =
let (chosen, next) = fresh used index
in ((chosen, text) :) <$> assign (chosen : used) next rest
| otherwise = ((map toLower key, text) :) <$> assign used index rest
-- \| 跳过已有的自动约束名称
fresh used index
| chosen `elem` used = fresh used (index + 1)
| otherwise = (chosen, index + 1)
where
chosen = "domain_check_" ++ show (index :: Int)
-- | 在纯目录中安装候选领域定义
install :: String -> DomainDefinition -> Database -> Database
install name definition db = (key, table{tableCols = domainColumns definition}) : filter ((/= key) . fst) db
where
key = domainTableName name
table = fromMaybe (Table key [] [] Nothing Nothing Nothing) (lookup key db)
-- | 沿目录解析领域链并拒绝循环
chain :: Database -> ColumnType -> Either String [(String, DomainDefinition)]
chain db = go []
where
-- \| 逐层读取当前定义而非缓存属性
go seen (CDomain name _)
| name `elem` seen = Left ("cyclic domain definition: " ++ name)
| otherwise = do
table <- maybe (Left ("unknown domain: " ++ name)) Right (lookup (domainTableName name) db)
definition@(DomainDefinition base _) <- domainDefinition table
rest <- go (name : seen) (columnType base)
Right ((name, definition) : rest)
go _ _ = Right []
-- | 合并列声明与继承的非空和默认值
effectiveColumns :: Database -> [(String, Column)] -> Either String [(String, Column)]
effectiveColumns db = mapM effective
where
-- \| 保留最近默认值并收紧可空属性
effective (name, column) = do
definitions <- chain db (columnType column)
let bases = [base | (_, DomainDefinition base _) <- definitions]
Right
( name
, column
{ columnNullable = columnNullable column && all columnNullable bases
, columnDefault = foldl (<|>) (columnDefault column) (map columnDefault bases)
}
)
-- | 绑定领域 VALUE 到独立的单值布局
domainCheckPlan :: Database -> [(String, Column)] -> Either String [(String, Int, Expr)]
domainCheckPlan db columns = concat <$> mapM checks (zip [0 ..] columns)
where
-- \| 按基础领域到外层领域绑定条件
checks (slot, (_, column)) = do
definitions <- chain db (columnType column)
mapM
(bind slot column)
[ (name ++ "." ++ key, text)
| (name, DomainDefinition _ constraints) <- reverse definitions
, (key, text) <- constraints
]
-- \| 拒绝聚合及子查询并固定类型槽位
bind slot column (name, text) = do
expression <- parseExpression text
bound <- prepareDomainCheckExpression (columnType column) expression
Right (name, slot, bound)
-- | SQL CHECK 的 UNKNOWN 与 TRUE 都通过
runDomainChecks :: [(String, Int, Expr)] -> Row -> Either String ()
runDomainChecks plan row = mapM_ check plan
where
-- \| 在单值作用域执行一条领域约束
check (name, slot, expression) = case drop slot row of
(_, value) : _ -> do
result <- evalExpr expression [("value", value)]
case result of
VBool True -> Right ()
VNull -> Right ()
VBool False -> Left ("CHECK on DOMAIN " ++ name ++ " failed")
_ -> Left ("CHECK on DOMAIN " ++ name ++ " is not boolean")
[] -> Left "missing bound domain value"
-- | 检查有效默认值是否满足领域约束
validateDomainDefaults :: Database -> [(String, Column)] -> Either String ()
validateDomainDefaults db columns = do
effective <- effectiveColumns db columns
mapM_ validate [entry | entry@(_, column) <- effective, CDomain{} <- [columnType column]]
where
-- \| 默认值不改变已有行但必须可赋值
validate entry@(name, column) = case columnDefault column of
Nothing -> Right ()
Just value -> do
coerced <- coerceValue (columnType column) value
checkNull name column coerced
plan <- domainCheckPlan db [entry]
runDomainChecks plan [(name, coerced)]
-- | 拒绝领域或列要求非空时的 NULL
checkNull :: String -> Column -> Value -> Either String ()
checkNull name column value
| value == VNull && not (columnNullable column) = Left ("DOMAIN NOT NULL: " ++ name)
| otherwise = Right ()
-- | 检查领域本身的定义及继承默认值
validateOwn :: Database -> String -> Either String ()
validateOwn db name = do
table <- maybe (Left ("unknown domain: " ++ name)) Right (lookup (domainTableName name) db)
DomainDefinition base _ <- domainDefinition table
let columns = [("value", plainColumn (CDomain name (columnType base)))]
_ <- domainCheckPlan db columns
validateDomainDefaults db columns
-- | 在完整快照上校验并生成领域替换定义
prepareDomainAlter :: String -> DomainAlter -> Database -> Either String [(String, Column)]
prepareDomainAlter rawName action db = do
table <- maybe (Left ("unknown domain: " ++ name)) Right (lookup (domainTableName name) db)
definition <- domainDefinition table >>= change
normalized <- canonicalDomain db name definition
let candidate = install name normalized db
mapM_ (validateDependent candidate) [key | (key, _) <- candidate, isDomainTable key]
mapM_ (validateTable candidate) [(key, value) | (key, value) <- candidate, not (isDomainTable key)]
Right (domainColumns normalized)
where
name = map toLower rawName
-- \| 修改单个属性且保留其他约束
change (DomainDefinition base checks) = case action of
DomainDefault value -> Right (DomainDefinition base{columnDefault = value} checks)
DomainNullable nullable -> Right (DomainDefinition base{columnNullable = nullable} checks)
DomainAddCheck key text -> Right (DomainDefinition base (checks ++ [(fromMaybe "" key, text)]))
DomainDropCheck key
| map toLower key `elem` map fst checks -> Right (DomainDefinition base (filter ((/= map toLower key) . fst) checks))
| otherwise -> Left ("unknown domain constraint: " ++ key)
-- \| 判断传递引用是否包含目标领域
affected candidate ty = any ((== name) . fst) <$> chain candidate ty
-- \| 校验依赖领域的继承默认值
validateDependent candidate key = do
let domain = drop (length (domainTableName "")) key
applies <- affected candidate (CDomain domain CStr)
when applies (validateOwn candidate domain)
-- \| 只校验受领域变化影响的列与数据
validateTable candidate (_, table) = do
selected <-
mapM
( \entry@(_, column) -> do
applies <- affected candidate (columnType column)
Right [entry | applies]
)
(tableCols table)
let columns = concat selected
effective <- effectiveColumns candidate columns
validateDomainDefaults candidate columns
plan <- domainCheckPlan candidate columns
mapM_ (validateRow effective plan) (tableRows table)
-- \| 按列布局检查非空与全部继承条件
validateRow columns plan row = do
selected <- orderRow (map fst columns) row
values <-
mapM
( \((key, column), (_, value)) -> do
coerced <- coerceValue (columnType column) value
checkNull key column coerced
Right (key, coerced)
)
(zip columns selected)
runDomainChecks plan values
chusql-core/engine/src/ChuSQL/Core/Engine/Error.hs
+5 -1
View File
@@ -33,8 +33,12 @@ data ErrorRule = ErrorRule
-- | 唯一的错误识别表，顺序即优先级
errorRules :: [ErrorRule]
errorRules =
[ ErrorRule "no database selected" NoDatabaseError "no_database"
[ ErrorRule "deadlock_detected:" StorageError "deadlock_detected"
, ErrorRule "lock_timeout:" StorageError "lock_timeout"
, ErrorRule "serialization_failure:" CatalogError "serialization_failure"
, ErrorRule "no database selected" NoDatabaseError "no_database"
, ErrorRule "administrator required" PermissionError "forbidden"
, ErrorRule "serialization_failure: domain snapshot changed" CatalogError "serialization_failure"
, ErrorRule "bound query object changed" CatalogError "serialization_failure"
, ErrorRule "bound query schema changed" CatalogError "serialization_failure"
, ErrorRule "unknown table" CatalogError "not_found"
chusql-core/engine/src/ChuSQL/Core/Engine/Runtime/Binding.hs
+59 -27
View File
@@ -1,23 +1,43 @@
module ChuSQL.Core.Engine.Runtime.Binding
( Expression (..), TypedExpression, expressionType, bindExpression, evaluateExpression
, Plan (..), TypedPlan, planSchema, bindPlan, executePlan
) where
module ChuSQL.Core.Engine.Runtime.Binding (
Expression (..),
TypedExpression,
expressionType,
bindExpression,
evaluateExpression,
Plan (..),
TypedPlan,
planSchema,
bindPlan,
executePlan,
) where
import ChuSQL.Core.Engine.Runtime.Types
import ChuSQL.Core.Engine.Runtime.Functions
import ChuSQL.Core.Engine.Runtime.Types
import Control.Monad (filterM)
import Data.List (nub, sort)
-- 仅依据 schema 绑定表达式与查询，执行已解析的类型和函数身份。
-- | 表达未绑定的类型化语言
data Expression = Literal TypeExpr RuntimeValue | Reference String | Apply String [Expression]
| ListLiteral TypeExpr [Expression] | NothingLiteral TypeExpr | JustLiteral Expression
| TupleLiteral [Expression] deriving (Show, Eq)
data Expression
= Literal TypeExpr RuntimeValue
| Reference String
| Apply String [Expression]
| ListLiteral TypeExpr [Expression]
| NothingLiteral TypeExpr
| JustLiteral Expression
| TupleLiteral [Expression]
deriving (Show, Eq)
-- | 保存绑定后的表达式节点
data ExpressionNode = Constant RuntimeValue | BoundReference Int String | BoundApply FunctionId [TypedExpression]
| BoundList [TypedExpression] | BoundNothing | BoundJust TypedExpression | BoundTuple [TypedExpression]
data ExpressionNode
= Constant RuntimeValue
| BoundReference Int String
| BoundApply FunctionId [TypedExpression]
| BoundList [TypedExpression]
| BoundNothing
| BoundJust TypedExpression
| BoundTuple [TypedExpression]
deriving (Show, Eq)
-- | 保存表达式的确定类型
@@ -56,9 +76,9 @@ bindExpression types functions env expression = case expression of
bound <- mapM bind arguments
compound (tupleType (map (typeExpression . expressionType) bound)) (BoundTuple bound)
where
-- | 绑定当前作用域的子表达式
-- \| 绑定当前作用域的子表达式
bind = bindExpression types functions env
-- | 校验并构造复合表达式
-- \| 校验并构造复合表达式
compound ty node = do
tid <- descriptorId <$> resolveType types ty
Right (TypedExpression tid node)
@@ -80,16 +100,24 @@ evaluateExpression types functions row (TypedExpression tid node) = do
validateValue types tid value
Right value
where
-- | 执行当前行的子表达式
-- \| 执行当前行的子表达式
evaluate = evaluateExpression types functions row
-- | 表达未绑定的关系计划
data Plan = TableScan String | SingleRow | Projection [(String, Expression)] Plan
| Selection Expression Plan deriving (Show, Eq)
data Plan
= TableScan String
| SingleRow
| Projection [(String, Expression)] Plan
| Selection Expression Plan
deriving (Show, Eq)
-- | 保存绑定后的关系节点
data PlanNode = BoundScan String | BoundSingleRow | BoundProjection [(String, TypedExpression)] TypedPlan
| BoundSelection TypedExpression TypedPlan deriving (Show, Eq)
data PlanNode
= BoundScan String
| BoundSingleRow
| BoundProjection [(String, TypedExpression)] TypedPlan
| BoundSelection TypedExpression TypedPlan
deriving (Show, Eq)
-- | 保存输出 schema 和关系节点
data TypedPlan = TypedPlan [(String, TypeId)] PlanNode deriving (Show, Eq)
@@ -122,13 +150,13 @@ bindPlan types functions catalog plan = case plan of
then Right (TypedPlan (planSchema child) (BoundSelection predicate child))
else Left "selection predicate must be Bool"
where
-- | 绑定子计划
-- \| 绑定子计划
bind = bindPlan types functions catalog
-- | 解析 schema 列的类型身份
-- \| 解析 schema 列的类型身份
bindColumn (name, ty) = do
tid <- descriptorId <$> resolveType types ty
Right (name, tid)
-- | 绑定投影项
-- \| 绑定投影项
bindItem env (name, value) = do
bound <- bindExpression types functions env value
Right (name, bound)
@@ -140,8 +168,12 @@ uniqueNames names
| otherwise = Left "duplicate schema column"
-- | 执行绑定计划并校验存储返回值
executePlan :: TypeRegistry -> FunctionRegistry -> (String -> Either String [[(String, RuntimeValue)]])
-> TypedPlan -> Either String [[(String, RuntimeValue)]]
executePlan ::
TypeRegistry ->
FunctionRegistry ->
(String -> Either String [[(String, RuntimeValue)]]) ->
TypedPlan ->
Either String [[(String, RuntimeValue)]]
executePlan types functions scan (TypedPlan output node) = do
rows <- case node of
BoundScan name -> scan name
@@ -155,23 +187,23 @@ executePlan types functions scan (TypedPlan output node) = do
mapM_ (validateRow output) rows
Right [[(name, value) | (name, _) <- output, Just value <- [lookup name row]] | row <- rows]
where
-- | 计算一行的投影项
-- \| 计算一行的投影项
project items row = mapM (projectItem row) items
-- | 计算命名输出项
-- \| 计算命名输出项
projectItem row (name, expression) = do
value <- evaluateExpression types functions row expression
Right (name, value)
-- | 计算选择条件
-- \| 计算选择条件
keep condition row = do
value <- evaluateExpression types functions row condition
case value of
RBool result -> Right result
_ -> Left "bound predicate returned a non-Bool value"
-- | 校验存储行的完整 schema
-- \| 校验存储行的完整 schema
validateRow columns row
| sort (map fst row) /= sort (map fst columns) = Left "storage row schema mismatch"
| otherwise = mapM_ (validateColumn row) columns
-- | 按列名校验返回值的类型
-- \| 按列名校验返回值的类型
validateColumn row (name, tid) = case lookup name row of
Just value -> validateValue types tid value
Nothing -> Left ("missing bound column: " ++ name)
chusql-core/engine/src/ChuSQL/Core/Engine/Runtime/Dependencies.hs
+52 -32
View File
@@ -1,7 +1,7 @@
module ChuSQL.Core.Engine.Runtime.Dependencies (queryTables, domainDependencies, typeDependencies, validateDependencies) where
import ChuSQL.Core.Engine.Syntax.AST
import ChuSQL.Core.Model (Database, ColumnType (..), columnType, tableCols, tableIdentity, tableSchemaVersion, domainTableName, lastTablePart)
import ChuSQL.Core.Model (ColumnType (..), Database, columnType, domainTableName, lastTablePart, tableCols, tableIdentity, tableSchemaVersion)
import Data.List (nub)
-- 收集读写语句及嵌套表达式的表依赖，供执行期校验目录。
@@ -10,7 +10,7 @@ import Data.List (nub)
validateDependencies :: Database -> Database -> Either String ()
validateDependencies expected current = mapM_ validate expected
where
-- | 数据和统计变化不使结构绑定失效
-- \| 数据和统计变化不使结构绑定失效
validate (name, table) = case lookup name current of
Just actual | tableIdentity actual /= tableIdentity table -> Left ("bound query object changed: " ++ name)
Just actual | tableSchemaVersion actual /= tableSchemaVersion table -> Left ("bound query schema changed: " ++ name)
@@ -19,47 +19,67 @@ validateDependencies expected current = mapM_ validate expected
-- | 收集读写目标、来源及子查询依赖
queryTables :: Statement -> [String]
queryTables statement = nub (case statement of
SelectDistinct child -> queryTables child
Select items source condition _ having ordering _ -> sourceTables source
++ concatMap (expressionTables . snd) items ++ concatMap expressionTables condition
++ concatMap expressionTables having
++ concatMap (maybe [] expressionTables . orderKeyExpression . fst) ordering
Insert target _ rows -> target : concatMap (concatMap expressionTables) rows
Update target assignments condition -> target : concatMap (expressionTables . snd) assignments
++ concatMap expressionTables condition
Delete target condition -> target : concatMap expressionTables condition
DropTable target -> [target]
CreateIndex target _ -> [target]
DropIndex target _ -> [target]
DropColumn target _ -> [target]
AddColumn target _ -> [target]
RenameColumn target _ _ -> [target]
AlterColumnType target _ _ -> [target]
AlterColumnDefault target _ _ -> [target]
AlterColumnNull target _ _ -> [target]
_ -> [])
queryTables statement =
nub
( case statement of
SelectDistinct child -> queryTables child
Select items source condition _ having ordering _ ->
sourceTables source
++ concatMap (expressionTables . snd) items
++ concatMap expressionTables condition
++ concatMap expressionTables having
++ concatMap (maybe [] expressionTables . orderKeyExpression . fst) ordering
Insert target _ rows -> target : concatMap (concatMap expressionTables) rows
Update target assignments condition ->
target
: concatMap (expressionTables . snd) assignments
++ concatMap expressionTables condition
Delete target condition -> target : concatMap expressionTables condition
DropTable target -> [target]
CreateIndex target _ -> [target]
DropIndex target _ -> [target]
DropColumn target _ -> [target]
AddColumn target _ -> [target]
RenameColumn target _ _ -> [target]
AlterColumnType target _ _ -> [target]
AlterColumnDefault target _ _ -> [target]
AlterColumnNull target _ _ -> [target]
_ -> []
)
-- | 收集来源列依赖的领域定义
domainDependencies :: Database -> [String] -> [String]
domainDependencies catalog sources = typeDependencies catalog
[columnType column | (name, table) <- catalog, name `elem` sources
, (_, column) <- tableCols table]
domainDependencies catalog sources =
typeDependencies
catalog
[ columnType column
| (name, table) <- catalog
, name `elem` sources
, (_, column) <- tableCols table
]
-- | 收集声明类型依赖的领域目录
typeDependencies :: Database -> [ColumnType] -> [String]
typeDependencies catalog types = collect [] (concatMap typeDomains types)
where
-- | 递归展开定义并防止重复访问
-- \| 递归展开定义并防止重复访问
collect seen [] = seen
collect seen (domain : remaining) =
let definitions = [(name, table) | (name, table) <- catalog
, lastTablePart name == domainTableName domain, name `notElem` seen]
let definitions =
[ (name, table)
| (name, table) <- catalog
, lastTablePart name == domainTableName domain
, name `notElem` seen
]
names = map fst definitions
nested = [child | (_, table) <- definitions, (_, column) <- tableCols table
, child <- typeDomains (columnType column)]
in collect (seen ++ names) (remaining ++ nested)
-- | 保留嵌套领域的每层依赖
nested =
[ child
| (_, table) <- definitions
, (_, column) <- tableCols table
, child <- typeDomains (columnType column)
]
in collect (seen ++ names) (remaining ++ nested)
-- \| 保留嵌套领域的每层依赖
typeDomains (CDomain name base) = name : typeDomains base
typeDomains _ = []
chusql-core/engine/src/ChuSQL/Core/Engine/Runtime/Functions.hs
+33 -16
View File
@@ -1,10 +1,16 @@
module ChuSQL.Core.Engine.Runtime.Functions
( FunctionId (..), FunctionSignature (..), FunctionRegistry
, builtinFunctions, resolveFunction, resolveNullableFunction, functionSignature, invokeFunction
) where
module ChuSQL.Core.Engine.Runtime.Functions (
FunctionId (..),
FunctionSignature (..),
FunctionRegistry,
builtinFunctions,
resolveFunction,
resolveNullableFunction,
functionSignature,
invokeFunction,
) where
import Control.Monad (zipWithM_)
import ChuSQL.Core.Engine.Runtime.Types
import Control.Monad (zipWithM_)
import Data.Char (toLower)
-- 内建函数的固定身份、签名解析和类型化调用。
@@ -14,9 +20,12 @@ newtype FunctionId = FunctionId Int deriving (Show, Eq, Ord)
-- | 描述已解析的函数签名
data FunctionSignature = FunctionSignature
{ signatureId :: FunctionId, signatureName :: String
, signatureArguments :: [TypeId], signatureResult :: TypeId
} deriving (Show, Eq)
{ signatureId :: FunctionId
, signatureName :: String
, signatureArguments :: [TypeId]
, signatureResult :: TypeId
}
deriving (Show, Eq)
-- | 保存函数签名和实现
newtype FunctionRegistry = FunctionRegistry [(FunctionSignature, [RuntimeValue] -> Either String RuntimeValue)]
@@ -27,9 +36,12 @@ builtinFunctions types = do
int <- descriptorId <$> resolveType types intType
bool <- descriptorId <$> resolveType types boolType
string <- descriptorId <$> resolveType types stringType
Right (FunctionRegistry
[(FunctionSignature (FunctionId 1) "length" [string] int, stringLength)
,(FunctionSignature (FunctionId 2) "not" [bool] bool, booleanNot)])
Right
( FunctionRegistry
[ (FunctionSignature (FunctionId 1) "length" [string] int, stringLength)
, (FunctionSignature (FunctionId 2) "not" [bool] bool, booleanNot)
]
)
-- | 按名称和完整参数类型解析函数
resolveFunction :: FunctionRegistry -> String -> [TypeId] -> Either String FunctionSignature
@@ -38,14 +50,17 @@ resolveFunction registry name arguments = resolveNullableFunction registry name
-- | 按可空 SQL 参数解析确定签名
resolveNullableFunction :: FunctionRegistry -> String -> [Maybe TypeId] -> Either String FunctionSignature
resolveNullableFunction (FunctionRegistry functions) name arguments =
case [sig | (sig, _) <- functions, map toLower (signatureName sig) == map toLower name
, length arguments == length (signatureArguments sig)
, and (zipWith matches arguments (signatureArguments sig))] of
case [ sig
| (sig, _) <- functions
, map toLower (signatureName sig) == map toLower name
, length arguments == length (signatureArguments sig)
, and (zipWith matches arguments (signatureArguments sig))
] of
[sig] -> Right sig
[] -> Left ("no matching function signature: " ++ name)
_ -> Left ("ambiguous function signature: " ++ name)
where
-- | 匹配具体参数或未定型 NULL
-- \| 匹配具体参数或未定型 NULL
matches Nothing _ = True
matches (Just supplied) expected = supplied == expected
@@ -62,7 +77,9 @@ invokeFunction types (FunctionRegistry functions) fid values =
[(sig, implementation)]
| length values == length (signatureArguments sig) -> do
zipWithM_
(validateValue types) (signatureArguments sig) values
(validateValue types)
(signatureArguments sig)
values
result <- implementation values
validateValue types (signatureResult sig) result
Right result
chusql-core/engine/src/ChuSQL/Core/Engine/Runtime/Query.hs
+28 -18
View File
@@ -1,25 +1,33 @@
module ChuSQL.Core.Engine.Runtime.Query
( BoundQuery, QueryType (..), queryTypes, bindQuery, executeQueryM
) where
module ChuSQL.Core.Engine.Runtime.Query (
BoundQuery,
QueryType (..),
queryTypes,
bindQuery,
executeQueryM,
) where
import ChuSQL.Core.Engine.Algebra.Eval (evalRelOpM)
import ChuSQL.Core.Engine.Algebra.Op (RelOp)
import ChuSQL.Core.Engine.Algebra.Optimize (optimize)
import ChuSQL.Core.Engine.Algebra.Planner (translate)
import ChuSQL.Core.Engine.Semantic (prepare, querySchema)
import ChuSQL.Core.Engine.Syntax.AST (Statement)
import ChuSQL.Core.Engine.Storage (MonadStorage (schema, withSchemaGuard))
import ChuSQL.Core.Engine.Runtime.Dependencies (domainDependencies, queryTables, validateDependencies)
import ChuSQL.Core.Engine.Runtime.Relation (bindRelOpColumns)
import ChuSQL.Core.Engine.Runtime.SQL (sqlType)
import ChuSQL.Core.Engine.Runtime.Types (TypeId)
import ChuSQL.Core.Engine.Runtime.Relation (bindRelOpColumns)
import ChuSQL.Core.Engine.Runtime.Dependencies (queryTables, domainDependencies, validateDependencies)
import ChuSQL.Core.Engine.Semantic (prepare, querySchema)
import ChuSQL.Core.Engine.Storage (MonadStorage (schema, withSchemaGuard))
import ChuSQL.Core.Engine.Syntax.AST (Statement)
import ChuSQL.Core.Model
-- SQL 查询绑定边界：保存解析后的类型、函数身份及输出 schema。
-- | 保存运行时或兼容列类型
data QueryType = RuntimeType TypeId | CompatibilityType ColumnType
| DomainType String QueryType | NullType deriving (Show, Eq)
data QueryType
= RuntimeType TypeId
| CompatibilityType ColumnType
| DomainType String QueryType
| NullType
deriving (Show, Eq)
-- | 保存仅由 schema 绑定的执行计划
data BoundQuery = BoundQuery Database [String] [(String, Column)] [(String, QueryType)] RelOp
@@ -31,7 +39,7 @@ queryTypes (BoundQuery _ _ _ types _) = types
-- | 绑定 SQL 查询并优化确定的计划
bindQuery :: Database -> Statement -> Either String BoundQuery
bindQuery catalog statement = do
let schemaOnly = [(name, table {tableRows = []}) | (name, table) <- catalog]
let schemaOnly = [(name, table{tableRows = []}) | (name, table) <- catalog]
resolved <- prepare schemaOnly statement
columns <- querySchema schemaOnly resolved
plan <- translate resolved
@@ -41,11 +49,11 @@ bindQuery catalog statement = do
dependencies = sources ++ domainDependencies schemaOnly sources
Right (BoundQuery schemaOnly dependencies columns types bound)
where
-- | 解析输出列的运行时身份
-- \| 解析输出列的运行时身份
typeOf (name, column) = do
ty <- bindingType (columnType column)
Right (name, ty)
-- | 保留阶段外类型的兼容表示
-- \| 保留阶段外类型的兼容表示
bindingType CNull = Right NullType
bindingType (CDomain name base) = DomainType name <$> bindingType base
bindingType CDate = Right (CompatibilityType CDate)
@@ -54,22 +62,24 @@ bindQuery catalog statement = do
bindingType ty = RuntimeType <$> sqlType ty
-- | 执行绑定查询并校验输出类型
executeQueryM :: MonadStorage m => BoundQuery -> m (Either String [Row])
executeQueryM :: (MonadStorage m) => BoundQuery -> m (Either String [Row])
executeQueryM (BoundQuery catalog dependencies columns _ plan) = do
current <- schema
case current >>= validateDependencies guardedTables of
Left err -> pure (Left err)
Right () -> do
result <- withSchemaGuard guardedTables
(evalRelOpM catalog plan)
result <-
withSchemaGuard
guardedTables
(evalRelOpM catalog plan)
pure (result >>= mapM validateRow)
where
guardedTables = [(name, table) | (name, table) <- catalog, name `elem` dependencies]
-- | 校验完整输出行而不重新解析函数
-- \| 校验完整输出行而不重新解析函数
validateRow row
| map fst row /= map fst columns = Left "bound query output schema mismatch"
| otherwise = mapM validateColumn (zip columns row)
-- | 校验输出列的物理值
-- \| 校验输出列的物理值
validateColumn ((name, column), (_, value))
| columnType column == CNull, value == VNull = Right (name, value)
| valueFits (columnType column) value = Right (name, value)
chusql-core/engine/src/ChuSQL/Core/Engine/Runtime/Relation.hs
+10 -8
View File
@@ -20,14 +20,14 @@ bindWriteColumns db statement = case statement of
Delete table <$> mapM (bindSlots columns []) condition
_ -> Right statement
where
-- | 保留写入目标并绑定右侧表达式
-- \| 保留写入目标并绑定右侧表达式
bindItem columns (name, expression) = (,) name <$> bindSlots columns [] expression
-- | 按绑定布局排列一行并拒绝缺列
orderRow :: [String] -> Row -> Either String Row
orderRow columns row = mapM column columns
where
-- | 保留列名并取得物理值
-- \| 保留列名并取得物理值
column name = case lookup name row of
Just value -> Right (name, value)
Nothing -> Left ("missing bound column: " ++ name)
@@ -84,8 +84,10 @@ bindSlots columns outer expression = case expression of
ColumnSlot slot name -> ColumnSlot <$> rebindSlot columns slot name "unknown local column: " <*> pure name
OuterColumnSlot width slot name
| width < 0 -> Left ("bound column position mismatch: " ++ name)
| otherwise -> OuterColumnSlot (length columns)
<$> rebindSlot outer slot name "unknown column: " <*> pure name
| otherwise ->
OuterColumnSlot (length columns)
<$> rebindSlot outer slot name "unknown column: "
<*> pure name
FunctionCall name arguments -> FunctionCall name <$> mapM bind arguments
BoundFunction fid tid arguments -> BoundFunction fid tid <$> mapM bind arguments
Construct ty arguments -> Construct ty <$> mapM bind arguments
@@ -121,7 +123,7 @@ bindSlots columns outer expression = case expression of
bind = bindSlots columns outer
-- \| 查找布局中首次出现的列
slots names name = [slot | (slot, key) <- zip [0 ..] names, key == name]
-- | 保留有效位置，仅允许唯一列重定位
-- \| 保留有效位置，仅允许唯一列重定位
rebindSlot names slot name missing
| slot < 0 = Left ("bound column position mismatch: " ++ name)
| take 1 (drop slot names) == [name] = Right slot
@@ -133,7 +135,7 @@ bindSlots columns outer expression = case expression of
outerReference name = case slots outer name of
slot : _ -> Right (OuterColumnSlot (length columns) slot name)
[] -> Left ("unknown column: " ++ name)
-- | 将子查询自由变量绑定到父输入布局
-- \| 将子查询自由变量绑定到父输入布局
bindSubquery query = do
references <- case query of
Subquery _ names -> mapM reference names
@@ -141,12 +143,12 @@ bindSlots columns outer expression = case expression of
| map fst references == names -> mapM rebind references
| otherwise -> Left "bound subquery references mismatch"
Right (BoundSubquery (subqueryStatement query) (subqueryRefs query) references)
-- | 补回限定名之前先绑定真实物理列
-- \| 补回限定名之前先绑定真实物理列
reference name = do
let physical = if name `elem` (columns ++ outer) then name else reverse (takeWhile (/= '.') (reverse name))
boundReference <- bind (Col physical)
Right (name, boundReference)
-- | 重绑定时保留局部或外层身份
-- \| 重绑定时保留局部或外层身份
rebind (name, boundReference) = (,) name <$> bind boundReference
-- \| 绑定两个操作数
binary constructor a b = constructor <$> bind a <*> bind b
chusql-core/engine/src/ChuSQL/Core/Engine/Runtime/SQL.hs
+4 -2
View File
@@ -1,9 +1,9 @@
module ChuSQL.Core.Engine.Runtime.SQL (resolveSQLFunction, sqlResultType, invokeSQLFunction, sqlType, constructSQLValue) where
import Control.Monad (zipWithM_)
import ChuSQL.Core.Engine.Runtime.Functions
import ChuSQL.Core.Engine.Runtime.Types
import ChuSQL.Core.Model (ColumnType (..), Value (..))
import Control.Monad (zipWithM_)
-- SQL 现有标量表示与已解析运行时函数之间的类型边界。
@@ -54,7 +54,9 @@ invokeSQLFunction fid expected values = do
then Left "function argument count mismatch"
else do
zipWithM_
validateArgument (signatureArguments sig) values
validateArgument
(signatureArguments sig)
values
if VNull `elem` values
then Right VNull
else do
chusql-core/engine/src/ChuSQL/Core/Engine/Semantic.hs
+93 -45
View File
@@ -1,15 +1,16 @@
{-# LANGUAGE TupleSections #-}
module ChuSQL.Core.Engine.Semantic (check, prepare, prepareRowExpression, prepareCheckExpression, querySchema) where
import Control.Monad (void)
module ChuSQL.Core.Engine.Semantic (check, prepare, prepareRowExpression, prepareCheckExpression, prepareDomainCheckExpression, querySchema) where
import ChuSQL.Core.Engine.Algebra.Expr (bareColumns, colsInExpr, hasAggregate, hasSubquery)
import ChuSQL.Core.Engine.Runtime.Functions (FunctionSignature (..), builtinFunctions, functionSignature)
import ChuSQL.Core.Engine.Runtime.SQL (resolveSQLFunction, sqlResultType, sqlType)
import ChuSQL.Core.Engine.Runtime.Relation (bindSlots)
import ChuSQL.Core.Engine.Runtime.SQL (resolveSQLFunction, sqlResultType, sqlType)
import ChuSQL.Core.Engine.Runtime.Types
import ChuSQL.Core.Engine.Syntax.AST
import ChuSQL.Core.Engine.Syntax.Parser (parseExpression)
import ChuSQL.Core.Model
import Control.Monad (void)
import Data.Char (isAlphaNum, isAscii, toLower)
import Data.List (intercalate, isInfixOf, isPrefixOf, isSuffixOf, nub)
@@ -46,7 +47,7 @@ data Scope = Scope
scopeAt :: String -> Scope -> String -> Either String (String, Column)
scopeAt place scope c = search (scopeFrames scope)
where
-- | 在最近的可见列层停止查找
-- \| 在最近的可见列层停止查找
search [] = Left (missingColumn place c (concat (scopeFrames scope)))
search (columns : rest) = case columnAt place columns c of
Right entry -> Right entry
@@ -116,7 +117,12 @@ checkGrouping env keys items having
| null keys && null having && not (any (hasAggregate . snd) items) = Right ()
| otherwise = do
checkColumns "GROUP BY" env keys
mapM_ (checkRuntimeColumn "GROUP BY" env) keys
mapM_
( \column -> do
(_, definition) <- scopeAt "GROUP BY" env column
void (requireHashing "GROUP BY" (InferType (columnType definition)))
)
keys
mapM_ okCol (nub (concatMap (bareColumns . snd) items ++ concatMap bareColumns having))
where
-- \| 这一列能不能出现在分组查询里
@@ -172,12 +178,12 @@ inferExpr db place env (Neg a) = do
InferType x
| numericType x -> Right t
| otherwise -> Left (place ++ ": operator needs TInt on both sides, got TInt and " ++ typeLabel x)
inferExpr db place env (Gt a b) = comparison db place env a b
inferExpr db place env (Lt a b) = comparison db place env a b
inferExpr db place env (Eq a b) = comparison db place env a b
inferExpr db place env (GtE a b) = comparison db place env a b
inferExpr db place env (LtE a b) = comparison db place env a b
inferExpr db place env (NotEq a b) = comparison db place env a b
inferExpr db place env (Gt a b) = comparison OrdCapability db place env a b
inferExpr db place env (Lt a b) = comparison OrdCapability db place env a b
inferExpr db place env (Eq a b) = comparison EqCapability db place env a b
inferExpr db place env (GtE a b) = comparison OrdCapability db place env a b
inferExpr db place env (LtE a b) = comparison OrdCapability db place env a b
inferExpr db place env (NotEq a b) = comparison EqCapability db place env a b
inferExpr db place env (And a b) = boolean db place env a b
inferExpr db place env (Or a b) = boolean db place env a b
inferExpr db place env (IsNull a) = inferExpr db place env a >> Right (InferType CBool)
@@ -215,12 +221,15 @@ inferExpr db place env (InList a es _) = do
mapM_ (itemType place ta) ts
Right (InferType CBool)
inferExpr _ _ _ (ExistsSub _ _) = Right (InferType CBool)
inferExpr db place env (QuantCmp _ a sq q) = do
inferExpr db place env (QuantCmp operator a sq q) = do
ta <- inferExpr db place env a
cols <- inferSubqueryOutput db env sq
case cols of
[(_, t)]
| compatible ta t -> Right (InferType CBool)
| compatible ta t -> do
let capability = if operator `elem` [CmpEq, CmpNe] then EqCapability else OrdCapability
mapM_ (requireCapability capability place) [ta, t]
Right (InferType CBool)
| otherwise -> Left (place ++ ": " ++ quantLabel q ++ " subquery column has type " ++ renderType t ++ ", expected " ++ renderType ta)
_ -> Left (place ++ ": " ++ quantLabel q ++ " subquery must return exactly one column")
@@ -241,7 +250,7 @@ inferSubqueryOutput _ _ BoundSubquery{} = Left "execution subquery cannot be use
inferSubqueryOutput db outerScope sq = case subqueryStatement sq of
SelectDistinct q -> do
columns <- inferSubqueryOutput db outerScope sq{subqueryStatement = q}
mapM_ (requireOrdering "DISTINCT" . snd) columns
mapM_ (requireHashing "DISTINCT" . snd) columns
_ <- distinctOrdering q
Right columns
q@Select{selectItems = items} -> do
@@ -295,12 +304,14 @@ numericOperand place want other t = case t of
_ -> Left (place ++ ": operator needs " ++ renderType want ++ " on both sides, got " ++ renderType other ++ " and " ++ renderType t)
-- | 比较两边的类型检查
comparison :: Database -> String -> Scope -> Expr -> Expr -> Either String InferredType
comparison db place env a b = do
comparison :: TypeCapability -> Database -> String -> Scope -> Expr -> Expr -> Either String InferredType
comparison capability db place env a b = do
ta <- inferExpr db place env a
tb <- inferExpr db place env b
if compatible ta tb
then Right (InferType CBool)
then do
mapM_ (requireCapability capability place) [ta, tb]
Right (InferType CBool)
else Left (place ++ ": both sides of = must have the same type, got " ++ renderType ta ++ " and " ++ renderType tb)
-- | 两类能不能比，类型族规则来自 Model
@@ -362,7 +373,8 @@ prepare db q = do
let visible = filter (not . isDomainTable . fst) db
resolved <- resolveStatement visible typed
case resolved of
CreateDomain name base -> do
CreateDomain name (DomainDefinition column _) -> do
let base = columnType column
checkDomainName name
checkTypeParameters base
case base of
@@ -373,6 +385,9 @@ prepare db q = do
if any ((== domainTableName name) . fst) db
then Left ("domain already exists: " ++ name)
else Right ()
AlterDomain name _ -> do
_ <- maybe (Left ("unknown domain: " ++ name)) Right (lookup (domainTableName name) db)
Right ()
DropDomain name -> do
_ <- maybe (Left ("unknown domain: " ++ name)) Right (lookup (domainTableName name) db)
if any (any (usesDomain name . columnType . snd) . tableCols . snd) db
@@ -418,7 +433,9 @@ ddlTargets statement = case statement of
-- | 从 schema 解析列上的命名类型
resolveDomainTypes :: Database -> Statement -> Either String Statement
resolveDomainTypes db statement = case statement of
CreateDomain name base -> CreateDomain name <$> resolveDomainType base
CreateDomain name (DomainDefinition column checks) -> do
ty <- resolveDomainType (columnType column)
Right (CreateDomain name (DomainDefinition column{columnType = ty} checks))
CreateTable name cols -> CreateTable name <$> mapM resolveColumnType cols
AddColumn name col -> AddColumn name <$> resolveColumnType col
AlterColumnType name col ty -> AlterColumnType name col <$> resolveDomainType ty
@@ -491,9 +508,11 @@ checkResolvedWith db outer q = case q of
else Right ()
CreateIndex tbl col -> do
t <- lookupTable db tbl
if col `elem` tableCols' t
then checkRuntimeColumn "CREATE INDEX" (Scope (tableCols t) []) col
else Left ("CREATE INDEX: unknown column: " ++ col)
case lookup col (tableCols t) of
Nothing -> Left ("CREATE INDEX: unknown column: " ++ col)
Just column
| typeClassOf (columnType column) == RuntimeClass -> Left "CREATE INDEX: runtime index key encoding is not registered"
| otherwise -> Right ()
DropIndex tbl col -> do
t <- lookupTable db tbl
if col `elem` tableCols' t
@@ -579,10 +598,12 @@ checkResolvedWith db outer q = case q of
UseDatabase name -> checkDatabaseName name
ShowDatabases -> Right ()
CreateDomain{} -> Right ()
AlterDomain{} -> Right ()
DropDomain{} -> Right ()
ShowDomains -> Right ()
-- \| 事务控制由会话层执行，语义层不做列检查
BeginTransaction -> Right ()
BeginSerializableTransaction -> Right ()
CommitTransaction -> Right ()
RollbackTransaction -> Right ()
-- \| 保存点名字只是会话层的记号，不做列检查
@@ -660,7 +681,7 @@ checkChecks columns = mapM_ checkOne
Just text -> case parseExpression text of
Left err -> Left ("CHECK: " ++ firstLine err)
Right e -> void (prepareCheckExpression columns e)
-- | 只保留解析错误的第一行
-- \| 只保留解析错误的第一行
firstLine = takeWhile (/= '\n')
-- | 账号名上限
@@ -825,6 +846,21 @@ prepareCheckExpression columns expression = do
checkBool [] "CHECK" (Scope columns []) resolved
bindSlots (map fst columns) [] resolved
-- | 在单值作用域规范化领域 VALUE 的大小写
prepareDomainCheckExpression :: ColumnType -> Expr -> Either String Expr
prepareDomainCheckExpression ty expression = do
checkNoAggregate "CHECK" expression
if hasSubquery expression then Left "CHECK: subqueries are not allowed here" else Right ()
let names = nub ("value" : colsInExpr expression)
columns = [(name, plainColumn ty) | name <- names, map toLower name == "value"]
resolved <- resolveExpr [] columns [] valueName expression
prepareCheckExpression [("value", plainColumn ty)] resolved
where
-- \| 领域条件只允许引用 VALUE
valueName name
| map toLower name == "value" = Right "value"
| otherwise = Left ("unknown column in DOMAIN CHECK: " ++ name)
-- | 将推导类型转换为可空参数类型
inferredArgument :: InferredType -> Maybe ColumnType
inferredArgument InferNull = Nothing
@@ -856,8 +892,14 @@ statementColumns stmt = case stmt of
SelectDistinct q -> statementColumns q
Select items fromC mWhere groupBy having orderBy _ -> do
ordering <- mapM (orderExpression items . fst) orderBy
Right (concatMap (colsInExpr . snd) items ++ fromColumns fromC
++ concatMap colsInExpr mWhere ++ groupBy ++ concatMap colsInExpr having ++ concatMap colsInExpr ordering)
Right
( concatMap (colsInExpr . snd) items
++ fromColumns fromC
++ concatMap colsInExpr mWhere
++ groupBy
++ concatMap colsInExpr having
++ concatMap colsInExpr ordering
)
_ -> Right []
where
-- \| 取 FROM 里出现的列
@@ -909,11 +951,12 @@ nameIn env place c = fst <$> columnAt place env c
scopedColumnAt :: String -> [(String, Column)] -> EnclosingScopes -> String -> Either String (String, Column)
scopedColumnAt place inner outer c
| length (filter (== '.') c) == 1
, columns : _ <- filter ownsSource (inner : outer) = columnAt place columns c
, columns : _ <- filter ownsSource (inner : outer) =
columnAt place columns c
| otherwise = scopeAt place (Scope inner outer) c
where
qualifier = takeWhile (/= '.') c ++ "."
-- | 判断当前层是否声明该来源限定名
-- \| 判断当前层是否声明该来源限定名
ownsSource = any (isPrefixOf qualifier . fst)
-- | 单表保留原表头，连接使用唯一前缀
@@ -927,18 +970,21 @@ checkOrder db env items column = do
expression <- orderExpression items column
void (inferExpr db "ORDER BY" env expression >>= requireOrdering "ORDER BY")
-- | 校验列已有排序能力
checkRuntimeColumn :: String -> Scope -> String -> Either String ()
checkRuntimeColumn place env column = do
(_, definition) <- scopeAt place env column
void (requireOrdering place (InferType (columnType definition)))
-- | 拒绝尚未注册能力的复合类型
requireOrdering :: String -> InferredType -> Either String InferredType
requireOrdering place inferred@(InferType ty)
| typeClassOf ty == RuntimeClass = Left (place ++ ": runtime type capability is not registered")
-- | 统一检查已注册能力并保留 NULL 传播
requireCapability :: TypeCapability -> String -> InferredType -> Either String InferredType
requireCapability capability place inferred@(InferType ty)
| not (hasTypeCapability capability ty) = Left (place ++ ": type capability is not registered: " ++ show capability)
| otherwise = Right inferred
requireOrdering _ inferred = Right inferred
requireCapability _ _ inferred = Right inferred
-- | 只允许已提供排序能力的类型
requireOrdering :: String -> InferredType -> Either String InferredType
requireOrdering = requireCapability OrdCapability
-- | 分组与去重同时要求相等和哈希能力
requireHashing :: String -> InferredType -> Either String InferredType
requireHashing place inferred =
requireCapability EqCapability place inferred >> requireCapability HashCapability place inferred
-- | 解析投影、条件、排序与连接引用
resolveQuery :: Database -> EnclosingScopes -> Statement -> [(String, Expr)] -> Either String Statement
@@ -957,11 +1003,13 @@ resolveQuery db outer q items = do
qualifiedReference c =
qualifiedReferenceIn db (selectFrom q) c
|| ( any
(any
(\ (key, _)
-> length (filter (== '.') key) == 1
&& ("." ++ key) `isSuffixOf` c))
outer
( any
( \(key, _) ->
length (filter (== '.') key) == 1
&& ("." ++ key) `isSuffixOf` c
)
)
outer
&& (not (any (elem '.' . fst) db) || any (\(table, _) -> (table ++ ".") `isPrefixOf` c) db)
)
-- \| 换算一个投影项：带别名时保留别名做输出列名
@@ -981,13 +1029,13 @@ resolveQuery db outer q items = do
condition <- mapM (resolveExpr db env outer (name "WHERE")) (selectWhere q)
grouping <- mapM (name "GROUP BY") (selectGroupBy q)
having <- mapM (resolveExpr db env outer (name "HAVING")) (selectHaving q)
-- | 整项名称可用输出别名，复合表达式只查来源
-- \| 整项名称可用输出别名，复合表达式只查来源
let orderKey key@(OrderColumn column)
| column `elem` map fst projected = key <$ orderExpression projected key
| otherwise = OrderColumn <$> name "ORDER BY" column
orderKey key@(OrderPosition _) = key <$ orderPosition (length projected) key
orderKey (OrderExpression expression) = OrderExpression <$> resolveExpr db env outer (name "ORDER BY") expression
ordering <- mapM (\(key, direction) -> (, direction) <$> orderKey key) (selectOrderBy q)
ordering <- mapM (\(key, direction) -> (,direction) <$> orderKey key) (selectOrderBy q)
pure (Select projected source condition grouping having ordering (selectLimit q))
-- | 收集没有别名遮蔽的来源表名
chusql-core/engine/src/ChuSQL/Core/Engine/Storage.hs
+4
View File
@@ -118,6 +118,10 @@ class (Monad m) => MonadStorage m where
-- \| ALTER 用：整表换列定义与全部行
replaceSchema :: String -> [(String, Column)] -> [Row] -> m (Either String ())
-- | 校验完整快照后原子修改领域定义
alterDomain :: String -> (Database -> Either String [(String, Column)]) -> m (Either String ())
alterDomain _ _ = pure (Left "atomic domain alteration is unavailable in this storage")
-- \| 取全库快照（表名 + 列 + 全部行）
snapshot :: m (Either String Database)
chusql-core/engine/src/ChuSQL/Core/Engine/Storage/IPC.hs
+82 -53
View File
@@ -1,5 +1,5 @@
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE ImportQualifiedPost #-}
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE OverloadedStrings #-}
module ChuSQL.Core.Engine.Storage.IPC (
@@ -29,10 +29,10 @@ module ChuSQL.Core.Engine.Storage.IPC (
doDescribeTable,
doDropTable,
doListCatalog,
databaseSnapshot,
snapshotDatabase,
) where
import Data.Foldable (for_)
import Control.Monad (when)
import ChuSQL.Core.Engine.Parallel (parallelShardLimit, poolRun, workerPool)
import ChuSQL.Core.Engine.Storage
import ChuSQL.Core.Engine.Storage.FFI (closeStorage, openStorage, storageRequest)
@@ -41,16 +41,43 @@ import ChuSQL.Core.Protocol (Account (..), Request (..), Response (..), SchemaCo
import Control.Concurrent (getNumCapabilities)
import Control.Concurrent.MVar (MVar, modifyMVar_, newMVar, putMVar, readMVar, takeMVar, withMVar)
import Control.Exception (bracket, mask_)
import Control.Monad (when)
import Data.Aeson (eitherDecodeStrict, encode)
import Data.Aeson qualified as A
import Data.Aeson.KeyMap qualified as KM
import Data.ByteString qualified as BS
import Data.ByteString.Lazy qualified as BL
import Data.Foldable (for_)
import Data.Maybe (fromMaybe)
import System.IO.Unsafe (unsafePerformIO)
-- IPC 存储实现：把 MonadStorage 的操作变成发给存储库的 JSON 请求。
-- 复用相同提交点的不可变快照版本。
databaseSnapshot :: Maybe (String, Database) -> Env -> IO (Either String (String, Database))
databaseSnapshot previous env = do
response <- ask env "database_snapshot" (ReqDatabaseSnapshot (fst <$> previous)) Just
pure $ case response of
Left err -> Left err
Right (RespSnapshotUnchanged checkpoint) -> case previous of
Just cached | fst cached == checkpoint -> Right cached
_ -> Left "snapshot version does not match cached database"
Right (RespDomainSnapshot infos rows checkpoint) -> do
db <- snapshotDatabase infos rows
Right (checkpoint, db)
Right (RespError err) -> Left err
_ -> Left "unexpected database snapshot response"
-- 从同一版本的结构和行集构建引擎数据库。
snapshotDatabase :: [TableInfo] -> [(String, [Row])] -> Either String Database
snapshotDatabase infos rows = mapM load infos
where
-- 布局、对象身份与数据必须来自同一版本。
load info = do
values <- maybe (Left "incomplete database snapshot") Right (lookup (tiTable info) rows)
let name = tiTable info
Right (name, Table name (schemaToColumns (tiColumns info)) values (Just (metaOf info)) (tiIdentity info) (tiSchemaVersion info))
-- | 一条会话的存储环境：当前库名随会话走，不共享
data Env = Env
{ envDatabase :: Maybe String
@@ -146,15 +173,16 @@ closeConnection = modifyMVar_ linkRef $ \current -> withMVar (snd linkUsers) $ \
-- | 发一条请求并解码应答；链路没装就报错，不崩
sendRequest :: Request -> IO Response
sendRequest req = withStorageLink $ \case
Nothing -> pure (RespError "storage not configured")
Just link
-> do raw <- slRequest link (BL.toStrict (encode req))
case raw of
Left e -> pure (RespError ("storage error: " ++ e))
Right bytes
-> case eitherDecodeStrict (BS.dropWhileEnd (== 13) bytes) of
Left e -> pure (RespError ("decode error: " ++ e))
Right resp -> pure resp
Nothing -> pure (RespError "storage not configured")
Just link ->
do
raw <- slRequest link (BL.toStrict (encode req))
case raw of
Left e -> pure (RespError ("storage error: " ++ e))
Right bytes ->
case eitherDecodeStrict (BS.dropWhileEnd (== 13) bytes) of
Left e -> pure (RespError ("decode error: " ++ e))
Right resp -> pure resp
-- | 原样转发一条载荷给存储库
sendRawRequest :: BS.ByteString -> IO (Either String BS.ByteString)
@@ -244,22 +272,22 @@ okOnly _ = Nothing
-- | 发 Scan
doScan :: Env -> String -> IO (Either String [Row])
doScan env t = ask env "scan" (ReqScan t) $ \case
RespRows rows -> Just rows
_ -> Nothing
RespRows rows -> Just rows
_ -> Nothing
-- | 发带投影的 Scan
doScanColumns :: Env -> String -> [String] -> IO (Either String [Row])
doScanColumns env t cols = ask env "scan" (ReqScanColumns t cols) $ \case
RespRows rows
-> Just
[[(c, v) | c <- cols, Just v <- [lookup c row]] | row <- rows]
_ -> Nothing
RespRows rows ->
Just
[[(c, v) | c <- cols, Just v <- [lookup c row]] | row <- rows]
_ -> Nothing
-- | 发分片 Scan：只取第 shard 片
doScanShard :: Env -> String -> Maybe [String] -> Int -> Int -> IO (Either String [Row])
doScanShard env t cols shard shards = ask env "scan_shard" (ReqScanShard t cols shard shards) $ \case
RespRows rows -> Just rows
_ -> Nothing
RespRows rows -> Just rows
_ -> Nothing
-- | 发 Insert（索引由存储层维护）
doInsert :: Env -> String -> Row -> IO (Either String ())
@@ -276,21 +304,21 @@ doDeleteKeys env t ks = ask env "delete_keys" (ReqDeleteKeys t ks) okOnly
-- | 发 ListTables
doListTables :: Env -> IO (Either String [String])
doListTables env = ask env "list_tables" ReqListTables $ \case
RespTables ts -> Just ts
_ -> Nothing
RespTables ts -> Just ts
_ -> Nothing
-- | 一次拿全库 schema
doListCatalog :: Env -> IO (Either String [TableInfo])
doListCatalog env = ask env "list_catalog" ReqListCatalog $ \case
RespCatalog xs -> Just xs
_ -> Nothing
RespCatalog xs -> Just xs
_ -> Nothing
-- | 按某一列的索引取一行（键可以是整数或字符串）
doLookupByColumn :: Env -> String -> String -> Value -> IO (Either String IndexResult)
doLookupByColumn env t c k = ask env "lookup_by_index" (ReqLookupByColumn t c k) $ \case
RespRows rows -> Just (IndexRows rows)
RespNoIndex -> Just NoIndex
_ -> Nothing
RespRows rows -> Just (IndexRows rows)
RespNoIndex -> Just NoIndex
_ -> Nothing
-- | 范围扫描：没有可用索引就回 Nothing，让上层退回扫描
doScanRange ::
@@ -301,15 +329,15 @@ doScanRange ::
Maybe (Value, Bool) ->
IO (Either String (Maybe [Row]))
doScanRange env t c lo hi = ask env "range_by_index" (ReqRangeByIndex t c lo hi) $ \case
RespRows rows -> Just (Just rows)
RespNoIndex -> Just Nothing
_ -> Nothing
RespRows rows -> Just (Just rows)
RespNoIndex -> Just Nothing
_ -> Nothing
-- | 发 DescribeTable
doDescribeTable :: Env -> String -> IO (Either String TableInfo)
doDescribeTable env t = ask env "describe_table" (ReqDescribeTable t) $ \case
RespSchema info -> Just info
_ -> Nothing
RespSchema info -> Just info
_ -> Nothing
-- | 发 DropTable
doDropTable :: Env -> String -> IO (Either String ())
@@ -353,7 +381,7 @@ instance MonadStorage IPCStorage where
withSchemaGuard dependencies action = IPCStorage $ \env ->
runIPCStorageIn action env{envSchemaGuard = Just dependencies}
-- | 写前握手，旧存储不能先写入后才报错
-- \| 写前握手，旧存储不能先写入后才报错
withWriteSchemaGuard dependencies action = IPCStorage $ \env -> case dependencies of
[] -> pure (Left "bound write requires a target schema guard")
(target, _) : _ -> do
@@ -374,8 +402,8 @@ instance MonadStorage IPCStorage where
-- \| 列出现有库
listDatabases = IPCStorage $ \env -> ask env "list_databases" (ReqDatabase "list_databases" "") $ \case
RespTables names -> Just names
_ -> Nothing
RespTables names -> Just names
_ -> Nothing
-- \| 发 Scan
scan t = IPCStorage (`doScan` t)
@@ -430,30 +458,31 @@ instance MonadStorage IPCStorage where
-- \| 删一列
dropColumn t c = IPCStorage $ \env -> ask env "drop_column" (ReqDropColumn t c) okOnly
-- \| 列表 + 逐表扫描拼库
snapshot = IPCStorage $ \env -> do
result <- doListCatalog env
case result of
-- \| 校验一致快照并复查提交点后发布领域
alterDomain name validate = IPCStorage $ \env -> do
snapshotResult <- ask env "domain_snapshot" ReqDomainSnapshot $ \case
RespDomainSnapshot infos rows checkpoint -> Just (infos, rows, checkpoint)
_ -> Nothing
case snapshotResult of
Left err -> pure (Left err)
Right xs -> fmap sequence (mapM (loadEntry env) xs)
Right (infos, rows, checkpoint) -> case mapM (loadDomainEntry rows) infos >>= validate of
Left err -> pure (Left err)
Right cols -> ask env "alter_domain" (ReqAlterDomain name (map toWire cols) checkpoint) okOnly
where
-- \| 读取表数据并保留扫描错误
loadEntry env info = do
rows <- doScan env (tiTable info)
pure $
fmap
( \rs ->
( tiTable info
, Table (tiTable info) (schemaToColumns (tiColumns info)) rs (Just (metaOf info)) (tiIdentity info) (tiSchemaVersion info)
)
)
rows
-- \| 保留快照身份并拒绝缺失的表数据
loadDomainEntry rows info = do
values <- maybe (Left "incomplete domain snapshot") Right (lookup (tiTable info) rows)
let key = tiTable info
Right (key, Table key (schemaToColumns (tiColumns info)) values (Just (metaOf info)) (tiIdentity info) (tiSchemaVersion info))
-- 在存储读门内获取同一版本的完整快照。
snapshot = IPCStorage $ \env -> fmap (fmap snd) (databaseSnapshot Nothing env)
-- \| 只问数据字典要结构，设了当前库只留它的表
schema = IPCStorage $ \env -> do
result <- ask env "all_catalogs" ReqAllCatalog $ \case
RespCatalog infos -> Just infos
_ -> Nothing
RespCatalog infos -> Just infos
_ -> Nothing
pure (fmap (concatMap (visibleTables (envDatabase env))) result)
-- | 线上表信息转引擎侧统计
chusql-core/engine/src/ChuSQL/Core/Engine/Storage/Memory.hs
+6 -1
View File
@@ -30,7 +30,7 @@ instance MonadStorage MemoryStorage where
-- \| 纯状态视图在求值期间不受外部修改
withSchemaGuard _ action = action
-- | 纯状态写入不受外部结构变更影响
-- \| 纯状态写入不受外部结构变更影响
withWriteSchemaGuard _ action = action
-- \| 全表扫描
@@ -81,6 +81,11 @@ instance MonadStorage MemoryStorage where
db
)
-- \| 在同一状态转换内校验并发布领域
alterDomain name validate = MemoryStorage $ \db -> case validate db of
Left err -> Right (Left err, db)
Right cols -> runMemoryStorage (replaceSchema name cols []) db
-- \| 原样返回当前库
snapshot = MemoryStorage $ \db -> Right (Right db, db)
chusql-core/engine/src/ChuSQL/Core/Engine/Syntax/AST.hs
+55 -40
View File
@@ -1,15 +1,26 @@
module ChuSQL.Core.Engine.Syntax.AST (Statement (..), FromClause (..), JoinKind (..), Expr (..), Subquery (..), Quantifier (..), CompareOp (..), SortDir (..), OrderKey (..), makeSelect, distinctOrdering, orderPosition, orderExpression, orderKeyExpression) where
module ChuSQL.Core.Engine.Syntax.AST (Statement (..), DomainDefinition (..), DomainAlter (..), FromClause (..), JoinKind (..), Expr (..), Subquery (..), Quantifier (..), CompareOp (..), SortDir (..), OrderKey (..), makeSelect, distinctOrdering, orderPosition, orderExpression, orderKeyExpression) where
import ChuSQL.Core.Model (Column (..), ColumnType, Value (..))
import ChuSQL.Core.Engine.Runtime.Types (TypeId, TypeExpr, RuntimeValue)
import ChuSQL.Core.Engine.Runtime.Functions (FunctionId)
import ChuSQL.Core.Engine.Runtime.Types (RuntimeValue, TypeExpr, TypeId)
import ChuSQL.Core.Model (Column (..), ColumnType, Value (..))
-- 语法树：语句、数据来源、条件表达式、排序方向。
-- | 领域的基础列属性及具名检查条件
data DomainDefinition = DomainDefinition Column [(String, String)] deriving (Show, Eq)
-- | 领域约束及默认值的结构变更
data DomainAlter
= DomainDefault (Maybe Value)
| DomainNullable Bool
| DomainAddCheck (Maybe String) String
| DomainDropCheck String
deriving (Show, Eq)
data Statement
= Select
-- | 输出名称与投影表达式，普通列用 Col
{ selectItems :: [(String, Expr)]
-- ^ 输出名称与投影表达式，普通列用 Col
, selectFrom :: FromClause
, selectWhere :: Maybe Expr
, selectGroupBy :: [String]
@@ -22,7 +33,8 @@ data Statement
| Delete String (Maybe Expr)
| Update String [(String, Expr)] (Maybe Expr)
| CreateTable String [(String, Column)]
| CreateDomain String ColumnType
| CreateDomain String DomainDefinition
| AlterDomain String DomainAlter
| DropDomain String
| ShowDomains
| DropTable String
@@ -40,35 +52,37 @@ data Statement
| ShowGrants
| ShowRoles
| DropUser String
-- | 新角色：权限的载体，用户靠成员资格继承它的权限
| CreateRole String
-- | 删角色：连同它的授权与成员一起删
| DropRole String
-- | GRANT 权限 ON 表 TO 角色，末位表示带 WITH GRANT OPTION
| GrantPrivileges [String] String String Bool
-- | REVOKE 权限 ON 表 FROM 角色
| RevokePrivileges [String] String String
| -- | 新角色：权限的载体，用户靠成员资格继承它的权限
CreateRole String
| -- | 删角色：连同它的授权与成员一起删
DropRole String
| -- | GRANT 权限 ON 表 TO 角色，末位表示带 WITH GRANT OPTION
GrantPrivileges [String] String String Bool
| -- | REVOKE 权限 ON 表 FROM 角色
RevokePrivileges [String] String String
| RevokeGrantOption [String] String String
-- | GRANT 角色 TO 用户：给用户加角色
| GrantRole String [String]
-- | REVOKE <角色> FROM <用户>[, ...]
| RevokeRole String [String]
| -- | GRANT 角色 TO 用户：给用户加角色
GrantRole String [String]
| -- | REVOKE <角色> FROM <用户>[, ...]
RevokeRole String [String]
| CreateDatabase String
| DropDatabase String
| UseDatabase String
| ShowDatabases
-- | 开事务：BEGIN / START TRANSACTION
| BeginTransaction
-- | 提交事务
| CommitTransaction
-- | 回滚事务
| RollbackTransaction
-- | 建保存点：SAVEPOINT 名字
| Savepoint String
-- | 回滚到保存点：ROLLBACK TO [SAVEPOINT] 名字
| RollbackToSavepoint String
-- | 释放保存点：RELEASE [SAVEPOINT] 名字
| ReleaseSavepoint String
| -- | 开事务：BEGIN / START TRANSACTION
BeginTransaction
| -- | 串行化快照事务
BeginSerializableTransaction
| -- | 提交事务
CommitTransaction
| -- | 回滚事务
RollbackTransaction
| -- | 建保存点：SAVEPOINT 名字
Savepoint String
| -- | 回滚到保存点：ROLLBACK TO [SAVEPOINT] 名字
RollbackToSavepoint String
| -- | 释放保存点：RELEASE [SAVEPOINT] 名字
ReleaseSavepoint String
deriving (Show, Eq)
-- | 连接类型：内连接与左外连接
@@ -86,15 +100,16 @@ data FromClause
deriving (Show, Eq)
-- | 子查询：语句，加上它对外层列的引用（自由变量，语义检查时填）
data Subquery = Subquery
{ subqueryStatement :: Statement
, subqueryRefs :: [String]
}
data Subquery
= Subquery
{ subqueryStatement :: Statement
, subqueryRefs :: [String]
}
| BoundSubquery
{ subqueryStatement :: Statement
, subqueryRefs :: [String]
, subquerySlots :: [(String, Expr)]
}
{ subqueryStatement :: Statement
, subqueryRefs :: [String]
, subquerySlots :: [(String, Expr)]
}
deriving (Show, Eq)
-- | 量词：ANY 至少一行成立，ALL 每一行都成立
@@ -219,9 +234,9 @@ resolveDistinctOrdering :: Statement -> Either String [(OrderKey, SortDir)]
resolveDistinctOrdering q = mapM resolve (selectOrderBy q)
where
items = case q of
Select { selectItems = projected } -> projected
Select{selectItems = projected} -> projected
_ -> []
-- | 排序键必须出现在去重结果中
-- \| 排序键必须出现在去重结果中
resolve (key, direction) = do
position <- orderPosition (length items) key
case position of
@@ -230,7 +245,7 @@ resolveDistinctOrdering q = mapM resolve (selectOrderBy q)
[slot] -> Right (OrderPosition slot, direction)
[] -> Left "DISTINCT: ORDER BY column must appear in the select list"
_ -> Left "DISTINCT: ambiguous ORDER BY column"
-- | 名称或完整表达式必须对应去重输出项
-- \| 名称或完整表达式必须对应去重输出项
matches (OrderColumn column) label expression = label == column || expression == Col column
matches (OrderExpression key) _ expression = key == expression
matches (OrderPosition _) _ _ = False
chusql-core/engine/src/ChuSQL/Core/Engine/Syntax/Parser.hs
+170 -84
View File
@@ -1,16 +1,16 @@
{-# LANGUAGE TupleSections #-}
{-# LANGUAGE ImportQualifiedPost #-}
{-# LANGUAGE TupleSections #-}
module ChuSQL.Core.Engine.Syntax.Parser (parseStatement, parseExpression) where
import Data.Functor (($>))
import ChuSQL.Core.Model (Column (..), ColumnType (..), Value (..), plainColumn)
import ChuSQL.Core.Engine.Builtin (Builtin (..), builtinName, builtinNames, builtinNode, compareNode)
import ChuSQL.Core.Engine.Syntax.AST
import ChuSQL.Core.Engine.Runtime.Types
import ChuSQL.Core.Engine.Syntax.AST
import ChuSQL.Core.Model (Column (..), ColumnType (..), Value (..), plainColumn)
import Control.Monad (void)
import Control.Monad.Combinators.Expr (Operator (..), makeExprParser)
import Data.Char (isAlpha, isAlphaNum, isHexDigit, isSpace, toLower, toUpper)
import Data.Functor (($>))
import Data.List (intercalate, isInfixOf)
import Data.Maybe (fromMaybe)
import Data.Void (Void)
@@ -115,8 +115,9 @@ operatorTable =
[ [Prefix (Neg <$ symbol "-")]
, [InfixL (Mul <$ symbol "*"), InfixL (Div <$ symbol "/")]
, [InfixL (Add <$ symbol "+"), InfixL (Sub <$ symbol "-")]
, [ InfixN comparisonOp ]
, [ Postfix inPredicate
, [InfixN comparisonOp]
,
[ Postfix inPredicate
, Postfix (try quantifiedCmp)
, Postfix (IsNotNull <$ try (keyword "is" *> keyword "not" *> keyword "null"))
, Postfix (IsNull <$ (keyword "is" *> keyword "null"))
@@ -210,22 +211,29 @@ subquerySelect = Subquery <$> queryStatement <*> pure []
-- | 读取带具体类型参数的构造器
constructorCall :: Parser Expr
constructorCall = do
cid <- (TypeConstructorId 5 <$ keyword "list") <|> (TypeConstructorId 6 <$ keyword "maybe")
<|> (TypeConstructorId 7 <$ keyword "tuple")
cid <-
(TypeConstructorId 5 <$ keyword "list")
<|> (TypeConstructorId 6 <$ keyword "maybe")
<|> (TypeConstructorId 7 <$ keyword "tuple")
arguments <- between (symbol "<") (symbol ">") (sepBy runtimeTypeP (symbol ","))
values <- between (symbol "(") (symbol ")") (sepBy expr (symbol ","))
pure (Construct (TypeApply cid arguments) values)
-- | 解析运行时逻辑类型语法
runtimeTypeP :: Parser TypeExpr
runtimeTypeP = choice
[ intType <$ keyword "int", doubleType <$ keyword "double", boolType <$ keyword "bool"
, stringType <$ keyword "string", stringType <$ keyword "str", stringType <$ keyword "text"
, listType <$> between (symbol "[") (symbol "]") runtimeTypeP
, listType <$> (keyword "list" *> runtimeTypeP)
, maybeType <$> (keyword "maybe" *> runtimeTypeP)
, tupleType <$> between (symbol "(") (symbol ")") (sepBy runtimeTypeP (symbol ","))
]
runtimeTypeP =
choice
[ intType <$ keyword "int"
, doubleType <$ keyword "double"
, boolType <$ keyword "bool"
, stringType <$ keyword "string"
, stringType <$ keyword "str"
, stringType <$ keyword "text"
, listType <$> between (symbol "[") (symbol "]") runtimeTypeP
, listType <$> (keyword "list" *> runtimeTypeP)
, maybeType <$> (keyword "maybe" *> runtimeTypeP)
, tupleType <$> between (symbol "(") (symbol ")") (sepBy runtimeTypeP (symbol ","))
]
-- | 读取待绑定的标量函数调用
scalarFunctionCall :: Parser Expr
@@ -238,11 +246,11 @@ scalarFunctionCall = do
aggregateCall :: Parser Expr
aggregateCall = choice (countAll : map named builtinNames)
where
-- | COUNT(*)，没有参数
-- \| COUNT(*)，没有参数
countAll = CountAll <$ try (keyword (builtinName BCountAll) *> between (symbol "(") (symbol ")") (symbol "*"))
-- | 表里每一个带参数的内置函数
-- \| 表里每一个带参数的内置函数
named (name, builtin) = builtinNode builtin <$> aggArg name
-- | 读聚合函数的参数
-- \| 读聚合函数的参数
aggArg name = try (keyword name *> between (symbol "(") (symbol ")") expr)
-- | 只认字面量，给 DEFAULT 用
@@ -252,14 +260,11 @@ literalValue =
[ VNull <$ keyword "null"
, VBool True <$ keyword "TRUE"
, VBool False <$ keyword "FALSE"
, try (negateLit <$> floatLit)
, try (symbol "-" *> (VFloat . negate <$> try floatLit <|> VInt . negate <$> integer))
, try (VFloat <$> floatLit)
, VInt <$> integer
, VFloat <$> try floatLit
, VStr <$> stringLit
]
where
-- | 负浮点字面量
negateLit d = VFloat (negate d)
-- | 一个排序键：输出序号、列名或表达式
orderItem :: Parser (OrderKey, SortDir)
@@ -365,8 +370,10 @@ tableRef = do
-- | 连接类型：内连接 / 左外连接
joinKind :: Parser JoinKind
joinKind =
try ((keyword "left" *> optional (keyword "outer") *> keyword "join")
$> LeftJoin)
try
( (keyword "left" *> optional (keyword "outer") *> keyword "join")
$> LeftJoin
)
<|> try ((keyword "inner" *> keyword "join") $> InnerJoin)
<|> (keyword "join" $> InnerJoin)
@@ -430,9 +437,10 @@ createTableStatement = do
-- | 读 DROP TABLE
dropTableStatement :: Parser Statement
dropTableStatement = do keyword "drop"
keyword "table"
DropTable <$> tableName
dropTableStatement = do
keyword "drop"
keyword "table"
DropTable <$> tableName
-- | 读 CREATE INDEX（列名即索引名）
createIndexStatement :: Parser Statement
@@ -491,10 +499,11 @@ alterTableStatement = do
-- | 读 CREATE USER，口令是明文
createUserStatement :: Parser Statement
createUserStatement = do keyword "create"
keyword "user"
name <- userName
CreateUser name <$> identifiedBy
createUserStatement = do
keyword "create"
keyword "user"
name <- userName
CreateUser name <$> identifiedBy
-- | 读 IDENTIFIED BY '口令'
identifiedBy :: Parser String
@@ -513,15 +522,19 @@ alterUserStatement = do
-- | 读身份属性开关
identityAttribute :: Parser (String, Bool)
identityAttribute = choice
[ ("allow_sudo_auth", True) <$ keyword "allow_sudo_auth"
, ("allow_sudo_auth", False) <$ keyword "noallow_sudo_auth"
, ("system_catalog_manager", True) <$ keyword "system_catalog_manager"
, ("system_catalog_manager", False) <$ keyword "nosystem_catalog_manager"
, ("login", True) <$ keyword "login", ("login", False) <$ keyword "nologin"
, ("superuser", True) <$ keyword "superuser", ("superuser", False) <$ keyword "nosuperuser"
, ("enabled", True) <$ keyword "enabled", ("enabled", False) <$ keyword "disabled"
]
identityAttribute =
choice
[ ("allow_sudo_auth", True) <$ keyword "allow_sudo_auth"
, ("allow_sudo_auth", False) <$ keyword "noallow_sudo_auth"
, ("system_catalog_manager", True) <$ keyword "system_catalog_manager"
, ("system_catalog_manager", False) <$ keyword "nosystem_catalog_manager"
, ("login", True) <$ keyword "login"
, ("login", False) <$ keyword "nologin"
, ("superuser", True) <$ keyword "superuser"
, ("superuser", False) <$ keyword "nosuperuser"
, ("enabled", True) <$ keyword "enabled"
, ("enabled", False) <$ keyword "disabled"
]
-- | 读 ALTER ROLE 属性或口令
alterRoleStatement :: Parser Statement
@@ -533,9 +546,10 @@ alterRoleStatement = do
-- | 读 DROP USER
dropUserStatement :: Parser Statement
dropUserStatement = do keyword "drop"
keyword "user"
DropUser <$> userName
dropUserStatement = do
keyword "drop"
keyword "user"
DropUser <$> userName
-- | 读 CREATE ROLE
createRoleStatement :: Parser Statement
@@ -566,9 +580,11 @@ privilegeName =
-- | ON 后面的授权对象：* 或一张表
grantObject :: Parser String
grantObject = try (("database:" ++) <$> (keyword "database" *> identifier))
<|> try ((++ ".*") <$> (identifier <* symbol "." <* symbol "*"))
<|> ("*" <$ symbol "*") <|> (optional (keyword "table") *> tableName)
grantObject =
try (("database:" ++) <$> (keyword "database" *> identifier))
<|> try ((++ ".*") <$> (identifier <* symbol "." <* symbol "*"))
<|> ("*" <$ symbol "*")
<|> (optional (keyword "table") *> tableName)
-- | 可选的 WITH GRANT OPTION
grantOption :: Parser Bool
@@ -603,6 +619,64 @@ revokeStatement = do
, if optionOnly then fail "GRANT OPTION FOR requires object privileges" else RevokeRole <$> userName <*> (keyword "from" *> sepBy1 userName (symbol ","))
]
-- | 读取领域及其独立约束
createDomainStatement :: Parser Statement
createDomainStatement = do
name <- keyword "create" *> keyword "domain" *> identifier
ty <- keyword "as" *> columnTypeP
modifiers <- many domainConstraint
pure (CreateDomain name (foldl (flip ($)) (DomainDefinition (plainColumn ty) []) modifiers))
-- | 读取领域属性或 CHECK 条件
domainConstraint :: Parser (DomainDefinition -> DomainDefinition)
domainConstraint =
choice
[ try (keyword "not" *> keyword "null") $> domainNull False
, keyword "null" $> domainNull True
, keyword "default" *> (domainDefault . Just <$> literalValue)
, do
name <- optional (keyword "constraint" *> identifier)
text <- keyword "check" *> checkText
pure (domainCheck (fromMaybe "" name) text)
]
-- | 设置领域非空声明
domainNull :: Bool -> DomainDefinition -> DomainDefinition
domainNull nullable (DomainDefinition column checks) = DomainDefinition column{columnNullable = nullable} checks
-- | 设置领域默认值
domainDefault :: Maybe Value -> DomainDefinition -> DomainDefinition
domainDefault value (DomainDefinition column checks) = DomainDefinition column{columnDefault = value} checks
-- | 增加领域检查条件
domainCheck :: String -> String -> DomainDefinition -> DomainDefinition
domainCheck name text (DomainDefinition column checks) = DomainDefinition column (checks ++ [(name, text)])
-- | 读取领域约束与默认值变更
alterDomainStatement :: Parser Statement
alterDomainStatement = do
name <- keyword "alter" *> keyword "domain" *> identifier
action <-
choice
[ keyword "set"
*> choice
[ DomainDefault . Just <$> (keyword "default" *> literalValue)
, DomainNullable False <$ (keyword "not" *> keyword "null")
]
, keyword "drop"
*> choice
[ DomainDefault Nothing <$ keyword "default"
, DomainNullable True <$ (keyword "not" *> keyword "null")
, DomainDropCheck <$> (keyword "constraint" *> identifier)
]
, keyword "add"
*> ( DomainAddCheck
<$> optional (keyword "constraint" *> identifier)
<*> (keyword "check" *> checkText)
)
]
pure (AlterDomain name action)
-- | 建表时的一列：类型后面跟若干约束
columnDef :: Parser (String, Column)
columnDef = do
@@ -615,18 +689,24 @@ columnDef = do
columnConstraint :: Parser (Column -> Column)
columnConstraint =
choice
[ try ((keyword "not" *> keyword "null")
$> (\ c -> c {columnNullable = False}))
, keyword "null" $> (\ c -> c {columnNullable = True})
[ try
( (keyword "not" *> keyword "null")
$> (\c -> c{columnNullable = False})
)
, keyword "null" $> (\c -> c{columnNullable = True})
, keyword "auto_increment"
$>
(\ c -> c {columnAutoIncrement = True, columnNullable = False})
, try ((keyword "primary" *> keyword "key")
$>
(\ c
-> c {columnPrimaryKey = True, columnUnique = True,
columnNullable = False}))
, keyword "unique" $> (\ c -> c {columnUnique = True})
$> (\c -> c{columnAutoIncrement = True, columnNullable = False})
, try
( (keyword "primary" *> keyword "key")
$> ( \c ->
c
{ columnPrimaryKey = True
, columnUnique = True
, columnNullable = False
}
)
)
, keyword "unique" $> (\c -> c{columnUnique = True})
, keyword "default" *> ((\v c -> c{columnDefault = Just v}) <$> literalValue)
, keyword "check" *> ((\s c -> c{columnCheck = Just s}) <$> checkText)
]
@@ -669,13 +749,14 @@ columnTypeP =
-- | 解析复合列的稳定类型身份
runtimeColumnType :: Parser ColumnType
runtimeColumnType = do
expression <- choice
[ listType <$> between (symbol "[") (symbol "]") runtimeTypeP
, listType <$> (keyword "list" *> runtimeTypeP)
, maybeType <$> (keyword "maybe" *> runtimeTypeP)
, tupleType <$> between (symbol "(") (symbol ")") (sepBy runtimeTypeP (symbol ","))
, stringType <$ keyword "string"
]
expression <-
choice
[ listType <$> between (symbol "[") (symbol "]") runtimeTypeP
, listType <$> (keyword "list" *> runtimeTypeP)
, maybeType <$> (keyword "maybe" *> runtimeTypeP)
, tupleType <$> between (symbol "(") (symbol ")") (sepBy runtimeTypeP (symbol ","))
, stringType <$ keyword "string"
]
case resolveType builtinTypes expression of
Left err -> fail err
Right descriptor -> pure (CRuntime (descriptorId descriptor))
@@ -734,7 +815,7 @@ queryStatement = withStatement <|> selectStatement
expandWith :: [(String, [String], Statement)] -> Statement -> Either String Statement
expandWith defs main = go [] defs
where
-- | 前面的 CTE 已进环境，接着展开本体
-- \| 前面的 CTE 已进环境，接着展开本体
go env [] = expandStatement env main
go env ((name, cols, body) : rest) = do
body' <- expandStatement env body
@@ -749,14 +830,15 @@ cteKey = map toLower
renameColumns :: String -> [String] -> Statement -> Either String Statement
renameColumns _ [] q = Right q
renameColumns name cols (SelectDistinct q) = SelectDistinct <$> renameColumns name cols q
renameColumns name cols q@Select { selectItems = items } = do
renameColumns name cols q@Select{selectItems = items} = do
checkCteArity name cols (map snd items)
ordering <- if all (\(label, expression) -> expression == Col label) items
then Right (selectOrderBy q)
else mapM renameOrder (selectOrderBy q)
Right q { selectItems = zip cols (map snd items), selectOrderBy = ordering }
ordering <-
if all (\(columnName, expression) -> expression == Col columnName) items
then Right (selectOrderBy q)
else mapM renameOrder (selectOrderBy q)
Right q{selectItems = zip cols (map snd items), selectOrderBy = ordering}
where
-- | 列清单同步改写内层排序输出别名
-- \| 列清单同步改写内层排序输出别名
renameOrder (OrderColumn column, direction) = case [new | ((old, _), new) <- zip items cols, old == column] of
[] -> Right (OrderColumn column, direction)
[new] -> Right (OrderColumn new, direction)
@@ -774,16 +856,16 @@ checkCteArity name cols out
expandStatement :: [(String, Statement)] -> Statement -> Either String Statement
expandStatement env stmt = case stmt of
SelectDistinct q -> SelectDistinct <$> expandStatement env q
q@Select { selectFrom = fromC, selectItems = items, selectWhere = mWhere } -> do
q@Select{selectFrom = fromC, selectItems = items, selectWhere = mWhere} -> do
fromC' <- expandClause env fromC
items' <- traverse (\(name, e) -> fmap (name,) (expandExpr env e)) items
w <- traverse (expandExpr env) mWhere
having <- traverse (expandExpr env) (selectHaving q)
ordering <- mapM expandOrder (selectOrderBy q)
Right q { selectFrom = fromC', selectItems = items', selectWhere = w, selectHaving = having, selectOrderBy = ordering }
Right q{selectFrom = fromC', selectItems = items', selectWhere = w, selectHaving = having, selectOrderBy = ordering}
_ -> Right stmt
where
-- | 展开排序表达式中的 CTE 子查询
-- \| 展开排序表达式中的 CTE 子查询
expandOrder (OrderExpression expression, direction) = (,) . OrderExpression <$> expandExpr env expression <*> pure direction
expandOrder ordering = Right ordering
@@ -832,10 +914,10 @@ expandExpr env e = case e of
QuantCmp op x sq q -> (\x' sq' -> QuantCmp op x' sq' q) <$> go x <*> goSub sq
_ -> Right e
where
-- | 继续看子表达式
-- \| 继续看子表达式
go = expandExpr env
-- | 继续看子查询里的语句
goSub sq = (\stmt -> sq { subqueryStatement = stmt }) <$> expandStatement env (subqueryStatement sq)
-- \| 继续看子查询里的语句
goSub sq = (\stmt -> sq{subqueryStatement = stmt}) <$> expandStatement env (subqueryStatement sq)
-- | 读 INSERT，可一次插多行
insertStatement :: Parser Statement
@@ -878,15 +960,18 @@ updateStatement = do
-- | 事务控制语句
transactionStatement :: Parser Statement
transactionStatement =
try (BeginTransaction <$ (keyword "start" *> keyword "transaction"))
<|> (BeginTransaction <$ (keyword "begin" *> optional (keyword "transaction")))
(beginMode =<< (try (keyword "start" *> keyword "transaction") <|> (keyword "begin" *> void (optional (keyword "transaction")))))
<|> (CommitTransaction <$ (keyword "commit" *> optional (keyword "transaction")))
<|> try (RollbackToSavepoint <$> rollbackToName)
<|> (RollbackTransaction <$ (keyword "rollback" *> optional (keyword "transaction")))
<|> (Savepoint <$> (keyword "savepoint" *> identifier))
<|> (ReleaseSavepoint <$> (keyword "release" *> optional (keyword "savepoint") *> identifier))
where
-- | ROLLBACK TO [SAVEPOINT] 名字
-- 解析可选的串行化隔离级别。
beginMode _ = do
mode <- optional (keyword "isolation" *> keyword "level" *> keyword "serializable")
pure (maybe BeginTransaction (const BeginSerializableTransaction) mode)
-- \| ROLLBACK TO [SAVEPOINT] 名字
rollbackToName =
keyword "rollback" *> keyword "to" *> optional (keyword "savepoint") *> identifier
@@ -897,7 +982,7 @@ parseStatement input =
Left err -> Left (safeParseError input err)
Right q -> Right q
where
-- | 按关键字分派到各语句解析器
-- \| 按关键字分派到各语句解析器
statementP =
withStatement
<|> transactionStatement
@@ -907,7 +992,8 @@ parseStatement input =
<|> (UseDatabase <$> (keyword "use" *> identifier))
<|> try (ShowDatabases <$ (keyword "show" *> keyword "databases"))
<|> try (ShowDomains <$ (keyword "show" *> keyword "domains"))
<|> try (CreateDomain <$> (keyword "create" *> keyword "domain" *> identifier) <*> (keyword "as" *> columnTypeP))
<|> try createDomainStatement
<|> try alterDomainStatement
<|> try (DropDomain <$> (keyword "drop" *> keyword "domain" *> identifier))
<|> insertStatement
<|> deleteStatement
chusql-core/engine/test/NormalizationSpec.hs
+57 -19
View File
@@ -19,49 +19,87 @@ spec = do
`shouldBe` True
attributeClosure [] ["a"] `shouldBe` ["a"]
it "accepts BCNF with a declared key" $ do
fmap (map verdict . findings) (analyzeNormalization schema
(Declaration [["a"]] [] True (Just True)))
fmap
(map verdict . findings)
( analyzeNormalization
schema
(Declaration [["a"]] [] True (Just True))
)
`shouldBe` Right (replicate 4 Satisfied)
it "detects partial dependencies of composite keys" $ do
fmap (map verdict . findings) (analyzeNormalization schema
(Declaration [["a", "b"]] [Dependency ["a"] ["c"]] True (Just True)))
fmap
(map verdict . findings)
( analyzeNormalization
schema
(Declaration [["a", "b"]] [Dependency ["a"] ["c"]] True (Just True))
)
`shouldBe` Right [Satisfied, Violated, Violated, Violated]
it "detects transitive dependencies and a preserving split" $ do
let analysis = analyzeNormalization schema
(Declaration [["a"]] [Dependency ["b"] ["c"]] True (Just True))
let analysis =
analyzeNormalization
schema
(Declaration [["a"]] [Dependency ["b"] ["c"]] True (Just True))
fmap (map verdict . findings) analysis
`shouldBe` Right [Satisfied, Satisfied, Violated, Violated]
fmap suggestions analysis
`shouldBe` Right [Decomposition [["b", "c"], ["a", "b"]] Satisfied Satisfied]
it "distinguishes 3NF from BCNF and detects dependency loss" $ do
let analysis = analyzeNormalization schema
(Declaration [["a", "b"]] [Dependency ["c"] ["b"]] True (Just True))
let analysis =
analyzeNormalization
schema
(Declaration [["a", "b"]] [Dependency ["c"] ["b"]] True (Just True))
fmap (map verdict . findings) analysis
`shouldBe` Right [Satisfied, Satisfied, Satisfied, Violated]
fmap suggestions analysis
`shouldBe` Right [Decomposition [["b", "c"], ["a", "c"]] Satisfied Violated]
it "does not certify incomplete dependencies" $ do
fmap (map verdict . findings) (analyzeNormalization schema
(Declaration [["a"]] [] False (Just True)))
fmap
(map verdict . findings)
( analyzeNormalization
schema
(Declaration [["a"]] [] False (Just True))
)
`shouldBe` Right [Satisfied, Insufficient, Insufficient, Insufficient]
it "requires an atomicity declaration" $ do
fmap (map verdict . findings) (analyzeNormalization schema
(Declaration [] [] True Nothing)) `shouldBe` Right (replicate 4 Insufficient)
fmap
(map verdict . findings)
( analyzeNormalization
schema
(Declaration [] [] True Nothing)
)
`shouldBe` Right (replicate 4 Insufficient)
it "propagates a declared 1NF violation" $ do
fmap (map verdict . findings) (analyzeNormalization schema
(Declaration [] [] True (Just False))) `shouldBe` Right (replicate 4 Violated)
fmap
(map verdict . findings)
( analyzeNormalization
schema
(Declaration [] [] True (Just False))
)
`shouldBe` Right (replicate 4 Violated)
it "rejects unknown attributes" $ do
analyzeNormalization schema (Declaration [] [Dependency ["z"] ["a"]] True (Just True))
`shouldBe` Left "normalization: unknown or duplicate dependency attributes"
it "rejects nonminimal declared candidate keys" $ do
analyzeNormalization schema (Declaration [["a", "b"]]
[Dependency ["a"] ["b", "c"]] True (Just True))
analyzeNormalization
schema
( Declaration
[["a", "b"]]
[Dependency ["a"] ["b", "c"]]
True
(Just True)
)
`shouldBe` Left "normalization: declared candidate key is not minimal"
it "includes the empty candidate key for constant relations" $ do
fmap candidateKeys (analyzeNormalization schema
(Declaration [] [Dependency [] ["a", "b", "c"]] True (Just True))) `shouldBe` Right [[]]
fmap
candidateKeys
( analyzeNormalization
schema
(Declaration [] [Dependency [] ["a", "b", "c"]] True (Just True))
)
`shouldBe` Right [[]]
it "rejects excessive exhaustive searches" $ do
analyzeNormalization [(show n, TInt) | n <- [1 :: Int .. 13]]
analyzeNormalization
[(show n, TInt) | n <- [1 :: Int .. 13]]
(Declaration [] [] True (Just True))
`shouldBe` Left "normalization: exhaustive analysis supports at most 12 columns"
where
chusql-core/engine/test/RuntimeSpec.hs
+550 -196
View File
File diff suppressed because it is too large. Load diff
chusql-core/engine/test/Spec.hs
+543 -180
View File
File diff suppressed because it is too large. Load diff
chusql-core/model/src/ChuSQL/Core/Model.hs
+85 -79
View File
@@ -1,57 +1,58 @@
{-# LANGUAGE PatternSynonyms #-}
module ChuSQL.Core.Model
( ColumnType (..)
, Column (..)
, pattern TInt
, pattern TStr
, pattern TBool
, TypeClass (..)
, Value (..)
, Row
, TableMeta (..)
, Histogram (..)
, Table (..)
, TableIdentity (..)
, Database
, histogramOf
, histogramFor
, colNames
, colType
, lookupTable
, resolveTable
, lastTablePart
, deriveQualifier
, allColumns
, qualify
, unqualify
, resolveColumn
, plainColumn
, typeLabel
, typeName
, parseColumnType
, domainTableName
, isDomainTable
, typeClassOf
, assignable
, comparableTypes
, integerType
, numericType
, valueFits
, coerceValue
, canonicalValue
, compareValue
, valuesEqual
, hashValue
) where
module ChuSQL.Core.Model (
ColumnType (..),
Column (..),
pattern TInt,
pattern TStr,
pattern TBool,
TypeClass (..),
Value (..),
Row,
TableMeta (..),
Histogram (..),
Table (..),
TableIdentity (..),
Database,
histogramOf,
histogramFor,
colNames,
colType,
lookupTable,
resolveTable,
lastTablePart,
deriveQualifier,
allColumns,
qualify,
unqualify,
resolveColumn,
plainColumn,
typeLabel,
typeName,
parseColumnType,
domainTableName,
isDomainTable,
typeClassOf,
assignable,
comparableTypes,
hasTypeCapability,
integerType,
numericType,
valueFits,
coerceValue,
canonicalValue,
compareValue,
valuesEqual,
hashValue,
) where
import ChuSQL.Core.Runtime
import qualified Data.ByteString.Lazy.Char8 as BS
import Data.Char (isDigit, isSpace, toLower)
import Data.Hashable (Hashable (..))
import Data.List (isPrefixOf, isSuffixOf, stripPrefix)
import Data.Maybe (fromMaybe, mapMaybe)
import Data.Word (Word64)
import ChuSQL.Core.Runtime
import qualified Data.ByteString.Lazy.Char8 as BS
-- 数据模型：列类型、列约束、值、行、表、数据库，外加查表小工具。
@@ -94,20 +95,20 @@ plainColumn t = Column t True Nothing False False False Nothing
-- | 普通整型列的匹配模式
pattern TInt :: Column
pattern TInt <- Column CInt _ _ _ _ _ _
where
TInt = plainColumn CInt
where
TInt = plainColumn CInt
-- | 普通字符串列的匹配模式
pattern TStr :: Column
pattern TStr <- Column CStr _ _ _ _ _ _
where
TStr = plainColumn CStr
where
TStr = plainColumn CStr
-- | 普通布尔列的匹配模式
pattern TBool :: Column
pattern TBool <- Column CBool _ _ _ _ _ _
where
TBool = plainColumn CBool
where
TBool = plainColumn CBool
-- | 错误消息里用的类型名
typeLabel :: ColumnType -> String
@@ -192,11 +193,11 @@ parseColumnType raw
("blob", []) -> Just CBlob
_ -> either (const Nothing) (Just . CRuntime . descriptorId) (parseTypeExpr raw >>= resolveType builtinTypes)
where
-- | 去掉空白的类型名
-- \| 去掉空白的类型名
cleaned = map toLower (filter (not . isSpace) raw)
-- | 基础名与参数段
-- \| 基础名与参数段
(base, rest) = span (/= '(') cleaned
-- | 括号里的参数表
-- \| 括号里的参数表
args = case stripSuffixMaybe ")" (drop 1 rest) of
Nothing -> []
Just inner -> map (filter (not . isSpace)) (splitOn ',' inner)
@@ -265,6 +266,8 @@ typeClassOf CRuntime{} = RuntimeClass
-- | 同大类可赋值；日期/时间列接受字符串字面量
assignable :: ColumnType -> ColumnType -> Bool
assignable (CDomain _ base) source = assignable base source
assignable target (CDomain _ base) = assignable target base
assignable target source
| source == CNull = True
| target == source = True
@@ -284,19 +287,29 @@ integerType _ = False
-- | 比较用的类型族：同大类可比，日期时间与文本互比
comparableTypes :: ColumnType -> ColumnType -> Bool
comparableTypes (CDomain _ base) other = comparableTypes base other
comparableTypes other (CDomain _ base) = comparableTypes other base
comparableTypes a b
| typeClassOf a == RuntimeClass || typeClassOf b == RuntimeClass = False
| a == CNull || b == CNull = True
| typeClassOf a == RuntimeClass || typeClassOf b == RuntimeClass = a == b && hasTypeCapability EqCapability a
| otherwise = a == CNull || b == CNull || sameClass || crossTemporal
where
-- | 两个类型同属一个大类
-- \| 两个类型同属一个大类
sameClass = typeClassOf a == typeClassOf b
-- | 时间与文本放在一起比
-- \| 时间与文本放在一起比
crossTemporal = (temporal a && textual b) || (textual a && temporal b)
-- | 是不是时间类型
-- \| 是不是时间类型
temporal t = typeClassOf t == TemporalClass
-- | 是不是文本类型
-- \| 是不是文本类型
textual t = typeClassOf t == TextClass
-- | 领域沿用基础类型的已注册值运算能力
hasTypeCapability :: TypeCapability -> ColumnType -> Bool
hasTypeCapability capability (CDomain _ base) = hasTypeCapability capability base
hasTypeCapability capability (CRuntime tid) =
either (const False) (elem capability . descriptorCapabilities) (describeType builtinTypes tid)
hasTypeCapability _ _ = True
-- | 是不是数值类型
numericType :: ColumnType -> Bool
numericType t = typeClassOf t == NumericClass
@@ -380,9 +393,9 @@ toDecimal p s v
VFloat d -> rounded d
_ -> Left ("cannot put this value into " ++ typeLabel (CDecimal p s))
where
-- | 缩放因子
-- \| 缩放因子
scale = (10 :: Double) ^ s
-- | 按比例四舍五入
-- \| 按比例四舍五入
rounded d
| isNaN d || isInfinite d || isInfinite (d * scale) = Left "decimal value is not finite or exceeds precision"
| abs result >= (10 :: Double) ^ (p - s) = Left "decimal value exceeds precision"
@@ -443,7 +456,6 @@ validTimeText t = case splitOn ':' t of
allOf :: Int -> String -> Bool
allOf n s = length s == n && all isDigit s
-- | 引擎里的一行值
data Value
= VNull
@@ -480,7 +492,7 @@ compareValue a b = case (canonicalValue a, canonicalValue b) of
(VFloat x, VFloat y) -> compare x y
(VStr x, VStr y) -> compare x y
(VBool x, VBool y) -> compare x y
(VRuntime tx x, VRuntime ty y) -> compare (encodeType tx, show (canonicalRuntime x)) (encodeType ty, show (canonicalRuntime y))
(VRuntime tx x, VRuntime ty y) -> compare (tx, x) (ty, y)
(x, y) -> compare (valueRank x) (valueRank y)
-- | 族的先后，只用来给异族值定序
@@ -504,7 +516,7 @@ hashValue v = case canonicalValue v of
VFloat d -> hashWithSalt 1 d
VStr t -> hashWithSalt 2 t
VBool b -> hashWithSalt 3 b
VRuntime tid value -> hashWithSalt 4 (BS.unpack (encodeType tid), show (canonicalRuntime value))
VRuntime tid value -> hashWithSalt 4 (BS.unpack (encodeType tid), value)
-- | 解析现有标量列的逻辑类型
scalarTypeExpression :: ColumnType -> Maybe TypeExpr
@@ -526,14 +538,6 @@ scalarRuntimeValue (VBool b) = Just (RBool b)
scalarRuntimeValue (VStr text) = Just (RString text)
scalarRuntimeValue _ = Nothing
-- | 统一复合值中的浮点零关键字
canonicalRuntime :: RuntimeValue -> RuntimeValue
canonicalRuntime (RDouble n) | n == 0 = RDouble 0
canonicalRuntime (RList values) = RList (map canonicalRuntime values)
canonicalRuntime (RMaybe value) = RMaybe (fmap canonicalRuntime value)
canonicalRuntime (RTuple values) = RTuple (map canonicalRuntime values)
canonicalRuntime value = value
-- | 值按类型打散搅拌
instance Hashable Value where
hashWithSalt s = hashWithSalt s . hashValue
@@ -635,11 +639,11 @@ resolveTable db name = case lookup name db of
| [(key, table)] <- suffixed -> Right (key, table)
| otherwise -> Left (miss suffixed)
where
-- | 引用里去掉库前缀的那一段
-- \| 引用里去掉库前缀的那一段
bare = lastTablePart name
-- | schema 里以这个表名收尾的键：`sales.users` 能推出裸表名引用
-- \| schema 里以这个表名收尾的键：`sales.users` 能推出裸表名引用
suffixed = [(key, table) | (key, table) <- db, ('.' : name) `isSuffixOf` key]
-- | 一个候选都没有就是查无此表，多于一个就是有歧义
-- \| 一个候选都没有就是查无此表，多于一个就是有歧义
miss [] = "unknown table: " ++ name
miss _ = "ambiguous table: " ++ name
@@ -664,6 +668,8 @@ resolveColumn name env = case [(k, v) | (k, v) <- env, matches k] of
[] -> Left ("unknown column: " ++ name)
_ -> Left ("ambiguous column: " ++ name)
where
-- | 裸名匹配末段，库限定名只补一层前缀
matches k = k == name || ('.' : name) `isSuffixOf` k
|| (length (filter (== '.') name) == 2 && length (filter (== '.') k) == 1 && ('.' : k) `isSuffixOf` name)
-- \| 裸名匹配末段，库限定名只补一层前缀
matches k =
k == name
|| ('.' : name) `isSuffixOf` k
|| (length (filter (== '.') name) == 2 && length (filter (== '.') k) == 1 && ('.' : k) `isSuffixOf` name)
chusql-core/model/src/ChuSQL/Core/Protocol.hs
+30
View File
@@ -105,6 +105,14 @@ data Request
| ReqDropIndex String String
| ReqDropColumn String String
| ReqReplaceSchema String [SchemaColumn] [Row]
| ReqDomainSnapshot
| ReqDatabaseSnapshot (Maybe String)
| ReqMvccBegin Bool
| ReqMvccLock String [TxnOp] [String]
| ReqMvccPreview String
| ReqMvccCommit String String
| ReqMvccRollback String
| ReqAlterDomain String [SchemaColumn] String
| ReqListCatalog
| ReqApplyTransaction [TxnOp]
@@ -253,6 +261,14 @@ instance ToJSON Request where
, "columns" .= cols
, "rows" .= map rowToJSON rs
]
toJSON (ReqMvccBegin serializable) = object ["method" .= ("mvcc_begin" :: T.Text), "serializable" .= serializable]
toJSON (ReqMvccLock token ops dependencies) = object ["method" .= ("mvcc_lock" :: T.Text), "transaction" .= token, "ops" .= ops, "reads" .= dependencies]
toJSON (ReqMvccPreview token) = object ["method" .= ("mvcc_preview" :: T.Text), "transaction" .= token]
toJSON (ReqMvccCommit token checkpoint) = object ["method" .= ("mvcc_commit" :: T.Text), "transaction" .= token, "checkpoint" .= checkpoint, "ops" .= ([] :: [TxnOp])]
toJSON (ReqMvccRollback token) = object ["method" .= ("mvcc_rollback" :: T.Text), "transaction" .= token]
toJSON ReqDomainSnapshot = object ["method" .= ("domain_snapshot" :: T.Text)]
toJSON (ReqDatabaseSnapshot checkpoint) = object ["method" .= ("database_snapshot" :: T.Text), "checkpoint" .= checkpoint]
toJSON (ReqAlterDomain name columns checkpoint) = object ["method" .= ("alter_domain" :: T.Text), "table" .= name, "columns" .= columns, "checkpoint" .= checkpoint]
toJSON ReqListCatalog = object ["method" .= ("list_catalog" :: T.Text)]
toJSON (ReqApplyTransaction ops) =
object
@@ -418,6 +434,9 @@ data Response
| RespOk
| RespError String
| RespCatalog [TableInfo]
| RespDomainSnapshot [TableInfo] [(String, [Row])] String
| RespSnapshotUnchanged String
| RespMvccSnapshot String [TableInfo] [(String, [Row])] String
-- | JSON 解回响应
instance FromJSON Response where
@@ -432,6 +451,17 @@ instance FromJSON Response where
"tables" -> RespTables <$> o .: "tables"
"error" -> RespError <$> o .: "message"
"schema" -> RespSchema <$> parseJSON (A.Object o)
"snapshot_unchanged" -> RespSnapshotUnchanged <$> o .: "checkpoint"
"mvcc_snapshot" -> do
schemas <- o .: "schemas"
tables <- o .: "rows" :: Parser [(String, [A.Value])]
rows <- mapM (\(name, values) -> (,) name <$> mapM rowFromJSON values) tables
RespMvccSnapshot <$> o .: "transaction" <*> pure schemas <*> pure rows <*> o .: "checkpoint"
"domain_snapshot" -> do
schemas <- o .: "schemas"
tables <- o .: "rows" :: Parser [(String, [A.Value])]
rows <- mapM (\(name, values) -> (,) name <$> mapM rowFromJSON values) tables
RespDomainSnapshot schemas rows <$> o .: "checkpoint"
"catalog" -> do
xs <- o .: "schemas" :: Parser [A.Value]
RespCatalog <$> mapM parseJSON xs
chusql-core/model/src/ChuSQL/Core/Runtime.hs
+110 -38
View File
@@ -1,19 +1,45 @@
module ChuSQL.Core.Runtime
( TypeConstructorId (..), TypeId, TypeExpr (..), Kind (..)
, TypeRegistry, TypeDescriptor (..), PhysicalType (..), RuntimeValue (..)
, builtinTypes, resolveType, describeType, typeExpression
, intType, doubleType, boolType, stringType, listType, maybeType, tupleType, renderTypeExpr, parseTypeExpr, renderRuntimeValue
, validateValue, encodeValue, decodeValue, encodeType, decodeType, runtimeToJSON, runtimeFromJSON
) where
module ChuSQL.Core.Runtime (
TypeConstructorId (..),
TypeId,
TypeExpr (..),
Kind (..),
TypeRegistry,
TypeDescriptor (..),
TypeCapability (..),
PhysicalType (..),
RuntimeValue (..),
builtinTypes,
resolveType,
describeType,
typeExpression,
intType,
doubleType,
boolType,
stringType,
listType,
maybeType,
tupleType,
renderTypeExpr,
parseTypeExpr,
renderRuntimeValue,
validateValue,
encodeValue,
decodeValue,
encodeType,
decodeType,
runtimeToJSON,
runtimeFromJSON,
) where
import Control.Monad (zipWithM, zipWithM_)
import Data.Aeson (Value (..), eitherDecode, encode)
import Data.Aeson.Types (Parser, parseEither)
import qualified Data.ByteString.Lazy as BS
import Data.Char (isAlphaNum, toLower)
import Data.Hashable (Hashable (..))
import Data.Scientific (floatingOrInteger)
import qualified Data.Text as T
import qualified Data.Vector as V
import Data.Char (isAlphaNum, toLower)
import qualified Text.ParserCombinators.ReadP as P
-- 运行时类型身份、构造器注册及版本化物理编码。
@@ -31,30 +57,66 @@ newtype TypeId = TypeId TypeExpr deriving (Show, Eq, Ord)
data Kind = TypeKind | TypeArrow Kind Kind | ProductKind deriving (Show, Eq)
-- | 描述独立的物理表示
data PhysicalType = SignedInteger | FloatingPoint | Boolean | Utf8 | Sequence PhysicalType
| Optional PhysicalType | Product [PhysicalType] deriving (Show, Eq)
data PhysicalType
= SignedInteger
| FloatingPoint
| Boolean
| Utf8
| Sequence PhysicalType
| Optional PhysicalType
| Product [PhysicalType]
deriving (Show, Eq)
-- | 类型实例可提供的值运算
data TypeCapability = EqCapability | OrdCapability | HashCapability deriving (Show, Eq)
-- | 保存类型实例描述
data TypeDescriptor = TypeDescriptor
{ descriptorId :: TypeId, descriptorName :: String, descriptorPhysical :: PhysicalType
} deriving (Show, Eq)
{ descriptorId :: TypeId
, descriptorName :: String
, descriptorPhysical :: PhysicalType
, descriptorCapabilities :: [TypeCapability]
}
deriving (Show, Eq)
-- | 保存构造器元数据
newtype TypeRegistry = TypeRegistry [(TypeConstructorId, String, Kind)] deriving (Show, Eq)
-- | 表达运行时值
data RuntimeValue = RInt Int | RDouble Double | RBool Bool | RString String
| RList [RuntimeValue] | RMaybe (Maybe RuntimeValue) | RTuple [RuntimeValue]
deriving (Show, Eq)
data RuntimeValue
= RInt Int
| RDouble Double
| RBool Bool
| RString String
| RList [RuntimeValue]
| RMaybe (Maybe RuntimeValue)
| RTuple [RuntimeValue]
deriving (Show, Eq, Ord)
-- | 递归哈希结构并统一浮点正负零
instance Hashable RuntimeValue where
-- \| 根据构造器和子值混合哈希
hashWithSalt salt value = case value of
RInt n -> hashWithSalt (hashWithSalt salt (1 :: Int)) n
RDouble n -> hashWithSalt (hashWithSalt salt (2 :: Int)) (if n == 0 then 0 else n)
RBool flag -> hashWithSalt (hashWithSalt salt (3 :: Int)) flag
RString text -> hashWithSalt (hashWithSalt salt (4 :: Int)) text
RList values -> hashWithSalt (hashWithSalt salt (5 :: Int)) values
RMaybe optional -> hashWithSalt (hashWithSalt salt (6 :: Int)) optional
RTuple values -> hashWithSalt (hashWithSalt salt (7 :: Int)) values
-- | 恢复固定编号的内建构造器
builtinTypes :: TypeRegistry
builtinTypes = TypeRegistry
[(TypeConstructorId 1, "Int", TypeKind), (TypeConstructorId 2, "Double", TypeKind)
,(TypeConstructorId 3, "Bool", TypeKind), (TypeConstructorId 4, "String", TypeKind)
,(TypeConstructorId 5, "List", TypeArrow TypeKind TypeKind)
,(TypeConstructorId 6, "Maybe", TypeArrow TypeKind TypeKind)
,(TypeConstructorId 7, "Tuple", ProductKind)]
builtinTypes =
TypeRegistry
[ (TypeConstructorId 1, "Int", TypeKind)
, (TypeConstructorId 2, "Double", TypeKind)
, (TypeConstructorId 3, "Bool", TypeKind)
, (TypeConstructorId 4, "String", TypeKind)
, (TypeConstructorId 5, "List", TypeArrow TypeKind TypeKind)
, (TypeConstructorId 6, "Maybe", TypeArrow TypeKind TypeKind)
, (TypeConstructorId 7, "Tuple", ProductKind)
]
-- | 构造整数类型表达式
intType :: TypeExpr
@@ -96,7 +158,7 @@ renderTypeExpr ty@(TypeApply cid arguments) = case (cid, arguments) of
(TypeConstructorId 7, fields) -> "(" ++ joinTypes fields ++ ")"
_ -> show ty
where
-- | 连接元组字段类型
-- \| 连接元组字段类型
joinTypes [] = ""
joinTypes [field] = renderTypeExpr field
joinTypes (field : fields) = renderTypeExpr field ++ ", " ++ joinTypes fields
@@ -107,25 +169,28 @@ parseTypeExpr source = case P.readP_to_S (P.skipSpaces *> expression <* P.skipSp
[(ty, "")] -> Right ty
_ -> Left "invalid runtime type expression"
where
-- | 解析类型原子和构造器应用
expression = (intType <$ token "int") P.<++ (doubleType <$ token "double")
P.<++ (boolType <$ token "bool") P.<++ (stringType <$ token "string")
P.<++ (listType <$> between "[" "]" expression)
P.<++ (listType <$> (token "list" *> expression))
P.<++ (maybeType <$> (token "maybe" *> expression))
P.<++ (tupleType <$> between "(" ")" (P.sepBy expression (symbol ",")))
-- | 解析具有名称边界的类型关键字
-- \| 解析类型原子和构造器应用
expression =
(intType <$ token "int")
P.<++ (doubleType <$ token "double")
P.<++ (boolType <$ token "bool")
P.<++ (stringType <$ token "string")
P.<++ (listType <$> between "[" "]" expression)
P.<++ (listType <$> (token "list" *> expression))
P.<++ (maybeType <$> (token "maybe" *> expression))
P.<++ (tupleType <$> between "(" ")" (P.sepBy expression (symbol ",")))
-- \| 解析具有名称边界的类型关键字
token word = do
mapM_ character word
rest <- P.look
case rest of
c : _ | isAlphaNum c || c == '_' -> P.pfail
_ -> P.skipSpaces
-- | 不区分大小写匹配关键字字符
-- \| 不区分大小写匹配关键字字符
character c = P.satisfy ((== c) . toLower)
-- | 解析类型分隔符
-- \| 解析类型分隔符
symbol text = P.string text <* P.skipSpaces
-- | 解析括号包围的类型
-- \| 解析括号包围的类型
between open close = P.between (symbol open) (symbol close)
-- | 渲染运行时值的可读文本
@@ -165,7 +230,12 @@ resolveType registry@(TypeRegistry constructors) expression@(TypeApply cid args)
(TypeConstructorId 6, TypeArrow TypeKind TypeKind, [child]) -> Right (Optional child)
(TypeConstructorId 7, ProductKind, fields) -> Right (Product fields)
_ -> Left ("invalid type constructor application: " ++ name)
Right (TypeDescriptor (TypeId expression) name physical)
let capabilities =
[ capability
| capability <- [EqCapability, OrdCapability, HashCapability]
, all (elem capability . descriptorCapabilities) children
]
Right (TypeDescriptor (TypeId expression) name physical capabilities)
-- | 按身份查询类型描述
describeType :: TypeRegistry -> TypeId -> Either String TypeDescriptor
@@ -216,10 +286,12 @@ decodeValue registry expected bytes = do
json <- eitherDecode bytes
(expression, payload) <- parseEither parseEnvelope json
descriptor <- resolveType registry expression
if descriptorId descriptor /= expected then Left "encoded value type identity mismatch" else do
value <- parseEither (parseValue (descriptorPhysical descriptor)) payload
validateValue registry expected value
Right value
if descriptorId descriptor /= expected
then Left "encoded value type identity mismatch"
else do
value <- parseEither (parseValue (descriptorPhysical descriptor)) payload
validateValue registry expected value
Right value
-- | 编码版本化类型身份
encodeType :: TypeId -> BS.ByteString
chusql-core/storage/src/btree.rs
+104 -20
View File
@@ -22,7 +22,8 @@ const SLOT: usize = 2;
const CELL_FIXED: usize = 2 + 8;
/// 键长上限：节点里的格子必须放得进一页
pub const MAX_KEY_BYTES: usize = 64;
pub const MAX_KEY_BYTES: usize = 128;
const LEGACY_KEY_BYTES: usize = 64;
/// 一个节点最多几个键
fn max_keys(order: usize) -> usize {
@@ -31,7 +32,7 @@ fn max_keys(order: usize) -> usize {
/// 一个格子最多占多少字节
fn max_cell_bytes() -> usize {
CELL_FIXED + MAX_KEY_BYTES
CELL_FIXED + LEGACY_KEY_BYTES
}
/// 这个 order 放得进一页吗
@@ -44,6 +45,16 @@ pub fn max_order(page_size: usize) -> usize {
page_size.saturating_sub(HEADER) / (SLOT + max_cell_bytes()) + 1
}
/// 按页容量和分支数计算有效键长
pub fn key_limit(page_size: usize, order: usize) -> usize {
if order < 2 {
return 0;
}
(page_size.saturating_sub(HEADER) / (order - 1))
.saturating_sub(SLOT + CELL_FIXED)
.min(MAX_KEY_BYTES)
}
/// 读一个 u8
fn read_u8(p: &Page, off: usize) -> u8 {
p.data[off]
@@ -66,7 +77,12 @@ fn write_u16(p: &mut Page, off: usize, v: u16) {
/// 读一个小端 u32
fn read_u32(p: &Page, off: usize) -> u32 {
u32::from_le_bytes([p.data[off], p.data[off + 1], p.data[off + 2], p.data[off + 3]])
u32::from_le_bytes([
p.data[off],
p.data[off + 1],
p.data[off + 2],
p.data[off + 3],
])
}
/// 写一个小端 u32
@@ -91,7 +107,7 @@ fn write_meta(page: &mut Page, page_size: usize, order: usize, root: PageId) {
page.zero();
page.data[0..4].copy_from_slice(META_MAGIC);
write_u32(page, OFF_PAGE_SIZE, page_size as u32);
write_u16(page, OFF_MAX_KEY, MAX_KEY_BYTES as u16);
write_u16(page, OFF_MAX_KEY, key_limit(page_size, order) as u16);
write_u64(page, OFF_ROOT, root);
write_u16(page, OFF_ORDER, order as u16);
}
@@ -115,6 +131,13 @@ fn check_meta(page: &Page, page_size: usize, order: usize) -> io::Result<()> {
),
));
}
let stored_key = read_u16(page, OFF_MAX_KEY) as usize;
if stored_key < LEGACY_KEY_BYTES || stored_key > key_limit(page_size, order) {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"unsupported index key limit",
));
}
Ok(())
}
@@ -169,13 +192,19 @@ fn put_node(p: &mut Page, node: &Node) -> io::Result<()> {
/// 从一页读出节点
fn get_node(p: &Page) -> io::Result<Node> {
if p.data.len() < HEADER {
return Err(io::Error::new(io::ErrorKind::InvalidData, "truncated index node header"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"truncated index node header",
));
}
let kind = read_u8(p, OFF_KIND);
let count = read_u16(p, OFF_COUNT) as usize;
let slots_end = HEADER + count * SLOT;
if !matches!(kind, NODE_LEAF | NODE_INTERNAL) || slots_end > p.data.len() {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index node kind or slot count"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid index node kind or slot count",
));
}
let trailer = read_u64(p, OFF_NEXT);
let mut keys = Vec::with_capacity(count);
@@ -184,30 +213,55 @@ fn get_node(p: &Page) -> io::Result<Node> {
for i in 0..count {
let off = read_u16(p, HEADER + i * SLOT) as usize;
if off < slots_end || off + CELL_FIXED > p.data.len() {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index cell offset"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid index cell offset",
));
}
let klen = read_u16(p, off) as usize;
if klen > MAX_KEY_BYTES || off + CELL_FIXED + klen > p.data.len() {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index key length"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid index key length",
));
}
if off + CELL_FIXED + klen > cell_end {
return Err(io::Error::new(io::ErrorKind::InvalidData, "overlapping index cells"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"overlapping index cells",
));
}
cell_end = off;
keys.push(p.data[off + 2..off + 2 + klen].to_vec());
values.push(read_u64(p, off + 2 + klen));
}
if keys.windows(2).any(|pair| pair[0] > pair[1]) {
return Err(io::Error::new(io::ErrorKind::InvalidData, "index keys are not sorted"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"index keys are not sorted",
));
}
if kind == NODE_INTERNAL {
values.push(trailer);
if values.contains(&META_PAGE) {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index child page"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid index child page",
));
}
Ok(Node { kind, next: 0, keys, values })
Ok(Node {
kind,
next: 0,
keys,
values,
})
} else {
Ok(Node { kind, next: trailer, keys, values })
Ok(Node {
kind,
next: trailer,
keys,
values,
})
}
}
@@ -244,7 +298,12 @@ fn split_node(node: &mut Node) -> (Vec<u8>, Node) {
node.keys.pop();
(
sep,
Node { kind: NODE_INTERNAL, next: 0, keys: right_keys, values: right_values },
Node {
kind: NODE_INTERNAL,
next: 0,
keys: right_keys,
values: right_values,
},
)
}
}
@@ -287,20 +346,26 @@ impl DiskBTree {
}
/// 键太长直接报错，避免写坏节点
fn check_key(&self, key: &[u8]) -> io::Result<()> {
if key.len() > MAX_KEY_BYTES {
pub(crate) fn check_key(key: &[u8], page_size: usize, order: usize) -> io::Result<()> {
let limit = key_limit(page_size, order);
if key.len() > limit {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!(
"index key is {} bytes, at most {} are supported",
key.len(),
MAX_KEY_BYTES
limit
),
));
}
Ok(())
}
/// 按当前索引配置预检键长
pub(crate) fn validate_key(&self, key: &[u8]) -> io::Result<()> {
Self::check_key(key, self.file.page_size(), self.order)
}
/// 读根页页号；空树是 None
fn root(&mut self) -> io::Result<Option<PageId>> {
let r = self.file.with_page(META_PAGE, |p| read_u64(p, OFF_ROOT))?;
@@ -331,6 +396,11 @@ impl DiskBTree {
self.file.update_page(id, |p| put_node(p, node))?
}
/// 设置后续索引页的提交点
pub(crate) fn set_lsn(&mut self, lsn: u64) {
self.file.set_lsn(lsn);
}
/// 清空整棵树
pub fn clear(&mut self) -> io::Result<()> {
let page_size = self.file.page_size();
@@ -362,14 +432,26 @@ impl DiskBTree {
/// 按 key 取全部位置（重复键按值升序）
pub fn get_all(&mut self, key: &[u8]) -> io::Result<Vec<u64>> {
let mut values: Vec<_> = self.scan(Some(key), Some(key))?.into_iter().map(|(_, value)| value).collect();
let mut values: Vec<_> = self
.scan(Some(key), Some(key))?
.into_iter()
.map(|(_, value)| value)
.collect();
values.sort_unstable();
Ok(values)
}
/// 插入一个键值对（键可重复）
pub fn insert(&mut self, key: &[u8], value: u64) -> io::Result<()> {
self.check_key(key)?;
self.validate_key(key)?;
if key.len() > LEGACY_KEY_BYTES {
let limit = key_limit(self.file.page_size(), self.order);
let mut meta = self.file.read_page(META_PAGE)?;
if read_u16(&meta, OFF_MAX_KEY) as usize != limit {
write_u16(&mut meta, OFF_MAX_KEY, limit as u16);
self.file.write_page(&meta)?;
}
}
match self.root()? {
None => {
let node = Node {
@@ -412,7 +494,9 @@ impl DiskBTree {
} else {
let child = node.values[child_slot(&node.keys, key)];
if let Some((sep, right_id)) = self.insert_rec(child, key, value)? {
let at = node.keys.partition_point(|k| k.as_slice() <= sep.as_slice());
let at = node
.keys
.partition_point(|k| k.as_slice() <= sep.as_slice());
node.keys.insert(at, sep);
node.values.insert(at + 1, right_id);
}
chusql-core/storage/src/catalog.rs
+134 -41
View File
@@ -37,6 +37,8 @@ pub struct TableSchema {
#[serde(default)]
pub object_id: u64,
#[serde(default)]
pub file_generation: String,
#[serde(default)]
// 0 表示旧目录尚无可验证的结构版本。
pub schema_version: u64,
#[serde(default)]
@@ -78,6 +80,13 @@ impl Catalog {
return Ok(Catalog::default());
}
let mut catalog: Self = serde_json::from_slice(&bytes).map_err(io::Error::other)?;
if catalog
.tables
.values()
.any(|schema| !valid_generation(&schema.file_generation))
{
return Err(io::Error::other("invalid physical file generation"));
}
catalog.upgrade_object_ids()?;
Ok(catalog)
}
@@ -94,7 +103,9 @@ impl Catalog {
}
for schema in self.tables.values_mut() {
if schema.object_id == 0 {
self.object_high_water = self.object_high_water.checked_add(1)
self.object_high_water = self
.object_high_water
.checked_add(1)
.ok_or_else(|| io::Error::other("catalog object id exhausted"))?;
schema.object_id = self.object_high_water;
}
@@ -102,9 +113,37 @@ impl Catalog {
Ok(())
}
/// 切换物理文件代次而不改变逻辑结构版本
pub(crate) fn set_generation(
&mut self,
table: &str,
object: u64,
source: &str,
target: &str,
) -> io::Result<()> {
if !valid_generation(target) {
return Err(io::Error::other("invalid physical file generation"));
}
let schema = self
.tables
.get_mut(table)
.ok_or_else(|| io::Error::other("migration table missing"))?;
if schema.object_id != object
|| (schema.file_generation != source && schema.file_generation != target)
{
return Err(io::Error::other(
"serialization_failure: migration object changed",
));
}
schema.file_generation = target.to_owned();
Ok(())
}
/// 分配不复用的对象编号
pub fn reserve_object_id(&mut self) -> io::Result<u64> {
self.object_high_water = self.object_high_water.checked_add(1)
self.object_high_water = self
.object_high_water
.checked_add(1)
.ok_or_else(|| io::Error::other("catalog object id exhausted"))?;
Ok(self.object_high_water)
}
@@ -123,7 +162,11 @@ impl Catalog {
file.sync_all()?;
drop(file);
std::fs::rename(pending, &path)?;
std::fs::OpenOptions::new().read(true).write(true).open(path)?.sync_all()
std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(path)?
.sync_all()
}
/// 一张表的 schema
@@ -149,7 +192,12 @@ impl Catalog {
}
/// 按日志中的编号建立表
pub fn create_table_with_id(&mut self, table: &str, columns: Vec<SchemaColumn>, object_id: Option<u64>) -> io::Result<()> {
pub fn create_table_with_id(
&mut self,
table: &str,
columns: Vec<SchemaColumn>,
object_id: Option<u64>,
) -> io::Result<()> {
if self.tables.contains_key(table) {
return Err(io::Error::new(
io::ErrorKind::AlreadyExists,
@@ -160,13 +208,17 @@ impl Catalog {
Some(id) if id == 0 || self.tables.values().any(|schema| schema.object_id == id) => {
return Err(io::Error::other("invalid catalog object id"));
}
Some(id) => { self.object_high_water = self.object_high_water.max(id); id }
Some(id) => {
self.object_high_water = self.object_high_water.max(id);
id
}
None => self.reserve_object_id()?,
};
self.tables.insert(
table.to_string(),
TableSchema {
object_id,
file_generation: String::new(),
schema_version: 1,
system: false,
columns,
@@ -182,7 +234,9 @@ impl Catalog {
/// 标记内部账号表
pub fn mark_system(&mut self, table: &str) -> io::Result<()> {
let entry = self.tables.get_mut(table)
let entry = self
.tables
.get_mut(table)
.ok_or_else(|| io::Error::other("missing system table"))?;
entry.system = true;
Ok(())
@@ -306,7 +360,10 @@ impl Catalog {
));
}
let version = next_schema_version(entry)?;
let mut entry = self.tables.remove(from).ok_or_else(|| io::Error::other("missing rename source"))?;
let mut entry = self
.tables
.remove(from)
.ok_or_else(|| io::Error::other("missing rename source"))?;
entry.schema_version = version;
self.seen.remove(from);
self.tables.insert(to.to_string(), entry);
@@ -319,13 +376,19 @@ impl Catalog {
}
/// 用日志中的目标版本替换列定义
pub fn set_columns_version(&mut self, table: &str, columns: Vec<SchemaColumn>, version: Option<u64>) -> io::Result<()> {
let entry = self
.tables
.get_mut(table)
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?;
pub fn set_columns_version(
&mut self,
table: &str,
columns: Vec<SchemaColumn>,
version: Option<u64>,
) -> io::Result<()> {
let entry = self.tables.get_mut(table).ok_or_else(|| {
io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table))
})?;
if let Some(version) = version {
if version == 0 { return Err(io::Error::other("invalid WAL schema version")); }
if version == 0 {
return Err(io::Error::other("invalid WAL schema version"));
}
entry.schema_version = version;
} else if entry.columns != columns {
entry.schema_version = next_schema_version(entry)?;
@@ -342,7 +405,11 @@ impl Catalog {
pub fn normalize_types(&mut self) -> io::Result<bool> {
let mut changed = false;
for entry in self.tables.values_mut() {
if entry.columns.iter().any(|column| normalize_type(&column.ty) != column.ty) {
if entry
.columns
.iter()
.any(|column| normalize_type(&column.ty) != column.ty)
{
entry.schema_version = next_schema_version(entry)?;
}
for column in entry.columns.iter_mut() {
@@ -362,13 +429,19 @@ impl Catalog {
}
/// 用日志目标版本删除列定义
pub fn remove_column_version(&mut self, table: &str, column: &str, version: Option<u64>) -> io::Result<bool> {
let entry = self
.tables
.get_mut(table)
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?;
pub fn remove_column_version(
&mut self,
table: &str,
column: &str,
version: Option<u64>,
) -> io::Result<bool> {
let entry = self.tables.get_mut(table).ok_or_else(|| {
io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table))
})?;
if let Some(version) = version {
if version == 0 { return Err(io::Error::other("invalid WAL schema version")); }
if version == 0 {
return Err(io::Error::other("invalid WAL schema version"));
}
entry.schema_version = version;
} else if entry.columns.iter().any(|c| c.name == column) {
entry.schema_version = next_schema_version(entry)?;
@@ -388,10 +461,9 @@ impl Catalog {
/// 记住这一版已整理过（隐藏列名要留住）
pub fn mark_compacted(&mut self, table: &str) -> io::Result<()> {
let entry = self
.tables
.get_mut(table)
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?;
let entry = self.tables.get_mut(table).ok_or_else(|| {
io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table))
})?;
entry.compacted = true;
self.seen.remove(table);
Ok(())
@@ -399,7 +471,9 @@ impl Catalog {
/// 这张表还有隐藏列字节吗
pub fn has_dropped_columns(&self, table: &str) -> bool {
self.tables.get(table).is_some_and(|e| !e.dropped_columns.is_empty())
self.tables
.get(table)
.is_some_and(|e| !e.dropped_columns.is_empty())
}
/// 还有隐藏列字节待整理的表
@@ -413,10 +487,9 @@ impl Catalog {
/// 记下一条索引定义
pub fn add_index(&mut self, table: &str, column: &str) -> io::Result<()> {
let entry = self
.tables
.get_mut(table)
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?;
let entry = self.tables.get_mut(table).ok_or_else(|| {
io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table))
})?;
if entry.indexes.iter().any(|i| i.column == column) {
return Err(io::Error::new(
io::ErrorKind::AlreadyExists,
@@ -431,10 +504,9 @@ impl Catalog {
/// 去掉一条索引定义
pub fn remove_index(&mut self, table: &str, column: &str) -> io::Result<()> {
let entry = self
.tables
.get_mut(table)
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?;
let entry = self.tables.get_mut(table).ok_or_else(|| {
io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table))
})?;
match entry.indexes.iter().position(|i| i.column == column) {
None => Err(io::Error::new(
io::ErrorKind::NotFound,
@@ -450,11 +522,20 @@ impl Catalog {
/// 把行里"字典还不知道的列"补进来；返回有没有变化
fn add_missing_columns(entry: &mut TableSchema, row: &Row) -> io::Result<bool> {
let additions: Vec<_> = row.iter().filter(|(name, _)|
!entry.dropped_columns.contains(*name) && !entry.columns.iter().any(|c| &c.name == *name))
.filter_map(|(name, value)| infer_type(value).map(|ty| SchemaColumn {
name: name.clone(), ty, ..Default::default()
})).collect();
let additions: Vec<_> = row
.iter()
.filter(|(name, _)| {
!entry.dropped_columns.contains(*name)
&& !entry.columns.iter().any(|c| &c.name == *name)
})
.filter_map(|(name, value)| {
infer_type(value).map(|ty| SchemaColumn {
name: name.clone(),
ty,
..Default::default()
})
})
.collect();
if additions.is_empty() {
return Ok(false);
}
@@ -465,7 +546,9 @@ fn add_missing_columns(entry: &mut TableSchema, row: &Row) -> io::Result<bool> {
/// 递增结构版本，耗尽时拒绝变更
fn next_schema_version(entry: &TableSchema) -> io::Result<u64> {
entry.schema_version.checked_add(1)
entry
.schema_version
.checked_add(1)
.ok_or_else(|| io::Error::other("catalog schema version exhausted"))
}
@@ -576,12 +659,22 @@ fn hist_add(stat: &mut ColumnStat, v: &serde_json::Value) {
/// 从直方图里减掉一个值；值在区间外就不动
fn hist_remove(stat: &mut ColumnStat, v: &serde_json::Value) {
let Some(x) = v.as_f64() else { return };
let (Some(lo), Some(hi)) = (stat.lo, stat.hi) else { return };
let (Some(lo), Some(hi)) = (stat.lo, stat.hi) else {
return;
};
if x < lo || x > hi || stat.hist.len() != HIST_BUCKETS {
return;
}
let index = hist_index(lo, hi, x);
if let Some(bucket) = (0..stat.hist.len()).filter(|&i| stat.hist[i] > 0).min_by_key(|&i| i.abs_diff(index)) {
if let Some(bucket) = (0..stat.hist.len())
.filter(|&i| stat.hist[i] > 0)
.min_by_key(|&i| i.abs_diff(index))
{
stat.hist[bucket] -= 1;
}
}
/// 文件代次仅允许空值或固定长度十六进制
pub(crate) fn valid_generation(value: &str) -> bool {
value.is_empty() || (value.len() == 32 && value.bytes().all(|byte| byte.is_ascii_hexdigit()))
}
chusql-core/storage/src/config.rs
+13 -7
View File
@@ -33,7 +33,9 @@ pub fn default_config_path() -> PathBuf {
{
let appdata = std::env::var_os("APPDATA").filter(|v| !v.is_empty());
if let Some(appdata) = appdata {
return PathBuf::from(appdata).join(APP_DIR_NAME).join(CONFIG_FILE_NAME);
return PathBuf::from(appdata)
.join(APP_DIR_NAME)
.join(CONFIG_FILE_NAME);
}
}
#[cfg(not(windows))]
@@ -62,7 +64,9 @@ pub fn default_data_dir() -> PathBuf {
}
let roaming = std::env::var_os("APPDATA").filter(|v| !v.is_empty());
if let Some(roaming) = roaming {
return PathBuf::from(roaming).join(APP_DIR_NAME).join(DATA_DIR_NAME);
return PathBuf::from(roaming)
.join(APP_DIR_NAME)
.join(DATA_DIR_NAME);
}
PathBuf::from(APP_DIR_NAME).join(DATA_DIR_NAME)
}
@@ -210,9 +214,7 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
})?,
};
let file_origin = Origin::File(
path.clone().unwrap_or_else(default_config_path),
);
let file_origin = Origin::File(path.clone().unwrap_or_else(default_config_path));
let (page_size, page_origin) = pick(
file.page.and_then(|p| p.size),
@@ -250,8 +252,12 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
let log_files = log_files.trim().to_string();
let log_level = log_level.trim();
let log_level = Level::parse(log_level)
.ok_or_else(|| format!("log.level is not one of off/error/warn/info/debug: {}", log_level))?;
let log_level = Level::parse(log_level).ok_or_else(|| {
format!(
"log.level is not one of off/error/warn/info/debug: {}",
log_level
)
})?;
if data_dir.is_empty() {
return Err("storage.data_dir must not be empty".to_string());
chusql-core/storage/src/ffi.rs
+58 -15
View File
@@ -1,6 +1,6 @@
use std::cell::RefCell;
use std::ffi::{c_char, CStr};
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::ffi::{CStr, c_char};
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::ptr;
use std::slice;
@@ -42,7 +42,11 @@ pub unsafe extern "C" fn chusql_storage_open(config_path: *const c_char) -> *mut
let text = unsafe { CStr::from_ptr(config_path) }
.to_str()
.map_err(|_| "config path is not valid UTF-8".to_string())?;
if text.is_empty() { None } else { Some(text.to_string()) }
if text.is_empty() {
None
} else {
Some(text.to_string())
}
};
Storage::open(path.as_deref()).map(Box::new)
}));
@@ -78,27 +82,57 @@ pub unsafe extern "C" fn chusql_storage_close(handle: *mut Storage) {
/// 输出指针可写且不与输入重叠；响应使用 free 释放。
#[unsafe(no_mangle)]
pub unsafe extern "C" fn chusql_storage_maintenance(
config_path: *const c_char, request: *const u8, request_len: usize,
out: *mut *mut u8, out_len: *mut usize,
config_path: *const c_char,
request: *const u8,
request_len: usize,
out: *mut *mut u8,
out_len: *mut usize,
) -> i32 {
if !out.is_null() { unsafe { *out = ptr::null_mut(); } }
if !out_len.is_null() { unsafe { *out_len = 0; } }
if request.is_null() || out.is_null() || out_len.is_null() || request_len > isize::MAX as usize {
if !out.is_null() {
unsafe {
*out = ptr::null_mut();
}
}
if !out_len.is_null() {
unsafe {
*out_len = 0;
}
}
if request.is_null() || out.is_null() || out_len.is_null() || request_len > isize::MAX as usize
{
set_error("invalid maintenance argument");
return ERR;
}
let result = catch_unwind(AssertUnwindSafe(|| {
let path = if config_path.is_null() { None } else { Some(unsafe { CStr::from_ptr(config_path) }.to_str().map_err(|e| e.to_string())?) };
let text = std::str::from_utf8(unsafe { slice::from_raw_parts(request, request_len) }).map_err(|e| e.to_string())?;
let path = if config_path.is_null() {
None
} else {
Some(
unsafe { CStr::from_ptr(config_path) }
.to_str()
.map_err(|e| e.to_string())?,
)
};
let text = std::str::from_utf8(unsafe { slice::from_raw_parts(request, request_len) })
.map_err(|e| e.to_string())?;
Storage::maintenance(path, text)
}));
let response = match result {
Ok(Ok(response)) => response,
Ok(Err(err)) => { set_error(err); return ERR; }
Err(_) => { set_error("panic in system maintenance"); return ERR; }
Ok(Err(err)) => {
set_error(err);
return ERR;
}
Err(_) => {
set_error("panic in system maintenance");
return ERR;
}
};
let buffer = response.into_bytes().into_boxed_slice();
unsafe { *out_len = buffer.len(); *out = Box::into_raw(buffer) as *mut u8; }
unsafe {
*out_len = buffer.len();
*out = Box::into_raw(buffer) as *mut u8;
}
OK
}
@@ -121,7 +155,12 @@ pub unsafe extern "C" fn chusql_storage_request(
if !out_len.is_null() {
unsafe { *out_len = 0 };
}
if handle.is_null() || request.is_null() || out.is_null() || out_len.is_null() || request_len > isize::MAX as usize {
if handle.is_null()
|| request.is_null()
|| out.is_null()
|| out_len.is_null()
|| request_len > isize::MAX as usize
{
set_error("invalid argument");
return ERR;
}
@@ -178,6 +217,10 @@ pub unsafe extern "C" fn chusql_storage_last_error(out_len: *mut usize) -> *cons
if !out_len.is_null() {
unsafe { *out_len = bytes.len() };
}
if bytes.is_empty() { ptr::null() } else { bytes.as_ptr() }
if bytes.is_empty() {
ptr::null()
} else {
bytes.as_ptr()
}
})
}
chusql-core/storage/src/heap.rs
+83 -15
View File
@@ -1,8 +1,8 @@
use serde::Deserializer;
use serde::de::{IgnoredAny, MapAccess, Visitor};
use std::collections::HashSet;
use std::io;
use std::path::Path;
use serde::de::{IgnoredAny, MapAccess, Visitor};
use serde::Deserializer;
use crate::btree::DiskBTree;
use crate::page::PageId;
@@ -11,12 +11,18 @@ use crate::protocol::Row;
// 堆表：行按插入顺序进页，支持全表扫描、按索引取行、行级删除。
pub(crate) const DUPLICATE_ID: &str = "duplicate value on the id index";
const HEADER_SIZE: usize = 2;
const SLOT_SIZE: usize = 8;
const SLOT_DIR_START: usize = HEADER_SIZE;
/// 未选字段只校验并跳过，不分配值。
fn decode_row(bytes: &[u8], columns: Option<&[String]>, hidden: &HashSet<String>) -> io::Result<Row> {
fn decode_row(
bytes: &[u8],
columns: Option<&[String]>,
hidden: &HashSet<String>,
) -> io::Result<Row> {
struct RowVisitor<'a>(Option<&'a [String]>, &'a HashSet<String>);
impl<'de> Visitor<'de> for RowVisitor<'_> {
type Value = Row;
@@ -38,7 +44,9 @@ fn decode_row(bytes: &[u8], columns: Option<&[String]>, hidden: &HashSet<String>
}
}
let mut decoder = serde_json::Deserializer::from_slice(bytes);
let row = decoder.deserialize_map(RowVisitor(columns, hidden)).map_err(io::Error::other)?;
let row = decoder
.deserialize_map(RowVisitor(columns, hidden))
.map_err(io::Error::other)?;
decoder.end().map_err(io::Error::other)?;
Ok(row)
}
@@ -93,7 +101,7 @@ fn row_int(row: &Row, column: &str) -> Option<i64> {
}
/// i64 键 → 8 字节大端（保序，负数也在前）
fn key_of(value: i64) -> [u8; 8] {
pub(crate) fn key_of(value: i64) -> [u8; 8] {
(value as u64 ^ (1u64 << 63)).to_be_bytes()
}
@@ -108,10 +116,23 @@ fn value_key(v: &serde_json::Value) -> Option<Vec<u8>> {
}
/// 取行里某一列的索引键；NULL 与缺列不入索引
fn row_key(row: &Row, column: &str) -> Option<Vec<u8>> {
pub(crate) fn row_key(row: &Row, column: &str) -> Option<Vec<u8>> {
row.get(column).and_then(value_key)
}
/// 按实际键编码校验一列的索引容量
pub(crate) fn check_index_key(
row: &Row,
column: &str,
page_size: usize,
order: usize,
) -> io::Result<()> {
if let Some(key) = row_key(row, column) {
DiskBTree::check_key(&key, page_size, order)?;
}
Ok(())
}
pub struct HeapPage;
impl HeapPage {
@@ -312,7 +333,7 @@ impl HeapTable {
Ok(()) => {}
Err(err) if err.kind() == io::ErrorKind::InvalidData => {
// 格式不兼容（例如上一版落的索引文件）：删掉重开
let _ = std::fs::remove_file(p);
crate::page_wal::remove(p)?;
t.attach_index(col, p, page_size, btree_order, pool_size)?;
}
Err(err) => return Err(err),
@@ -373,6 +394,7 @@ impl HeapTable {
let Some(k) = row_key(&r, column) else {
continue;
};
DiskBTree::check_key(&k, page_size, btree_order)?;
// 行身份索引仍然唯一；其余列允许重复值
if column == "id" && !seen.insert(k.clone()) {
return Err(io::Error::new(
@@ -450,10 +472,11 @@ impl HeapTable {
let Some(k) = row_key(row, &idx.column) else {
continue;
};
idx.tree.validate_key(&k)?;
if idx.column == "id" && idx.tree.get(&k)?.is_some() {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"duplicate value on the id index".to_string(),
DUPLICATE_ID.to_string(),
));
}
pending.push((i, k));
@@ -511,6 +534,16 @@ impl HeapTable {
Ok((page_id, slot))
}
/// 读取身份索引中同键行数供提交预检
pub(crate) fn id_key_count(&mut self, key: &[u8]) -> io::Result<usize> {
let index = self
.indexes
.iter_mut()
.find(|index| index.column == "id")
.ok_or_else(|| io::Error::other("missing id index"))?;
Ok(index.tree.get_all(key)?.len())
}
/// 按 `id` 取一行
pub fn get_by_key(&mut self, key: i64) -> io::Result<Option<Row>> {
self.get_by_column_key("id", key)
@@ -577,13 +610,21 @@ impl HeapTable {
let mut rows = Vec::new();
for (key, pos) in pairs {
if let Some((bound, inclusive)) = &lo_bound {
let outside = if *inclusive { &key < bound } else { &key <= bound };
let outside = if *inclusive {
&key < bound
} else {
&key <= bound
};
if outside {
continue;
}
}
if let Some((bound, inclusive)) = &hi_bound {
let outside = if *inclusive { &key > bound } else { &key >= bound };
let outside = if *inclusive {
&key > bound
} else {
&key >= bound
};
if outside {
continue;
}
@@ -699,7 +740,10 @@ impl HeapTable {
}
/// 全表扫描，带位置与列投影
fn scan_projected_with_positions(&mut self, columns: Option<&[String]>) -> io::Result<Vec<(PageId, u16, Row)>> {
fn scan_projected_with_positions(
&mut self,
columns: Option<&[String]>,
) -> io::Result<Vec<(PageId, u16, Row)>> {
let n = self.file.num_pages()?;
self.scan_pages_with_positions(columns, 0, n)
}
@@ -729,6 +773,14 @@ impl HeapTable {
/// 整表改写并重建所有索引
pub fn replace_all(&mut self, rows: &[Row]) -> io::Result<()> {
for row in rows {
Self::encode_row(row, self.file.page_size())?;
for index in &self.indexes {
if let Some(key) = row_key(row, &index.column) {
index.tree.validate_key(&key)?;
}
}
}
self.file.truncate()?;
for idx in &mut self.indexes {
idx.tree.clear()?;
@@ -745,6 +797,14 @@ impl HeapTable {
Ok(())
}
/// 同时标记堆页与索引页的提交点
pub(crate) fn set_lsn(&mut self, lsn: u64) {
self.file.set_lsn(lsn);
for index in &mut self.indexes {
index.tree.set_lsn(lsn);
}
}
/// 把脏页写回（索引也一起）
pub fn flush(&mut self) -> io::Result<()> {
for idx in &mut self.indexes {
@@ -767,11 +827,19 @@ impl HeapTable {
}
/// 当前缓存页数
pub fn cached_pages(&self) -> usize { self.file.cached_pages() }
pub fn cached_pages(&self) -> usize {
self.file.cached_pages()
}
/// 命中次数
pub fn hits(&self) -> u64 { self.file.hits() }
pub fn hits(&self) -> u64 {
self.file.hits()
}
/// 未命中次数
pub fn misses(&self) -> u64 { self.file.misses() }
pub fn misses(&self) -> u64 {
self.file.misses()
}
/// 命中率
pub fn hit_rate(&self) -> f64 { self.file.hit_rate() }
pub fn hit_rate(&self) -> f64 {
self.file.hit_rate()
}
}
chusql-core/storage/src/lib.rs
+2
View File
@@ -6,7 +6,9 @@ pub mod config;
pub mod ffi;
pub mod heap;
pub mod log;
mod mvcc;
pub mod page;
mod page_wal;
pub mod protocol;
pub mod runtime;
pub mod wal;
chusql-core/storage/src/log.rs
+13 -3
View File
@@ -1,7 +1,7 @@
use std::fmt;
use std::io::{IsTerminal, Write};
use std::sync::atomic::{AtomicU8, Ordering};
use std::sync::OnceLock;
use std::sync::atomic::{AtomicU8, Ordering};
// 极简日志：等级用原子变量，关掉的等级只花一次原子读。
@@ -123,8 +123,18 @@ fn streams_to_stderr(level: Level) -> bool {
/// 按目标流是否终端决定是否着色
fn color_enabled(to_stderr: bool) -> bool {
let cached = if to_stderr { &COLOR_STDERR } else { &COLOR_STDOUT };
*cached.get_or_init(|| if to_stderr { std::io::stderr().is_terminal() } else { std::io::stdout().is_terminal() })
let cached = if to_stderr {
&COLOR_STDERR
} else {
&COLOR_STDOUT
};
*cached.get_or_init(|| {
if to_stderr {
std::io::stderr().is_terminal()
} else {
std::io::stdout().is_terminal()
}
})
}
/// 拼一行（纯函数，可测）
chusql-core/storage/src/mvcc.rs
+1172
View File
File diff suppressed because it is too large. Load diff
chusql-core/storage/src/page.rs
+50 -33
View File
@@ -3,6 +3,8 @@ use std::fs::{File, OpenOptions};
use std::io::{self, Read, Seek, SeekFrom, Write};
use std::path::Path;
use crate::page_wal::PageWal;
// 页管理：定长页的读写，外加一层页缓冲池。
pub type PageId = u64;
@@ -33,6 +35,7 @@ impl Page {
#[derive(Debug)]
pub struct PageFile {
file: File,
journal: PageWal,
page_size: usize,
capacity: usize,
cache: HashMap<PageId, Page>,
@@ -55,14 +58,16 @@ impl PageFile {
page_size: usize,
capacity: usize,
) -> io::Result<Self> {
let file = OpenOptions::new()
let mut file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(path)?;
.open(path.as_ref())?;
let journal = PageWal::open(path.as_ref(), page_size, &mut file)?;
Ok(PageFile {
file,
journal,
page_size,
capacity,
cache: HashMap::new(),
@@ -74,6 +79,16 @@ impl PageFile {
})
}
/// 标记后续页修改所属的提交点
pub fn set_lsn(&mut self, lsn: u64) {
self.journal.set_lsn(lsn);
}
/// 读取已记录的页提交 LSN
pub fn page_lsn(&self, id: PageId) -> Option<u64> {
self.journal.page_lsn(id)
}
/// 页大小
pub fn page_size(&self) -> usize {
self.page_size
@@ -109,7 +124,6 @@ impl PageFile {
}
}
/// 标记为最近使用
fn touch(&mut self, id: PageId) {
self.clock += 1;
@@ -127,27 +141,28 @@ impl PageFile {
return Ok(false);
};
if self.dirty.contains(&victim) {
let p = self.cache[&victim].clone();
self.write_raw(&p)?;
self.dirty.remove(&victim);
let pages: Vec<_> = self
.dirty
.iter()
.filter_map(|id| self.cache.get(id).cloned())
.collect();
let images: Vec<_> = pages
.iter()
.map(|page| (page.id, page.data.as_slice()))
.collect();
self.journal.prepare(&images)?;
for page in &pages {
self.write_image(page)?;
}
self.dirty.clear();
}
self.cache.remove(&victim);
self.last_used.remove(&victim);
Ok(true)
}
/// 直接写文件
fn write_raw(&mut self, page: &Page) -> io::Result<()> {
if page.data.len() != self.page_size {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!(
"page has {} bytes, this file uses {}",
page.data.len(),
self.page_size
),
));
}
/// 将已记录镜像的页写入数据文件
fn write_image(&mut self, page: &Page) -> io::Result<()> {
self.file
.seek(SeekFrom::Start(page.id * self.page_size as u64))?;
self.file.write_all(&page.data)
@@ -156,12 +171,12 @@ impl PageFile {
/// 直接从文件读
fn read_raw(&mut self, id: PageId) -> io::Result<Page> {
let mut data = vec![0u8; self.page_size];
self.file.seek(SeekFrom::Start(id * self.page_size as u64))?;
self.file
.seek(SeekFrom::Start(id * self.page_size as u64))?;
self.file.read_exact(&mut data)?;
Ok(Page { id, data })
}
/// 读一页（走缓存）
pub fn read_page(&mut self, id: PageId) -> io::Result<Page> {
self.note_access(id)?;
@@ -241,26 +256,27 @@ impl PageFile {
));
}
let on_disk = len / size;
let cached_max = self
.cache
.keys()
.copied()
.max()
.map(|m| m + 1)
.unwrap_or(0);
let cached_max = self.cache.keys().copied().max().map(|m| m + 1).unwrap_or(0);
Ok(on_disk.max(cached_max))
}
/// 把脏页写回磁盘
pub fn flush(&mut self) -> io::Result<()> {
let dirty: Vec<PageId> = self.dirty.iter().copied().collect();
for id in dirty {
if let Some(p) = self.cache.get(&id) {
let p = p.clone();
self.write_raw(&p)?;
}
let pages: Vec<Page> = self
.dirty
.iter()
.filter_map(|id| self.cache.get(id).cloned())
.collect();
let images: Vec<_> = pages
.iter()
.map(|page| (page.id, page.data.as_slice()))
.collect();
self.journal.prepare(&images)?;
for page in &pages {
self.write_image(page)?;
}
self.file.sync_all()?;
self.journal.checkpoint()?;
self.dirty.clear();
Ok(())
}
@@ -272,6 +288,7 @@ impl PageFile {
/// 清空文件
pub fn truncate(&mut self) -> io::Result<()> {
self.journal.reset()?;
self.cache.clear();
self.last_used.clear();
self.dirty.clear();
chusql-core/storage/src/page_wal.rs
+294
View File
@@ -0,0 +1,294 @@
use std::collections::BTreeMap;
use std::fs::{File, OpenOptions};
use std::io::{self, Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
// 全页预写镜像修复撕裂写，页 LSN 在数据同步后原子发布。
const HEADER: usize = 40;
const RESET: u64 = u64::MAX;
#[derive(Debug, Serialize, Deserialize)]
struct State {
version: u32,
page_size: usize,
sequence: u64,
pages: BTreeMap<u64, u64>,
}
#[derive(Debug)]
pub(crate) struct PageWal {
path: PathBuf,
state: State,
lsn: u64,
pending: bool,
failed: bool,
file: Option<File>,
}
/// 为页文件生成独立的元数据路径
fn sidecar(path: &Path, suffix: &str) -> PathBuf {
let mut name = path.as_os_str().to_os_string();
name.push(suffix);
PathBuf::from(name)
}
/// 计算帧校验和以拒绝损坏的完整镜像
fn checksum(bytes: &[u8]) -> u64 {
bytes.iter().fold(0xcbf29ce484222325, |hash, byte| {
(hash ^ u64::from(*byte)).wrapping_mul(0x100000001b3)
})
}
/// 同步目录中的创建、替换与删除
fn sync_parent(path: &Path) -> io::Result<()> {
#[cfg(unix)]
File::open(
path.parent()
.filter(|p| !p.as_os_str().is_empty())
.unwrap_or(Path::new(".")),
)?
.sync_all()?;
Ok(())
}
impl PageWal {
/// 加载页版本并先恢复未完成的全页写入
pub(crate) fn open(path: &Path, page_size: usize, data: &mut File) -> io::Result<Self> {
let metadata = sidecar(path, ".pages");
let state = match std::fs::read(&metadata) {
Ok(bytes) => serde_json::from_slice::<State>(&bytes).map_err(io::Error::other)?,
Err(error) if error.kind() == io::ErrorKind::NotFound => State {
version: 1,
page_size,
sequence: 0,
pages: BTreeMap::new(),
},
Err(error) => return Err(error),
};
if state.version != 1 || state.page_size != page_size {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"unsupported page LSN metadata",
));
}
let mut journal = Self {
path: path.to_path_buf(),
state,
lsn: 0,
pending: false,
failed: false,
file: None,
};
journal.recover(data)?;
Ok(journal)
}
/// 设置后续脏页所属的逻辑 WAL 提交点
pub(crate) fn set_lsn(&mut self, lsn: u64) {
self.lsn = lsn;
}
/// 读取已写镜像记录的页 LSN
pub(crate) fn page_lsn(&self, page: u64) -> Option<u64> {
self.state.pages.get(&page).copied()
}
/// 延迟创建日志并同步目录项
fn handle(&mut self) -> io::Result<&mut File> {
if self.file.is_none() {
let path = sidecar(&self.path, ".page-wal");
let file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&path)?;
sync_parent(&path)?;
self.file = Some(file);
}
Ok(self.file.as_mut().expect("opened above"))
}
/// 持久化一批镜像后才允许修改数据页
pub(crate) fn prepare(&mut self, pages: &[(u64, &[u8])]) -> io::Result<()> {
if self.failed {
return Err(io::Error::other("page journal requires recovery"));
}
if pages.iter().any(|(page, bytes)| {
(*page == RESET && !bytes.is_empty())
|| (*page != RESET
&& (bytes.len() != self.state.page_size
|| page.checked_mul(self.state.page_size as u64).is_none()))
}) {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"invalid page image size or offset",
));
}
if pages.is_empty() {
return Ok(());
}
self.failed = true;
for (page, bytes) in pages {
let sequence = self
.state
.sequence
.checked_add(1)
.ok_or_else(|| io::Error::other("page write sequence exhausted"))?;
let lsn = self
.lsn
.max(self.state.pages.get(page).copied().unwrap_or(0));
let mut frame = Vec::with_capacity(HEADER + bytes.len());
frame.extend_from_slice(b"CPW1");
frame.extend_from_slice(&sequence.to_le_bytes());
frame.extend_from_slice(&lsn.to_le_bytes());
frame.extend_from_slice(&page.to_le_bytes());
frame.extend_from_slice(&(bytes.len() as u32).to_le_bytes());
let mut checked = frame.clone();
checked.extend_from_slice(bytes);
frame.extend_from_slice(&checksum(&checked).to_le_bytes());
frame.extend_from_slice(bytes);
let file = self.handle()?;
file.seek(SeekFrom::End(0))?;
file.write_all(&frame)?;
self.state.sequence = sequence;
if *page == RESET {
self.state.pages.clear();
} else {
self.state.pages.insert(*page, lsn);
}
self.pending = true;
}
self.handle()?.sync_all()?;
self.failed = false;
Ok(())
}
/// 先记录截断以免重放旧页复活已删除数据
pub(crate) fn reset(&mut self) -> io::Result<()> {
self.prepare(&[(RESET, &[])])
}
/// 数据同步后发布页版本并清理镜像日志
pub(crate) fn checkpoint(&mut self) -> io::Result<()> {
if self.failed {
return Err(io::Error::other("page journal requires recovery"));
}
if !self.pending {
return Ok(());
}
self.failed = true;
let temporary = sidecar(&self.path, ".pages.pending");
let metadata = sidecar(&self.path, ".pages");
let mut file = File::create(&temporary)?;
serde_json::to_writer(&mut file, &self.state).map_err(io::Error::other)?;
file.sync_all()?;
drop(file);
std::fs::rename(&temporary, &metadata)?;
sync_parent(&metadata)?;
self.handle()?.set_len(0)?;
self.handle()?.sync_all()?;
self.pending = false;
self.failed = false;
Ok(())
}
/// 验证日志帧后重放比页检查点更新的镜像
fn recover(&mut self, data: &mut File) -> io::Result<()> {
let path = sidecar(&self.path, ".page-wal");
let mut file = match File::open(path) {
Ok(file) => file,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error),
};
if file.metadata()?.len() == 0 {
return Ok(());
}
let checkpoint = self.state.sequence;
let mut previous = 0;
loop {
let mut header = [0; HEADER];
match file.read_exact(&mut header) {
Ok(()) => (),
Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => break,
Err(error) => return Err(error),
}
let sequence = u64::from_le_bytes(header[4..12].try_into().expect("fixed header"));
let lsn = u64::from_le_bytes(header[12..20].try_into().expect("fixed header"));
let page = u64::from_le_bytes(header[20..28].try_into().expect("fixed header"));
let len = u32::from_le_bytes(header[28..32].try_into().expect("fixed header")) as usize;
if &header[..4] != b"CPW1"
|| sequence <= previous
|| (page == RESET && len != 0)
|| (page != RESET && len != self.state.page_size)
{
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid page image header",
));
}
let mut bytes = vec![0; len];
match file.read_exact(&mut bytes) {
Ok(()) => (),
Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => break,
Err(error) => return Err(error),
}
let mut checked = header[..32].to_vec();
checked.extend_from_slice(&bytes);
if checksum(&checked)
!= u64::from_le_bytes(header[32..40].try_into().expect("fixed header"))
{
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"page image checksum mismatch",
));
}
previous = sequence;
if sequence <= checkpoint {
continue;
}
if page == RESET {
data.set_len(0)?;
self.state.pages.clear();
} else {
let offset = page
.checked_mul(self.state.page_size as u64)
.ok_or_else(|| io::Error::other("page offset overflow"))?;
data.seek(SeekFrom::Start(offset))?;
data.write_all(&bytes)?;
self.state.pages.insert(page, lsn);
}
self.state.sequence = sequence;
}
data.sync_all()?;
self.pending = true;
self.checkpoint()
}
}
/// 删除页文件及其版本与镜像文件
pub(crate) fn remove(path: &Path) -> io::Result<()> {
for suffix in [".page-wal", ".pages", ".pages.pending", ""] {
match std::fs::remove_file(sidecar(path, suffix)) {
Ok(()) => (),
Err(error) if error.kind() == io::ErrorKind::NotFound => (),
Err(error) => return Err(error),
}
}
sync_parent(path)
}
/// 改名时保留对应页版本与恢复材料
pub(crate) fn rename(from: &Path, to: &Path) -> io::Result<()> {
for suffix in [".page-wal", ".pages", ".pages.pending", ""] {
match std::fs::rename(sidecar(from, suffix), sidecar(to, suffix)) {
Ok(()) => (),
Err(error) if error.kind() == io::ErrorKind::NotFound => (),
Err(error) => return Err(error),
}
}
sync_parent(to)
}
chusql-core/storage/src/protocol.rs
+195 -42
View File
@@ -94,19 +94,40 @@ pub struct IndexWire {
#[derive(Debug, Serialize, Deserialize)]
#[serde(tag = "method", rename_all = "snake_case")]
pub enum Request {
CatalogManage { command: Box<Request> },
CatalogManage {
command: Box<Request>,
},
AccountsList,
IdentityInitialize { administrator: String },
RoleCreate { user: String },
IdentityAlter { user: String, can_login: Option<bool>, is_superuser: Option<bool>, enabled: Option<bool>, system_catalog_manager: Option<bool>, allow_sudo_auth: Option<bool> },
AccountCreate { user: String, password_hash: String },
AccountReset { user: String, password_hash: String },
IdentityInitialize {
administrator: String,
},
RoleCreate {
user: String,
},
IdentityAlter {
user: String,
can_login: Option<bool>,
is_superuser: Option<bool>,
enabled: Option<bool>,
system_catalog_manager: Option<bool>,
allow_sudo_auth: Option<bool>,
},
AccountCreate {
user: String,
password_hash: String,
},
AccountReset {
user: String,
password_hash: String,
},
AccountLogin {
user: String,
#[serde(default)]
at: Option<String>,
},
AccountDrop { user: String },
AccountDrop {
user: String,
},
/// 引导系统目录：建系统表，给了 user 就同时补首个管理员
BootstrapSystem {
#[serde(default)]
@@ -115,14 +136,18 @@ pub enum Request {
password_hash: Option<String>,
},
SystemStatus,
BootstrapTypes { types: Vec<Row> },
BootstrapTypes {
types: Vec<Row>,
},
Ping,
Scan {
table: String,
#[serde(default)]
columns: Option<Vec<String>>,
},
ValidateWrite { table: String },
ValidateWrite {
table: String,
},
/// 分片扫描：第 shard 片只读总页数按 shards 均分后的那一段
ScanShard {
table: String,
@@ -177,6 +202,37 @@ pub enum Request {
table: String,
},
ListCatalog,
DomainSnapshot,
MvccBegin {
#[serde(default)]
serializable: bool,
},
MvccLock {
transaction: String,
ops: Vec<StorageOp>,
#[serde(default)]
reads: Vec<String>,
},
MvccPreview {
transaction: String,
},
MvccCommit {
transaction: String,
checkpoint: String,
ops: Vec<StorageOp>,
},
MvccRollback {
transaction: String,
},
DatabaseSnapshot {
#[serde(default)]
checkpoint: Option<String>,
},
AlterDomain {
table: String,
columns: Vec<SchemaColumn>,
checkpoint: String,
},
CreateIndex {
table: String,
column: String,
@@ -204,7 +260,7 @@ pub enum Request {
}
/// 事务提交里的一条写操作
#[derive(Debug, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "op", rename_all = "snake_case")]
pub enum StorageOp {
/// 追加写入的行（提交端已先删掉要覆盖的行）
@@ -215,37 +271,89 @@ pub enum StorageOp {
Replace { table: String, rows: Vec<Row> },
}
/// 检查存储表名和列名的字符集
pub(crate) fn valid_storage_name(name: &str) -> bool {
!name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')
}
/// 检查一行顶层字段名
fn valid_row_columns(row: &Row) -> bool {
row.keys().all(|name| valid_storage_name(name))
}
impl Request {
/// 拒绝可能影响文件路径的列名
/// 校验列定义与写入载荷的顶层字段名
pub fn valid_columns(&self) -> bool {
let valid = |name: &str| !name.is_empty()
&& name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_');
match self {
Self::CreateIndex { column, .. } | Self::DropIndex { column, .. }
| Self::DropColumn { column, .. } | Self::LookupByIndex { column, .. }
| Self::RangeByIndex { column, .. } => valid(column),
Self::CreateTable { columns, .. } => columns.iter().all(|c| valid(&c.name)),
Self::ReplaceSchema { columns, .. } => columns.iter().all(|c| valid(&c.name)),
Self::CreateIndex { column, .. }
| Self::DropIndex { column, .. }
| Self::DropColumn { column, .. }
| Self::LookupByIndex { column, .. }
| Self::RangeByIndex { column, .. } => valid_storage_name(column),
Self::CreateTable { columns, .. } | Self::AlterDomain { columns, .. } => {
columns.iter().all(|c| valid_storage_name(&c.name))
}
Self::ReplaceSchema { columns, rows, .. } => {
columns.iter().all(|c| valid_storage_name(&c.name))
&& rows.iter().all(valid_row_columns)
}
Self::Insert { row, .. } => valid_row_columns(row),
Self::InsertBatch { rows, .. } | Self::ReplaceAll { rows, .. } => {
rows.iter().all(valid_row_columns)
}
Self::ApplyTransaction { ops }
| Self::MvccLock { ops, .. }
| Self::MvccCommit { ops, .. } => ops.iter().all(|op| match op {
StorageOp::Upsert { rows, .. } | StorageOp::Replace { rows, .. } => {
rows.iter().all(valid_row_columns)
}
StorageOp::Delete { .. } => true,
}),
_ => true,
}
}
/// 普通表操作的统一访问边界
pub fn table(&self) -> Option<&str> {
match self {
Self::Scan { table, .. } | Self::ScanShard { table, .. } | Self::ValidateWrite { table }
Self::Scan { table, .. }
| Self::ScanShard { table, .. }
| Self::ValidateWrite { table }
| Self::Insert { table, .. }
| Self::InsertBatch { table, .. } | Self::DeleteKeys { table, .. }
| Self::LookupByIndex { table, .. } | Self::ReplaceAll { table, .. }
| Self::InsertBatch { table, .. }
| Self::DeleteKeys { table, .. }
| Self::LookupByIndex { table, .. }
| Self::ReplaceAll { table, .. }
| Self::RangeByIndex { table, .. }
| Self::DescribeTable { table } | Self::CreateTable { table, .. }
| Self::DropTable { table } | Self::CreateIndex { table, .. }
| Self::DropIndex { table, .. } | Self::DropColumn { table, .. }
| Self::DescribeTable { table }
| Self::CreateTable { table, .. }
| Self::DropTable { table }
| Self::CreateIndex { table, .. }
| Self::DropIndex { table, .. }
| Self::DropColumn { table, .. }
| Self::Compact { table }
| Self::ReplaceSchema { table, .. } => Some(table),
Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList | Self::CatalogManage { .. }
| Self::IdentityInitialize { .. } | Self::RoleCreate { .. } | Self::IdentityAlter { .. }
| Self::AccountCreate { .. } | Self::AccountReset { .. } | Self::AccountLogin { .. }
| Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus
| Self::ReplaceSchema { table, .. }
| Self::AlterDomain { table, .. } => Some(table),
Self::Ping
| Self::ListTables
| Self::ListCatalog
| Self::DomainSnapshot
| Self::MvccPreview { .. }
| Self::MvccBegin { .. }
| Self::MvccLock { .. }
| Self::MvccCommit { .. }
| Self::MvccRollback { .. }
| Self::DatabaseSnapshot { .. }
| Self::AccountsList
| Self::CatalogManage { .. }
| Self::IdentityInitialize { .. }
| Self::RoleCreate { .. }
| Self::IdentityAlter { .. }
| Self::AccountCreate { .. }
| Self::AccountReset { .. }
| Self::AccountLogin { .. }
| Self::AccountDrop { .. }
| Self::BootstrapSystem { .. }
| Self::SystemStatus
| Self::BootstrapTypes { .. }
| Self::ApplyTransaction { .. } => None,
}
@@ -284,14 +392,26 @@ fn default_id_column() -> String {
#[derive(Debug, Serialize, Deserialize)]
#[serde(tag = "status", rename_all = "snake_case")]
pub enum Response {
Guarded { response: Box<Response> },
Accounts { accounts: Vec<Account> },
Guarded {
response: Box<Response>,
},
Accounts {
accounts: Vec<Account>,
},
Pong,
Rows { rows: Vec<Row> },
Tables { tables: Vec<String> },
Rows {
rows: Vec<Row>,
},
Tables {
tables: Vec<String>,
},
Schema {
table: String,
#[serde(default, serialize_with = "serialize_version", deserialize_with = "deserialize_version")]
#[serde(
default,
serialize_with = "serialize_version",
deserialize_with = "deserialize_version"
)]
schema_version: u64,
columns: Vec<SchemaColumn>,
#[serde(default)]
@@ -309,9 +429,30 @@ pub enum Response {
#[serde(default)]
last_lsn: u64,
},
Error { message: String },
Catalog { schemas: Vec<TableSchemaWire> },
Object { database_id: u64, object_id: Option<u64> },
Error {
message: String,
},
Catalog {
schemas: Vec<TableSchemaWire>,
},
DomainSnapshot {
schemas: Vec<TableSchemaWire>,
rows: Vec<(String, Vec<Row>)>,
checkpoint: String,
},
MvccSnapshot {
transaction: String,
schemas: Vec<TableSchemaWire>,
rows: Vec<(String, Vec<Row>)>,
checkpoint: String,
},
SnapshotUnchanged {
checkpoint: String,
},
Object {
database_id: u64,
object_id: Option<u64>,
},
}
#[derive(Debug, Deserialize)]
@@ -330,7 +471,11 @@ pub struct TableSchemaWire {
pub database_id: u64,
#[serde(default)]
pub object_id: u64,
#[serde(default, serialize_with = "serialize_version", deserialize_with = "deserialize_version")]
#[serde(
default,
serialize_with = "serialize_version",
deserialize_with = "deserialize_version"
)]
pub schema_version: u64,
pub columns: Vec<SchemaColumn>,
#[serde(default)]
@@ -342,7 +487,10 @@ pub struct TableSchemaWire {
}
/// 将结构版本编码成精确十进制字符串
fn serialize_version<S: serde::Serializer>(version: &u64, serializer: S) -> Result<S::Ok, S::Error> {
fn serialize_version<S: serde::Serializer>(
version: &u64,
serializer: S,
) -> Result<S::Ok, S::Error> {
serializer.serialize_str(&version.to_string())
}
@@ -350,9 +498,14 @@ fn serialize_version<S: serde::Serializer>(version: &u64, serializer: S) -> Resu
fn deserialize_version<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result<u64, D::Error> {
let value = serde_json::Value::deserialize(deserializer)?;
match value {
serde_json::Value::String(s) if !s.is_empty() && s.len() <= 20 && s.bytes().all(|c| c.is_ascii_digit()) =>
s.parse().map_err(serde::de::Error::custom),
serde_json::Value::Number(n) => n.as_u64().ok_or_else(|| serde::de::Error::custom("invalid schema version")),
serde_json::Value::String(s)
if !s.is_empty() && s.len() <= 20 && s.bytes().all(|c| c.is_ascii_digit()) =>
{
s.parse().map_err(serde::de::Error::custom)
}
serde_json::Value::Number(n) => n
.as_u64()
.ok_or_else(|| serde::de::Error::custom("invalid schema version")),
_ => Err(serde::de::Error::custom("invalid schema version")),
}
}
chusql-core/storage/src/runtime.rs
+2727 -541
View File
File diff suppressed because it is too large. Load diff
chusql-core/storage/src/wal.rs
+129 -28
View File
@@ -26,12 +26,30 @@ const OP_CREATE_TABLE: u8 = 11;
const OP_DROP_TABLE: u8 = 12;
const OP_CREATE_INDEX: u8 = 13;
const OP_DROP_INDEX: u8 = 14;
const OP_MIGRATE: u8 = 15;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub enum WalOp {
HideColumn { table: String, column: String, schema_version: Option<u64> },
Compact { table: String, rows: Vec<Row> },
Accounts { accounts: Vec<Account>, removed: Vec<String>, migrate_roles: bool },
Migrate {
table: String,
object_id: u64,
source: String,
target: String,
},
HideColumn {
table: String,
column: String,
schema_version: Option<u64>,
},
Compact {
table: String,
rows: Vec<Row>,
},
Accounts {
accounts: Vec<Account>,
removed: Vec<String>,
migrate_roles: bool,
},
Insert {
table: String,
row: Row,
@@ -80,6 +98,28 @@ pub enum WalOp {
Commit,
}
impl WalOp {
/// 返回单表操作的目标，账号组另行展开
pub(crate) fn table(&self) -> Option<&str> {
match self {
Self::Migrate { table, .. }
| Self::HideColumn { table, .. }
| Self::Compact { table, .. }
| Self::Insert { table, .. }
| Self::InsertBatch { table, .. }
| Self::DeleteKeys { table, .. }
| Self::ReplaceAll { table, .. }
| Self::DropColumn { table, .. }
| Self::ReplaceSchema { table, .. }
| Self::CreateTable { table, .. }
| Self::DropTable { table }
| Self::CreateIndex { table, .. }
| Self::DropIndex { table, .. } => Some(table),
Self::Accounts { .. } | Self::Commit => None,
}
}
}
#[derive(Debug, Serialize, Deserialize)]
struct HideColumnPayload {
column: String,
@@ -255,13 +295,11 @@ impl Wal {
/// 读当前 WAL 的第一条操作；空则 None
pub fn read(&self) -> io::Result<Option<WalOp>> {
let scan = self.read_all()?;
Ok(scan.records.into_iter().find_map(|(_, op)| {
if op == WalOp::Commit {
None
} else {
Some(op)
}
}))
Ok(scan.records.into_iter().find_map(
|(_, op)| {
if op == WalOp::Commit { None } else { Some(op) }
},
))
}
/// 读全部记录，撕裂或损坏的尾巴按长度截断
@@ -287,9 +325,12 @@ impl Wal {
let mut frames = Vec::new();
let mut offset = 0usize;
while offset + 4 <= buf.len() {
let body_len =
u32::from_le_bytes([buf[offset], buf[offset + 1], buf[offset + 2], buf[offset + 3]])
as usize;
let body_len = u32::from_le_bytes([
buf[offset],
buf[offset + 1],
buf[offset + 2],
buf[offset + 3],
]) as usize;
let end = offset + 4 + body_len;
if end > buf.len() {
break;
@@ -324,7 +365,10 @@ impl Wal {
}
let bytes = std::fs::read(&path)?;
if bytes.len() != 8 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid WAL checkpoint length"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid WAL checkpoint length",
));
}
let mut raw = [0u8; 8];
raw.copy_from_slice(&bytes[..8]);
@@ -342,10 +386,14 @@ impl Wal {
if let Some((lsn, _, _, _)) = frames.last() {
base = base.max(*lsn);
}
*slot = base.checked_add(1).ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
*slot = base
.checked_add(1)
.ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
}
let lsn = *slot;
*slot = slot.checked_add(1).ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
*slot = slot
.checked_add(1)
.ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
Ok(lsn)
}
@@ -392,6 +440,7 @@ fn truncate(f: &mut File) -> io::Result<()> {
/// 把操作编成字节
fn encode(op: &WalOp, lsn: u64) -> io::Result<Vec<u8>> {
let (op_type, table, payload) = match op {
WalOp::Migrate { table, .. } => (OP_MIGRATE, table.clone(), serde_json::to_vec(op).map_err(io::Error::other)?),
WalOp::HideColumn { table, column, schema_version } => (OP_HIDE_COLUMN, table.clone(),
serde_json::to_vec(&serde_json::json!({"column":column,"schema_version":schema_version})).map_err(io::Error::other)?),
WalOp::Compact { table, rows } => {
@@ -487,6 +536,25 @@ fn decode_body(body: &[u8]) -> Result<(u64, WalOp), String> {
let payload = &body[11 + table_len..];
let op = match op_type {
OP_MIGRATE => {
let op: WalOp = serde_json::from_slice(payload)
.map_err(|e| format!("bad migration payload: {e}"))?;
match &op {
WalOp::Migrate {
table: name,
source,
target,
..
} if name == &table
&& crate::catalog::valid_generation(source)
&& !target.is_empty()
&& crate::catalog::valid_generation(target) =>
{
op
}
_ => return Err("invalid migration WAL".into()),
}
}
OP_COMMIT => WalOp::Commit,
OP_HIDE_COLUMN => {
let value: serde_json::Value = serde_json::from_slice(payload)
@@ -498,28 +566,61 @@ fn decode_body(body: &[u8]) -> Result<(u64, WalOp), String> {
.map_err(|e| format!("bad hide_column payload: {e}"))?;
(p.column, p.schema_version)
};
WalOp::HideColumn { table, column, schema_version }
WalOp::HideColumn {
table,
column,
schema_version,
}
}
OP_COMPACT => {
let rows: Vec<Row> = serde_json::from_slice(payload)
.map_err(|e| format!("bad compact payload: {e}"))?;
let rows: Vec<Row> =
serde_json::from_slice(payload).map_err(|e| format!("bad compact payload: {e}"))?;
WalOp::Compact { table, rows }
}
OP_ACCOUNTS => {
if table != USERS_TABLE { return Err("invalid account WAL table".into()); }
let value: serde_json::Value = serde_json::from_slice(payload).map_err(|e| format!("bad account payload: {e}"))?;
if table != USERS_TABLE {
return Err("invalid account WAL table".into());
}
let value: serde_json::Value =
serde_json::from_slice(payload).map_err(|e| format!("bad account payload: {e}"))?;
let (accounts, removed, migrate_roles) = if value.is_array() {
(serde_json::from_value(value).map_err(|e| format!("bad account payload: {e}"))?, Vec::new(), false)
(
serde_json::from_value(value)
.map_err(|e| format!("bad account payload: {e}"))?,
Vec::new(),
false,
)
} else {
(serde_json::from_value(value.get("accounts").cloned().ok_or("missing account payload")?).map_err(|e| format!("bad account payload: {e}"))?,
serde_json::from_value(value.get("removed").cloned().ok_or("missing identity cleanup")?).map_err(|e| format!("bad account cleanup: {e}"))?,
value.get("migrate_roles").and_then(serde_json::Value::as_bool).ok_or("missing role migration flag")?)
(
serde_json::from_value(
value
.get("accounts")
.cloned()
.ok_or("missing account payload")?,
)
.map_err(|e| format!("bad account payload: {e}"))?,
serde_json::from_value(
value
.get("removed")
.cloned()
.ok_or("missing identity cleanup")?,
)
.map_err(|e| format!("bad account cleanup: {e}"))?,
value
.get("migrate_roles")
.and_then(serde_json::Value::as_bool)
.ok_or("missing role migration flag")?,
)
};
WalOp::Accounts { accounts, removed, migrate_roles }
WalOp::Accounts {
accounts,
removed,
migrate_roles,
}
}
OP_INSERT => {
let row: Row =
serde_json::from_slice(payload).map_err(|e| format!("bad insert payload: {}", e))?;
let row: Row = serde_json::from_slice(payload)
.map_err(|e| format!("bad insert payload: {}", e))?;
WalOp::Insert { table, row }
}
OP_INSERT_BATCH => {
chusql-core/storage/tests/btree_test.rs
+125 -12
View File
@@ -1,4 +1,4 @@
use chusql_core_storage::btree::{fits_in_page, DiskBTree, MAX_KEY_BYTES};
use chusql_core_storage::btree::{DiskBTree, MAX_KEY_BYTES, fits_in_page};
use chusql_core_storage::config::{DEFAULT_BTREE_ORDER, DEFAULT_PAGE_SIZE};
use chusql_core_storage::page::DEFAULT_POOL_SIZE;
use std::io::{ErrorKind, Seek, SeekFrom, Write};
@@ -16,20 +16,28 @@ fn malformed_node_fields_return_invalid_data() {
tree.insert(b"y", 2).unwrap();
}
let original = std::fs::read(&path).unwrap();
let cell = u16::from_le_bytes([original[DEFAULT_PAGE_SIZE + 11], original[DEFAULT_PAGE_SIZE + 12]]) as usize;
let cell = u16::from_le_bytes([
original[DEFAULT_PAGE_SIZE + 11],
original[DEFAULT_PAGE_SIZE + 12],
]) as usize;
for (name, offset, bytes) in [
("slot count", 1, vec![255, 255]),
("node kind", 0, vec![2]),
("cell overlaps slots", 11, vec![0, 0]),
("cell beyond page", 11, vec![255, 15]),
("overlapping cells", 13, (cell as u16).to_le_bytes().to_vec()),
(
"overlapping cells",
13,
(cell as u16).to_le_bytes().to_vec(),
),
("key length", cell, vec![255, 255]),
("key order", cell + 2, vec![b'z']),
("zero child", 0, vec![1]),
] {
std::fs::write(&path, &original).unwrap();
let mut file = std::fs::OpenOptions::new().write(true).open(&path).unwrap();
file.seek(SeekFrom::Start((DEFAULT_PAGE_SIZE + offset) as u64)).unwrap();
file.seek(SeekFrom::Start((DEFAULT_PAGE_SIZE + offset) as u64))
.unwrap();
file.write_all(&bytes).unwrap();
drop(file);
let mut tree = open(dir.path(), "corrupt.idx");
@@ -42,7 +50,13 @@ fn malformed_node_fields_return_invalid_data() {
/// 按默认参数打开指定索引文件
fn open(dir: &std::path::Path, name: &str) -> DiskBTree {
DiskBTree::open(dir.join(name), DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).unwrap()
DiskBTree::open(
dir.join(name),
DEFAULT_PAGE_SIZE,
DEFAULT_BTREE_ORDER,
DEFAULT_POOL_SIZE,
)
.unwrap()
}
/// 整数键编码成保序的 8 字节（与堆表一致）
@@ -101,12 +115,24 @@ fn reopen_persists() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("t.idx");
{
let mut t = DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).unwrap();
let mut t = DiskBTree::open(
&path,
DEFAULT_PAGE_SIZE,
DEFAULT_BTREE_ORDER,
DEFAULT_POOL_SIZE,
)
.unwrap();
for i in 0..50 {
t.insert(&key(i), i as u64).unwrap();
}
}
let mut t = DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).unwrap();
let mut t = DiskBTree::open(
&path,
DEFAULT_PAGE_SIZE,
DEFAULT_BTREE_ORDER,
DEFAULT_POOL_SIZE,
)
.unwrap();
for i in 0..50 {
assert_eq!(t.get(&key(i)).unwrap(), Some(i as u64));
}
@@ -148,12 +174,26 @@ fn config_mismatch_errors() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("t.idx");
{
let mut t = DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).unwrap();
let mut t = DiskBTree::open(
&path,
DEFAULT_PAGE_SIZE,
DEFAULT_BTREE_ORDER,
DEFAULT_POOL_SIZE,
)
.unwrap();
t.insert(&key(1), 1).unwrap();
}
assert!(DiskBTree::open(&path, DEFAULT_PAGE_SIZE, 8, DEFAULT_POOL_SIZE).is_err());
assert!(DiskBTree::open(&path, 8192, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).is_err());
assert!(DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).is_ok());
assert!(
DiskBTree::open(
&path,
DEFAULT_PAGE_SIZE,
DEFAULT_BTREE_ORDER,
DEFAULT_POOL_SIZE
)
.is_ok()
);
}
/// 不是索引文件要报错
@@ -162,7 +202,15 @@ fn not_an_index_file_errors() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("junk.idx");
std::fs::write(&path, vec![7u8; DEFAULT_PAGE_SIZE]).unwrap();
assert!(DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).is_err());
assert!(
DiskBTree::open(
&path,
DEFAULT_PAGE_SIZE,
DEFAULT_BTREE_ORDER,
DEFAULT_POOL_SIZE
)
.is_err()
);
}
/// order 放不进一页要报错
@@ -182,7 +230,10 @@ fn delete_removes_only_that_key() {
t.insert(&key(i), i as u64 + 1).unwrap();
}
assert!(t.delete(&key(50), 51).unwrap(), "deleting an existing entry should return true");
assert!(
t.delete(&key(50), 51).unwrap(),
"deleting an existing entry should return true"
);
assert_eq!(t.get(&key(50)).unwrap(), None);
assert_eq!(t.get(&key(49)).unwrap(), Some(50));
assert_eq!(t.get(&key(51)).unwrap(), Some(52));
@@ -262,7 +313,11 @@ fn splits_keep_keys_sorted() {
let all: Vec<Vec<u8>> = t.iter_all().unwrap().into_iter().map(|(k, _)| k).collect();
assert_eq!(all, expected);
for k in &expected {
assert!(t.get(k).unwrap().is_some(), "missing {:?}", String::from_utf8_lossy(k));
assert!(
t.get(k).unwrap().is_some(),
"missing {:?}",
String::from_utf8_lossy(k)
);
}
}
@@ -314,3 +369,61 @@ fn clear_resets_the_tree() {
t.insert(b"c", 3).unwrap();
assert_eq!(t.get(b"c").unwrap(), Some(3));
}
/// 长键跨页后可重启查询并保留旧配置
#[test]
fn long_keys_split_reopen_and_preserve_dense_configuration() {
use chusql_core_storage::btree::key_limit;
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("long.idx");
let mut tree = DiskBTree::open(&path, 512, 4, DEFAULT_POOL_SIZE).unwrap();
for n in 0..80_u64 {
let key = format!("{n:03}{}", "a".repeat(125));
tree.insert(key.as_bytes(), n).unwrap();
}
drop(tree);
let mut tree = DiskBTree::open(&path, 512, 4, DEFAULT_POOL_SIZE).unwrap();
assert_eq!(tree.scan(None, None).unwrap().len(), 80);
for n in 0..80_u64 {
let key = format!("{n:03}{}", "a".repeat(125));
assert_eq!(tree.get(key.as_bytes()).unwrap(), Some(n));
}
let dense_path = dir.path().join("dense.idx");
let mut dense = DiskBTree::open(&dense_path, 512, 7, DEFAULT_POOL_SIZE).unwrap();
let limit = key_limit(512, 7);
assert!((64..MAX_KEY_BYTES).contains(&limit));
dense.insert(&vec![b'a'; limit], 1).unwrap();
assert_eq!(
dense.insert(&vec![b'b'; limit + 1], 2).unwrap_err().kind(),
ErrorKind::InvalidInput
);
assert_eq!(dense.get(&vec![b'a'; limit]).unwrap(), Some(1));
}
/// 旧文件头可无损打开并首次写入长键
#[test]
fn legacy_key_header_upgrades_only_when_long_keys_are_written() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("legacy.idx");
let mut tree = open(dir.path(), "legacy.idx");
tree.insert(b"old", 1).unwrap();
drop(tree);
let mut bytes = std::fs::read(&path).unwrap();
bytes[8..10].copy_from_slice(&64_u16.to_le_bytes());
std::fs::write(&path, &bytes).unwrap();
let mut tree = open(dir.path(), "legacy.idx");
assert_eq!(tree.get(b"old").unwrap(), Some(1));
drop(tree);
assert_eq!(std::fs::read(&path).unwrap(), bytes);
let mut tree = open(dir.path(), "legacy.idx");
let key = "author/".to_string() + &"p".repeat(80);
tree.insert(key.as_bytes(), 2).unwrap();
drop(tree);
let mut tree = open(dir.path(), "legacy.idx");
assert_eq!(tree.get(key.as_bytes()).unwrap(), Some(2));
assert_eq!(tree.get(b"old").unwrap(), Some(1));
assert_eq!(
&std::fs::read(&path).unwrap()[8..10],
&128_u16.to_le_bytes()
);
}
chusql-core/storage/tests/catalog_test.rs
+83 -31
View File
@@ -33,9 +33,14 @@ fn schema_versions_track_structure_and_survive_reload() {
fn legacy_schema_version_remains_unknown_until_change() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("catalog.json");
std::fs::write(&path, json!({"tables": {
"t": {"object_id": 42, "columns": []}
}}).to_string()).unwrap();
std::fs::write(
&path,
json!({"tables": {
"t": {"object_id": 42, "columns": []}
}})
.to_string(),
)
.unwrap();
let mut c = Catalog::load(&path).unwrap();
assert_eq!(c.describe("t").unwrap().schema_version, 0);
c.save(&path).unwrap();
@@ -51,10 +56,15 @@ fn legacy_schema_version_remains_unknown_until_change() {
fn exhausted_schema_version_preserves_structure() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("catalog.json");
std::fs::write(&path, json!({"tables": {
"t": {"object_id": 42, "schema_version": u64::MAX,
"columns": [{"name": "id", "ty": "int"}]}
}}).to_string()).unwrap();
std::fs::write(
&path,
json!({"tables": {
"t": {"object_id": 42, "schema_version": u64::MAX,
"columns": [{"name": "id", "ty": "int"}]}
}})
.to_string(),
)
.unwrap();
let mut c = Catalog::load(&path).unwrap();
assert!(c.remove_column("t", "id").is_err());
assert!(c.set_columns("t", vec![]).is_err());
@@ -90,20 +100,42 @@ fn object_ids_survive_reload_and_are_not_reused() {
fn legacy_object_ids_upgrade_without_collisions() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("catalog.json");
std::fs::write(&path, json!({"tables": {
"a": {"columns": [], "object_id": 42},
"b": {"columns": []}
}}).to_string()).unwrap();
std::fs::write(
&path,
json!({"tables": {
"a": {"columns": [], "object_id": 42},
"b": {"columns": []}
}})
.to_string(),
)
.unwrap();
let catalog = Catalog::load(&path).unwrap();
assert_eq!(catalog.describe("a").unwrap().object_id, 42);
assert_eq!(catalog.describe("b").unwrap().object_id, 43);
catalog.save(&path).unwrap();
assert_eq!(Catalog::load(&path).unwrap().describe("b").unwrap().object_id, 43);
std::fs::write(&path, json!({"tables": {
"a": {"columns": [], "object_id": 42},
"b": {"columns": [], "object_id": 42}
}}).to_string()).unwrap();
assert!(Catalog::load(&path).unwrap_err().to_string().contains("duplicate"));
assert_eq!(
Catalog::load(&path)
.unwrap()
.describe("b")
.unwrap()
.object_id,
43
);
std::fs::write(
&path,
json!({"tables": {
"a": {"columns": [], "object_id": 42},
"b": {"columns": [], "object_id": 42}
}})
.to_string(),
)
.unwrap();
assert!(
Catalog::load(&path)
.unwrap_err()
.to_string()
.contains("duplicate")
);
}
/// 编号耗尽明确失败且不建立对象
@@ -115,8 +147,6 @@ fn exhausted_object_ids_fail_without_creating_table() {
assert!(catalog.describe("t").is_none());
}
/// 用键值对构造一行
fn row(pairs: &[(&str, serde_json::Value)]) -> Row {
let mut m = Row::new();
@@ -128,7 +158,8 @@ fn row(pairs: &[(&str, serde_json::Value)]) -> Row {
/// 空文件得到空字典，未知表返回 None
#[test]
fn empty_file_gives_empty_catalog_and_unknown_table_returns_none() { let dir = tempfile::tempdir().unwrap();
fn empty_file_gives_empty_catalog_and_unknown_table_returns_none() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("catalog.json");
let c = Catalog::load(&path).unwrap();
assert_eq!(c.table_names().len(), 0);
@@ -142,7 +173,8 @@ fn ensure_then_describe() {
let path = dir.path().join("catalog.json");
let mut c = Catalog::load(&path).unwrap();
c.ensure_columns("users", &row(&[("id", json!(1)), ("name", json!("Alice"))])).unwrap();
c.ensure_columns("users", &row(&[("id", json!(1)), ("name", json!("Alice"))]))
.unwrap();
c.save(&path).unwrap();
let c2 = Catalog::load(&path).unwrap();
@@ -164,8 +196,10 @@ fn second_insert_does_not_duplicate_columns() {
let path = dir.path().join("catalog.json");
let mut c = Catalog::load(&path).unwrap();
c.ensure_columns("users", &row(&[("id", json!(1))])).unwrap();
c.ensure_columns("users", &row(&[("id", json!(2))])).unwrap();
c.ensure_columns("users", &row(&[("id", json!(1))]))
.unwrap();
c.ensure_columns("users", &row(&[("id", json!(2))]))
.unwrap();
assert_eq!(c.describe("users").unwrap().columns.len(), 1);
}
@@ -176,9 +210,11 @@ fn remove_column_drops_definition_stats_and_index() {
let path = dir.path().join("catalog.json");
let mut c = Catalog::load(&path).unwrap();
c.ensure_columns("t", &row(&[("id", json!(1)), ("code", json!(5))])).unwrap();
c.ensure_columns("t", &row(&[("id", json!(1)), ("code", json!(5))]))
.unwrap();
c.add_index("t", "code").unwrap();
c.rebuild_stats("t", &[row(&[("id", json!(1)), ("code", json!(5))])]).unwrap();
c.rebuild_stats("t", &[row(&[("id", json!(1)), ("code", json!(5))])])
.unwrap();
let removed = c.remove_column("t", "code").unwrap();
assert!(removed, "first remove should hit the column");
@@ -186,7 +222,10 @@ fn remove_column_drops_definition_stats_and_index() {
assert_eq!(s.columns.len(), 1);
assert_eq!(s.columns[0].name, "id");
assert!(s.indexes.is_empty(), "index on code should be gone");
assert!(!s.stats.contains_key("code"), "stats for code should be gone");
assert!(
!s.stats.contains_key("code"),
"stats for code should be gone"
);
let again = c.remove_column("t", "code").unwrap();
assert!(!again);
@@ -217,7 +256,8 @@ fn stats_track_a_histogram_for_numeric_columns() {
assert!(name.hist.is_empty(), "字符串列不进直方图");
assert_eq!(name.lo, None);
c.record_insert("t", &row(&[("id", json!(1000)), ("name", json!("x"))])).unwrap();
c.record_insert("t", &row(&[("id", json!(1000)), ("name", json!("x"))]))
.unwrap();
let s = c.describe("t").unwrap();
let id = s.stats.get("id").unwrap();
assert_eq!(id.lo, Some(0.0));
@@ -240,7 +280,9 @@ fn stats_track_a_histogram_for_numeric_columns() {
fn duplicate_deletion_keeps_distinct_until_last_occurrence() {
let mut catalog = Catalog::default();
let duplicate = row(&[("id", json!(1)), ("v", json!(7))]);
catalog.rebuild_stats("t", &[duplicate.clone(), duplicate.clone()]).unwrap();
catalog
.rebuild_stats("t", &[duplicate.clone(), duplicate.clone()])
.unwrap();
catalog.record_delete("t", std::slice::from_ref(&duplicate));
assert_eq!(catalog.describe("t").unwrap().stats["v"].distinct, 1);
catalog.record_delete("t", &[duplicate]);
@@ -251,7 +293,9 @@ fn duplicate_deletion_keeps_distinct_until_last_occurrence() {
#[test]
fn capped_distinct_keeps_histogram_and_other_columns_accurate() {
let mut catalog = Catalog::default();
let rows: Vec<Row> = (0..5000).map(|n| row(&[("id", json!(n)), ("v", json!(n % 2))])).collect();
let rows: Vec<Row> = (0..5000)
.map(|n| row(&[("id", json!(n)), ("v", json!(n % 2))]))
.collect();
catalog.rebuild_stats("t", &rows).unwrap();
let schema = catalog.describe("t").unwrap();
assert!(schema.stats["id"].capped);
@@ -277,7 +321,9 @@ fn rebuilt_histogram_has_exact_buckets_in_any_order() {
let stat = &forward.describe("t").unwrap().stats["v"];
assert_eq!(stat.hist, vec![10; 16]);
assert_eq!(stat.hist, reverse.describe("t").unwrap().stats["v"].hist);
forward.record_insert("t", &row(&[("v", json!(10000))])).unwrap();
forward
.record_insert("t", &row(&[("v", json!(10000))]))
.unwrap();
forward.record_delete("t", &[row(&[("v", json!(10000))])]);
forward.rebuild_stats("t", &rows[80..]).unwrap();
let repaired = &forward.describe("t").unwrap().stats["v"];
@@ -316,7 +362,13 @@ fn legacy_catalog_loads_and_normalizes() {
assert_eq!(s.columns.len(), 2);
assert!(s.columns[0].nullable, "老列默认可空");
assert!(c.normalize_types().unwrap());
let types: Vec<&str> = c.describe("legacy").unwrap().columns.iter().map(|col| col.ty.as_str()).collect();
let types: Vec<&str> = c
.describe("legacy")
.unwrap()
.columns
.iter()
.map(|col| col.ty.as_str())
.collect();
assert_eq!(types, vec!["int", "str"]);
assert!(!c.normalize_types().unwrap(), "第二次没有可改的");
}
chusql-core/storage/tests/config_test.rs
+21 -6
View File
@@ -36,8 +36,7 @@ fn defaults_without_file() {
fn file_overrides_only_written_keys() {
let _guard = env_guard();
let path = PathBuf::from("t.toml");
let loaded = config::resolve(Some("[page]\nsize = 8192\n"), Some(path.clone()))
.unwrap();
let loaded = config::resolve(Some("[page]\nsize = 8192\n"), Some(path.clone())).unwrap();
assert_eq!(loaded.config.page_size, 8192);
assert_eq!(loaded.config.btree_order, DEFAULT_BTREE_ORDER);
@@ -45,9 +44,17 @@ fn file_overrides_only_written_keys() {
assert_eq!(loaded.config.log_files, PathBuf::from(DEFAULT_LOG_FILES));
assert_eq!(loaded.config_path, Some(path.clone()));
let page = loaded.origins.iter().find(|(n, _, _)| *n == "page.size").unwrap();
let page = loaded
.origins
.iter()
.find(|(n, _, _)| *n == "page.size")
.unwrap();
assert_eq!(page.2, Origin::File(path));
let order = loaded.origins.iter().find(|(n, _, _)| *n == "btree.order").unwrap();
let order = loaded
.origins
.iter()
.find(|(n, _, _)| *n == "btree.order")
.unwrap();
assert_eq!(order.2, Origin::Default);
}
@@ -110,7 +117,11 @@ fn default_config_path_follows_system_convention() {
let path = config::default_config_path();
assert_eq!(path.file_name().unwrap(), "settings.toml");
assert_eq!(path.parent().unwrap().file_name().unwrap(), "ChuSQL");
assert!(path.ends_with(format!("ChuSQL{}settings.toml", std::path::MAIN_SEPARATOR)), "got {}", path.display());
assert!(
path.ends_with(format!("ChuSQL{}settings.toml", std::path::MAIN_SEPARATOR)),
"got {}",
path.display()
);
// 本机 APPDATA 存在时必须是 %APPDATA%\ChuSQL\settings.toml
#[cfg(windows)]
@@ -168,7 +179,11 @@ fn explicit_missing_path_is_an_error() {
"got {}",
err
);
assert!(err.contains("chusql-no-such-config-9f21.toml"), "got {}", err);
assert!(
err.contains("chusql-no-such-config-9f21.toml"),
"got {}",
err
);
}
/// 不给 --config 且无配置文件时用内置默认
chusql-core/storage/tests/ffi_test.rs
+64 -10
View File
@@ -26,16 +26,40 @@ fn maintenance_rejects_invalid_arguments_and_resets_outputs() {
let mut len = 99;
// SAFETY: 输出均可写；无效输入应在任何读取之前被拒绝。
unsafe {
assert_eq!(chusql_storage_maintenance(ptr::null(), ptr::null(), 0, &mut out, &mut len), -1);
assert_eq!(
chusql_storage_maintenance(ptr::null(), ptr::null(), 0, &mut out, &mut len),
-1
);
assert!(out.is_null());
assert_eq!(len, 0);
len = 99;
assert_eq!(chusql_storage_maintenance(ptr::null(), input.as_ptr(), input.len(), ptr::null_mut(), &mut len), -1);
assert_eq!(
chusql_storage_maintenance(
ptr::null(),
input.as_ptr(),
input.len(),
ptr::null_mut(),
&mut len
),
-1
);
assert_eq!(len, 0);
out = input.as_ptr() as *mut u8;
assert_eq!(chusql_storage_maintenance(ptr::null(), input.as_ptr(), input.len(), &mut out, ptr::null_mut()), -1);
assert_eq!(
chusql_storage_maintenance(
ptr::null(),
input.as_ptr(),
input.len(),
&mut out,
ptr::null_mut()
),
-1
);
assert!(out.is_null());
assert_eq!(chusql_storage_maintenance(ptr::null(), input.as_ptr(), usize::MAX, &mut out, &mut len), -1);
assert_eq!(
chusql_storage_maintenance(ptr::null(), input.as_ptr(), usize::MAX, &mut out, &mut len),
-1
);
}
assert!(out.is_null());
assert_eq!(len, 0);
@@ -47,20 +71,36 @@ fn maintenance_rejects_invalid_arguments_and_resets_outputs() {
fn request_validates_input_and_returns_owned_response() -> Result<(), Box<dyn std::error::Error>> {
let temp = tempfile::tempdir()?;
let config = temp.path().join("settings.toml");
std::fs::write(&config, format!("[storage]\ndata_dir = '{}'\nlog_files = '{}'\n[log]\nlevel = 'off'\n", temp.path().join("data").display(), temp.path().join("logs").display()))?;
std::fs::write(
&config,
format!(
"[storage]\ndata_dir = '{}'\nlog_files = '{}'\n[log]\nlevel = 'off'\n",
temp.path().join("data").display(),
temp.path().join("logs").display()
),
)?;
let storage = Storage::open(config.to_str())?;
let input = br#"{"method":"ping"}"#;
let mut out = ptr::null_mut();
let mut len = 0;
// SAFETY: 句柄和实际请求有效；超长长度必须在构造切片前被拒绝。
unsafe {
assert_eq!(chusql_storage_request(&storage, input.as_ptr(), usize::MAX, &mut out, &mut len), -1);
assert_eq!(
chusql_storage_request(&storage, input.as_ptr(), usize::MAX, &mut out, &mut len),
-1
);
assert!(out.is_null());
assert_eq!(len, 0);
assert_eq!(last_error(), "invalid argument");
assert_eq!(chusql_storage_request(&storage, [0xff].as_ptr(), 1, &mut out, &mut len), -1);
assert_eq!(
chusql_storage_request(&storage, [0xff].as_ptr(), 1, &mut out, &mut len),
-1
);
assert!(last_error().starts_with("request is not valid UTF-8:"));
assert_eq!(chusql_storage_request(&storage, input.as_ptr(), input.len(), &mut out, &mut len), 0);
assert_eq!(
chusql_storage_request(&storage, input.as_ptr(), input.len(), &mut out, &mut len),
0
);
assert!(!out.is_null());
let response = slice::from_raw_parts(out, len).to_vec();
chusql_storage_free(out, len);
@@ -71,7 +111,10 @@ fn request_validates_input_and_returns_owned_response() -> Result<(), Box<dyn st
let config = CString::new(config.to_str().unwrap())?;
// SAFETY: 路径与输出有效；失败不产生需要释放的缓冲。
unsafe {
assert_eq!(chusql_storage_maintenance(config.as_ptr(), [0xff].as_ptr(), 1, &mut out, &mut len), -1);
assert_eq!(
chusql_storage_maintenance(config.as_ptr(), [0xff].as_ptr(), 1, &mut out, &mut len),
-1
);
}
assert!(out.is_null());
assert_eq!(len, 0);
@@ -79,7 +122,18 @@ fn request_validates_input_and_returns_owned_response() -> Result<(), Box<dyn st
let maintenance = br#"{"mode":"reset","user":"root","password_hash":"hash","types":[{"name":"int","base_type":"int","parameterized":false}]}"#;
// SAFETY: 维护仅使用本测试临时目录；成功响应复制后由配套函数释放。
unsafe {
assert_eq!(chusql_storage_maintenance(config.as_ptr(), maintenance.as_ptr(), maintenance.len(), &mut out, &mut len), 0, "{}", last_error());
assert_eq!(
chusql_storage_maintenance(
config.as_ptr(),
maintenance.as_ptr(),
maintenance.len(),
&mut out,
&mut len
),
0,
"{}",
last_error()
);
assert!(!out.is_null());
let response = slice::from_raw_parts(out, len).to_vec();
chusql_storage_free(out, len);
chusql-core/storage/tests/heap_test.rs
+135 -37
View File
@@ -25,20 +25,61 @@ fn delete_propagates_index_read_errors() {
let original = row(&[("id", json!(1))]);
t.insert_row(&original).unwrap();
t.flush().unwrap();
let file = std::fs::OpenOptions::new().write(true).open(&index).unwrap();
let file = std::fs::OpenOptions::new()
.write(true)
.open(&index)
.unwrap();
file.set_len(DEFAULT_PAGE_SIZE as u64).unwrap();
let err = t.delete_by_keys(&[1]).unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
assert_eq!(t.scan().unwrap(), vec![original]);
}
/// 索引键失败不得留下堆行或清空旧表
#[test]
fn invalid_index_keys_leave_heap_and_indexes_unchanged() {
let dir = tempfile::tempdir().unwrap();
let mut table = HeapTable::open(dir.path().join("t.db"), DEFAULT_PAGE_SIZE, 1).unwrap();
let original = row(&[("id", json!(1)), ("name", json!("old"))]);
table.insert_row(&original).unwrap();
table
.build_index(
"name",
&dir.path().join("t.name.idx"),
DEFAULT_PAGE_SIZE,
4,
1,
)
.unwrap();
let invalid = row(&[("id", json!(2)), ("name", json!("x".repeat(129)))]);
assert!(table.insert_row(&invalid).is_err());
assert_eq!(table.scan().unwrap(), vec![original.clone()]);
assert!(table.replace_all(&[invalid]).is_err());
assert_eq!(table.scan().unwrap(), vec![original.clone()]);
assert_eq!(
table.get_by_string_key("name", "old").unwrap(),
Some(original)
);
assert!(
table
.replace_all(&[row(&[("name", json!("x".repeat(10_000)))])])
.is_err()
);
assert_eq!(table.scan().unwrap().len(), 1);
}
/// 跨页投影与分片保持行序及隐藏列
#[test]
fn projected_shards_preserve_order_and_hidden_columns() {
let dir = tempfile::tempdir().unwrap();
let mut t = HeapTable::open(dir.path().join("t.db"), 256, 1).unwrap();
for i in 0..40 {
t.insert_row(&row(&[("id", json!(i)), ("name", json!("alice")), ("secret", json!(i))])).unwrap();
t.insert_row(&row(&[
("id", json!(i)),
("name", json!("alice")),
("secret", json!(i)),
]))
.unwrap();
}
t.set_hidden_columns(["secret".to_string()].into_iter().collect());
let columns = vec!["id".to_string(), "secret".to_string()];
@@ -47,9 +88,15 @@ fn projected_shards_preserve_order_and_hidden_columns() {
let pages = t.num_pages().unwrap();
assert!(pages > 1);
let mut shards = t.scan_columns_pages(Some(&columns), 0, pages / 2).unwrap();
shards.extend(t.scan_columns_pages(Some(&columns), pages / 2, pages + 1).unwrap());
shards.extend(
t.scan_columns_pages(Some(&columns), pages / 2, pages + 1)
.unwrap(),
);
assert_eq!(shards, expected);
assert_eq!(t.read_at(0, 0).unwrap().unwrap(), row(&[("id", json!(0)), ("name", json!("alice"))]));
assert_eq!(
t.read_at(0, 0).unwrap().unwrap(),
row(&[("id", json!(0)), ("name", json!("alice"))])
);
}
/// 扫描与点读均返回行解码错误
@@ -78,15 +125,20 @@ fn empty_index_file_is_rebuilt_from_rows() {
let data = dir.path().join("t.db");
let index = dir.path().join("t.idx");
{
let mut t = HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
let mut t = HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE)
.unwrap();
for i in 1..=5 {
t.insert_row(&row(&[("id", json!(i))])).unwrap();
}
t.flush().unwrap();
}
std::fs::write(&index, b"").unwrap();
let mut t = HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
assert!(t.get_by_column_key("id", 3).unwrap().is_some(), "空索引文件要从堆数据重建");
let mut t =
HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
assert!(
t.get_by_column_key("id", 3).unwrap().is_some(),
"空索引文件要从堆数据重建"
);
assert!(t.get_by_column_key("id", 9).unwrap().is_none());
}
@@ -97,14 +149,24 @@ fn string_index_looks_up_by_value() {
let data = dir.path().join("s.db");
let index = dir.path().join("s.idx");
let mut t = HeapTable::open(&data, DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap();
t.insert_row(&row(&[("id", json!(1)), ("name", json!("alice"))])).unwrap();
t.insert_row(&row(&[("id", json!(2)), ("name", json!("bob"))])).unwrap();
t.insert_row(&row(&[("id", json!(3)), ("name", json!(null))])).unwrap();
t.build_index("name", &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
t.insert_row(&row(&[("id", json!(1)), ("name", json!("alice"))]))
.unwrap();
t.insert_row(&row(&[("id", json!(2)), ("name", json!("bob"))]))
.unwrap();
t.insert_row(&row(&[("id", json!(3)), ("name", json!(null))]))
.unwrap();
t.build_index("name", &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE)
.unwrap();
assert_eq!(t.get_by_string_key("name", "bob").unwrap().unwrap()["id"], json!(2));
assert_eq!(
t.get_by_string_key("name", "bob").unwrap().unwrap()["id"],
json!(2)
);
assert!(t.get_by_string_key("name", "dave").unwrap().is_none());
assert!(t.get_by_string_key("name", "").unwrap().is_none(), "NULL 不入索引");
assert!(
t.get_by_string_key("name", "").unwrap().is_none(),
"NULL 不入索引"
);
}
/// 允许重复值建索引：点查把重复的行都取回来
@@ -114,9 +176,12 @@ fn duplicate_values_are_indexed_and_all_returned() {
let data = dir.path().join("d.db");
let index = dir.path().join("d.idx");
let mut t = HeapTable::open(&data, DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap();
t.insert_row(&row(&[("id", json!(1)), ("name", json!("alice"))])).unwrap();
t.insert_row(&row(&[("id", json!(2)), ("name", json!("alice"))])).unwrap();
t.build_index("name", &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
t.insert_row(&row(&[("id", json!(1)), ("name", json!("alice"))]))
.unwrap();
t.insert_row(&row(&[("id", json!(2)), ("name", json!("alice"))]))
.unwrap();
t.build_index("name", &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE)
.unwrap();
let mut ids: Vec<i64> = t
.get_all_by_string_key("name", "alice")
@@ -218,9 +283,15 @@ fn replace_all_rebuilds_index() {
])
.unwrap();
assert!(t.get_by_key(1).unwrap().is_none(), "old key must be invalidated");
assert!(
t.get_by_key(1).unwrap().is_none(),
"old key must be invalidated"
);
assert!(t.get_by_key(2).unwrap().is_none());
let got = t.get_by_key(7).unwrap().expect("new row should be indexed by id");
let got = t
.get_by_key(7)
.unwrap()
.expect("new row should be indexed by id");
assert_eq!(got["name"], json!("Zoe"));
assert_eq!(t.scan().unwrap().len(), 2);
}
@@ -239,8 +310,11 @@ fn delete_by_keys_removes_rows() {
.unwrap();
for i in 1..=5 {
t.insert_row(&row(&[("id", json!(i)), ("name", json!(format!("u{}", i)))]))
.unwrap();
t.insert_row(&row(&[
("id", json!(i)),
("name", json!(format!("u{}", i))),
]))
.unwrap();
}
let deleted = t.delete_by_keys(&[2, 4, 99]).unwrap();
@@ -254,7 +328,10 @@ fn delete_by_keys_removes_rows() {
.collect();
assert_eq!(ids, vec![1, 3, 5], "remaining rows must keep their order");
assert!(t.get_by_key(2).unwrap().is_none(), "index entry should be deleted too");
assert!(
t.get_by_key(2).unwrap().is_none(),
"index entry should be deleted too"
);
assert!(t.get_by_key(4).unwrap().is_none());
assert!(t.get_by_key(3).unwrap().is_some());
@@ -284,10 +361,7 @@ fn insert_rows_batch() {
assert_eq!(t.scan().unwrap().len(), 20);
assert_eq!(t.get_by_key(20).unwrap().unwrap()["name"], json!("u20"));
let dup = vec![
row(&[("id", json!(100))]),
row(&[("id", json!(100))]),
];
let dup = vec![row(&[("id", json!(100))]), row(&[("id", json!(100))])];
assert!(t.insert_rows(&dup).is_err());
}
@@ -320,16 +394,23 @@ fn secondary_index_is_maintained() {
)
.unwrap();
assert!(t.has_index("code"));
assert_eq!(t.get_by_column_key("code", 7002).unwrap().unwrap()["id"], json!(2));
assert_eq!(
t.get_by_column_key("code", 7002).unwrap().unwrap()["id"],
json!(2)
);
t.insert_row(&row(&[("id", json!(3)), ("code", json!(7003))]))
.unwrap();
assert_eq!(t.get_by_column_key("code", 7003).unwrap().unwrap()["id"], json!(3));
assert_eq!(
t.get_by_column_key("code", 7003).unwrap().unwrap()["id"],
json!(3)
);
t.delete_by_keys(&[2]).unwrap();
assert!(t.get_by_column_key("code", 7002).unwrap().is_none());
t.insert_row(&row(&[("id", json!(4)), ("code", json!(7001))])).unwrap();
t.insert_row(&row(&[("id", json!(4)), ("code", json!(7001))]))
.unwrap();
let mut dup_ids: Vec<i64> = t
.get_all_by_column_key("code", 7001)
.unwrap()
@@ -339,7 +420,12 @@ fn secondary_index_is_maintained() {
dup_ids.sort_unstable();
assert_eq!(dup_ids, vec![1, 4], "duplicate index values keep both rows");
let mut t2 = HeapTable::open(dir.path().join("d.db"), DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap();
let mut t2 = HeapTable::open(
dir.path().join("d.db"),
DEFAULT_PAGE_SIZE,
DEFAULT_POOL_SIZE,
)
.unwrap();
t2.insert_rows(&[
row(&[("id", json!(1)), ("age", json!(30))]),
row(&[("id", json!(2)), ("age", json!(30))]),
@@ -360,7 +446,6 @@ fn secondary_index_is_maintained() {
);
}
/// 整表整理要收回删行留下的页内空洞
#[test]
fn compact_pages_reclaims_dead_space() {
@@ -370,7 +455,8 @@ fn compact_pages_reclaims_dead_space() {
let pad = "x".repeat(200);
for i in 0..10 {
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap();
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))]))
.unwrap();
}
let pages = t.num_pages().unwrap();
@@ -395,7 +481,8 @@ fn deletes_compact_pages_so_space_is_reused() {
let pad = "y".repeat(80);
for i in 0..30 {
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap();
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))]))
.unwrap();
}
assert_eq!(t.num_pages().unwrap(), 1, "30 行要先塞进一页");
@@ -404,7 +491,8 @@ fn deletes_compact_pages_so_space_is_reused() {
t.delete_by_keys(&keys).unwrap();
for i in 100..119 {
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap();
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))]))
.unwrap();
}
assert_eq!(t.num_pages().unwrap(), 1, "删掉的空间要能被复用");
assert_eq!(t.scan().unwrap().len(), 30);
@@ -416,11 +504,13 @@ fn compact_pages_keeps_index_positions_valid() {
let dir = tempfile::tempdir().unwrap();
let data = dir.path().join("t.db");
let index = dir.path().join("t.idx");
let mut t = HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
let mut t =
HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
let pad = "z".repeat(80);
for i in 0..30 {
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap();
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))]))
.unwrap();
}
let keys: Vec<i64> = (5..25).collect();
t.delete_by_keys(&keys).unwrap();
@@ -428,9 +518,17 @@ fn compact_pages_keeps_index_positions_valid() {
for i in [0, 1, 2, 3, 4, 25, 26, 27, 28, 29] {
let found = t.get_by_column_key("id", i).unwrap();
assert_eq!(found.as_ref().map(|r| r["pad"].clone()), Some(json!(pad)), "整理后 id={} 仍要能点查", i);
assert_eq!(
found.as_ref().map(|r| r["pad"].clone()),
Some(json!(pad)),
"整理后 id={} 仍要能点查",
i
);
}
for i in 5..25 {
assert!(t.get_by_column_key("id", i).unwrap().is_none(), "删掉的行不该再查出来");
assert!(
t.get_by_column_key("id", i).unwrap().is_none(),
"删掉的行不该再查出来"
);
}
}
chusql-core/storage/tests/page_test.rs
+144
View File
@@ -142,3 +142,147 @@ fn dirty_page_flushed_on_drop() {
let p = pf2.read_page(0).unwrap();
assert_eq!(p.data[0], 42);
}
/// 已同步镜像恢复撕裂页且页 LSN 持久保留
#[test]
fn full_page_image_repairs_torn_write_before_read() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("torn.db");
let metadata = dir.path().join("torn.db.pages");
let journal = dir.path().join("torn.db.page-wal");
let mut file = PageFile::with_options(&path, 512, 1).unwrap();
file.set_lsn(7);
file.write_page(&Page::new(0, 512)).unwrap();
file.flush().unwrap();
let old_metadata = std::fs::read(&metadata).unwrap();
file.set_lsn(13);
let mut changed = Page::new(0, 512);
changed.data.fill(0x5a);
file.write_page(&changed).unwrap();
file.append_page().unwrap(); // 淘汰页零前镜像已同步
let image = std::fs::read(&journal).unwrap();
assert!(!image.is_empty());
drop(file);
std::fs::write(&metadata, old_metadata).unwrap();
std::fs::write(&journal, &image).unwrap();
std::fs::write(&path, [0x11; 213]).unwrap();
let mut reopened = PageFile::open(&path, 512).unwrap();
assert_eq!(reopened.read_page(0).unwrap().data, changed.data);
assert_eq!(reopened.page_lsn(0), Some(13));
drop(reopened);
assert_eq!(std::fs::metadata(&journal).unwrap().len(), 0);
let mut reopened = PageFile::open(&path, 512).unwrap();
assert_eq!(reopened.read_page(0).unwrap().data, changed.data);
assert_eq!(reopened.page_lsn(0), Some(13));
}
/// 撕裂日志尾不发布，完整损坏帧明确拒绝
#[test]
fn page_image_tail_and_corruption_are_distinguished() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("tail.db");
let metadata = dir.path().join("tail.db.pages");
let journal = dir.path().join("tail.db.page-wal");
let mut file = PageFile::with_options(&path, 512, 1).unwrap();
file.write_page(&Page::new(0, 512)).unwrap();
file.flush().unwrap();
let old_metadata = std::fs::read(&metadata).unwrap();
let old_data = std::fs::read(&path).unwrap();
file.set_lsn(20);
let mut changed = Page::new(0, 512);
changed.data.fill(9);
file.write_page(&changed).unwrap();
file.append_page().unwrap();
let image = std::fs::read(&journal).unwrap();
drop(file);
for end in [1, 39, image.len() - 1] {
std::fs::write(&metadata, &old_metadata).unwrap();
std::fs::write(&path, &old_data).unwrap();
std::fs::write(&journal, &image[..end]).unwrap();
let mut reopened = PageFile::open(&path, 512).unwrap();
assert_eq!(reopened.read_page(0).unwrap().data, vec![0; 512]);
}
let mut corrupt = image;
corrupt[40] ^= 1;
std::fs::write(&metadata, &old_metadata).unwrap();
std::fs::write(&journal, &corrupt).unwrap();
assert!(
PageFile::open(&path, 512)
.unwrap_err()
.to_string()
.contains("checksum")
);
assert_eq!(std::fs::read(&journal).unwrap(), corrupt);
}
/// 截断标记阻止旧页重放后复活
#[test]
fn truncate_clears_page_versions_and_survives_reopen() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("reset.db");
let mut file = PageFile::open(&path, 512).unwrap();
file.set_lsn(5);
for _ in 0..3 {
file.append_page().unwrap();
}
file.flush().unwrap();
file.set_lsn(8);
file.truncate().unwrap();
file.append_page().unwrap();
file.flush().unwrap();
assert_eq!(file.page_lsn(0), Some(8));
assert_eq!(file.page_lsn(2), None);
drop(file);
let mut file = PageFile::open(&path, 512).unwrap();
assert_eq!(file.num_pages().unwrap(), 1);
assert_eq!(file.page_lsn(0), Some(8));
assert_eq!(file.page_lsn(2), None);
}
/// 镜像写入失败后禁止析构或重试绕过恢复
#[test]
fn page_journal_error_fences_further_flushes() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("fenced.db");
let journal = dir.path().join("fenced.db.page-wal");
std::fs::write(&path, [0; 512]).unwrap();
let mut file = PageFile::open(&path, 512).unwrap();
let mut page = Page::new(0, 512);
page.data.fill(7);
file.write_page(&page).unwrap();
std::fs::create_dir(&journal).unwrap();
assert!(file.flush().is_err());
std::fs::remove_dir(&journal).unwrap();
assert!(
file.flush()
.unwrap_err()
.to_string()
.contains("requires recovery")
);
drop(file);
let mut file = PageFile::open(&path, 512).unwrap();
assert_eq!(file.read_page(0).unwrap().data, vec![0; 512]);
}
/// 小缓存淘汰不会反复记录同一批未改页
#[test]
fn eviction_batches_images_without_pool_sized_write_amplification() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("bounded.db");
let journal = dir.path().join("bounded.db.page-wal");
let mut file = PageFile::with_options(&path, 512, 2).unwrap();
file.set_lsn(9);
for id in 0..100 {
let mut page = Page::new(id, 512);
page.data.fill(id as u8);
file.write_page(&page).unwrap();
}
assert!(std::fs::metadata(&journal).unwrap().len() <= 100 * (512 + 40));
file.flush().unwrap();
drop(file);
let mut file = PageFile::open(&path, 512).unwrap();
for id in 0..100 {
assert_eq!(file.read_page(id).unwrap().data, vec![id as u8; 512]);
assert_eq!(file.page_lsn(id), Some(9));
}
}
chusql-core/storage/tests/runtime_test.rs
+3062 -569
View File
File diff suppressed because it is too large. Load diff
chusql-core/storage/tests/wal_test.rs
+68 -17
View File
@@ -1,5 +1,5 @@
use chusql_core_storage::protocol::Row;
use chusql_core_storage::wal::{committed_ops, records_after, Wal, WalOp};
use chusql_core_storage::wal::{Wal, WalOp, committed_ops, records_after};
use serde_json::json;
// WAL 测试：追加、读取、清空、截断尾巴、LSN 与提交标记。
@@ -11,12 +11,32 @@ fn schema_versions_round_trip_and_legacy_hide_decodes() {
let path = dir.path().join("wal.log");
let wal = Wal::new(&path);
let ops = [
WalOp::HideColumn { table: "t".into(), column: "name".into(), schema_version: Some(u64::MAX) },
WalOp::DropColumn { table: "t".into(), column: "name".into(), rows: vec![], schema_version: Some(3) },
WalOp::ReplaceSchema { table: "t".into(), columns: vec![], rows: vec![], schema_version: Some(4) },
WalOp::HideColumn {
table: "t".into(),
column: "name".into(),
schema_version: Some(u64::MAX),
},
WalOp::DropColumn {
table: "t".into(),
column: "name".into(),
rows: vec![],
schema_version: Some(3),
},
WalOp::ReplaceSchema {
table: "t".into(),
columns: vec![],
rows: vec![],
schema_version: Some(4),
},
];
wal.append_group(&ops).unwrap();
assert_eq!(committed_ops(&wal.read_all().unwrap().records).into_iter().map(|(_, op)| op).collect::<Vec<_>>(), ops);
assert_eq!(
committed_ops(&wal.read_all().unwrap().records)
.into_iter()
.map(|(_, op)| op)
.collect::<Vec<_>>(),
ops
);
drop(wal);
let mut body = 1_u64.to_le_bytes().to_vec();
body.push(8);
@@ -26,9 +46,14 @@ fn schema_versions_round_trip_and_legacy_hide_decodes() {
let mut frame = (body.len() as u32).to_le_bytes().to_vec();
frame.extend_from_slice(&body);
std::fs::write(&path, frame).unwrap();
assert_eq!(Wal::new(&path).read().unwrap(), Some(WalOp::HideColumn {
table: "t".into(), column: "name".into(), schema_version: None,
}));
assert_eq!(
Wal::new(&path).read().unwrap(),
Some(WalOp::HideColumn {
table: "t".into(),
column: "name".into(),
schema_version: None,
})
);
}
/// 建表编号完整往返且兼容旧载荷
@@ -37,7 +62,11 @@ fn create_table_ids_round_trip_and_legacy_payloads_decode() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("wal.log");
let wal = Wal::new(&path);
let op = WalOp::CreateTable { table: "t".into(), columns: vec![], object_id: Some(42) };
let op = WalOp::CreateTable {
table: "t".into(),
columns: vec![],
object_id: Some(42),
};
wal.append(&op).unwrap();
assert_eq!(wal.read_all().unwrap().records[0].1, op);
drop(wal);
@@ -50,11 +79,16 @@ fn create_table_ids_round_trip_and_legacy_payloads_decode() {
frame.extend_from_slice(&body);
std::fs::write(&path, frame).unwrap();
let wal = Wal::new(&path);
assert_eq!(wal.read_all().unwrap().records[0].1,
WalOp::CreateTable { table: "t".into(), columns: vec![], object_id: None });
assert_eq!(
wal.read_all().unwrap().records[0].1,
WalOp::CreateTable {
table: "t".into(),
columns: vec![],
object_id: None
}
);
}
/// 用键值对构造一行
fn row(pairs: &[(&str, serde_json::Value)]) -> Row {
let mut m = Row::new();
@@ -167,7 +201,10 @@ fn every_record_gets_an_increasing_lsn() {
let scan = wal.read_all().unwrap();
let lsns: Vec<u64> = scan.records.iter().map(|(lsn, _)| *lsn).collect();
assert_eq!(lsns, vec![1, 2, 3]);
assert_eq!(scan.good_len as u64, std::fs::metadata(&path).unwrap().len());
assert_eq!(
scan.good_len as u64,
std::fs::metadata(&path).unwrap().len()
);
let reopened = Wal::new(&path);
reopened
@@ -254,7 +291,10 @@ fn trailing_torn_frame_can_be_truncated() {
assert!(scan.good_len < full.len() - 3);
wal.truncate_to(scan.good_len).unwrap();
assert_eq!(std::fs::metadata(&path).unwrap().len(), scan.good_len as u64);
assert_eq!(
std::fs::metadata(&path).unwrap().len(),
scan.good_len as u64
);
assert_eq!(wal.read_all().unwrap().records.len(), 1);
}
@@ -360,9 +400,21 @@ fn lsn_continues_from_the_checkpoint() {
#[test]
fn records_after_keeps_only_the_tail() {
let records = vec![
(1u64, WalOp::Insert { table: "t".into(), row: row(&[("id", json!(1))]) }),
(
1u64,
WalOp::Insert {
table: "t".into(),
row: row(&[("id", json!(1))]),
},
),
(2, WalOp::Commit),
(3, WalOp::Insert { table: "t".into(), row: row(&[("id", json!(2))]) }),
(
3,
WalOp::Insert {
table: "t".into(),
row: row(&[("id", json!(2))]),
},
),
(4, WalOp::Commit),
];
let kept = records_after(&records, 2);
@@ -371,4 +423,3 @@ fn records_after_keeps_only_the_tail() {
assert_eq!(records_after(&records, 4).len(), 0);
assert_eq!(records_after(&records, 0).len(), 4);
}