修复错误,优化性能,实现类型系统,优化安装卸载过程

This commit is contained in:
LIm1tZz7 committed 2026-10-05 00:15:01 +08:00
1 parent fd5f320360
commit 48eec484b7
104 files changed
+4127 -845

No files matched your search

+4 -3
View File
@@ -2,9 +2,10 @@ cabal-version: 2.4
name: chusql-core-engine
version: 1.0.0
synopsis: ChuSQL 执行引擎:SQL 词法与语法分析、表达式求值
description: ChuSQL 服务端执行引擎。负责 SQL 词法分析与 AST 构建、
WHERE 条件表达式求值、关系代数运算与内存表管理。
synopsis: ChuSQL SQL parsing, query planning, and execution engine
description: Provides SQL tokenization, syntax and semantic analysis, query
planning and optimization, expression evaluation, and relational
algebra execution, with in-memory and persistent storage support.
build-type: Simple
library
@@ -99,6 +99,15 @@ runStatementUncheckedM db (Update tbl assigns mWhere) =
runStatementUncheckedM _ (CreateTable name cols) = do
result <- createTable name cols
pure (result >> Right [])
runStatementUncheckedM _ (CreateDomain name base) = do
result <- createTable (domainTableName name) [("base", plainColumn base)]
pure (result >> Right [])
runStatementUncheckedM _ (DropDomain name) = do
result <- dropTable (domainTableName name)
pure (result >> Right [])
runStatementUncheckedM db ShowDomains = pure (Right
[[("domain", VStr (drop (length (domainTableName "")) name)), ("base_type", VStr (typeName (columnType base)))]
| (name, table) <- db, isDomainTable name, Just base <- [lookup "base" (tableCols table)]])
runStatementUncheckedM _ (DropTable name) = do
result <- dropTable name
pure (result >> Right [])
@@ -123,6 +132,8 @@ runStatementUncheckedM db (AlterColumnNull tbl col nullable) =
alterTable db tbl (\table -> setColumnNullable col nullable (tableCols table) (tableRows table))
runStatementUncheckedM _ CreateUser{} = pure (Left "CREATE USER is executed by the account service")
runStatementUncheckedM _ AlterUser{} = pure (Left "ALTER USER is executed by the account service")
runStatementUncheckedM _ AlterIdentity{} = pure (Left "ALTER ROLE is executed by the account service")
runStatementUncheckedM _ ShowRoles = pure (Left "SHOW ROLES is executed by the account service")
runStatementUncheckedM _ DropUser{} = pure (Left "DROP USER is executed by the account service")
runStatementUncheckedM _ CreateRole{} = pure (Left "CREATE ROLE is executed by the privilege service")
runStatementUncheckedM _ DropRole{} = pure (Left "DROP ROLE is executed by the privilege service")
@@ -135,27 +135,33 @@ filterPairsM ::
[Row] ->
[Row] ->
m (Either String [Row])
filterPairsM db outer kind ncols cond lrows rrows = go lrows []
filterPairsM db outer kind ncols cond lrows rrows = go lrows [] HM.empty
where
refs = subqueryRefsIn cond
-- | 逐左行配对,左连接没配上就补 NULL
go [] acc = pure (Right (reverse acc))
go (l : ls) acc = do
hits <- keep l rrows []
go [] acc _ = pure (Right (reverse acc))
go (l : ls) acc cache = do
hits <- keep l rrows [] cache
case hits of
Left e -> pure (Left e)
Right hs -> do
Right (hs, nextCache) -> do
let hs' = case (kind, hs) of
(LeftJoin, []) -> [l ++ nullRow]
_ -> hs
go ls (reverse hs' ++ acc)
go ls (reverse hs' ++ acc) nextCache
-- | 拿一行右行去比条件
keep _ [] acc = pure (Right (reverse acc))
keep l (r : rs) acc = do
ok <- evalCondForRowM db cond (l ++ r ++ outer)
case ok of
keep _ [] acc cache = pure (Right (reverse acc, cache))
keep l (r : rs) acc cache = do
let env = l ++ r ++ outer
key = bindingKey refs env
resolved <- case HM.lookup key cache of
Just expression -> pure (Right expression)
Nothing -> substSubqueries db cond env
case resolved of
Left e -> pure (Left e)
Right True -> keep l rs ((l ++ r) : acc)
Right False -> keep l rs acc
Right expression -> case evalCondForRow expression env of
Left e -> pure (Left e)
Right matches -> keep l rs (if matches then (l ++ r) : acc else acc) (HM.insert key expression cache)
nullRow = [(c, VNull) | c <- ncols]
-- | 索引连接:右表是裸 Scan 时逐左行点查
@@ -197,6 +203,7 @@ floatType :: ColumnType -> Bool
floatType CFloat = True
floatType CDouble = True
floatType (CDecimal _ _) = True
floatType (CDomain _ base) = floatType base
floatType _ = False
-- | 是不是浮点值
@@ -249,7 +256,14 @@ evalAggregate outer keys aggs rows = do
-- | 绑定键:子查询引用的外层列在这一行上的取值
bindingKey :: [String] -> Row -> String
bindingKey refs env = show [(c, lookup c env) | c <- refs]
bindingKey refs env = show [(c, bindingValue c env) | c <- refs]
-- | 查已解析外层引用对应的物理列值
bindingValue :: String -> Row -> Maybe Value
bindingValue column env = case lookup column env of
Just value -> Just value
Nothing | '.' `elem` column -> lookup (reverse (takeWhile (/= '.') (reverse column))) [(key, value) | (key, value) <- env, '.' `notElem` key]
Nothing -> Nothing
-- | 每个绑定留第一行当样本,去掉重复绑定
bindingSamples :: [String] -> Row -> [Row] -> [(String, Row)]
@@ -414,7 +428,14 @@ substSubqueries db e env = case e of
runSubqueryRows :: (MonadStorage m) => Database -> Subquery -> Row -> m (Either String [Row])
runSubqueryRows db sq outer = case translate (subqueryStatement sq) of
Left e -> pure (Left e)
Right op -> evalRelOpIn db outer (optimize db op)
Right op -> case mapM bind (subqueryRefs sq) of
Left err -> pure (Left err)
Right bindings -> evalRelOpIn db (bindings ++ outer) (optimize db op)
where
-- | 给外层物理列补回已解析的限定名
bind column = case bindingValue column outer of
Just value -> Right (column, value)
Nothing -> Left ("unknown outer column: " ++ column)
-- | 单子求值:Scan/点查/范围查都问存储
evalRelOpM :: (MonadStorage m) => Database -> RelOp -> m (Either String [Row])
@@ -1,7 +1,7 @@
module ChuSQL.Core.Engine.Algebra.JoinOrder (reorderJoins) where
import ChuSQL.Core.Engine.Algebra.Cost (pointRows, rangeRows, selectivityOf, tableMetaOf)
import ChuSQL.Core.Engine.Algebra.Expr (colsInExpr, hasDivision)
import ChuSQL.Core.Engine.Algebra.Expr (colsInExpr, hasDivision, hasSubquery)
import ChuSQL.Core.Engine.Algebra.Op (RelOp (..), relOpCols)
import ChuSQL.Core.Engine.Syntax.AST (Expr (..), JoinKind (..))
import ChuSQL.Core.Model
@@ -46,8 +46,13 @@ leafRows db op = case op of
Scan _ t _ -> rowCountOf db t
Lookup _ t c _ -> pointRows <$> tableMetaOf db t <*> pure c
Range _ t c lo hi -> (\m -> rangeRows m c lo hi) <$> tableMetaOf db t
Filter p x -> scaleRows db p x <$> leafRows db x
Filter p x | not (hasDivision p || hasSubquery p) -> scaleRows db p x <$> leafRows db x
Project _ x -> leafRows db x
Compute items x | all (\(_, e) -> not (hasDivision e || hasSubquery e)) items -> leafRows db x
Sort _ x -> leafRows db x
Limit n x -> min (max 0 n) <$> leafRows db x
Derived _ x -> leafRows db x
Unit -> Just 1
_ -> Nothing
-- | 表统计里的行数
@@ -117,8 +117,16 @@ pushProject db need (Join kind l r c)
pushProject db need leaf@(Scan a t _)
| needsAll need = leaf
| all (`elem` need) (relOpCols db leaf) = leaf
| otherwise = Scan a t (Just (nub (map (unqualify a) need)))
pushProject _ _ (Derived a x) = Derived a x
| otherwise = Scan a t (Just (nub [column | requested <- need, let column = physicalColumn requested, column `elem` available]))
where
available = map (unqualify a) (relOpCols db (Scan a t Nothing))
-- | 把扫描限定列换成存储物理列名
physicalColumn column = unqualify (Just (deriveQualifier a t)) (unqualify a column)
pushProject db need (Derived a x)
| needsAll need || all (`elem` wanted) (relOpCols db x) = Derived a (pushProject db ["*"] x)
| otherwise = Derived a (Project wanted (pushProject db wanted x))
where
wanted = nub (map (unqualify a) need)
pushProject db need leaf@(Lookup _ _ _ _) = pushProjectLeaf db need leaf
pushProject db need leaf@(Range _ _ _ _ _) = pushProjectLeaf db need leaf
@@ -1,8 +1,9 @@
module ChuSQL.Core.Engine.Algebra.Planner (translate) where
import ChuSQL.Core.Engine.Algebra.Expr (aggregatesIn)
import ChuSQL.Core.Engine.Algebra.Expr (aggregatesIn, colsInExpr)
import ChuSQL.Core.Engine.Algebra.Op
import ChuSQL.Core.Engine.Syntax.AST
import Data.List (nub)
-- 计划生成:把语句翻译成算子树。
@@ -91,5 +92,16 @@ translate q = case q of
SelectExpr items fromC mWhere groupBy orderBy mLimit -> do
plan0 <- source fromC mWhere
let (plan, items') = grouped groupBy items plan0
Right (Compute items' (limited orderBy mLimit plan))
Right $ if any (\(c, _) -> c `elem` map fst items') orderBy
then orderedProjection items' orderBy mLimit plan
else Compute items' (limited orderBy mLimit plan)
_ -> Left "only SELECT is supported by translate"
-- | 在排序前计算别名并保留输出顺序
orderedProjection :: [(String, Expr)] -> [(String, SortDir)] -> Maybe Int -> RelOp -> RelOp
orderedProjection items orderBy mLimit plan =
Compute items (limited ordering mLimit (Compute (carried ++ keys) plan))
where
carried = [(column, Col column) | column <- nub (concatMap (colsInExpr . snd) items)]
keys = [("$order" ++ show n, maybe (Col c) id (lookup c items)) | (n, (c, _)) <- zip [1 :: Int ..] orderBy]
ordering = [(key, direction) | ((key, _), (_, direction)) <- zip keys orderBy]
@@ -36,6 +36,8 @@ errorRules =
[ ErrorRule "no database selected" NoDatabaseError "no_database"
, ErrorRule "administrator required" PermissionError "forbidden"
, ErrorRule "unknown table" CatalogError "not_found"
, ErrorRule "unknown domain" CatalogError "not_found"
, ErrorRule "domain is still used" CatalogError "query_error"
, ErrorRule "unknown column" CatalogError "query_error"
, ErrorRule "unknown database" CatalogError "not_found"
, ErrorRule "unknown role" CatalogError "not_found"
@@ -72,5 +74,7 @@ errorCode message = maybe "query_error" ruleCode (errorLike message)
-- | 账号服务对外的错误码
accountErrorCode :: String -> String
accountErrorCode "account already exists" = "conflict"
accountErrorCode "identity already exists" = "conflict"
accountErrorCode "the last enabled login superuser cannot be removed" = "bad_request"
accountErrorCode "unknown account" = "not_found"
accountErrorCode _ = "storage_error"
@@ -5,7 +5,7 @@ import ChuSQL.Core.Model
import ChuSQL.Core.Engine.Syntax.AST
import ChuSQL.Core.Engine.Syntax.Parser (parseExpression)
import Data.Char (isAlphaNum, isAscii, toLower)
import Data.List (intercalate, isInfixOf, isPrefixOf, nub)
import Data.List (intercalate, isInfixOf, isPrefixOf, isSuffixOf, nub)
-- 语义检查:表和列在不在、类型对不对,全部在执行前查。
@@ -53,10 +53,15 @@ checkFrom db outer (FromJoin _ left right cond) = do
checkBool db "ON" (Scope env outer) cond
Right env
-- | 派生表里 NULL 字面量列没有具体类型,按字符串处理
-- | 保留派生表输出的类型或未定型 NULL
derivedType :: InferredType -> ColumnType
derivedType (InferType t) = t
derivedType InferNull = CStr
derivedType InferNull = CNull
-- | 将列类型还原为表达式推导类型
inferredColumnType :: ColumnType -> InferredType
inferredColumnType CNull = InferNull
inferredColumnType ty = InferType ty
-- | 合并两层列,本层优先且不重复
scopedEnv :: [(String, Column)] -> [(String, Column)] -> [(String, Column)]
@@ -113,7 +118,7 @@ renderType (InferType t) = typeLabel t
-- | 推导表达式的类型(子查询也在这里整条查一遍)
inferExpr :: Database -> String -> Scope -> Expr -> Either String InferredType
inferExpr _ place env (Col c) = InferType . columnType . snd <$> scopeAt place env c
inferExpr _ place env (Col c) = inferredColumnType . columnType . snd <$> scopeAt place env c
inferExpr _ _ _ LitNull = Right InferNull
inferExpr _ _ _ (LitInt _) = Right (InferType CInt)
inferExpr _ _ _ (LitFloat _) = Right (InferType CFloat)
@@ -213,7 +218,7 @@ inferSubqueryOutput db outerScope sq = case subqueryStatement sq of
-- | 查这一列的推导类型
columnTypeOf env c = do
(_, col) <- scopeAt "SELECT" (Scope env enclosing) c
Right (c, InferType (columnType col))
Right (c, inferredColumnType (columnType col))
-- | 聚合的参数:列要在,且不能嵌套聚合
aggregateArg :: Database -> String -> Scope -> Expr -> Either String InferredType
@@ -282,7 +287,7 @@ boolean db place env a b = do
where
-- | 挨个查操作数是不是布尔
boolOperand _ _ InferNull = Right ()
boolOperand _ _ (InferType CBool) = Right ()
boolOperand _ _ (InferType ty) | typeClassOf ty == BooleanClass = Right ()
boolOperand p other t = Left (p ++ ": operator needs TBool on both sides, got " ++ renderType other ++ " and " ++ renderType t)
-- | 要求条件能算出布尔
@@ -292,7 +297,7 @@ checkBool db place env e = do
t <- inferExpr db place env e
case t of
InferNull -> Right ()
InferType CBool -> Right ()
InferType x | typeClassOf x == BooleanClass -> Right ()
InferType x -> Left (place ++ ": condition must be a boolean, got " ++ typeLabel x)
-- | 检查赋值列和类型
@@ -321,10 +326,88 @@ check db q = () <$ prepare db q
-- | 执行前解析列名并检查类型
prepare :: Database -> Statement -> Either String Statement
prepare db q = do
resolved <- resolveStatement db q
checkResolved db resolved
typed <- resolveDomainTypes db q
let visible = filter (not . isDomainTable . fst) db
resolved <- resolveStatement visible typed
case resolved of
CreateDomain name base -> do
checkDomainName name
checkTypeParameters base
case base of
CVarchar n | n <= 0 -> Left "CREATE DOMAIN: length must be positive"
CChar n | n <= 0 -> Left "CREATE DOMAIN: length must be positive"
CDecimal p s | p <= 0 || p > 308 || s < 0 || s > p -> Left "CREATE DOMAIN: invalid precision or scale"
_ -> Right ()
if any ((== domainTableName name) . fst) db
then Left ("domain already exists: " ++ name)
else Right ()
DropDomain name -> do
_ <- maybe (Left ("unknown domain: " ++ name)) Right (lookup (domainTableName name) db)
if any (any (usesDomain name . columnType . snd) . tableCols . snd) db
then Left ("domain is still used by a column: " ++ name)
else Right ()
ShowDomains -> Right ()
_ -> do
if any (isDomainTable . lastTablePart) (ddlTargets resolved)
then Left "reserved domain catalog"
else Right ()
checkResolved visible resolved
pure resolved
-- | 检查命名类型的名称
checkDomainName :: String -> Either String ()
checkDomainName name
| null name || length name > 48 = Left "CREATE DOMAIN: name must contain 1 to 48 characters"
| any (\c -> not (isAscii c && (isAlphaNum c || c == '_'))) name = Left "CREATE DOMAIN: invalid name"
| Just _ <- parseColumnType name = Left "CREATE DOMAIN: built-in type name is reserved"
| "__system_" `isPrefixOf` map toLower name = Left "CREATE DOMAIN: reserved name"
| otherwise = Right ()
-- | 判断列类型是否引用指定命名类型
usesDomain :: String -> ColumnType -> Bool
usesDomain name (CDomain other base) = map toLower name == map toLower other || usesDomain name base
usesDomain _ _ = False
-- | 列出结构语句的目标表
ddlTargets :: Statement -> [String]
ddlTargets statement = case statement of
CreateTable name _ -> [name]
DropTable name -> [name]
CreateIndex name _ -> [name]
DropIndex name _ -> [name]
AddColumn name _ -> [name]
DropColumn name _ -> [name]
RenameColumn name _ _ -> [name]
AlterColumnType name _ _ -> [name]
AlterColumnDefault name _ _ -> [name]
AlterColumnNull name _ _ -> [name]
_ -> []
-- | 从 schema 解析列上的命名类型
resolveDomainTypes :: Database -> Statement -> Either String Statement
resolveDomainTypes db statement = case statement of
CreateDomain name base -> CreateDomain name <$> resolveType base
CreateTable name cols -> CreateTable name <$> mapM resolveColumnType cols
AddColumn name col -> AddColumn name <$> resolveColumnType col
AlterColumnType name col ty -> AlterColumnType name col <$> resolveType ty
_ -> Right statement
where
-- | 解析一列的类型
resolveColumnType (name, col) = do
ty <- resolveType (columnType col)
Right (name, col {columnType = ty})
-- | 从类型目录取基础类型
resolveType = resolveWith []
-- | 递归解析类型并拒绝循环定义
resolveWith seen (CDomain name _)
| map toLower name `elem` seen = Left ("cyclic domain definition: " ++ name)
| otherwise = do
table <- maybe (Left ("unknown domain: " ++ name)) Right (lookup (domainTableName name) db)
base <- maybe (Left ("invalid domain catalog: " ++ name)) Right (lookup "base" (tableCols table))
resolved <- resolveWith (map toLower name : seen) (columnType base)
Right (CDomain (map toLower name) resolved)
resolveWith _ ty = Right ty
-- | 检查已解析的语句
checkResolved :: Database -> Statement -> Either String ()
checkResolved db = checkResolvedWith db []
@@ -350,8 +433,8 @@ checkResolvedWith db outer q = case q of
env <- checkFrom db outer fromC
let full = Scope env outer
mapM_ (\item -> () <$ inferExpr db "SELECT" full (snd item)) items
checkGrouping full groupBy items
checkColumns "ORDER BY" full (map fst orderBy)
checkGrouping full groupBy (items ++ [(c, Col c) | (c, _) <- orderBy, c `notElem` map fst items])
mapM_ (checkOrder full items . fst) orderBy
mapM_ (checkNoAggregate "WHERE") mWhere
mapM_ (checkBool db "WHERE" full) mWhere
Insert tbl cols rows -> do
@@ -420,6 +503,7 @@ checkResolvedWith db outer q = case q of
AlterColumnType tbl col ty -> do
t <- lookupTable db tbl
c <- requireColumn "ALTER COLUMN" t col
checkTypeParameters ty
if columnAutoIncrement c && not (integerType ty)
then Left "ALTER COLUMN: AUTO_INCREMENT column must stay an integer type"
else Right ()
@@ -435,6 +519,12 @@ checkResolvedWith db outer q = case q of
else Right ()
CreateUser name password -> checkAccount name password
AlterUser name password -> checkAccount name password
AlterIdentity name attributes -> do
checkRoleName name
if length (nub (map fst attributes)) == length attributes
then Right ()
else Left "duplicate identity attribute"
ShowRoles -> Right ()
DropUser name -> checkUserName name
CreateRole name -> checkRoleName name
DropRole name -> checkRoleName name
@@ -456,6 +546,9 @@ checkResolvedWith db outer q = case q of
DropDatabase name -> checkDatabaseName name
UseDatabase name -> checkDatabaseName name
ShowDatabases -> Right ()
CreateDomain{} -> Right ()
DropDomain{} -> Right ()
ShowDomains -> Right ()
-- | 事务控制由会话层执行,语义层不做列检查
BeginTransaction -> Right ()
CommitTransaction -> Right ()
@@ -485,6 +578,7 @@ checkInsertTargets t cols = mapM_ checkTarget (tableCols t)
-- | 检查一列定义是否合法(names 给 CHECK 用)
checkColumnDefs :: [String] -> [(String, Column)] -> Either String ()
checkColumnDefs names cols = do
mapM_ (checkTypeParameters . columnType . snd) cols
mapM_ (\(_, c) -> mapM_ (checkDefault c) (columnDefault c)) cols
mapM_ (\(_, c) -> checkAutoIncrement c) cols
if length [() | (_, c) <- cols, columnAutoIncrement c] > 1
@@ -492,6 +586,16 @@ checkColumnDefs names cols = do
else Right ()
checkChecks names cols
-- | 校验列类型的长度和精度参数
checkTypeParameters :: ColumnType -> Either String ()
checkTypeParameters (CDomain _ base) = checkTypeParameters base
checkTypeParameters CNull = Left "NULL is only valid as an inferred query output"
checkTypeParameters (CDecimal p s)
| p <= 0 || p > 308 || s < 0 || s > p = Left "invalid decimal precision or scale"
checkTypeParameters (CVarchar n) | n <= 0 = Left "character length must be positive"
checkTypeParameters (CChar n) | n <= 0 = Left "character length must be positive"
checkTypeParameters _ = Right ()
-- | 自增列必须是整数列
checkAutoIncrement :: Column -> Either String ()
checkAutoIncrement c
@@ -502,8 +606,8 @@ checkAutoIncrement c
checkDefault :: Column -> Value -> Either String ()
checkDefault c v
| VNull <- v = Right ()
| valueFits (columnType c) v = Right ()
| otherwise = Left ("DEFAULT: value does not fit column type " ++ show c)
| valueFits (columnType c) v = () <$ coerceValue (columnType c) v
| otherwise = Left ("DEFAULT: value does not fit column type " ++ typeLabel (columnType c))
-- | CHECK 里的列必须存在
checkChecks :: [String] -> [(String, Column)] -> Either String ()
@@ -717,6 +821,13 @@ resolveSelect db q@Select{} = resolveQuery db [] q (map (\c -> (c, Col c)) (sele
resolveSelect db q@SelectExpr{} = resolveQuery db [] q (selectItems q)
resolveSelect _ q = Right q
-- | 检查排序输出别名或来源列
checkOrder :: Scope -> [(String, Expr)] -> String -> Either String ()
checkOrder env items column = case [e | (label, e) <- items, label == column] of
[] -> checkColumns "ORDER BY" env [column]
[_] -> Right ()
_ -> Left ("ORDER BY: ambiguous projection alias: " ++ column)
-- | 解析投影、条件、排序与连接引用
resolveQuery :: Database -> [(String, Column)] -> Statement -> [(String, Expr)] -> Either String Statement
resolveQuery db outer q items = do
@@ -727,7 +838,13 @@ resolveQuery db outer q items = do
FromTable Nothing tbl -> unqualify (Just (deriveQualifier Nothing tbl)) c
_ -> c
-- | 查一列并换算成物理名
name place c = physical . fst <$> scopedColumnAt place env outer c
name place c
| length (filter (== '.') c) == 2, not (qualifiedReference c) = Left (missingColumn place c env)
| otherwise = physical . fst <$> scopedColumnAt place env outer c
-- | 库限定列只引用来源中的真实表名
qualifiedReference c = qualifiedReferenceIn db (selectFrom q) c
|| (any (\(key, _) -> length (filter (== '.') key) == 1 && ("." ++ key) `isSuffixOf` c) outer
&& (not (any (elem '.' . fst) db) || any (\(table, _) -> (table ++ ".") `isPrefixOf` c) db))
-- | 换算一个投影项:带别名时保留别名做输出列名
output (label, Col c) = do
k <- name "SELECT" c
@@ -742,11 +859,29 @@ resolveQuery db outer q items = do
else mapM output items
condition <- mapM (resolveExpr db env outer (name "WHERE")) (selectWhere q)
grouping <- mapM (name "GROUP BY") (selectGroupBy q)
ordering <- mapM (\(c, d) -> (\k -> (k, d)) <$> name "ORDER BY" c) (selectOrderBy q)
-- | 优先解析唯一投影别名,再查来源列
let orderName c = case [label | (label, _) <- projected, label == c, qIsExpression] of
[] -> name "ORDER BY" c
[label] -> Right label
_ -> Left ("ORDER BY: ambiguous projection alias: " ++ c)
qIsExpression = case q of SelectExpr{} -> True; _ -> False
ordering <- mapM (\(c, d) -> (\k -> (k, d)) <$> orderName c) (selectOrderBy q)
pure $ case q of
Select{} -> Select (map fst projected) source condition grouping ordering (selectLimit q)
_ -> SelectExpr projected source condition grouping ordering (selectLimit q)
-- | 收集没有别名遮蔽的来源表名
sourceTableNames :: FromClause -> [String]
sourceTableNames (FromTable Nothing table) = [table]
sourceTableNames (FromJoin _ left right _) = sourceTableNames left ++ sourceTableNames right
sourceTableNames _ = []
-- | 库限定引用必须使用当前来源的表名
qualifiedReferenceIn :: Database -> FromClause -> String -> Bool
qualifiedReferenceIn db source c = any (\table -> (table ++ ".") `isPrefixOf` c) tables
where
tables = sourceTableNames source ++ [key | table <- sourceTableNames source, Right (key, _) <- [resolveTable db table]]
-- | 单表不加限定名,其余来源交给 resolveFrom
resolveSource :: Database -> [(String, Column)] -> FromClause -> Either String FromClause
resolveSource db outer fromC = case fromC of
@@ -769,7 +904,11 @@ resolveFrom db outer (FromJoin kind left right cond) = do
l <- resolveFrom db outer left
r <- resolveFrom db outer right
env <- checkFrom db outer (FromJoin kind l r cond)
c <- resolveExpr db env outer (fmap fst . columnAt "ON" env) cond
-- | 连接条件也拒绝不存在的库前缀
let joinName column
| length (filter (== '.') column) == 2, not (qualifiedReferenceIn db (FromJoin kind left right cond) column) = Left (missingColumn "ON" column env)
| otherwise = fst <$> columnAt "ON" env column
c <- resolveExpr db env outer joinName cond
pure (FromJoin kind l r c)
-- | 派生表的输出列名:列取裸名,表达式保留原标签
@@ -7,6 +7,7 @@ module ChuSQL.Core.Engine.Storage.FFI (
openStorage,
closeStorage,
storageRequest,
maintainStorage,
) where
import Control.Exception (SomeException, try)
@@ -40,6 +41,21 @@ foreign import ccall unsafe "chusql_storage_free"
foreign import ccall unsafe "chusql_storage_last_error"
cStorageLastError :: Ptr CSize -> IO (Ptr CChar)
foreign import ccall safe "chusql_storage_maintenance"
cStorageMaintenance :: CString -> Ptr CChar -> CSize -> Ptr (Ptr CChar) -> Ptr CSize -> IO CInt
-- | 离线修复系统目录并返回结果
maintainStorage :: FilePath -> BS.ByteString -> IO (Either String BS.ByteString)
maintainStorage path payload = withCString path $ \config ->
BS.useAsCStringLen payload $ \(input, size) -> alloca $ \outRef -> alloca $ \lenRef -> do
code <- cStorageMaintenance config input (fromIntegral size) outRef lenRef
if code /= 0 then Left <$> lastError else do
output <- peek outRef
len <- peek lenRef
bytes <- BS.packCStringLen (output, fromIntegral len)
cStorageFree output len
pure (Right bytes)
-- | 库版本(Rust 侧的 CARGO_PKG_VERSION)
storageVersion :: IO String
storageVersion = cStorageVersion >>= peekCString
@@ -2,6 +2,7 @@ module ChuSQL.Core.Engine.Storage.Memory (MemoryStorage (runMemoryStorage)) wher
import ChuSQL.Core.Model
import ChuSQL.Core.Engine.Storage
import qualified Data.HashSet as HS
-- 内存实现:Database 上的状态,错误通道是 Either String。
@@ -74,11 +75,32 @@ instance MonadStorage MemoryStorage where
-- \| 原样返回当前库
snapshot = MemoryStorage $ \db -> Right (db, db)
-- \| 返回实时行数与去重统计的无数据结构
schema = MemoryStorage $ \db -> Right (map describe db, db)
where
-- | 将存储数据汇总为结构统计
describe (name, table) =
(name, table {tableRows = [], tableMeta = Just metadata})
where
metadata = TableMeta (length (tableRows table)) distinct [] []
distinct = [columnStatistics column (tableRows table) | (column, _) <- tableCols table]
-- \| 改表结构:列定义与行一起换
replaceSchema name cols rows = MemoryStorage $ \db -> case lookup name db of
Nothing -> Right (Left ("unknown table: " ++ name), db)
Just tbl -> Right (Right (), replaceTable name tbl{tableCols = cols, tableRows = rows} db)
-- | 对列值去重并限制统计内存
columnStatistics :: String -> [Row] -> (String, Int, Bool)
columnStatistics column rows = go rows HS.empty
where
-- | 累积规范化值,达到上限后停止
go remaining values
| HS.size values >= 4096 = (column, 4096, True)
| otherwise = case remaining of
[] -> (column, HS.size values, False)
row : rest -> go rest (maybe values (\v -> HS.insert (canonicalValue v) values) (lookup column row))
-- | 用给定表替换同名表
replaceTable :: String -> Table -> Database -> Database
replaceTable name table =
@@ -25,6 +25,9 @@ data Statement
| Delete String (Maybe Expr)
| Update String [(String, Expr)] (Maybe Expr)
| CreateTable String [(String, Column)]
| CreateDomain String ColumnType
| DropDomain String
| ShowDomains
| DropTable String
| CreateIndex String String
| DropIndex String String
@@ -36,6 +39,8 @@ data Statement
| AlterColumnNull String String Bool
| CreateUser String String
| AlterUser String String
| AlterIdentity String [(String, Bool)]
| ShowRoles
| DropUser String
-- | 新角色:权限的载体,用户靠成员资格继承它的权限
| CreateRole String
@@ -8,7 +8,7 @@ import ChuSQL.Core.Engine.Syntax.AST
import Control.Monad (void)
import Control.Monad.Combinators.Expr (Operator (..), makeExprParser)
import Data.Char (isAlpha, isAlphaNum, isHexDigit, isSpace, toLower, toUpper)
import Data.List (intercalate)
import Data.List (intercalate, isInfixOf)
import Data.Maybe (fromMaybe)
import Data.Void (Void)
import Text.Megaparsec
@@ -475,8 +475,25 @@ alterUserStatement = do
keyword "alter"
keyword "user"
name <- userName
pw <- identifiedBy
return (AlterUser name pw)
(AlterUser name <$> identifiedBy) <|> (AlterIdentity name <$> some identityAttribute)
-- | 读身份属性开关
identityAttribute :: Parser (String, Bool)
identityAttribute = choice
[ ("system_catalog_manager", True) <$ keyword "system_catalog_manager"
, ("system_catalog_manager", False) <$ keyword "nosystem_catalog_manager"
, ("login", True) <$ keyword "login", ("login", False) <$ keyword "nologin"
, ("superuser", True) <$ keyword "superuser", ("superuser", False) <$ keyword "nosuperuser"
, ("enabled", True) <$ keyword "enabled", ("enabled", False) <$ keyword "disabled"
]
-- | 读 ALTER ROLE 属性或口令
alterRoleStatement :: Parser Statement
alterRoleStatement = do
keyword "alter"
keyword "role"
name <- userName
(AlterUser name <$> identifiedBy) <|> (AlterIdentity name <$> some identityAttribute)
-- | 读 DROP USER
dropUserStatement :: Parser Statement
@@ -599,6 +616,7 @@ columnTypeP =
, keyword "date" >> pure CDate
, keyword "timestamp" >> pure CTimestamp
, keyword "blob" >> pure CBlob
, (\name -> CDomain (map toLower name) CStr) <$> identifier
]
-- | DECIMAL 的精度与小数位
@@ -682,7 +700,14 @@ renameColumns name cols q@Select { selectCols = names } = do
)
renameColumns name cols q@SelectExpr { selectItems = items } = do
checkCteArity name cols (map snd items)
Right q { selectItems = zip cols (map snd items) }
ordering <- mapM renameOrder (selectOrderBy q)
Right q { selectItems = zip cols (map snd items), selectOrderBy = ordering }
where
-- | 列清单同步改写内层排序输出别名
renameOrder (column, direction) = case [new | ((old, _), new) <- zip items cols, old == column] of
[] -> Right (column, direction)
[new] -> Right (new, direction)
_ -> Left ("WITH: ambiguous ORDER BY alias: " ++ column)
renameColumns _ _ _ = Left "WITH: a CTE must be a SELECT"
-- | 列清单个数必须和本体输出一致
@@ -808,7 +833,7 @@ transactionStatement =
parseStatement :: String -> Either String Statement
parseStatement input =
case runParser (sc *> statementP <* sc <* eof) "<query>" input of
Left err -> Left (errorBundlePretty err)
Left err -> Left (safeParseError input err)
Right q -> Right q
where
-- | 按关键字分派到各语句解析器
@@ -819,12 +844,17 @@ parseStatement input =
<|> try (CreateDatabase <$> (keyword "create" *> keyword "database" *> identifier))
<|> try (DropDatabase <$> (keyword "drop" *> keyword "database" *> identifier))
<|> (UseDatabase <$> (keyword "use" *> identifier))
<|> (ShowDatabases <$ (keyword "show" *> keyword "databases"))
<|> try (ShowDatabases <$ (keyword "show" *> keyword "databases"))
<|> try (ShowDomains <$ (keyword "show" *> keyword "domains"))
<|> try (CreateDomain <$> (keyword "create" *> keyword "domain" *> identifier) <*> (keyword "as" *> columnTypeP))
<|> try (DropDomain <$> (keyword "drop" *> keyword "domain" *> identifier))
<|> insertStatement
<|> deleteStatement
<|> updateStatement
<|> try createUserStatement
<|> try alterUserStatement
<|> try alterRoleStatement
<|> try (ShowRoles <$ (keyword "show" *> keyword "roles"))
<|> try dropUserStatement
<|> try createRoleStatement
<|> try dropRoleStatement
@@ -840,9 +870,15 @@ parseStatement input =
parseExpression :: String -> Either String Expr
parseExpression input =
case runParser (sc *> expr <* sc <* eof) "<expr>" input of
Left err -> Left (errorBundlePretty err)
Left err -> Left (safeParseError input err)
Right e -> Right e
-- | 隐去含口令输入的解析诊断
safeParseError :: String -> ParseErrorBundle String Void -> String
safeParseError input errors
| any (`isInfixOf` map toLower input) ["password", "identified"] = "syntax error in credential statement (input redacted)"
| otherwise = errorBundlePretty errors
-- | 表名至多有一个库前缀。
tableName :: Parser String
tableName = do
+217 -1
View File
@@ -104,9 +104,91 @@ chainDB = [("a", chainA), ("b", chainB), ("c", chainC)]
chainDBWithoutStats :: Database
chainDBWithoutStats = [(n, t { tableMeta = Nothing }) | (n, t) <- chainDB]
-- | 顺序执行命名类型测试语句
domainSql :: Database -> [String] -> Either String (Database, [Row])
domainSql db [] = Right (db, [])
domainSql db [sql] = parseStatement sql >>= runStatement db
domainSql db (sql : rest) = do
statement <- parseStatement sql
(next, _) <- runStatement db statement
domainSql next rest
-- | 跑引擎层全部 hspec 用例
main :: IO ()
main = hspec $ do
describe "identity syntax" $ do
it "parses role and user attributes with their aliases" $ do
parseStatement "ALTER ROLE worker LOGIN SUPERUSER ENABLED" `shouldBe` Right (AlterIdentity "worker" [("login", True), ("superuser", True), ("enabled", True)])
parseStatement "ALTER USER worker NOLOGIN NOSUPERUSER DISABLED" `shouldBe` Right (AlterIdentity "worker" [("login", False), ("superuser", False), ("enabled", False)])
parseStatement "ALTER ROLE worker IDENTIFIED BY 'Pass123!'" `shouldBe` Right (AlterUser "worker" "Pass123!")
parseStatement "SHOW ROLES" `shouldBe` Right ShowRoles
it "rejects empty unknown and repeated attributes" $ do
parseStatement "ALTER ROLE worker" `shouldSatisfy` isLeft
parseStatement "ALTER USER worker ACTIVE" `shouldSatisfy` isLeft
prepare [] (AlterIdentity "worker" [("login", True), ("login", False)]) `shouldSatisfy` isLeft
describe "ChuSQL.Core.Engine (DOMAIN)" $ do
it "parses domain commands and named columns" $ do
parseStatement "CREATE DOMAIN amount AS DECIMAL(12,2)" `shouldBe` Right (CreateDomain "amount" (CDecimal 12 2))
parseStatement "DROP DOMAIN amount" `shouldBe` Right (DropDomain "amount")
parseStatement "SHOW DOMAINS" `shouldBe` Right ShowDomains
parseStatement "CREATE TABLE items (price amount)" `shouldBe` Right (CreateTable "items" [("price", plainColumn (CDomain "amount" CStr))])
it "keeps a named type while applying its base semantics" $ do
let result = domainSql [] ["CREATE DOMAIN label AS VARCHAR(3)", "CREATE TABLE items (name label)", "INSERT INTO items (name) VALUES ('abc')"]
case result of
Left err -> expectationFailure err
Right (db, _) -> do
fmap columnType (lookup "items" db >>= lookup "name" . tableCols) `shouldBe` Just (CDomain "label" (CVarchar 3))
fmap snd (domainSql db ["SELECT name FROM items"]) `shouldBe` Right [[("name", VStr "abc")]]
domainSql db ["INSERT INTO items (name) VALUES ('long')"] `shouldSatisfy` isLeft
it "rejects unknown, duplicate and built-in domains" $ do
mapM_ (\sql -> domainSql [] sql `shouldSatisfy` isLeft)
[ ["CREATE TABLE items (name missing)"]
, ["CREATE DOMAIN INT AS INT"]
, ["CREATE DOMAIN a AS INT", "CREATE DOMAIN A AS INT"]
, ["CREATE DOMAIN a AS missing"]
, ["CREATE DOMAIN bad AS VARCHAR(0)"]
]
it "resolves nested domains and preserves their names and base checks" $ do
let result = domainSql [] ["CREATE DOMAIN label AS VARCHAR(3)", "CREATE DOMAIN product_label AS label", "CREATE DOMAIN title AS product_label", "CREATE TABLE items (name title)", "INSERT INTO items (name) VALUES ('abc')"]
case result of
Left err -> expectationFailure err
Right (db, _) -> do
fmap columnType (lookup "items" db >>= lookup "name" . tableCols) `shouldBe` Just (CDomain "title" (CDomain "product_label" (CDomain "label" (CVarchar 3))))
fmap snd (domainSql db ["SELECT name FROM items"]) `shouldBe` Right [[("name", VStr "abc")]]
domainSql db ["INSERT INTO items (name) VALUES ('long')"] `shouldSatisfy` isLeft
domainSql db ["DROP DOMAIN label"] `shouldSatisfy` isLeft
it "protects domain dependencies even without user columns" $ do
domainSql [] ["CREATE DOMAIN a AS INT", "CREATE DOMAIN b AS a", "DROP DOMAIN a"] `shouldSatisfy` isLeft
fmap snd (domainSql [] ["CREATE DOMAIN a AS INT", "CREATE DOMAIN b AS a", "DROP DOMAIN b", "DROP DOMAIN a", "SHOW DOMAINS"]) `shouldBe` Right []
domainSql [] ["CREATE DOMAIN self AS self"] `shouldSatisfy` isLeft
it "uses nested domains in column additions and conversions" $ do
fmap snd (domainSql [] ["CREATE DOMAIN a AS INT", "CREATE DOMAIN b AS a", "CREATE TABLE items (id INT)", "INSERT INTO items (id) VALUES (2)", "ALTER TABLE items ADD COLUMN qty b", "ALTER TABLE items ALTER COLUMN id TYPE b", "SELECT id + 1 AS next, qty FROM items"]) `shouldBe` Right [[("next", VInt 3), ("qty", VNull)]]
it "rejects cyclic catalog definitions without recursing forever" $ do
case domainSql [] ["CREATE DOMAIN a AS INT", "CREATE DOMAIN b AS a"] of
Left err -> expectationFailure err
Right (db, _) -> do
let cyclic = map (\(name, table) -> if name == domainTableName "a" then (name, table {tableCols = [("base", plainColumn (CDomain "b" CStr))]}) else (name, table)) db
fmap snd (domainSql cyclic ["CREATE TABLE items (id b)"]) `shouldBe` Left "cyclic domain definition: b"
it "prevents dropping a domain until its columns are removed" $ do
domainSql [] ["CREATE DOMAIN code AS INT", "CREATE TABLE items (code code)", "DROP DOMAIN code"] `shouldSatisfy` isLeft
fmap snd (domainSql [] ["CREATE DOMAIN code AS INT", "CREATE TABLE items (code code)", "DROP TABLE items", "DROP DOMAIN code", "SHOW DOMAINS"]) `shouldBe` Right []
domainSql [] ["DROP DOMAIN missing"] `shouldSatisfy` isLeft
it "resolves domains in ADD COLUMN and ALTER COLUMN TYPE" $ do
fmap snd (domainSql [] ["CREATE DOMAIN code AS INT", "CREATE TABLE items (id INT)", "INSERT INTO items (id) VALUES (1)", "ALTER TABLE items ADD COLUMN code code", "ALTER TABLE items ALTER COLUMN id TYPE code", "SELECT id, code FROM items"]) `shouldBe` Right [[("id", VInt 1), ("code", VNull)]]
it "hides and protects the domain catalog from SQL" $ do
domainSql [] ["CREATE DOMAIN code AS INT", "SELECT * FROM __system_domain_code"] `shouldSatisfy` isLeft
domainSql [] ["CREATE DOMAIN code AS INT", "DROP TABLE __system_domain_code"] `shouldSatisfy` isLeft
domainSql [] ["CREATE TABLE __system_domain_fake (base INT)"] `shouldSatisfy` isLeft
it "round trips named types and compares using the base type" $ do
let ty = CDomain "amount" (CDecimal 12 2)
parseColumnType (typeName ty) `shouldBe` Just ty
comparableTypes ty CInt `shouldBe` True
integerType (CDomain "code" CInt) `shouldBe` True
coerceValue (CDomain "code" CInt) (VStr "bad") `shouldSatisfy` isLeft
it "uses numeric domain columns in expressions and sorting" $ do
fmap snd (domainSql [] ["CREATE DOMAIN quantity AS INT", "CREATE TABLE items (qty quantity)", "INSERT INTO items (qty) VALUES (2), (1)", "SELECT qty + 1 AS next FROM items ORDER BY qty"]) `shouldBe` Right [[("next", VInt 2)], [("next", VInt 3)]]
it "uses boolean domains as conditions and logical operands" $ do
fmap snd (domainSql [] ["CREATE DOMAIN flag AS BOOL", "CREATE TABLE items (active flag)", "INSERT INTO items (active) VALUES (TRUE), (FALSE), (NULL)", "SELECT active FROM items WHERE active AND TRUE"]) `shouldBe` Right [[("active", VBool True)]]
describe "ChuSQL.Core.Engine.Syntax.Parser" $ do
it "resolves table-qualified columns inside database-qualified tables" $ do
let db = [("sales.users", users)]
@@ -114,6 +196,11 @@ main = hspec $ do
`shouldBe` Right [[("name", VStr "Alice")]]
rowsOf (parseStatement "SELECT sales.users.name FROM sales.users WHERE sales.users.id = 1" >>= runStatement db)
`shouldBe` Right [[("name", VStr "Alice")]]
let related = [("sales.users", users), ("sales.orders", orders)]
rowsOf (parseStatement "SELECT sales.users.name FROM sales.users WHERE EXISTS (SELECT orders.id FROM sales.orders WHERE orders.user_id = sales.users.id)" >>= runStatement related)
`shouldBe` Right [[("name", VStr "Alice")], [("name", VStr "Bob")]]
rowsOf (parseStatement "SELECT sales.users.name FROM sales.users WHERE EXISTS (SELECT orders.id FROM sales.orders WHERE orders.user_id = intruder.users.id)" >>= runStatement related)
`shouldSatisfy` either (isInfixOf "unknown column") (const False)
it "derives the column prefix when the session schema is scoped to one database" $ do
let scoped = [("users", users)]
@@ -1110,6 +1197,32 @@ main = hspec $ do
`shouldSatisfy` isLeft
describe "ChuSQL.Core.Engine (ORDER BY)" $ do
it "sorts projection aliases with source keys and LIMIT" $ do
let query = "SELECT age + 1 AS next, name FROM users ORDER BY next DESC, name LIMIT 2"
rowsOf (parseStatement query >>= runStatement testDB)
`shouldBe` Right [[("next", VInt 31), ("name", VStr "Carol")], [("next", VInt 26), ("name", VStr "Alice")]]
sameResultAsUnoptimized query
it "prefers output aliases over same-named source columns" $
rowsOf (parseStatement "SELECT -age AS age FROM users ORDER BY age LIMIT 1" >>= runStatement testDB)
`shouldBe` Right [[("age", VInt (-30))]]
it "evaluates non-ordering projections after LIMIT" $ do
rowsOf (parseStatement "SELECT name AS who, 1 / (age - 25) AS value FROM users ORDER BY who DESC LIMIT 1" >>= runStatement testDB)
`shouldBe` Right [[("who", VStr "Carol"), ("value", VInt 0)]]
rowsOf (parseStatement "SELECT 1 / 0 AS bad FROM users ORDER BY bad LIMIT 0" >>= runStatement testDB)
`shouldSatisfy` isLeft
it "sorts aggregate aliases and rejects ungrouped source sort keys" $ do
rowsOf (parseStatement "SELECT user_id, count(*) AS total FROM orders GROUP BY user_id ORDER BY total DESC" >>= runStatement testDB)
`shouldBe` Right [[("user_id", VInt 1), ("total", VInt 2)], [("user_id", VInt 2), ("total", VInt 1)]]
(parseStatement "SELECT count(*) AS total FROM users ORDER BY age" >>= runStatement testDB)
`shouldSatisfy` isLeft
it "keeps ordering aliases when a CTE renames its outputs" $
rowsOf (parseStatement "WITH t(x) AS (SELECT age + 1 AS next FROM users ORDER BY next DESC LIMIT 1) SELECT x FROM t" >>= runStatement testDB)
`shouldBe` Right [[("t.x", VInt 31)]]
it "rejects duplicate ordering aliases without changing duplicate outputs" $ do
(parseStatement "SELECT age AS x, id AS x FROM users ORDER BY x" >>= runStatement testDB)
`shouldSatisfy` either (isInfixOf "ambiguous projection alias") (const False)
rowsOf (parseStatement "SELECT age AS x, id AS x, name AS who FROM users ORDER BY who LIMIT 1" >>= runStatement testDB)
`shouldBe` Right [[("x", VInt 25), ("x", VInt 1), ("who", VStr "Alice")]]
it "parses ORDER BY DESC" $ do
case parseStatement "SELECT name FROM users ORDER BY age DESC" of
Right q -> selectOrderBy q `shouldBe` [("age", Desc)]
@@ -1402,6 +1515,38 @@ main = hspec $ do
sameResultAsUnoptimized "SELECT u.name FROM users u WHERE EXISTS (SELECT * FROM orders o WHERE o.user_id = u.id)"
describe "ChuSQL.Core.Engine (子查询去相关)" $ do
it "reuses an uncorrelated JOIN subquery across all pairs" $ do
let query = "SELECT u.name, o.product FROM users u JOIN orders o ON u.id = o.user_id AND o.product IN (SELECT product FROM orders WHERE user_id = 1)"
(result, scans) = countedRun query
fmap (map (map snd)) result `shouldBe` Right [[VStr "Alice", VStr "Book"], [VStr "Alice", VStr "Cup"]]
scanTimes "orders" scans `shouldBe` 2
sameResultAsUnoptimized query
it "reuses JOIN subqueries by values from both sides" $ do
let query = "SELECT o.product FROM orders o JOIN users u ON o.user_id = u.id AND EXISTS (SELECT id FROM users x WHERE x.id = o.user_id AND x.age = u.age)"
(result, scans) = countedRun query
fmap (map (map snd)) result `shouldBe` Right [[VStr "Book"], [VStr "Pen"], [VStr "Cup"]]
scanTimes "users" scans `shouldBe` 7
sameResultAsUnoptimized query
it "keeps unmatched LEFT JOIN rows with a cached subquery" $ do
let query = "SELECT u.name, o.product FROM users u LEFT JOIN orders o ON u.id = o.user_id AND EXISTS (SELECT id FROM users WHERE id = 1)"
(result, scans) = countedRun query
fmap (map (map snd)) result `shouldBe` Right [[VStr "Alice", VStr "Book"], [VStr "Alice", VStr "Cup"], [VStr "Bob", VStr "Pen"], [VStr "Carol", VNull]]
scanTimes "users" scans `shouldBe` 2
sameResultAsUnoptimized query
it "does not evaluate JOIN subqueries when there are no pairs" $ do
let query = "SELECT u.name FROM users u LEFT JOIN (SELECT id FROM orders WHERE id < 0) o ON EXISTS (SELECT 1 / 0 FROM orders)"
(result, scans) = countedRun query
fmap (map (map snd)) result `shouldBe` Right [[VStr "Alice"], [VStr "Bob"], [VStr "Carol"]]
scanTimes "orders" scans `shouldBe` 1
sameResultAsUnoptimized query
it "propagates JOIN subquery errors before subsequent pairs" $ do
fst (countedRun "SELECT u.name FROM users u JOIN orders o ON EXISTS (SELECT 1 / 0 FROM orders)")
`shouldSatisfy` either (isInfixOf "division by zero") (const False)
it "runs a non-correlated subquery only once" $ do
let (result, scans) = countedRun "SELECT name FROM users WHERE id IN (SELECT user_id FROM orders)"
fmap (map (map snd)) result `shouldBe` Right [[VStr "Alice"], [VStr "Bob"]]
@@ -1603,6 +1748,28 @@ main = hspec $ do
relOp = Filter cond (Limit 2 (Scan Nothing "users" Nothing))
optimize testDB relOp `shouldBe` relOp
it "projects derived outputs without dropping expression errors" $ do
let original = Project ["d.name"] (Derived (Just "d") (Compute [("name", Col "name"), ("bad", Div (LitInt 1) (LitInt 0))] (Scan Nothing "users" Nothing)))
evalRelOp testDB (optimize testDB original) `shouldBe` evalRelOp testDB original
evalRelOp testDB original `shouldBe` Left "division by zero"
sameResultAsUnoptimized "SELECT d.name FROM (SELECT name, age FROM users) d WHERE d.age > 20 ORDER BY d.name"
it "supplies memory schema statistics without exposing rows" $ do
case runMemoryStorage (schema :: MemoryStorage Database) testDB of
Left err -> expectationFailure err
Right (structures, _) -> do
map (tableRows . snd) structures `shouldBe` [[], []]
fmap (fmap metaRowCount . tableMeta) (lookup "users" structures) `shouldBe` Just (Just 3)
it "uses derived schema estimates for safe join reordering" $ do
let query = "SELECT d.name FROM (SELECT id, name FROM a) d JOIN b ON d.id = b.a_id JOIN c ON b.id = c.b_id ORDER BY d.name"
case parseStatement query >>= prepare chainDB >>= translate of
Left err -> expectationFailure err
Right original -> do
let optimized = optimize chainDB original
planTables optimized `shouldBe` ["b", "c", "a"]
evalRelOp chainDB optimized `shouldBe` evalRelOp chainDB original
it "is idempotent after reaching a fixed point" $ do
case parseStatement "SELECT u.name, o.product FROM users u JOIN orders o ON u.id = o.user_id WHERE u.age > 20" of
Left err -> expectationFailure err
@@ -2034,6 +2201,20 @@ main = hspec $ do
runIPCStorageIn (scan "iso") alpha `shouldReturn` Right [[("id", VInt 1)]]
runIPCStorageIn (scan "iso") beta `shouldReturn` Right [[("id", VInt 2)]]
it "persists nested domains and named columns across restart" $ do
withTestSession $ \srv -> do
withServerEnv srv $ do
runIPCStorage (runStatementM (CreateDomain "label" (CVarchar 3))) `shouldReturn` Right []
runIPCStorage (runStatementM (CreateDomain "product_label" (CDomain "label" CStr))) `shouldReturn` Right []
runIPCStorage (runStatementM (CreateTable "typed" [("name", plainColumn (CDomain "product_label" CStr))])) `shouldReturn` Right []
runIPCStorage (runStatementM (Insert "typed" ["name"] [[LitStr "abc"]])) `shouldReturn` Right []
closeConnection
openSessionIn (dataDir srv)
withServerEnv srv $ do
runIPCStorage (runStatementM ShowDomains) `shouldReturn` Right [[("domain", VStr "label"), ("base_type", VStr "varchar(3)")], [("domain", VStr "product_label"), ("base_type", VStr "domain(label,varchar(3))")]]
runIPCStorage (runStatementM (Insert "typed" ["name"] [[LitStr "long"]])) >>= (`shouldSatisfy` isLeft)
runIPCStorage (runStatementM (DropDomain "label")) >>= (`shouldSatisfy` isLeft)
it "rows survive a restart on the same data directory" $ do
withTestSession $ \srv -> do
withServerEnv srv $ runIPCStorage (insert "persist" [("id", VInt 42)]) `shouldReturn` Right ()
@@ -2081,6 +2262,16 @@ main = hspec $ do
-- 存储层的日志现在直接打在本进程的输出里,没有单独的 debug.log 可读;
-- 「走索引而不是全表扫」由下面 CREATE INDEX 那组用例与 Rust 侧 runtime 测试覆盖。
it "keeps correlated outer references out of physical scan columns" $ do
withTestSession $ \srv -> withServerEnv srv $ do
runIPCStorage (runStatementM (CreateTable "outer_t" [("id", TInt), ("name", TStr)])) `shouldReturn` Right []
runIPCStorage (runStatementM (CreateTable "inner_t" [("id", TInt), ("user_id", TInt), ("unused", TStr)])) `shouldReturn` Right []
runIPCStorage (runStatementM (Insert "outer_t" ["id", "name"] [[LitInt 1, LitStr "one"], [LitInt 2, LitStr "two"]])) `shouldReturn` Right []
runIPCStorage (runStatementM (Insert "inner_t" ["id", "user_id", "unused"] [[LitInt 10, LitInt 1, LitStr "x"]])) `shouldReturn` Right []
case parseStatement "SELECT u.name FROM outer_t u WHERE EXISTS (SELECT id FROM inner_t o WHERE o.user_id = u.id)" of
Left err -> expectationFailure err
Right statement -> runIPCStorage (runStatementM statement) `shouldReturn` Right [[("u.name", VStr "one")]]
it "CREATE INDEX makes a query on that column use the index" $ do
withTestSession $ \srv -> withServerEnv srv $ do
_ <- runIPCStorage (runStatementM (CreateTable "ix_t" [("id", TInt), ("code", TInt)]))
@@ -2273,6 +2464,31 @@ main = hspec $ do
vals seeded "SELECT age * 2 FROM users WHERE name = 'Nine'" `shouldBe` Right [[VNull]]
describe "ChuSQL.Core.Engine (TYPES)" $ do
it "checks rounded decimal precision, finite values and domains" $ do
coerceValue (CDecimal 4 2) (VFloat 99.994) `shouldBe` Right (VFloat 99.99)
mapM_ (\value -> coerceValue (CDecimal 4 2) value `shouldSatisfy` isLeft)
[VFloat 99.995, VFloat (-99.995), VInt 100, VFloat (0 / 0), VFloat (1 / 0)]
coerceValue (CDecimal 2 0) (VFloat 2.5) `shouldBe` Right (VFloat 2)
coerceValue (CDomain "amount" (CDecimal 4 2)) (VInt 100) `shouldSatisfy` isLeft
coerceValue (CDecimal 309 0) (VInt 1) `shouldSatisfy` isLeft
it "rejects invalid decimal definitions and overflowing defaults" $ do
mapM_ (\query -> (parseStatement query >>= runStatement testDB) `shouldSatisfy` isLeft)
["CREATE TABLE bad (v DECIMAL(2,3))", "CREATE TABLE bad (v DECIMAL(309,0))", "CREATE TABLE bad (v DECIMAL(2,0) DEFAULT 100)"]
it "does not expose password input in syntax errors" $ do
let query = "ALTER USER alice IDENTIFIED BY 'never-expose-this' trailing"
parseStatement query `shouldSatisfy` either (\err -> "syntax" `isInfixOf` err && not ("never-expose-this" `isInfixOf` err)) (const False)
it "rejects arbitrary prefixes for unqualified schema columns" $ do
resolveColumn "intruder.id" [("id", TInt)] `shouldSatisfy` isLeft
resolveColumn "wrong.u.id" [("id", TInt)] `shouldSatisfy` isLeft
mapM_ (\query -> (parseStatement query >>= runStatement testDB) `shouldSatisfy` isLeft)
["SELECT wrong.users.id FROM users", "SELECT wrong.u.id FROM users u", "SELECT users.id FROM users JOIN orders ON wrong.users.id = orders.user_id"]
it "preserves untyped NULL through derived tables and CTEs" $ do
mapM_ (\query -> rowsOf (parseStatement query >>= runStatement testDB) `shouldBe` Right [[("answer", VNull)]])
[ "SELECT d.n + 1 AS answer FROM (SELECT NULL AS n) d"
, "SELECT d.n AND TRUE AS answer FROM (SELECT NULL AS n) d"
, "WITH t AS (SELECT NULL AS n) SELECT t.n + 1 AS answer FROM t"
, "SELECT d.n + 1 AS answer FROM (SELECT x.n FROM (SELECT NULL AS n) x) d"
]
-- vals:在给定库上跑 SQL 只取值
let vals db input = fmap (map (map snd)) (parseStatement input >>= rowsOf . runStatement db)
-- withTable:先建表,返回新库
@@ -2296,7 +2512,7 @@ main = hspec $ do
, ("k", plainColumn CBool)
]
)
parseStatement "CREATE TABLE t (a nope)" `shouldSatisfy` isLeft
fmap snd (domainSql [] ["CREATE TABLE t (a nope)"]) `shouldBe` Left "unknown domain: nope"
it "reads and writes float literals" $ do
vals testDB "SELECT 1.5 + 1" `shouldBe` Right [[VFloat 2.5]]
+5 -4
View File
@@ -2,10 +2,11 @@ cabal-version: 2.4
name: chusql-core-model
version: 1.0.0
synopsis: ChuSQL 共用数据模型与线上协议类型
description: ChuSQL 各进程共用的纯数据类型:列与值的模型、存储层线上协议
(请求/响应/表结构与账号)以及查询结果集。这里不放任何执行
逻辑,前端只依赖这个包就不会把引擎链进自己的二进制。
synopsis: Shared ChuSQL data models and wire protocol types
description: Defines shared column and value models, storage protocol requests
and responses, table schemas, accounts, and query results.
Contains data types without execution logic, allowing clients to
use the models without linking the query engine.
build-type: Simple
library
+73 -36
View File
@@ -29,6 +29,8 @@ module ChuSQL.Core.Model
, typeLabel
, typeName
, parseColumnType
, domainTableName
, isDomainTable
, typeClassOf
, assignable
, comparableTypes
@@ -44,7 +46,7 @@ module ChuSQL.Core.Model
import Data.Char (isDigit, isSpace, toLower)
import Data.Hashable (Hashable (..))
import Data.List (isSuffixOf, stripPrefix)
import Data.List (isPrefixOf, isSuffixOf, stripPrefix)
import Data.Maybe (fromMaybe, mapMaybe)
-- 数据模型:列类型、列约束、值、行、表、数据库,外加查表小工具。
@@ -64,6 +66,8 @@ data ColumnType
| CDate
| CTimestamp
| CBlob
| CNull
| CDomain String ColumnType
deriving (Show, Eq)
-- | 列定义:类型 + 约束;约束全部取默认值就是普通可空列。
@@ -115,6 +119,8 @@ typeLabel (CDecimal p s) = "TDecimal " ++ show p ++ " " ++ show s
typeLabel CDate = "TDate"
typeLabel CTimestamp = "TTimestamp"
typeLabel CBlob = "TBlob"
typeLabel CNull = "NULL"
typeLabel (CDomain name _) = name
-- | 线上/磁盘上的类型名
typeName :: ColumnType -> String
@@ -131,35 +137,50 @@ typeName (CDecimal p s) = "decimal(" ++ show p ++ "," ++ show s ++ ")"
typeName CDate = "date"
typeName CTimestamp = "timestamp"
typeName CBlob = "blob"
typeName CNull = "null"
typeName (CDomain name base) = "domain(" ++ name ++ "," ++ typeName base ++ ")"
-- | 生成类型定义的目录键
domainTableName :: String -> String
domainTableName name = "__system_domain_" ++ map toLower name
-- | 判断类型定义目录键
isDomainTable :: String -> Bool
isDomainTable name = "__system_domain_" `isPrefixOf` map toLower name
-- | 解析类型名,可带长度/精度参数;不认识的给 Nothing
parseColumnType :: String -> Maybe ColumnType
parseColumnType raw = case (base, args) of
("int", []) -> Just CInt
("integer", []) -> Just CInt
("bigint", []) -> Just CBigInt
("smallint", []) -> Just CSmallInt
("str", []) -> Just CStr
("text", []) -> Just CStr
("varchar", []) -> Just CStr
("varchar", [n]) -> CVarchar <$> positive n
("char", []) -> Just (CChar 1)
("char", [n]) -> CChar <$> positive n
("bool", []) -> Just CBool
("boolean", []) -> Just CBool
("float", []) -> Just CFloat
("real", []) -> Just CFloat
("double", []) -> Just CDouble
("decimal", []) -> Just (CDecimal 10 0)
("numeric", []) -> Just (CDecimal 10 0)
("decimal", [p]) -> flip CDecimal 0 <$> positive p
("numeric", [p]) -> flip CDecimal 0 <$> positive p
("decimal", [p, s]) -> CDecimal <$> positive p <*> plain s
("numeric", [p, s]) -> CDecimal <$> positive p <*> plain s
("date", []) -> Just CDate
("timestamp", []) -> Just CTimestamp
("blob", []) -> Just CBlob
_ -> Nothing
parseColumnType raw
| Just inner <- stripPrefix "domain(" raw
, Just body <- stripSuffixMaybe ")" inner
, (name, ',' : definition) <- break (== ',') body =
CDomain name <$> parseColumnType definition
| otherwise = case (base, args) of
("int", []) -> Just CInt
("integer", []) -> Just CInt
("bigint", []) -> Just CBigInt
("smallint", []) -> Just CSmallInt
("str", []) -> Just CStr
("text", []) -> Just CStr
("varchar", []) -> Just CStr
("varchar", [n]) -> CVarchar <$> positive n
("char", []) -> Just (CChar 1)
("char", [n]) -> CChar <$> positive n
("bool", []) -> Just CBool
("boolean", []) -> Just CBool
("float", []) -> Just CFloat
("real", []) -> Just CFloat
("double", []) -> Just CDouble
("decimal", []) -> Just (CDecimal 10 0)
("numeric", []) -> Just (CDecimal 10 0)
("decimal", [p]) -> flip CDecimal 0 <$> positive p
("numeric", [p]) -> flip CDecimal 0 <$> positive p
("decimal", [p, s]) -> CDecimal <$> positive p <*> plain s
("numeric", [p, s]) -> CDecimal <$> positive p <*> plain s
("date", []) -> Just CDate
("timestamp", []) -> Just CTimestamp
("blob", []) -> Just CBlob
_ -> Nothing
where
-- | 去掉空白的类型名
cleaned = map toLower (filter (not . isSpace) raw)
@@ -209,6 +230,7 @@ data TypeClass
| TextClass
| TemporalClass
| BooleanClass
| NullClass
deriving (Show, Eq)
-- | 整型大类判断
@@ -226,10 +248,13 @@ typeClassOf CBlob = TextClass
typeClassOf CDate = TemporalClass
typeClassOf CTimestamp = TemporalClass
typeClassOf CBool = BooleanClass
typeClassOf CNull = NullClass
typeClassOf (CDomain _ base) = typeClassOf base
-- | 同大类可赋值;日期/时间列接受字符串字面量
assignable :: ColumnType -> ColumnType -> Bool
assignable target source
| source == CNull = True
| target == source = True
| typeClassOf target == typeClassOf source = True
| typeClassOf target == TemporalClass && typeClassOf source == TextClass = True
@@ -240,11 +265,12 @@ integerType :: ColumnType -> Bool
integerType CInt = True
integerType CBigInt = True
integerType CSmallInt = True
integerType (CDomain _ base) = integerType base
integerType _ = False
-- | 比较用的类型族:同大类可比,日期时间与文本互比
comparableTypes :: ColumnType -> ColumnType -> Bool
comparableTypes a b = sameClass || crossTemporal
comparableTypes a b = a == CNull || b == CNull || sameClass || crossTemporal
where
-- | 两个类型同属一个大类
sameClass = typeClassOf a == typeClassOf b
@@ -261,6 +287,7 @@ numericType t = typeClassOf t == NumericClass
-- | 值能不能放进这一列(先不看可空)
valueFits :: ColumnType -> Value -> Bool
valueFits (CDomain _ base) value = valueFits base value
valueFits _ VNull = True
valueFits CInt (VInt _) = True
valueFits CBigInt (VInt _) = True
@@ -297,6 +324,8 @@ coerceValue t v = case t of
CDate -> toDate v
CTimestamp -> toTimestamp v
CBlob -> toText t v
CNull -> Left "a NULL output cannot contain a non-NULL value"
CDomain _ base -> coerceValue base v
-- | 整数收进整型列
toInt :: ColumnType -> Value -> Either String Value
@@ -315,15 +344,22 @@ toFloat t _ = Left ("cannot put this value into " ++ typeLabel t)
-- | DECIMAL 按小数位四舍五入
toDecimal :: Int -> Int -> Value -> Either String Value
toDecimal p s v = case v of
VInt n -> rounded (fromIntegral n)
VFloat d -> rounded d
_ -> Left ("cannot put this value into " ++ typeLabel (CDecimal p s))
toDecimal p s v
| p <= 0 || p > 308 || s < 0 || s > p = Left "invalid decimal precision or scale"
| otherwise = case v of
VInt n -> rounded (fromIntegral n)
VFloat d -> rounded d
_ -> Left ("cannot put this value into " ++ typeLabel (CDecimal p s))
where
-- | 缩放因子
scale = (10 :: Double) ^ max 0 s
scale = (10 :: Double) ^ s
-- | 按比例四舍五入
rounded d = Right (VFloat (fromIntegral (round (d * scale) :: Integer) / scale))
rounded d
| isNaN d || isInfinite d || isInfinite (d * scale) = Left "decimal value is not finite or exceeds precision"
| abs result >= (10 :: Double) ^ (p - s) = Left "decimal value exceeds precision"
| otherwise = Right (VFloat result)
where
result = fromIntegral (round (d * scale) :: Integer) / scale
-- | 字符串收进文本列
toText :: ColumnType -> Value -> Either String Value
@@ -562,5 +598,6 @@ resolveColumn name env = case [(k, v) | (k, v) <- env, matches k] of
[] -> Left ("unknown column: " ++ name)
_ -> Left ("ambiguous column: " ++ name)
where
-- | 裸名、补充了库前缀的写法,或 schema 键自带库前缀的写法
matches k = k == name || ('.' : name) `isSuffixOf` k || ('.' : k) `isSuffixOf` name
-- | 裸名匹配末段,库限定名只补一层前缀
matches k = k == name || ('.' : name) `isSuffixOf` k
|| (length (filter (== '.') name) == 2 && length (filter (== '.') k) == 1 && ('.' : k) `isSuffixOf` name)
+16 -1
View File
@@ -75,6 +75,10 @@ data Request
| ReqAllCatalog
| ReqInDatabase String Request
| ReqAccountsList
| ReqIdentityInitialize T.Text
| ReqRoleCreate T.Text
| ReqCatalogManage Request
| ReqIdentityAlter T.Text (Maybe Bool) (Maybe Bool) (Maybe Bool) (Maybe Bool)
| ReqAccountCreate T.Text T.Text
| ReqAccountReset T.Text T.Text
| ReqAccountLogin T.Text (Maybe T.Text)
@@ -134,7 +138,12 @@ instance ToJSON Request where
toJSON (ReqInDatabase name req) = case toJSON req of
A.Object fields -> A.Object (KM.insert "database" (A.String (T.pack name)) fields)
other -> other
toJSON (ReqCatalogManage command) = object ["method" .= ("catalog_manage" :: T.Text), "command" .= command]
toJSON ReqAccountsList = object ["method" .= ("accounts_list" :: T.Text)]
toJSON (ReqIdentityInitialize u) = object ["method" .= ("identity_initialize" :: T.Text), "administrator" .= u]
toJSON (ReqRoleCreate u) = object ["method" .= ("role_create" :: T.Text), "user" .= u]
toJSON (ReqIdentityAlter u login super enabled manager) = object ["method" .= ("identity_alter" :: T.Text), "user" .= u,
"can_login" .= login, "is_superuser" .= super, "enabled" .= enabled, "system_catalog_manager" .= manager]
toJSON (ReqAccountCreate u h) = object ["method" .= ("account_create" :: T.Text), "user" .= u, "password_hash" .= h]
toJSON (ReqAccountReset u h) = object ["method" .= ("account_reset" :: T.Text), "user" .= u, "password_hash" .= h]
toJSON (ReqAccountLogin u at) = object ["method" .= ("account_login" :: T.Text), "user" .= u, "at" .= at]
@@ -303,6 +312,10 @@ data Account = Account
, accountRevision :: Integer
, accountRegisteredAt :: T.Text
, accountLastLoginAt :: Maybe T.Text
, accountCanLogin :: Bool
, accountIsSuperuser :: Bool
, accountEnabled :: Bool
, accountSystemCatalogManager :: Bool
} deriving (Eq)
-- | 账号的展示:只露账号名与版本号
@@ -314,12 +327,14 @@ instance FromJSON Account where
parseJSON = withObject "Account" $ \o -> Account <$> o .: "id" <*> o .: "user"
<*> o .: "password_hash" <*> o .: "revision"
<*> o .:? "registered_at" .!= "" <*> o .:? "last_login_at"
<*> o .:? "can_login" .!= True <*> o .:? "is_superuser" .!= False <*> o .:? "enabled" .!= True <*> o .:? "system_catalog_manager" .!= False
-- | 账号编码成 JSON
instance ToJSON Account where
toJSON a = object ["id" .= accountId a, "user" .= accountUser a,
"password_hash" .= accountHash a, "revision" .= accountRevision a,
"registered_at" .= accountRegisteredAt a, "last_login_at" .= accountLastLoginAt a]
"registered_at" .= accountRegisteredAt a, "last_login_at" .= accountLastLoginAt a,
"can_login" .= accountCanLogin a, "is_superuser" .= accountIsSuperuser a, "enabled" .= accountEnabled a, "system_catalog_manager" .= accountSystemCatalogManager a]
-- | 存储层响应
data Response
@@ -1,7 +1,7 @@
# ChuSQL 存储服务配置示例
#
# 用法:安装时把本文件放到固定的用户配置目录(Windows 是 %APPDATA%\ChuSQL\chusql.toml,
# 其它平台是 ${XDG_CONFIG_HOME:-$HOME/.config}/ChuSQL/chusql.toml);
# 用法:安装时把本文件放到固定的用户配置目录(Windows 是 %APPDATA%\ChuSQL\settings.toml,
# 其它平台是 ${XDG_CONFIG_HOME:-$HOME/.config}/ChuSQL/settings.toml);
# 临时换一份就用命令行参数 --config <path>(没有环境变量通道了)。
#
# 每一项都可以删掉:没写的项回退到内置默认值(下面注释里的值就是默认值)。
@@ -28,6 +28,9 @@ pool_size = 1024
# Unix ${XDG_DATA_HOME:-$HOME/.local/share}/chusql/data,跟安装脚本装的位置一致);
# 要固定就写绝对路径,例如:
# data_dir = "/srv/chusql/data"
#
# 日志输出目录:启动器(csql-web)与安装脚本按它决定日志写哪;相对路径以启动目录为基准。
# log_files = "./logs"
[log]
# 日志等级:off / error / warn / info / debug。debug 会多打每个请求和连接。
+22 -9
View File
@@ -1,5 +1,5 @@
use std::collections::hash_map::DefaultHasher;
use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::hash::{Hash, Hasher};
use std::io;
use std::io::Write;
@@ -51,9 +51,11 @@ pub struct TableSchema {
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct Catalog {
#[serde(default)]
pub identity_high_water: i64,
tables: BTreeMap<String, TableSchema>,
#[serde(skip)]
seen: BTreeMap<String, HashSet<u64>>,
seen: BTreeMap<String, HashMap<u64, u64>>,
}
impl Catalog {
@@ -153,17 +155,25 @@ impl Catalog {
for r in rows {
for (name, v) in r {
let stat = entry.stats.entry(name.clone()).or_default();
hist_remove(stat, v);
if stat.capped {
continue;
}
if seen.remove(&fold(name, value_hash(v))) {
stat.distinct = stat.distinct.saturating_sub(1);
let key = fold(name, value_hash(v));
if let Some(count) = seen.get_mut(&key) {
*count = count.saturating_sub(1);
if *count == 0 {
seen.remove(&key);
stat.distinct = stat.distinct.saturating_sub(1);
}
}
hist_remove(stat, v);
}
}
if entry.row_count == 0 {
seen.clear();
for stat in entry.stats.values_mut() {
stat.distinct = 0;
stat.capped = false;
stat.lo = None;
stat.hi = None;
stat.hist.clear();
@@ -192,17 +202,18 @@ impl Catalog {
let seen = self.seen.entry(table.to_string()).or_default();
for (name, v) in row {
let stat = entry.stats.entry(name.clone()).or_default();
hist_add(stat, v);
if stat.capped {
continue;
}
if seen.insert(fold(name, value_hash(v))) {
let count = seen.entry(fold(name, value_hash(v))).or_insert(0);
*count += 1;
if *count == 1 {
stat.distinct += 1;
if stat.distinct >= STATS_DISTINCT_CAP {
stat.capped = true;
seen.clear();
}
}
hist_add(stat, v);
}
}
@@ -476,5 +487,7 @@ fn hist_remove(stat: &mut ColumnStat, v: &serde_json::Value) {
return;
}
let index = hist_index(lo, hi, x);
stat.hist[index] = stat.hist[index].saturating_sub(1);
if let Some(bucket) = (0..stat.hist.len()).filter(|&i| stat.hist[i] > 0).min_by_key(|&i| i.abs_diff(index)) {
stat.hist[bucket] -= 1;
}
}
+19 -4
View File
@@ -8,7 +8,7 @@ use crate::log::Level;
// 存储相关分区由本模块解析校验,其它层的分区一概不碰。
/// 配置文件名
pub const CONFIG_FILE_NAME: &str = "chusql.toml";
pub const CONFIG_FILE_NAME: &str = "settings.toml";
/// 配置目录名(随系统惯例放在用户配置目录下)
pub const APP_DIR_NAME: &str = "ChuSQL";
/// Unix 下的数据目录名(XDG 惯例小写,跟安装脚本的默认安装目录一致)
@@ -20,6 +20,8 @@ pub const DEFAULT_POOL_SIZE: usize = 1024;
pub const DEFAULT_PAGE_SIZE: usize = 4096;
pub const DEFAULT_BTREE_ORDER: usize = 4;
pub const DEFAULT_LOG_LEVEL: &str = "info";
/// 默认日志输出目录(相对启动目录)
pub const DEFAULT_LOG_FILES: &str = "./logs";
const MIN_PAGE_SIZE: usize = 512;
const MAX_PAGE_SIZE: usize = 65536;
@@ -91,6 +93,7 @@ pub struct Config {
pub btree_order: usize,
pub pool_size: usize,
pub data_dir: PathBuf,
pub log_files: PathBuf,
pub log_level: Level,
}
@@ -102,6 +105,7 @@ impl Default for Config {
btree_order: DEFAULT_BTREE_ORDER,
pool_size: DEFAULT_POOL_SIZE,
data_dir: default_data_dir(),
log_files: PathBuf::from(DEFAULT_LOG_FILES),
log_level: Level::Info,
}
}
@@ -161,6 +165,7 @@ struct FileBtree {
#[serde(deny_unknown_fields)]
struct FileStorage {
data_dir: Option<String>,
log_files: Option<String>,
}
#[derive(Debug, Default, Deserialize)]
@@ -229,6 +234,11 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
default_data_dir().to_string_lossy().into_owned(),
&file_origin,
);
let (log_files, log_files_origin) = pick(
file.storage.as_ref().and_then(|s| s.log_files.clone()),
DEFAULT_LOG_FILES.to_string(),
&file_origin,
);
// 管名已经没有了:存储不再是独立进程,管道/套接字那一层随 FFI 一起删掉
let (log_level, log_origin) = pick(
file.log.and_then(|l| l.level),
@@ -237,6 +247,7 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
);
let data_dir = data_dir.trim().to_string();
let log_files = log_files.trim().to_string();
let log_level = log_level.trim().to_string();
let log_level = Level::parse(&log_level)
@@ -245,6 +256,9 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
if data_dir.is_empty() {
return Err("storage.data_dir must not be empty".to_string());
}
if log_files.is_empty() {
return Err("storage.log_files must not be empty".to_string());
}
validate_layout(page_size, btree_order)?;
if pool_size == 0 {
return Err("buffer.pool_size must be at least 1".to_string());
@@ -254,6 +268,7 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
("btree.order", btree_order.to_string(), order_origin),
("buffer.pool_size", pool_size.to_string(), pool_origin),
("storage.data_dir", data_dir.clone(), dir_origin),
("storage.log_files", log_files.clone(), log_files_origin),
("log.level", log_level.name().to_string(), log_origin),
];
@@ -263,6 +278,7 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
btree_order,
pool_size,
data_dir: PathBuf::from(data_dir),
log_files: PathBuf::from(log_files),
log_level,
},
config_path: if text.is_some() { path } else { None },
@@ -318,15 +334,14 @@ size = 8192
data_dir = "../data"
[server]
host = "127.0.0.1"
listen_host = "127.0.0.1"
port = 7777
max_message = 1048576
[web]
port = 7778
user = "root"
"#;
let loaded = resolve(Some(text), Some(PathBuf::from("chusql.toml"))).unwrap();
let loaded = resolve(Some(text), Some(PathBuf::from("settings.toml"))).unwrap();
assert_eq!(loaded.config.page_size, 8192);
assert_eq!(loaded.config.data_dir, PathBuf::from("../data"));
}
+27
View File
@@ -71,6 +71,33 @@ pub unsafe extern "C" fn chusql_storage_close(handle: *mut Storage) {
let _ = catch_unwind(AssertUnwindSafe(|| unsafe { drop(Box::from_raw(handle)) }));
}
/// 离线维护系统目录并返回 JSON
///
/// # Safety
/// 输入为有效字符串;输出指针可写,响应使用 free 释放。
#[unsafe(no_mangle)]
pub unsafe extern "C" fn chusql_storage_maintenance(
config_path: *const c_char, request: *const u8, request_len: usize,
out: *mut *mut u8, out_len: *mut usize,
) -> i32 {
if request.is_null() || out.is_null() || out_len.is_null() { set_error("invalid maintenance argument"); return ERR; }
unsafe { *out = ptr::null_mut(); *out_len = 0; }
let result = catch_unwind(AssertUnwindSafe(|| {
let path = if config_path.is_null() { None } else { Some(unsafe { CStr::from_ptr(config_path) }.to_str().map_err(|e| e.to_string())?) };
let text = std::str::from_utf8(unsafe { slice::from_raw_parts(request, request_len) }).map_err(|e| e.to_string())?;
Storage::maintenance(path, text)
}));
let response = match result {
Ok(Ok(response)) => response,
Ok(Err(err)) => { set_error(err); return ERR; }
Err(_) => { set_error("panic in system maintenance"); return ERR; }
};
let mut buffer = response.into_bytes().into_boxed_slice();
unsafe { *out = buffer.as_mut_ptr(); *out_len = buffer.len(); }
std::mem::forget(buffer);
OK
}
/// 处理一条请求,响应缓冲须归还
///
/// # Safety
+22 -2
View File
@@ -6,6 +6,8 @@ use std::collections::HashMap;
pub type Row = HashMap<String, serde_json::Value>;
pub const USERS_TABLE: &str = "__system_users";
pub const IDENTITIES_TABLE: &str = "__system_identities";
pub const TYPES_TABLE: &str = "__system_types";
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Account {
@@ -17,6 +19,16 @@ pub struct Account {
pub registered_at: String,
#[serde(default)]
pub last_login_at: Option<String>,
#[serde(default = "default_true")]
pub can_login: bool,
#[serde(default)]
pub is_superuser: bool,
#[serde(default)]
pub system_catalog_manager: bool,
#[serde(default = "default_true")]
pub enabled: bool,
#[serde(default)]
pub identity_version: u8,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
@@ -80,7 +92,11 @@ pub struct IndexWire {
#[derive(Debug, Serialize, Deserialize)]
#[serde(tag = "method", rename_all = "snake_case")]
pub enum Request {
CatalogManage { command: Box<Request> },
AccountsList,
IdentityInitialize { administrator: String },
RoleCreate { user: String },
IdentityAlter { user: String, can_login: Option<bool>, is_superuser: Option<bool>, enabled: Option<bool>, system_catalog_manager: Option<bool> },
AccountCreate { user: String, password_hash: String },
AccountReset { user: String, password_hash: String },
AccountLogin {
@@ -97,6 +113,7 @@ pub enum Request {
password_hash: Option<String>,
},
SystemStatus,
BootstrapTypes { types: Vec<Row> },
Ping,
Scan {
table: String,
@@ -222,9 +239,11 @@ impl Request {
| Self::DropIndex { table, .. } | Self::DropColumn { table, .. }
| Self::Compact { table }
| Self::ReplaceSchema { table, .. } => Some(table),
Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList
Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList | Self::CatalogManage { .. }
| Self::IdentityInitialize { .. } | Self::RoleCreate { .. } | Self::IdentityAlter { .. }
| Self::AccountCreate { .. } | Self::AccountReset { .. } | Self::AccountLogin { .. }
| Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus
| Self::BootstrapTypes { .. }
| Self::ApplyTransaction { .. } => None,
}
}
@@ -245,7 +264,8 @@ pub const PRIVILEGE_TABLES: [&str; 4] = [
/// 是不是服务自己经请求通道读写的系统表
pub fn internal_table(table: &str) -> bool {
PRIVILEGE_TABLES.contains(&table.to_ascii_lowercase().as_str())
let name = table.to_ascii_lowercase();
PRIVILEGE_TABLES.contains(&name.as_str()) || name.starts_with("__system_domain_")
}
/// 是不是请求通道要拒绝的系统表
+529 -30
View File
@@ -8,8 +8,8 @@ use crate::config::{self, Config, Loaded};
use crate::heap::{scalar_int, HeapTable};
use crate::log;
use crate::protocol::{
blocked_table, reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row,
SchemaColumn, StorageOp, TableSchemaWire, USERS_TABLE,
blocked_table, reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row, IDENTITIES_TABLE,
SchemaColumn, StorageOp, TableSchemaWire, USERS_TABLE, TYPES_TABLE,
};
use crate::wal::{committed_ops, records_after, Wal, WalOp};
use crate::{log_debug, log_error, log_info, log_warn};
@@ -222,15 +222,41 @@ impl Server {
c.rebuild_stats(table, rows);
c.save(self.catalog_path())
}
WalOp::Accounts { accounts } => {
WalOp::Accounts { accounts, removed, migrate_roles } => {
let rows = account_rows(accounts)?;
self.tables.lock().map_err(|_| std::io::Error::other("table lock poisoned"))?.remove(USERS_TABLE);
for path in [self.table_path(USERS_TABLE), self.index_path(USERS_TABLE, "id")] {
std::fs::File::create(path)?.sync_all()?;
let identities = identity_rows(accounts)?;
for (table, records) in [(USERS_TABLE, &rows), (IDENTITIES_TABLE, &identities)] {
self.tables.lock().map_err(|_| std::io::Error::other("table lock poisoned"))?.remove(table);
for path in [self.table_path(table), self.index_path(table, "id")] {
std::fs::File::create(path)?.sync_all()?;
}
self.with_table(table, |t| { t.replace_all(records)?; t.flush() })?;
}
let legacy_roles = "__system_roles";
let has_roles = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?.describe(legacy_roles).is_some();
if has_roles && *migrate_roles {
self.with_existing_table(legacy_roles, |t| { t.replace_all(&[])?; t.flush() })?;
}
let mut cleaned = Vec::new();
for table in ["__system_grants", "__system_grant_options", "__system_members"] {
let exists = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?.describe(table).is_some();
if exists && !removed.is_empty() {
let rows = self.with_existing_table(table, |t| t.scan())?.into_iter().filter(|r|
!["role", "member"].iter().any(|field| r.get(*field).and_then(serde_json::Value::as_str).is_some_and(|name| removed.iter().any(|old| old == name)))).collect::<Vec<_>>();
self.with_existing_table(table, |t| { t.replace_all(&rows)?; t.flush() })?;
cleaned.push((table, rows));
}
}
self.with_table(USERS_TABLE, |t| { t.replace_all(&rows)?; t.flush() })?;
let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
declare_system_schema(&mut c, &rows)?;
c.rebuild_stats(IDENTITIES_TABLE, &identities);
c.set_columns(IDENTITIES_TABLE, identity_columns())?;
c.rebuild_stats(IDENTITIES_TABLE, &identities);
c.mark_system(IDENTITIES_TABLE)?;
c.add_index(IDENTITIES_TABLE, "user")?;
c.identity_high_water = c.identity_high_water.max(accounts.iter().map(|a| a.id).max().unwrap_or(0));
if has_roles && *migrate_roles { c.rebuild_stats(legacy_roles, &[]); }
for (table, records) in cleaned { c.rebuild_stats(table, &records); }
c.save(self.catalog_path())
}
WalOp::Insert { table, row } => {
@@ -284,6 +310,7 @@ impl Server {
self.with_table(table, |_t| Ok(()))?;
let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
c.create_table(table, columns.clone())?;
if table == TYPES_TABLE { c.mark_system(table)?; }
c.save(self.catalog_path())
}
WalOp::DropTable { table } => {
@@ -342,8 +369,67 @@ impl Server {
if self.recovery_required.load(Ordering::Acquire) {
return Err(std::io::Error::other("storage recovery required"));
}
self.validate_op(op)?;
self.group_via_wal(std::slice::from_ref(op))
let op = self.normalize_decimal_write(op)?;
self.validate_op(&op)?;
self.validate_domains(&op)?;
self.group_via_wal(std::slice::from_ref(&op))
}
/// 写日志前规范化定点数值
fn normalize_decimal_write(&self, op: &WalOp) -> std::io::Result<WalOp> {
let mut normalized = op.clone();
let catalog = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
let (table, rows): (&str, Vec<&mut Row>) = match &mut normalized {
WalOp::Insert { table, row } => (table, vec![row]),
WalOp::InsertBatch { table, rows } | WalOp::ReplaceAll { table, rows } => (table, rows.iter_mut().collect()),
WalOp::CreateTable { columns, .. } => {
for column in columns { decimal_parameters(&column.ty)?; }
return Ok(normalized);
}
WalOp::ReplaceSchema { columns, rows, .. } => {
for column in columns.iter() { decimal_parameters(&column.ty)?; }
for row in rows { normalize_decimal_row(row, columns)?; }
return Ok(normalized);
}
_ => return Ok(normalized),
};
if let Some(schema) = catalog.describe(table) {
for row in rows {
normalize_decimal_row(row, &schema.columns)?;
}
}
Ok(normalized)
}
/// 校验命名类型引用与删除依赖
fn validate_domains(&self, op: &WalOp) -> std::io::Result<()> {
let catalog = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
match op {
WalOp::CreateTable { columns, .. } | WalOp::ReplaceSchema { columns, .. } => {
for column in columns {
let mut ty = column.ty.as_str();
while let Some(body) = ty.strip_prefix("domain(").and_then(|s| s.strip_suffix(')')) {
let (name, base) = body.split_once(',').ok_or_else(|| std::io::Error::other("invalid domain type"))?;
let key = format!("__system_domain_{name}");
let definition = catalog.describe(&key).ok_or_else(|| std::io::Error::other(format!("unknown domain: {name}")))?;
if !definition.columns.iter().any(|c| c.name == "base" && c.ty == base) {
return Err(std::io::Error::other(format!("domain base type mismatch: {name}")));
}
ty = base;
}
}
}
WalOp::DropTable { table } => {
if let Some(name) = table.strip_prefix("__system_domain_") {
let prefix = format!("domain({name},");
if catalog.all_tables().iter().any(|(_, schema)| schema.columns.iter().any(|c| !schema.dropped_columns.contains(&c.name) && c.ty.contains(&prefix))) {
return Err(std::io::Error::other(format!("domain is still used by a column: {name}")));
}
}
}
_ => {}
}
Ok(())
}
/// 校验一条操作:命中被删的列就拒绝
@@ -406,6 +492,7 @@ impl Server {
rows: rows.clone(),
},
};
let wal_op = self.normalize_decimal_write(&wal_op)?;
self.validate_op(&wal_op)?;
wal_ops.push(wal_op);
}
@@ -420,13 +507,9 @@ impl Server {
std::fs::create_dir_all(&self.cfg.data_dir)?;
let wal = &self.wal;
let commit_lsn = wal.append_group(ops)?;
self.recovery_required.store(true, Ordering::Release);
for op in ops {
let result = self.apply_op(op);
if result.is_err() && matches!(op, WalOp::HideColumn { .. }) {
self.recovery_required.store(true, Ordering::Release);
return result;
}
result?;
self.apply_op(op)?;
}
for op in ops {
self.flush_op_tables(op)?;
@@ -434,6 +517,7 @@ impl Server {
self.flush_all_tables()?;
wal.write_checkpoint(commit_lsn)?;
wal.truncate_before(commit_lsn)?;
self.recovery_required.store(false, Ordering::Release);
log_debug!(core, "WAL committed group at LSN {}", commit_lsn);
Ok(())
}
@@ -479,15 +563,15 @@ impl Server {
fn recover_wal(&self) -> std::io::Result<()> {
std::fs::create_dir_all(&self.cfg.data_dir)?;
let wal = &self.wal;
let checkpoint = wal.read_checkpoint()?.unwrap_or(0);
let scan = wal.read_all()?;
if scan.records.is_empty() {
if scan.good_len > 0 {
if wal.path().exists() {
wal.truncate_to(scan.good_len)?;
}
return Ok(());
}
// 检查点以内的记录已经落盘,直接按边界截掉
let checkpoint = wal.read_checkpoint()?.unwrap_or(0);
let pending = records_after(&scan.records, checkpoint);
let records = committed_ops(&pending);
if records.is_empty() {
@@ -859,6 +943,12 @@ impl Server {
use std::io::{Error, ErrorKind};
let registered = {
let c = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?;
match c.describe(IDENTITIES_TABLE) {
Some(schema) if !schema.system => return Err(Error::new(ErrorKind::AlreadyExists, "reserved identity table collision")),
None if self.table_path(IDENTITIES_TABLE).exists() || self.index_path(IDENTITIES_TABLE, "id").exists() =>
return Err(Error::new(ErrorKind::AlreadyExists, "reserved identity file collision")),
_ => {}
}
c.describe(USERS_TABLE).map(|schema| schema.system)
};
match registered {
@@ -893,6 +983,21 @@ impl Server {
if !self.table_path(USERS_TABLE).exists() {
return Err(Error::other("missing account data"));
}
let unified = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe(IDENTITIES_TABLE).is_some();
if unified {
let credentials = self.with_existing_table(USERS_TABLE, |t| t.scan())?;
let identities = self.with_existing_table(IDENTITIES_TABLE, |t| t.scan())?;
return identities.into_iter().map(|mut row| {
for field in ["can_login", "is_superuser", "enabled"] {
if row.get(field).and_then(serde_json::Value::as_bool).is_none() { return Err(Error::other("invalid identity attributes")); }
}
let id = row.get("id").and_then(serde_json::Value::as_i64).ok_or_else(|| Error::other("invalid identity record"))?;
let credential = credentials.iter().find(|r| r.get("id").and_then(serde_json::Value::as_i64) == Some(id))
.ok_or_else(|| Error::other("missing identity credential"))?;
row.insert("password_hash".into(), credential.get("password_hash").cloned().ok_or_else(|| Error::other("invalid identity credential"))?);
decode_account_row(&row)
}).collect();
}
let mut accounts: Vec<Account> = self
.with_existing_table(USERS_TABLE, |t| t.scan())?
.iter()
@@ -919,9 +1024,11 @@ impl Server {
// 表里已经有行就不再补人:引导程序只在空表上播种,
// 免得有人拿 storage 透传绕过账号接口往里塞一个已知口令的账号
if accounts.is_empty() && !accounts.iter().any(|a| a.user == name) {
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0)
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0).max(self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.identity_high_water)
.checked_add(1).ok_or_else(|| Error::other("account id exhausted"))?;
accounts.push(new_account(id, &name, password_hash.unwrap_or_default().to_string())?);
let mut administrator = new_account(id, &name, password_hash.unwrap_or_default().to_string())?;
administrator.is_superuser = true;
accounts.push(administrator);
changed = true;
}
}
@@ -931,7 +1038,7 @@ impl Server {
for row in account_rows(&accounts)? {
HeapTable::encode_row(&row, self.cfg.page_size)?;
}
let op = WalOp::Accounts { accounts: accounts.clone() };
let op = WalOp::Accounts { accounts: accounts.clone(), removed: Vec::new(), migrate_roles: false };
self.recovery_required.store(true, Ordering::Release);
self.wal.truncate()?;
self.wal.append(&op)?;
@@ -952,17 +1059,93 @@ impl Server {
return Err(Error::other("system catalog is not initialized; run csql-bootstrap"));
}
let mut accounts = self.load_accounts()?;
let request = match request {
Request::CatalogManage { command } => {
let target = match command.as_ref() {
Request::AccountCreate { user, .. } | Request::AccountReset { user, .. } | Request::AccountDrop { user }
| Request::IdentityAlter { user, is_superuser: None, system_catalog_manager: None, .. } => Some(account_name(user)?),
Request::AccountsList => None,
_ => return Err(Error::other("superuser required for this catalog operation")),
};
if accounts.iter().any(|a| target.as_ref() == Some(&a.user) && (a.is_superuser || a.system_catalog_manager)) {
return Err(Error::other("superuser required to manage privileged identities"));
}
*command
}
other => other,
};
let mut names = std::collections::HashSet::new();
let mut ids = std::collections::HashSet::new();
for account in &mut accounts {
let name = account_name(&account.user)?;
if !names.insert(name.clone()) || account.id < 1 || !ids.insert(account.id) {
return Err(Error::other("invalid or duplicate identity record"));
}
if account.identity_version == 0 { account.user = name; }
}
let previous = accounts.clone();
let migrate_roles = matches!(request, Request::IdentityInitialize { .. });
if !migrate_roles && !matches!(request, Request::AccountsList) {
let legacy_exists = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe("__system_roles").is_some();
if legacy_exists && !self.with_existing_table("__system_roles", |t| t.scan())?.is_empty() {
return Err(Error::other("identity migration required"));
}
}
let had_superuser = accounts.iter().any(|a| a.enabled && a.can_login && a.is_superuser);
match request {
Request::AccountsList => return Ok(accounts),
Request::IdentityInitialize { administrator } => {
for account in &mut accounts {
if account.identity_version == 0 {
account.is_superuser = account.user == administrator.to_ascii_lowercase();
account.identity_version = 1;
}
}
let legacy_roles = "__system_roles";
let exists = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe(legacy_roles).is_some();
if exists {
for row in self.with_existing_table(legacy_roles, |t| t.scan())? {
let name = account_name(row.get("name").and_then(serde_json::Value::as_str).ok_or_else(|| Error::other("invalid legacy role"))?)?;
if accounts.iter().any(|a| a.user == name) { return Err(Error::other(format!("identity name collision: {name}"))); }
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0).max(self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.identity_high_water)
.checked_add(1).ok_or_else(|| Error::other("account id exhausted"))?;
let mut role = new_account(id, &name, String::new())?;
role.can_login = false;
accounts.push(role);
}
}
let unified = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe(IDENTITIES_TABLE).is_some();
if !accounts.is_empty() { require_superuser(&accounts)?; }
if unified && previous == accounts { return Ok(accounts); }
}
Request::AccountCreate { user, password_hash } => {
let name = account_name(&user)?;
if accounts.iter().any(|a| a.user == name) {
return Err(Error::new(ErrorKind::AlreadyExists, "account already exists"));
}
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0)
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0).max(self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.identity_high_water)
.checked_add(1).ok_or_else(|| Error::other("account id exhausted"))?;
accounts.push(new_account(id, &name, password_hash)?);
}
Request::RoleCreate { user } => {
let name = account_name(&user)?;
if accounts.iter().any(|a| a.user == name) { return Err(Error::new(ErrorKind::AlreadyExists, "identity already exists")); }
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0).max(self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.identity_high_water)
.checked_add(1).ok_or_else(|| Error::other("account id exhausted"))?;
let mut role = new_account(id, &name, String::new())?;
role.can_login = false;
accounts.push(role);
}
Request::IdentityAlter { user, can_login, is_superuser, enabled, system_catalog_manager } => {
let name = account_name(&user)?;
let account = accounts.iter_mut().find(|a| a.user == name).ok_or_else(|| Error::new(ErrorKind::NotFound, "unknown account"))?;
account.can_login = can_login.unwrap_or(account.can_login);
account.is_superuser = is_superuser.unwrap_or(account.is_superuser);
account.enabled = enabled.unwrap_or(account.enabled);
account.system_catalog_manager = system_catalog_manager.unwrap_or(account.system_catalog_manager);
account.revision = account.revision.checked_add(1).ok_or_else(|| Error::other("account revision exhausted"))?;
if had_superuser { require_superuser(&accounts)?; }
}
Request::AccountReset { user, password_hash } => {
let name = account_name(&user)?;
validate_account_hash(&password_hash)?;
@@ -976,6 +1159,7 @@ impl Server {
let stamp = at.unwrap_or_else(now_stamp);
let account = accounts.iter_mut().find(|a| a.user == name)
.ok_or_else(|| Error::new(ErrorKind::NotFound, "unknown account"))?;
if !account.enabled || !account.can_login { return Err(Error::other("identity cannot login")); }
account.last_login_at = Some(stamp);
}
Request::AccountDrop { user } => {
@@ -985,13 +1169,15 @@ impl Server {
if accounts.len() == before {
return Err(Error::new(ErrorKind::NotFound, "unknown account"));
}
if had_superuser { require_superuser(&accounts)?; }
}
_ => return Err(Error::other("unexpected account request")),
}
for row in account_rows(&accounts)? {
for row in account_rows(&accounts)?.into_iter().chain(identity_rows(&accounts)?) {
HeapTable::encode_row(&row, self.cfg.page_size)?;
}
let op = WalOp::Accounts { accounts: accounts.clone() };
let removed = previous.iter().filter(|old| !accounts.iter().any(|a| a.user == old.user)).map(|a| a.user.clone()).collect();
let op = WalOp::Accounts { accounts: accounts.clone(), removed, migrate_roles };
self.recovery_required.store(true, Ordering::Release);
self.wal.truncate()?;
self.wal.append(&op)?;
@@ -1001,6 +1187,53 @@ impl Server {
Ok(accounts)
}
/// 初始化并校验内置类型目录
fn bootstrap_types(&self, types: Vec<Row>) -> std::io::Result<Vec<Row>> {
use std::io::Error;
let _guard = self.write_lock.lock().map_err(|_| Error::other("write lock poisoned"))?;
if !self.account_table_ready()? { return Err(Error::other("system catalog is not initialized; run csql-bootstrap")); }
require_superuser(&self.load_accounts()?)?;
let columns = types_columns();
let existing = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe(TYPES_TABLE).cloned();
if existing.as_ref().is_some_and(|s| !s.system || s.columns != columns) { return Err(Error::other("reserved type catalog collision")); }
if existing.is_none() && (self.table_path(TYPES_TABLE).exists() || self.index_path(TYPES_TABLE, "id").exists()) {
return Err(Error::other("reserved type file collision"));
}
let mut rows = if existing.is_some() { self.with_existing_table(TYPES_TABLE, |t| t.scan())? } else { Vec::new() };
let mut names = std::collections::HashSet::new();
let mut ids = std::collections::HashSet::new();
for row in &rows {
let name = type_definition_name(row)?;
let id = row.get("id").and_then(serde_json::Value::as_i64).ok_or_else(|| Error::other("invalid type id"))?;
if id < 1 || !ids.insert(id) || !names.insert(name.to_string()) { return Err(Error::other("duplicate type catalog record")); }
}
let original = rows.clone();
let mut requested = std::collections::HashSet::new();
for mut definition in types {
let name = type_definition_name(&definition)?.to_string();
if definition.len() != 3 || !requested.insert(name.clone()) { return Err(Error::other("invalid or duplicate preinstalled type")); }
if let Some(saved) = rows.iter().find(|row| row.get("name").and_then(serde_json::Value::as_str) == Some(&name)) {
if saved.get("base_type") != definition.get("base_type") || saved.get("parameterized") != definition.get("parameterized") {
return Err(Error::other(format!("preinstalled type definition conflict: {name}")));
}
} else {
let id = rows.iter().filter_map(|r| r.get("id").and_then(serde_json::Value::as_i64)).max().unwrap_or(0)
.checked_add(1).ok_or_else(|| Error::other("type id exhausted"))?;
definition.insert("id".into(), serde_json::json!(id));
HeapTable::encode_row(&definition, self.cfg.page_size)?;
rows.push(definition);
}
}
if existing.is_some() && original == rows { return Ok(rows); }
let mut ops = Vec::new();
if existing.is_none() { ops.push(WalOp::CreateTable { table: TYPES_TABLE.into(), columns }); }
ops.push(WalOp::ReplaceAll { table: TYPES_TABLE.into(), rows: rows.clone() });
self.recovery_required.store(true, Ordering::Release);
self.group_via_wal(&ops)?;
self.recovery_required.store(false, Ordering::Release);
Ok(rows)
}
/// 一条请求翻成一条响应:先过读/写门再分发
fn handle_request(&self, req: Request) -> Response {
if concurrent_read(&req) {
@@ -1037,7 +1270,12 @@ impl Server {
}
}
match req {
request @ (Request::AccountsList | Request::AccountCreate { .. }
Request::BootstrapTypes { types } => match self.bootstrap_types(types) {
Ok(rows) => Response::Rows { rows },
Err(e) => Response::Error { message: e.to_string() },
},
request @ (Request::AccountsList | Request::CatalogManage { .. } | Request::AccountCreate { .. }
| Request::IdentityInitialize { .. } | Request::RoleCreate { .. } | Request::IdentityAlter { .. }
| Request::AccountReset { .. } | Request::AccountLogin { .. }
| Request::AccountDrop { .. }) => {
match self.accounts_request(request) {
@@ -1458,12 +1696,45 @@ impl Server {
}
}
/// 构造内置类型目录列定义
fn types_columns() -> Vec<SchemaColumn> {
vec![
SchemaColumn { name: "id".into(), ty: "int".into(), nullable: false, primary_key: true, ..Default::default() },
SchemaColumn { name: "name".into(), ty: "varchar(64)".into(), nullable: false, unique: true, ..Default::default() },
SchemaColumn { name: "base_type".into(), ty: "varchar(64)".into(), nullable: false, ..Default::default() },
SchemaColumn { name: "parameterized".into(), ty: "bool".into(), nullable: false, ..Default::default() },
]
}
/// 校验类型定义并读取名称
fn type_definition_name(row: &Row) -> std::io::Result<&str> {
let name = row.get("name").and_then(serde_json::Value::as_str).ok_or_else(|| std::io::Error::other("invalid type name"))?;
let base = row.get("base_type").and_then(serde_json::Value::as_str).ok_or_else(|| std::io::Error::other("invalid type definition"))?;
if name.is_empty() || name.len() > 48 || !name.bytes().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == b'_')
|| base.is_empty() || base.len() > 64 || row.get("parameterized").and_then(serde_json::Value::as_bool).is_none() {
return Err(std::io::Error::other("invalid type definition"));
}
Ok(name)
}
/// 账号列表转成行
fn account_rows(accounts: &[Account]) -> std::io::Result<Vec<Row>> {
accounts.iter().map(|a| Ok(HashMap::from([
("id".into(), serde_json::json!(a.id)),
("user".into(), serde_json::Value::String(a.user.clone())),
("password_hash".into(), serde_json::Value::String(a.password_hash.clone())),
]))).collect()
}
/// 身份元数据转成行
fn identity_rows(accounts: &[Account]) -> std::io::Result<Vec<Row>> {
accounts.iter().map(|a| Ok(HashMap::from([
("id".into(), serde_json::json!(a.id)),
("user".into(), serde_json::Value::String(a.user.clone())),
("can_login".into(), serde_json::json!(a.can_login)),
("is_superuser".into(), serde_json::json!(a.is_superuser)),
("system_catalog_manager".into(), serde_json::json!(a.system_catalog_manager)),
("enabled".into(), serde_json::json!(a.enabled)),
("identity_version".into(), serde_json::json!(a.identity_version)),
("registered_at".into(), serde_json::Value::String(a.registered_at.clone())),
("last_login_at".into(), match &a.last_login_at {
Some(stamp) => serde_json::Value::String(stamp.clone()),
@@ -1478,10 +1749,23 @@ fn users_columns() -> Vec<SchemaColumn> {
vec![
SchemaColumn { name: "id".into(), ty: "int".into(), nullable: false,
auto_increment: true, primary_key: true, ..Default::default() },
SchemaColumn { name: "user".into(), ty: "varchar(64)".into(), nullable: false,
unique: true, ..Default::default() },
SchemaColumn { name: "password_hash".into(), ty: "varchar(256)".into(),
nullable: false, ..Default::default() },
]
}
/// 系统身份目录结构
fn identity_columns() -> Vec<SchemaColumn> {
vec![
SchemaColumn { name: "id".into(), ty: "int".into(), nullable: false,
primary_key: true, ..Default::default() },
SchemaColumn { name: "user".into(), ty: "varchar(64)".into(), nullable: false,
unique: true, ..Default::default() },
SchemaColumn { name: "can_login".into(), ty: "bool".into(), nullable: false, ..Default::default() },
SchemaColumn { name: "is_superuser".into(), ty: "bool".into(), nullable: false, ..Default::default() },
SchemaColumn { name: "system_catalog_manager".into(), ty: "bool".into(), nullable: false, ..Default::default() },
SchemaColumn { name: "enabled".into(), ty: "bool".into(), nullable: false, ..Default::default() },
SchemaColumn { name: "identity_version".into(), ty: "int".into(), nullable: false, ..Default::default() },
SchemaColumn { name: "registered_at".into(), ty: "timestamp".into(),
nullable: false, ..Default::default() },
SchemaColumn { name: "last_login_at".into(), ty: "timestamp".into(),
@@ -1497,7 +1781,6 @@ fn declare_system_schema(catalog: &mut Catalog, rows: &[Row]) -> std::io::Result
catalog.set_columns(USERS_TABLE, users_columns())?;
catalog.rebuild_stats(USERS_TABLE, rows);
catalog.mark_system(USERS_TABLE)?;
catalog.add_index(USERS_TABLE, "user")?;
Ok(())
}
@@ -1519,6 +1802,11 @@ fn decode_account_row(row: &Row) -> std::io::Result<Account> {
revision: row.get("revision").and_then(serde_json::Value::as_u64).unwrap_or(1),
registered_at: row.get("registered_at").and_then(serde_json::Value::as_str).unwrap_or("").to_string(),
last_login_at: row.get("last_login_at").and_then(serde_json::Value::as_str).map(str::to_string),
can_login: row.get("can_login").and_then(serde_json::Value::as_bool).unwrap_or(true),
is_superuser: row.get("is_superuser").and_then(serde_json::Value::as_bool).unwrap_or(false),
system_catalog_manager: row.get("system_catalog_manager").and_then(serde_json::Value::as_bool).unwrap_or(false),
enabled: row.get("enabled").and_then(serde_json::Value::as_bool).unwrap_or(true),
identity_version: row.get("identity_version").and_then(serde_json::Value::as_u64).unwrap_or(0) as u8,
})
}
@@ -1589,9 +1877,20 @@ fn new_account(id: i64, user: &str, password_hash: String) -> std::io::Result<Ac
revision: 1,
registered_at: now_stamp(),
last_login_at: None,
can_login: true,
is_superuser: false,
system_catalog_manager: false,
enabled: true,
identity_version: 1,
})
}
/// 保留至少一个启用的登录管理员
fn require_superuser(accounts: &[Account]) -> std::io::Result<()> {
if accounts.iter().any(|a| a.can_login && a.enabled && a.is_superuser) { Ok(()) }
else { Err(std::io::Error::other("the last enabled login superuser cannot be removed")) }
}
/// 一张表的索引定义转线上格式
fn index_wires(schema: &crate::catalog::TableSchema) -> Vec<IndexWire> {
let mut out = vec![IndexWire {
@@ -1632,6 +1931,40 @@ impl Drop for Server {
}
}
/// 按列精度规范化一行定点数
fn normalize_decimal_row(row: &mut Row, columns: &[SchemaColumn]) -> std::io::Result<()> {
for column in columns {
if let Some((precision, scale)) = decimal_parameters(&column.ty)?
&& let Some(value) = row.get_mut(&column.name).filter(|v| !v.is_null()) {
let number = value.as_f64().ok_or_else(|| std::io::Error::other("invalid decimal value"))?;
let factor = 10f64.powi(scale);
let scaled = number * factor;
if !scaled.is_finite() { return Err(std::io::Error::other("decimal value is not finite or exceeds precision")); }
let rounded = scaled.round_ties_even() / factor;
if rounded.abs() >= 10f64.powi(precision - scale) { return Err(std::io::Error::other("decimal value exceeds precision")); }
*value = serde_json::json!(rounded);
}
}
Ok(())
}
/// 解析基础或命名定点类型参数
fn decimal_parameters(ty: &str) -> std::io::Result<Option<(i32, i32)>> {
let mut base = ty.trim();
while let Some(body) = base.strip_prefix("domain(").and_then(|s| s.strip_suffix(')')) {
base = body.split_once(',').ok_or_else(|| std::io::Error::other("invalid domain type"))?.1.trim();
}
let normalized = crate::catalog::normalize_type(base);
if normalized == "decimal" { return Ok(Some((10, 0))); }
if !normalized.starts_with("decimal(") { return Ok(None); }
let error = || std::io::Error::other("invalid decimal precision or scale");
let body = normalized.strip_prefix("decimal(").and_then(|s| s.strip_suffix(')')).ok_or_else(error)?;
let values = body.split(',').map(|s| s.trim().parse::<i32>()).collect::<Result<Vec<_>, _>>().map_err(|_| error())?;
let (precision, scale) = match values.as_slice() { [p] => (*p, 0), [p, s] => (*p, *s), _ => return Err(error()) };
if !(1..=308).contains(&precision) || scale < 0 || scale > precision { return Err(error()); }
Ok(Some((precision, scale)))
}
/// 系统库名:账号表住在这里,其中的表都是系统表。服务启动时只建它一个,
/// 别的库一律要显式建(没有默认工作库)。
const SYSTEM_DATABASE: &str = "system";
@@ -1793,9 +2126,11 @@ impl Databases {
value["table"] = serde_json::Value::String(table.to_string());
}
let request: Request = serde_json::from_value(value).map_err(Error::other)?;
let global = matches!(request, Request::AccountsList | Request::AccountCreate { .. }
let global = matches!(request, Request::AccountsList | Request::CatalogManage { .. } | Request::AccountCreate { .. }
| Request::IdentityInitialize { .. } | Request::RoleCreate { .. } | Request::IdentityAlter { .. }
| Request::AccountReset { .. } | Request::AccountLogin { .. }
| Request::AccountDrop { .. } | Request::BootstrapSystem { .. }
| Request::BootstrapTypes { .. }
| Request::SystemStatus | Request::Ping);
if !global && selected.is_empty() { return Err(Error::other("no database selected")); }
let server = if global || selected == SYSTEM_DATABASE { Arc::clone(&self.system) }
@@ -1838,21 +2173,185 @@ fn log_config(loaded: &Loaded) {
}
}
/// 获取数据目录的跨进程独占锁
fn lock_data_directory(cfg: &Config) -> std::io::Result<std::fs::File> {
std::fs::create_dir_all(&cfg.data_dir)?;
let file = std::fs::OpenOptions::new().read(true).write(true).create(true).truncate(false)
.open(cfg.data_dir.join(".chusql.lock"))?;
file.try_lock().map_err(|e| std::io::Error::other(format!("data directory is in use; stop all database processes: {e}")))?;
Ok(file)
}
/// 复制系统目录并拒绝链接文件
fn copy_system_directory(source: &std::path::Path, target: &std::path::Path) -> std::io::Result<()> {
std::fs::create_dir_all(target)?;
for entry in std::fs::read_dir(source)? {
let entry = entry?;
let ty = entry.file_type()?;
if ty.is_symlink() { return Err(std::io::Error::other("system directory contains a symbolic link")); }
if ty.is_dir() { copy_system_directory(&entry.path(), &target.join(entry.file_name()))?; }
else if ty.is_file() { std::fs::copy(entry.path(), target.join(entry.file_name()))?; }
else { return Err(std::io::Error::other("unsupported system directory entry")); }
}
Ok(())
}
/// 构造可重建的系统表结构
fn maintenance_schemas() -> Vec<(&'static str, Vec<SchemaColumn>)> {
let text = |name: &str, size: usize| SchemaColumn { name: name.into(), ty: format!("varchar({size})"), ..Default::default() };
vec![(USERS_TABLE, users_columns()), (IDENTITIES_TABLE, identity_columns()), (TYPES_TABLE, types_columns()),
("__system_roles", vec![text("name", 64)]),
("__system_grants", vec![text("role", 64), text("privilege", 16), text("object", 128)]),
("__system_grant_options", vec![text("role", 64), text("privilege", 16), text("object", 128)]),
("__system_members", vec![text("role", 64), text("member", 64)])]
}
/// 验证隔离副本并执行所选恢复策略
fn maintenance_candidate(cfg: Config, mode: &str, user: &str, hash: &str, types: Vec<Row>) -> Result<(), String> {
if mode == "reset" {
let seed = Server::new(cfg.clone()).map_err(|e| e.to_string())?;
seed.bootstrap_system(Some(user), Some(hash)).map_err(|e| e.to_string())?;
seed.bootstrap_types(types.clone()).map_err(|e| e.to_string())?;
}
let repair = mode == "repair";
let catalog_path = cfg.data_dir.join("catalog.json");
let schemas = maintenance_schemas();
let mut missing_data = Vec::new();
let mut catalog = match Catalog::load(&catalog_path) {
Ok(catalog) => catalog,
Err(_) => {
for entry in std::fs::read_dir(&cfg.data_dir).map_err(|e| e.to_string())? {
let entry = entry.map_err(|e| e.to_string())?;
if entry.path().extension().and_then(|s| s.to_str()) == Some("db") {
let name = entry.path().file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default();
if !schemas.iter().any(|(known, _)| *known == name) { return Err(format!("cannot reconstruct unknown system table schema: {name}")); }
}
}
Catalog::default()
}
};
if catalog.all_tables().is_empty() {
for entry in std::fs::read_dir(&cfg.data_dir).map_err(|e| e.to_string())? {
let path = entry.map_err(|e| e.to_string())?.path();
if path.extension().and_then(|s| s.to_str()) == Some("db") {
let name = path.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default();
if !schemas.iter().any(|(known, _)| *known == name) { return Err(format!("cannot reconstruct unknown system table schema: {name}")); }
}
}
}
for (name, columns) in &schemas {
let file = cfg.data_dir.join(format!("{name}.db"));
if let Some(schema) = catalog.describe(name).filter(|s| !file.exists() && s.row_count > 0) {
missing_data.push((*name, schema.row_count));
}
if repair && !file.exists() && catalog.describe(name).is_none_or(|s| s.row_count != 0) {
return Err(format!("repair would require initializing or recovering data in {name}"));
}
if catalog.describe(name).is_none() { catalog.create_table(name, columns.clone()).map_err(|e| e.to_string())?; }
else { catalog.set_columns(name, columns.clone()).map_err(|e| e.to_string())?; }
catalog.mark_system(name).map_err(|e| e.to_string())?;
}
catalog.save(&catalog_path).map_err(|e| e.to_string())?;
let server = Server::new(cfg).map_err(|e| e.to_string())?;
if repair {
let scan = server.wal.read_all().map_err(|e| e.to_string())?;
let checkpoint = server.wal.read_checkpoint().map_err(|e| e.to_string())?.unwrap_or(0);
if !committed_ops(&records_after(&scan.records, checkpoint)).is_empty() { return Err("repair would require replaying WAL data".into()); }
} else {
server.recover_wal().map_err(|e| e.to_string())?;
}
for (name, _) in &schemas { server.with_table(name, |t| t.flush()).map_err(|e| e.to_string())?; }
if !repair {
if mode == "recover" && server.load_accounts().map_err(|e| e.to_string())?.is_empty() && hash.is_empty() {
return Err("recover requires --password-stdin to initialize a missing administrator".into());
}
server.bootstrap_system(Some(user), Some(hash)).map_err(|e| e.to_string())?;
server.accounts_request(Request::IdentityInitialize { administrator: user.into() }).map_err(|e| e.to_string())?;
server.bootstrap_types(types.clone()).map_err(|e| e.to_string())?;
}
let accounts = server.load_accounts().map_err(|e| e.to_string())?;
require_superuser(&accounts).map_err(|e| e.to_string())?;
for (name, expected) in missing_data {
let rows = server.with_existing_table(name, |t| t.scan()).map_err(|e| e.to_string())?;
if (rows.len() as u64) < expected { return Err(format!("WAL cannot reconstruct all missing data in {name}; restore a backup or use reset")); }
}
if repair {
let saved = server.with_existing_table(TYPES_TABLE, |t| t.scan()).map_err(|e| e.to_string())?;
if types.iter().any(|wanted| !saved.iter().any(|row| wanted.iter().all(|(key, value)| row.get(key) == Some(value)))) {
return Err("repair would require changing preinstalled type data".into());
}
}
for (name, _) in &schemas { server.with_existing_table(name, |t| t.scan()).map_err(|e| e.to_string())?; }
{
let mut catalog = server.catalog.lock().map_err(|_| "catalog lock poisoned")?;
catalog.identity_high_water = catalog.identity_high_water.max(accounts.iter().map(|a| a.id).max().unwrap_or(0));
catalog.save(server.catalog_path()).map_err(|e| e.to_string())?;
}
server.rebuild_stats().map_err(|e| e.to_string())?;
server.flush_all_tables().map_err(|e| e.to_string())?;
Ok(())
}
/// 进程内存储句柄:一个数据目录一份状态,跨线程共享(内部各自加锁)。
pub struct Storage {
databases: Databases,
_directory_lock: std::fs::File,
}
impl Storage {
/// 离线修复或重建系统目录
pub fn maintenance(config_path: Option<&str>, request: &str) -> Result<String, String> {
let loaded = config::load(config_path)?;
let cfg = loaded.config;
let _lock = lock_data_directory(&cfg).map_err(|e| e.to_string())?;
let input: serde_json::Value = serde_json::from_str(request).map_err(|e| e.to_string())?;
let mode = input.get("mode").and_then(|v| v.as_str()).ok_or("missing maintenance mode")?;
if !["repair", "recover", "reset"].contains(&mode) { return Err("invalid maintenance mode".into()); }
let user = input.get("user").and_then(|v| v.as_str()).unwrap_or("root");
let hash = input.get("password_hash").and_then(|v| v.as_str()).unwrap_or("");
let types: Vec<Row> = serde_json::from_value(input.get("types").cloned().ok_or("missing type manifest")?).map_err(|e| e.to_string())?;
if types.is_empty() { return Err("type manifest must not be empty".into()); }
for row in &types { type_definition_name(row).map_err(|e| e.to_string())?; }
let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_err(|e| e.to_string())?.as_nanos();
let system = cfg.data_dir.join("system");
if system.is_symlink() { return Err("system directory cannot be a symbolic link".into()); }
let work = cfg.data_dir.join(format!("bootstrap-work-{stamp}"));
let backup = cfg.data_dir.join(format!("system-backup-{stamp}"));
let staged = work.join("system");
std::fs::create_dir_all(&staged).map_err(|e| e.to_string())?;
if mode != "reset" && system.exists() { copy_system_directory(&system, &staged).map_err(|e| e.to_string())?; }
let mut candidate_cfg = cfg.clone();
candidate_cfg.data_dir = staged.clone();
let result = maintenance_candidate(candidate_cfg, mode, user, hash, types);
if let Err(err) = result { return Err(format!("{err}; original system directory was not changed; candidate: {}. Try csql-bootstrap recover or reset", work.display())); }
if mode == "repair" && system.exists() {
for entry in std::fs::read_dir(&system).map_err(|e| e.to_string())? {
let entry = entry.map_err(|e| e.to_string())?;
if entry.path().extension().and_then(|s| s.to_str()) == Some("db")
&& std::fs::read(entry.path()).map_err(|e| e.to_string())? != std::fs::read(staged.join(entry.file_name())).map_err(|e| e.to_string())? {
return Err("repair would modify table data; original system directory was not changed. Use recover or reset".into());
}
}
}
if system.exists() { std::fs::rename(&system, &backup).map_err(|e| e.to_string())?; }
if let Err(err) = std::fs::rename(&staged, &system) {
if backup.exists() { std::fs::rename(&backup, &system).map_err(|rollback| format!("publish failed: {err}; rollback failed: {rollback}; backup: {}", backup.display()))?; }
return Err(format!("cannot publish repaired system directory: {err}"));
}
serde_json::to_string(&serde_json::json!({"status":"maintenance", "mode":mode, "backup":backup, "candidate":work,
"message":"System maintenance completed. Business databases were preserved."})).map_err(|e| e.to_string())
}
/// 打开数据目录并初始化存储
pub fn open(config_path: Option<&str>) -> Result<Storage, String> {
let loaded = config::load(config_path)?;
let directory_lock = lock_data_directory(&loaded.config).map_err(|e| e.to_string())?;
log::init();
log::set_level(loaded.config.log_level);
log_info!(core, "chusql-core-storage {} starting", env!("CARGO_PKG_VERSION"));
log_config(&loaded);
let databases = Databases::new(loaded.config).map_err(|e| e.to_string())?;
Ok(Storage { databases })
Ok(Storage { databases, _directory_lock: directory_lock })
}
/// 一行请求 JSON 换一行响应 JSON
+23 -13
View File
@@ -31,7 +31,7 @@ const OP_DROP_INDEX: u8 = 14;
pub enum WalOp {
HideColumn { table: String, column: String },
Compact { table: String, rows: Vec<Row> },
Accounts { accounts: Vec<Account> },
Accounts { accounts: Vec<Account>, removed: Vec<String>, migrate_roles: bool },
Insert {
table: String,
row: Row,
@@ -283,7 +283,7 @@ impl Wal {
frames.push((lsn, op, offset, end));
offset = end;
}
Err(_) => break,
Err(error) => return Err(io::Error::new(io::ErrorKind::InvalidData, error)),
}
}
Ok((frames, offset))
@@ -291,9 +291,13 @@ impl Wal {
/// 记录检查点:到此为止的 LSN 都已落盘
pub fn write_checkpoint(&self, lsn: u64) -> io::Result<()> {
let mut f = File::create(self.checkpoint_path())?;
let checkpoint = self.checkpoint_path();
let pending = checkpoint.with_extension("checkpoint.pending");
let mut f = File::create(&pending)?;
f.write_all(&lsn.to_le_bytes())?;
f.sync_all()
f.sync_all()?;
drop(f);
std::fs::rename(pending, checkpoint)
}
/// 读检查点 LSN
@@ -303,8 +307,8 @@ impl Wal {
return Ok(None);
}
let bytes = std::fs::read(&path)?;
if bytes.len() < 8 {
return Ok(None);
if bytes.len() != 8 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid WAL checkpoint length"));
}
let mut raw = [0u8; 8];
raw.copy_from_slice(&bytes[..8]);
@@ -322,10 +326,10 @@ impl Wal {
if let Some((lsn, _, _, _)) = frames.last() {
base = base.max(*lsn);
}
*slot = base + 1;
*slot = base.checked_add(1).ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
}
let lsn = *slot;
*slot += 1;
*slot = slot.checked_add(1).ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
Ok(lsn)
}
@@ -378,8 +382,8 @@ fn encode(op: &WalOp, lsn: u64) -> io::Result<Vec<u8>> {
let p = serde_json::to_vec(rows).map_err(io::Error::other)?;
(OP_COMPACT, table.clone(), p)
}
WalOp::Accounts { accounts } => (OP_ACCOUNTS, USERS_TABLE.to_string(),
serde_json::to_vec(accounts).map_err(io::Error::other)?),
WalOp::Accounts { accounts, removed, migrate_roles } => (OP_ACCOUNTS, USERS_TABLE.to_string(),
serde_json::to_vec(&serde_json::json!({"accounts":accounts,"removed":removed,"migrate_roles":migrate_roles})).map_err(io::Error::other)?),
WalOp::Insert { table, row } => {
let p = serde_json::to_vec(row).map_err(io::Error::other)?;
(OP_INSERT, table.clone(), p)
@@ -478,9 +482,15 @@ fn decode_body(body: &[u8]) -> Result<(u64, WalOp), String> {
}
OP_ACCOUNTS => {
if table != USERS_TABLE { return Err("invalid account WAL table".into()); }
let accounts = serde_json::from_slice(payload)
.map_err(|e| format!("bad account payload: {e}"))?;
WalOp::Accounts { accounts }
let value: serde_json::Value = serde_json::from_slice(payload).map_err(|e| format!("bad account payload: {e}"))?;
let (accounts, removed, migrate_roles) = if value.is_array() {
(serde_json::from_value(value).map_err(|e| format!("bad account payload: {e}"))?, Vec::new(), false)
} else {
(serde_json::from_value(value.get("accounts").cloned().ok_or("missing account payload")?).map_err(|e| format!("bad account payload: {e}"))?,
serde_json::from_value(value.get("removed").cloned().ok_or("missing identity cleanup")?).map_err(|e| format!("bad account cleanup: {e}"))?,
value.get("migrate_roles").and_then(serde_json::Value::as_bool).ok_or("missing role migration flag")?)
};
WalOp::Accounts { accounts, removed, migrate_roles }
}
OP_INSERT => {
let row: Row =
+30
View File
@@ -124,6 +124,36 @@ fn stats_track_a_histogram_for_numeric_columns() {
assert_eq!(id.hist.iter().sum::<u64>(), 0);
}
/// 重复值删除保留仍存在的不同值计数
#[test]
fn duplicate_deletion_keeps_distinct_until_last_occurrence() {
let mut catalog = Catalog::default();
let duplicate = row(&[("id", json!(1)), ("v", json!(7))]);
catalog.rebuild_stats("t", &[duplicate.clone(), duplicate.clone()]);
catalog.record_delete("t", std::slice::from_ref(&duplicate));
assert_eq!(catalog.describe("t").unwrap().stats["v"].distinct, 1);
catalog.record_delete("t", &[duplicate]);
assert_eq!(catalog.describe("t").unwrap().stats["v"].distinct, 0);
}
/// 去重封顶不影响直方图和其他列
#[test]
fn capped_distinct_keeps_histogram_and_other_columns_accurate() {
let mut catalog = Catalog::default();
let rows: Vec<Row> = (0..5000).map(|n| row(&[("id", json!(n)), ("v", json!(n % 2))])).collect();
catalog.rebuild_stats("t", &rows);
let schema = catalog.describe("t").unwrap();
assert!(schema.stats["id"].capped);
assert_eq!(schema.stats["id"].hist.iter().sum::<u64>(), 5000);
assert_eq!(schema.stats["v"].distinct, 2);
catalog.record_delete("t", &rows[..1000]);
let schema = catalog.describe("t").unwrap();
assert_eq!(schema.stats["id"].hist.iter().sum::<u64>(), 4000);
assert_eq!(schema.stats["v"].distinct, 2);
catalog.rebuild_stats("t", &rows[1000..]);
assert_eq!(catalog.describe("t").unwrap().stats["id"].distinct, 4000);
}
/// 老类型名并到新写法,参数保留,未知类型原样
#[test]
fn normalize_type_maps_legacy_names() {
+21 -8
View File
@@ -1,7 +1,9 @@
use std::path::PathBuf;
use std::sync::Mutex;
use chusql_core_storage::config::{self, Config, DEFAULT_BTREE_ORDER, DEFAULT_PAGE_SIZE, Origin};
use chusql_core_storage::config::{
self, Config, DEFAULT_BTREE_ORDER, DEFAULT_LOG_FILES, DEFAULT_PAGE_SIZE, Origin,
};
use chusql_core_storage::log::Level;
// 配置测试:默认值、文件覆盖、环境变量彻底无效、非法值报错。
@@ -25,6 +27,7 @@ fn defaults_without_file() {
assert_eq!(loaded.config.page_size, DEFAULT_PAGE_SIZE);
assert_eq!(loaded.config.btree_order, DEFAULT_BTREE_ORDER);
assert_eq!(loaded.config.data_dir, config::default_data_dir());
assert_eq!(loaded.config.log_files, PathBuf::from(DEFAULT_LOG_FILES));
assert_eq!(loaded.config.log_level, Level::Info);
}
@@ -39,6 +42,7 @@ fn file_overrides_only_written_keys() {
assert_eq!(loaded.config.page_size, 8192);
assert_eq!(loaded.config.btree_order, DEFAULT_BTREE_ORDER);
assert_eq!(loaded.config.data_dir, config::default_data_dir());
assert_eq!(loaded.config.log_files, PathBuf::from(DEFAULT_LOG_FILES));
assert_eq!(loaded.config_path, Some(path.clone()));
let page = loaded.origins.iter().find(|(n, _, _)| *n == "page.size").unwrap();
@@ -47,11 +51,11 @@ fn file_overrides_only_written_keys() {
assert_eq!(order.2, Origin::Default);
}
/// 五项全写都生效
/// 全部配置项及其来源生效
#[test]
fn file_all_keys() {
let _guard = env_guard();
let text = "[page]\nsize = 8192\n[btree]\norder = 32\n[buffer]\npool_size = 128\n[storage]\ndata_dir = \"mydata\"\n[log]\nlevel = \"debug\"\n";
let text = "[page]\nsize = 8192\n[btree]\norder = 32\n[buffer]\npool_size = 128\n[storage]\ndata_dir = \"mydata\"\nlog_files = \"mylogs\"\n[log]\nlevel = \"debug\"\n";
let path = PathBuf::from("t.toml");
let loaded = config::resolve(Some(text), Some(path.clone())).unwrap();
@@ -59,6 +63,7 @@ fn file_all_keys() {
assert_eq!(loaded.config.btree_order, 32);
assert_eq!(loaded.config.pool_size, 128);
assert_eq!(loaded.config.data_dir, PathBuf::from("mydata"));
assert_eq!(loaded.config.log_files, PathBuf::from("mylogs"));
assert_eq!(loaded.config.log_level, Level::Debug);
assert!(
loaded
@@ -103,16 +108,16 @@ fn env_vars_are_ignored_entirely() {
fn default_config_path_follows_system_convention() {
let _guard = env_guard();
let path = config::default_config_path();
assert_eq!(path.file_name().unwrap(), "chusql.toml");
assert_eq!(path.file_name().unwrap(), "settings.toml");
assert_eq!(path.parent().unwrap().file_name().unwrap(), "ChuSQL");
assert!(path.ends_with(format!("ChuSQL{}chusql.toml", std::path::MAIN_SEPARATOR)), "got {}", path.display());
assert!(path.ends_with(format!("ChuSQL{}settings.toml", std::path::MAIN_SEPARATOR)), "got {}", path.display());
// 本机 APPDATA 存在时必须是 %APPDATA%\ChuSQL\chusql.toml
// 本机 APPDATA 存在时必须是 %APPDATA%\ChuSQL\settings.toml
#[cfg(windows)]
if let Some(appdata) = std::env::var_os("APPDATA") {
assert_eq!(
path,
PathBuf::from(appdata).join("ChuSQL").join("chusql.toml")
PathBuf::from(appdata).join("ChuSQL").join("settings.toml")
);
}
}
@@ -186,7 +191,7 @@ fn load_without_explicit_path_uses_builtin_defaults() {
#[test]
fn server_section_belongs_to_another_layer() {
let _guard = env_guard();
let db_server = "[server]\nhost = \"0.0.0.0\"\nport = 7778\nmax_rows = 10\n";
let db_server = "[server]\nlisten_host = \"0.0.0.0\"\nport = 7778\nmax_rows = 10\n";
let loaded = config::resolve(Some(db_server), None).unwrap();
assert_eq!(loaded.config.data_dir, config::default_data_dir());
@@ -226,6 +231,14 @@ fn rejects_unknown_key() {
assert!(err.contains("bad config"), "got {}", err);
}
/// 空的日志目录要报错(否则日志会落到当前目录)
#[test]
fn rejects_empty_log_files() {
let _guard = env_guard();
let err = config::resolve(Some("[storage]\nlog_files = \" \"\n"), None).unwrap_err();
assert!(err.contains("storage.log_files"), "got {}", err);
}
/// 坏的页面尺寸只能来自文件
#[test]
fn rejects_bad_page_size_from_file() {
+434 -5
View File
@@ -10,6 +10,433 @@ thread_local! {
static CURRENT: RefCell<Option<Arc<Storage>>> = const { RefCell::new(None) };
}
/// 定点写入检查舍入和整批原子拒绝
#[test]
fn decimal_writes_enforce_precision_before_wal() -> Result<(), Box<dyn std::error::Error>> {
let (_server, data) = start_server();
let mut c = connect()?;
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"amounts","columns":[{"name":"id","ty":"int"},{"name":"value","ty":"decimal(4,2)"}]}))?;
request_ok(&mut c, serde_json::json!({"method":"insert","table":"amounts","row":{"id":1,"value":99.994}}))?;
let snapshot = send(&mut c, r#"{"method":"scan","table":"amounts"}"#);
let rows: serde_json::Value = serde_json::from_str(&snapshot)?;
assert_eq!(rows["rows"][0]["value"], serde_json::json!(99.99));
let checkpoint = std::fs::read(data.path().join("databases/main/wal.checkpoint"))?;
for value in [99.995, -99.995, 100.0] {
let response = send(&mut c, &serde_json::json!({"method":"insert_batch","table":"amounts","rows":[{"id":2,"value":1.2},{"id":3,"value":value}]}).to_string());
assert!(response.contains("exceeds precision"), "{response}");
let remaining: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"amounts"}"#))?;
assert_eq!(remaining, rows);
assert_eq!(std::fs::read(data.path().join("databases/main/wal.checkpoint"))?, checkpoint);
}
let invalid = send(&mut c, &serde_json::json!({"method":"create_table","table":"invalid","columns":[{"name":"value","ty":"decimal(2,3)"}]}).to_string());
assert!(invalid.contains("invalid decimal"), "{invalid}");
Ok(())
}
/// 检查点失败封闭请求并在重启恢复
#[test]
fn checkpoint_failure_requires_recovery_and_preserves_committed_rows() -> Result<(), Box<dyn std::error::Error>> {
let (server, data) = start_server();
let mut c = connect()?;
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"checkpointed","columns":[{"name":"id","ty":"int"}]}))?;
let pending = data.path().join("databases/main/wal.checkpoint.pending");
std::fs::create_dir(&pending)?;
let failed = send(&mut c, r#"{"method":"insert","table":"checkpointed","row":{"id":1}}"#);
assert!(failed.contains("error"), "{failed}");
assert!(send(&mut c, r#"{"method":"scan","table":"checkpointed"}"#).contains("recovery required"));
assert!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len() > 0);
drop(c);
drop(server);
std::fs::remove_dir(pending)?;
let _restarted = restart_at(data.path())?;
let mut c = connect()?;
let rows: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"checkpointed"}"#))?;
assert_eq!(rows["rows"], serde_json::json!([{"id":1}]));
Ok(())
}
/// 应用失败封闭读写并重放完整提交组
#[test]
fn committed_apply_failures_require_recovery() -> Result<(), Box<dyn std::error::Error>> {
for operation in [
serde_json::json!({"method":"create_table","table":"created","columns":[{"name":"id","ty":"int"}]}),
serde_json::json!({"method":"replace_all","table":"items","rows":[{"id":2}]}),
serde_json::json!({"method":"apply_transaction","ops":[{"op":"delete","table":"items","ids":[1]},{"op":"replace","table":"items","rows":[{"id":2}]}]}),
] {
let (server, data) = start_server();
let mut c = connect()?;
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"items","columns":[{"name":"id","ty":"int"}]}))?;
request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":1}}))?;
let directory = data.path().join("databases/main");
let checkpoint = std::fs::read(directory.join("wal.checkpoint"))?;
let pending = directory.join("catalog.json.pending");
std::fs::create_dir(&pending)?;
let failed = send(&mut c, &operation.to_string());
assert!(failed.contains("error"), "{operation}: {failed}");
for request in [
r#"{"method":"scan","table":"items"}"#,
r#"{"method":"insert","table":"items","row":{"id":3}}"#,
] {
let blocked = send(&mut c, request);
assert!(blocked.contains("recovery required"), "{blocked}");
}
assert_eq!(std::fs::read(directory.join("wal.checkpoint"))?, checkpoint);
assert!(std::fs::metadata(directory.join("wal.log"))?.len() > 0);
drop(c);
drop(server);
std::fs::remove_dir(pending)?;
let restarted = restart_at(data.path())?;
let mut c = connect()?;
let table = if operation["method"] == "create_table" { "created" } else { "items" };
let result: serde_json::Value = serde_json::from_str(&send(&mut c, &serde_json::json!({"method":"scan","table":table}).to_string()))?;
let expected = if table == "created" { serde_json::json!([]) } else { serde_json::json!([{"id":2}]) };
assert_eq!(result["rows"], expected, "{operation}");
assert_eq!(std::fs::metadata(directory.join("wal.log"))?.len(), 0);
drop(c);
drop(restarted);
}
Ok(())
}
/// 首帧撕裂恢复后新日志仍可重放
#[test]
fn torn_first_frame_is_removed_before_new_writes() -> Result<(), Box<dyn std::error::Error>> {
let (server, data) = start_server();
let mut c = connect()?;
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"torn","columns":[{"name":"id","ty":"int"}]}))?;
drop(c);
drop(server);
std::fs::write(data.path().join("databases/main/wal.log"), [20, 0, 0, 0, 1])?;
let restarted = restart_at(data.path())?;
assert_eq!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len(), 0);
let mut c = connect()?;
request_ok(&mut c, serde_json::json!({"method":"insert","table":"torn","row":{"id":1}}))?;
drop(c);
drop(restarted);
let _again = restart_at(data.path())?;
let mut c = connect()?;
let rows: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"torn"}"#))?;
assert_eq!(rows["rows"], serde_json::json!([{"id":1}]));
Ok(())
}
/// 校验预装类型幂等与冲突保护
#[test]
fn preinstalled_types_are_idempotent_hidden_and_conflict_safe() -> Result<(), Box<dyn std::error::Error>> {
let (_server, data) = start_server();
let mut c = connect()?;
let premature = send(&mut c, &type_seed_request().to_string());
assert!(premature.contains("login superuser"), "{premature}");
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"original-hash"}"#);
let accounts = send(&mut c, r#"{"method":"accounts_list"}"#);
let request = type_seed_request();
let seeded: serde_json::Value = serde_json::from_str(&send(&mut c, &request.to_string()))?;
assert_eq!(seeded["status"], "rows", "{seeded}");
assert_eq!(seeded["rows"].as_array().map(Vec::len), Some(2));
assert_eq!(serde_json::from_str::<serde_json::Value>(&send(&mut c, &request.to_string()))?, seeded);
let mut conflict = request;
conflict["types"][1]["base_type"] = "int".into();
let rejected = send(&mut c, &conflict.to_string());
assert!(rejected.contains("definition conflict: text"), "{rejected}");
assert_eq!(serde_json::from_str::<serde_json::Value>(&send(&mut c, &type_seed_request().to_string()))?, seeded);
assert_eq!(send(&mut c, r#"{"method":"accounts_list"}"#), accounts);
let catalog: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(data.path().join("system/catalog.json"))?)?;
assert_eq!(catalog["tables"]["__system_types"]["system"], true);
assert_eq!(catalog["tables"]["__system_types"]["row_count"], 2);
let denied = send(&mut c, r#"{"method":"scan","database":"system","table":"__system_types"}"#);
assert!(denied.contains("reserved system table"), "{denied}");
Ok(())
}
/// 类型预装失败保留日志并在重启后恢复
#[test]
fn preinstalled_types_recover_after_catalog_write_failure() -> Result<(), Box<dyn std::error::Error>> {
let (server, data) = start_server();
let mut c = connect()?;
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
std::fs::create_dir(data.path().join("system/catalog.json.pending"))?;
let failed = send(&mut c, &type_seed_request().to_string());
assert!(failed.contains("error"), "{failed}");
assert!(std::fs::metadata(data.path().join("system/wal.log"))?.len() > 0);
assert!(send(&mut c, &type_seed_request().to_string()).contains("recovery required"));
drop(c);
drop(server);
std::fs::remove_dir(data.path().join("system/catalog.json.pending"))?;
let _restarted = restart_at(data.path())?;
let mut c = connect()?;
let restored: serde_json::Value = serde_json::from_str(&send(&mut c, &type_seed_request().to_string()))?;
assert_eq!(restored["rows"].as_array().map(Vec::len), Some(2), "{restored}");
Ok(())
}
/// 类型目录不覆盖未登记的现有文件
#[test]
fn preinstalled_types_reject_reserved_file_collision() -> Result<(), Box<dyn std::error::Error>> {
let (_server, data) = start_server();
let mut c = connect()?;
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
let path = data.path().join("system/__system_types.db");
std::fs::write(&path, b"preserved")?;
let rejected = send(&mut c, &type_seed_request().to_string());
assert!(rejected.contains("reserved type file collision"), "{rejected}");
assert_eq!(std::fs::read(path)?, b"preserved");
Ok(())
}
/// 生成两项预装类型定义
fn type_seed_request() -> serde_json::Value {
serde_json::json!({"method":"bootstrap_types","types":[
{"name":"int","base_type":"int","parameterized":false},
{"name":"text","base_type":"str","parameterized":false}
]})
}
/// 引导保留已用身份编号
#[test]
fn bootstrap_respects_previous_identity_ids() -> Result<(), Box<dyn std::error::Error>> {
let (_server, _data) = start_server();
let mut c = connect()?;
send(&mut c, r#"{"method":"account_create","user":"alice","password_hash":"hash"}"#);
let deleted: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"account_drop","user":"alice"}"#))?;
assert_eq!(deleted["accounts"], serde_json::json!([]));
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
let seeded: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"accounts_list"}"#))?;
assert_eq!(seeded["accounts"][0]["id"], 2);
Ok(())
}
/// 旧账号日志迁移只提升初始管理员
#[test]
fn legacy_account_wal_migrates_without_promoting_other_names() -> Result<(), Box<dyn std::error::Error>> {
use chusql_core_storage::wal::Wal;
let (server, data) = start_server();
let mut c = connect()?;
request_ok(&mut c, serde_json::json!({"method":"create_table","database":"system","table":"__system_roles","columns":[{"name":"name","ty":"str"}]}))?;
request_ok(&mut c, serde_json::json!({"method":"insert","database":"system","table":"__system_roles","row":{"name":"reader"}}))?;
drop(c);
drop(server);
let path = data.path().join("system/wal.log");
let lsn = Wal::new(&path).read_checkpoint()?.unwrap_or(0) + 1;
let legacy = serde_json::json!([
{"id":7,"user":"owner","password_hash":"owner-hash","revision":3,"registered_at":"2020-01-01 00:00:00","last_login_at":null},
{"id":9,"user":"alice","password_hash":"alice-hash","revision":2,"registered_at":"2020-01-02 00:00:00","last_login_at":null}
]);
let table = b"__system_users";
let mut body = lsn.to_le_bytes().to_vec();
body.push(6);
body.extend_from_slice(&(table.len() as u16).to_le_bytes());
body.extend_from_slice(table);
body.extend_from_slice(&serde_json::to_vec(&legacy)?);
let mut frame = (body.len() as u32).to_le_bytes().to_vec();
frame.extend_from_slice(&body);
std::fs::write(&path, frame)?;
let _restarted = restart_at(data.path())?;
let mut c = connect()?;
let wrong = send(&mut c, r#"{"method":"identity_initialize","administrator":"missing"}"#);
assert!(wrong.contains("last enabled login superuser"), "{wrong}");
let migrated: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"identity_initialize","administrator":"owner"}"#))?;
assert_eq!(migrated["status"], "accounts", "{migrated}");
assert_eq!(migrated["accounts"][0]["id"], 7);
assert_eq!(migrated["accounts"][0]["is_superuser"], true);
assert_eq!(migrated["accounts"][0]["password_hash"], "owner-hash");
assert_eq!(migrated["accounts"][1]["is_superuser"], false);
assert_eq!(migrated["accounts"][2]["user"], "reader");
assert_eq!(migrated["accounts"][2]["id"], 10);
assert_eq!(serde_json::from_str::<serde_json::Value>(&send(&mut c, r#"{"method":"identity_initialize","administrator":"alice"}"#))?, migrated);
Ok(())
}
/// 身份属性重启保留且禁止删除最后管理员
#[test]
fn identity_attributes_survive_restart_and_protect_superuser() -> Result<(), Box<dyn std::error::Error>> {
let (server, data) = start_server();
let mut c = connect()?;
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
let created: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"role_create","user":"Reader"}"#))?;
assert_eq!(created["status"], "accounts", "{created}");
assert_eq!(created["accounts"][1]["can_login"], false);
assert_eq!(created["accounts"][1]["enabled"], true);
for change in [serde_json::json!({"enabled":false}), serde_json::json!({"can_login":false}), serde_json::json!({"is_superuser":false})] {
let mut request = change;
request["method"] = "identity_alter".into();
request["user"] = "root".into();
let denied = send(&mut c, &request.to_string());
assert!(denied.contains("last enabled login superuser"), "{denied}");
}
let collision = send(&mut c, r#"{"method":"account_create","user":"READER","password_hash":"hash"}"#);
assert!(collision.contains("already exists"), "{collision}");
let changed: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"identity_alter","user":"reader","can_login":true,"is_superuser":true}"#))?;
assert_eq!(changed["accounts"][1]["revision"], 2);
drop(c);
drop(server);
let _restarted = restart_at(data.path())?;
let mut c = connect()?;
let restored: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"accounts_list"}"#))?;
assert_eq!(restored, changed);
let denied = send(&mut c, r#"{"method":"scan","database":"system","table":"__system_identities"}"#);
assert!(denied.contains("reserved system table"), "{denied}");
Ok(())
}
/// 旧角色迁移与重名拒绝不破坏原数据
#[test]
fn identity_migration_preserves_roles_and_rejects_collisions() -> Result<(), Box<dyn std::error::Error>> {
let (_server, _data) = start_server();
let mut c = connect()?;
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
request_ok(&mut c, serde_json::json!({"method":"create_table","database":"system","table":"__system_roles","columns":[{"name":"name","ty":"str"}]}))?;
request_ok(&mut c, serde_json::json!({"method":"insert","database":"system","table":"__system_roles","row":{"name":"root"}}))?;
let before = send(&mut c, r#"{"method":"accounts_list"}"#);
let rejected = send(&mut c, r#"{"method":"identity_initialize","administrator":"root"}"#);
assert!(rejected.contains("identity name collision: root"), "{rejected}");
assert_eq!(send(&mut c, r#"{"method":"accounts_list"}"#), before);
request_ok(&mut c, serde_json::json!({"method":"replace_all","database":"system","table":"__system_roles","rows":[{"name":"reader"}]}))?;
let migrated: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"identity_initialize","administrator":"root"}"#))?;
assert_eq!(migrated["accounts"][1]["user"], "reader");
assert_eq!(migrated["accounts"][1]["can_login"], false);
assert_eq!(serde_json::from_str::<serde_json::Value>(&send(&mut c, r#"{"method":"identity_initialize","administrator":"reader"}"#))?, migrated);
Ok(())
}
/// 删除身份清理授权并在恢复后保留编号上界
#[test]
fn identity_drop_cleanup_replays_without_reusing_ids() -> Result<(), Box<dyn std::error::Error>> {
let (server, data) = start_server();
let mut c = connect()?;
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
send(&mut c, r#"{"method":"role_create","user":"reader"}"#);
for table in ["__system_grants", "__system_grant_options", "__system_members"] {
request_ok(&mut c, serde_json::json!({"method":"create_table","database":"system","table":table,"columns":[{"name":"role","ty":"str"},{"name":"member","ty":"str"}]}))?;
request_ok(&mut c, serde_json::json!({"method":"insert","database":"system","table":table,"row":{"role":"reader","member":"root"}}))?;
}
std::fs::create_dir(data.path().join("system/catalog.json.pending"))?;
let failed = send(&mut c, r#"{"method":"account_drop","user":"reader"}"#);
assert!(failed.contains("error"), "{failed}");
drop(c);
drop(server);
std::fs::remove_dir(data.path().join("system/catalog.json.pending"))?;
let _restarted = restart_at(data.path())?;
let mut c = connect()?;
for table in ["__system_grants", "__system_grant_options", "__system_members"] {
let rows: serde_json::Value = serde_json::from_str(&send(&mut c, &serde_json::json!({"method":"scan","database":"system","table":table}).to_string()))?;
assert_eq!(rows["rows"], serde_json::json!([]));
}
let recreated: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"role_create","user":"reader"}"#))?;
assert_eq!(recreated["accounts"][1]["id"], 3);
Ok(())
}
/// 命名类型引用与删除依赖在写入时校验
#[test]
fn domain_dependencies_are_checked_before_wal() -> Result<(), Box<dyn std::error::Error>> {
let (_srv, _data) = start_server();
let mut connection = connect()?;
let columns = serde_json::json!([{"name":"code","ty":"domain(code,int)"}]);
let missing = serde_json::json!({"method":"create_table","table":"typed","columns":columns});
let response: serde_json::Value = serde_json::from_str(&send(&mut connection, &missing.to_string()))?;
assert_eq!(response["status"], "error");
request_ok(&mut connection, serde_json::json!({"method":"create_table","table":"__system_domain_code","columns":[{"name":"base","ty":"int"}]}))?;
let forged = serde_json::json!({"method":"create_table","table":"forged","columns":[{"name":"code","ty":"domain(code,str)"}]});
let response: serde_json::Value = serde_json::from_str(&send(&mut connection, &forged.to_string()))?;
assert_eq!(response["status"], "error");
request_ok(&mut connection, missing)?;
let response: serde_json::Value = serde_json::from_str(&send(&mut connection, r#"{"method":"drop_table","table":"__system_domain_code"}"#))?;
assert_eq!(response["status"], "error");
request_ok(&mut connection, serde_json::json!({"method":"drop_table","table":"typed"}))?;
request_ok(&mut connection, serde_json::json!({"method":"drop_table","table":"__system_domain_code"}))?;
let response: serde_json::Value = serde_json::from_str(&send(&mut connection, r#"{"method":"create_table","table":"stale","columns":[{"name":"code","ty":"domain(code,int)"}]}"#))?;
assert_eq!(response["status"], "error");
Ok(())
}
/// 嵌套类型依赖与重启持久化
#[test]
fn nested_domains_preserve_dependencies_across_restart() -> Result<(), Box<dyn std::error::Error>> {
let (_srv, data) = start_server();
let mut connection = connect()?;
request_ok(&mut connection, serde_json::json!({"method":"create_table","table":"__system_domain_root","columns":[{"name":"base","ty":"int"}]}))?;
request_ok(&mut connection, serde_json::json!({"method":"create_table","table":"__system_domain_child","columns":[{"name":"base","ty":"domain(root,int)"}]}))?;
let rejected: serde_json::Value = serde_json::from_str(&send(&mut connection, r#"{"method":"drop_table","table":"__system_domain_root"}"#))?;
assert_eq!(rejected["status"], "error");
request_ok(&mut connection, serde_json::json!({"method":"create_table","table":"typed","columns":[{"name":"id","ty":"domain(child,domain(root,int))"}]}))?;
let forged: serde_json::Value = serde_json::from_str(&send(&mut connection, r#"{"method":"create_table","table":"forged","columns":[{"name":"id","ty":"domain(child,domain(root,str))"}]}"#))?;
assert_eq!(forged["status"], "error");
drop(connection);
CURRENT.with(|c| *c.borrow_mut() = None);
drop(_srv);
let _reopened = start_server_in(data.path());
let mut connection = connect()?;
let schema: serde_json::Value = serde_json::from_str(&send(&mut connection, r#"{"method":"describe_table","table":"typed"}"#))?;
assert_eq!(schema["status"], "schema");
assert_eq!(schema["columns"][0]["ty"], "domain(child,domain(root,int))");
request_ok(&mut connection, serde_json::json!({"method":"drop_table","table":"typed"}))?;
request_ok(&mut connection, serde_json::json!({"method":"drop_table","table":"__system_domain_child"}))?;
request_ok(&mut connection, serde_json::json!({"method":"drop_table","table":"__system_domain_root"}))?;
Ok(())
}
/// 重置与结构修复保留业务文件及系统数据
#[test]
fn offline_maintenance_preserves_data_and_rejects_live_storage() -> Result<(), Box<dyn std::error::Error>> {
let data = tempfile::tempdir()?;
let config = write_config(data.path());
let business = data.path().join("databases/business");
std::fs::create_dir_all(&business)?;
std::fs::write(business.join("sentinel"), b"business data")?;
let types = type_seed_request()["types"].clone();
let request = serde_json::json!({"mode":"reset","user":"root","password_hash":"original-hash","types":types});
Storage::maintenance(config.to_str(), &request.to_string())?;
let system = data.path().join("system");
let credentials = std::fs::read(system.join("__system_users.db"))?;
let mut catalog: serde_json::Value = serde_json::from_slice(&std::fs::read(system.join("catalog.json"))?)?;
catalog["tables"]["__system_identities"]["columns"] = serde_json::json!([]);
std::fs::write(system.join("catalog.json"), serde_json::to_vec(&catalog)?)?;
let repair = serde_json::json!({"mode":"repair","types":types});
Storage::maintenance(config.to_str(), &repair.to_string())?;
assert_eq!(std::fs::read(system.join("__system_users.db"))?, credentials);
assert_eq!(std::fs::read(business.join("sentinel"))?, b"business data");
let opened = Storage::open(config.to_str())?;
assert!(Storage::maintenance(config.to_str(), &request.to_string()).unwrap_err().contains("data directory is in use"));
drop(opened);
std::fs::remove_file(system.join("__system_users.db"))?;
let before = std::fs::read(system.join("catalog.json"))?;
let failure = Storage::maintenance(config.to_str(), &repair.to_string()).unwrap_err();
assert!(failure.contains("repair would require"));
assert_eq!(std::fs::read(system.join("catalog.json"))?, before);
assert!(!system.join("__system_users.db").exists());
Ok(())
}
/// 从账号 WAL 恢复缺失凭据文件
#[test]
fn offline_recovery_replays_account_wal() -> Result<(), Box<dyn std::error::Error>> {
let data = tempfile::tempdir()?;
let config = write_config(data.path());
let types = type_seed_request()["types"].clone();
Storage::maintenance(config.to_str(), &serde_json::json!({"mode":"reset","user":"root","password_hash":"original-hash","types":types}).to_string())?;
let storage = Storage::open(config.to_str())?;
let accounts: serde_json::Value = serde_json::from_str(&storage.request_line(r#"{"method":"accounts_list"}"#))?;
let saved = serde_json::from_value(accounts["accounts"].clone())?;
drop(storage);
let system = data.path().join("system");
let wal = chusql_core_storage::wal::Wal::new(system.join("wal.log"));
wal.append(&chusql_core_storage::wal::WalOp::Accounts { accounts: saved, removed: Vec::new(), migrate_roles: false })?;
wal.append(&chusql_core_storage::wal::WalOp::Commit)?;
drop(wal);
std::fs::remove_file(system.join("__system_users.db"))?;
Storage::maintenance(config.to_str(), &serde_json::json!({"mode":"recover","types":types}).to_string())?;
let storage = Storage::open(config.to_str())?;
let restored: serde_json::Value = serde_json::from_str(&storage.request_line(r#"{"method":"accounts_list"}"#))?;
assert_eq!(restored["accounts"], accounts["accounts"]);
drop(storage);
std::fs::write(system.join("catalog.json"), b"corrupted catalog")?;
Storage::maintenance(config.to_str(), &serde_json::json!({"mode":"recover","types":types}).to_string())?;
let storage = Storage::open(config.to_str())?;
let reconstructed: serde_json::Value = serde_json::from_str(&storage.request_line(r#"{"method":"accounts_list"}"#))?;
assert_eq!(reconstructed["accounts"], accounts["accounts"]);
Ok(())
}
/// 一次会话就是当前线程的存储句柄
struct Conn(Arc<Storage>);
@@ -800,7 +1227,7 @@ fn account_snapshot_replays_twice_without_duplicating_accounts() -> Result<(), B
for _ in 0..2 {
let wal = Wal::new(data.path().join("system/wal.log"));
wal.truncate()?;
wal.append(&WalOp::Accounts { accounts: accounts.clone() })?;
wal.append(&WalOp::Accounts { accounts: accounts.clone(), removed: Vec::new(), migrate_roles: false })?;
drop(wal);
std::fs::write(data.path().join("system/__system_users.idx"), b"torn index")?;
std::fs::write(data.path().join("system/__system_users.db"), b"torn heap")?;
@@ -1595,7 +2022,7 @@ fn stats_report_distinct_values() {
send(&mut c, r#"{"method":"delete_keys","table":"stat_t","keys":[1]}"#);
let r = send(&mut c, r#"{"method":"describe_table","table":"stat_t"}"#);
assert!(r.contains(r#""row_count":2"#), "after delete: {}", r);
assert!(r.contains(r#""name":"age","distinct":1"#), "after delete: {}", r);
assert!(r.contains(r#""name":"age","distinct":2"#), "remaining ages are still 30 and 41: {}", r);
}
/// 重开服务后索引仍在
@@ -1832,12 +2259,14 @@ fn account_table_uses_typed_columns_and_stamps_login() -> Result<(), Box<dyn std
assert!(account["last_login_at"].is_null(), "还没登录过: {}", account);
let catalog: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(data.path().join("system/catalog.json"))?)?;
let entry = &catalog["tables"]["__system_users"];
let credential_columns = &catalog["tables"]["__system_users"]["columns"];
assert_eq!(credential_columns.as_array().ok_or("missing credentials")?.iter().map(|c| c["name"].as_str().unwrap_or("")).collect::<Vec<_>>(), vec!["id", "password_hash"]);
let entry = &catalog["tables"]["__system_identities"];
let columns = entry["columns"].as_array().ok_or("missing columns")?;
let names: Vec<&str> = columns.iter().map(|col| col["name"].as_str().unwrap_or("")).collect();
assert_eq!(names, vec!["id", "user", "password_hash", "registered_at", "last_login_at", "revision"]);
assert_eq!(names, vec!["id", "user", "can_login", "is_superuser", "system_catalog_manager", "enabled", "identity_version", "registered_at", "last_login_at", "revision"]);
let types: Vec<&str> = columns.iter().map(|col| col["ty"].as_str().unwrap_or("")).collect();
assert_eq!(types, vec!["int", "varchar(64)", "varchar(256)", "timestamp", "timestamp", "int"]);
assert_eq!(types, vec!["int", "varchar(64)", "bool", "bool", "bool", "bool", "int", "timestamp", "timestamp", "int"]);
assert_eq!(columns[0]["primary_key"], true, "id 是主键");
assert_eq!(columns[0]["nullable"], false);
assert_eq!(columns[1]["unique"], true, "用户名唯一");
+25
View File
@@ -207,6 +207,31 @@ fn trailing_torn_frame_can_be_truncated() {
assert_eq!(wal.read_all().unwrap().records.len(), 1);
}
/// 完整损坏帧返回错误而非截断成功
#[test]
fn complete_corrupt_frame_is_rejected() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("wal.log");
std::fs::write(&path, [1, 0, 0, 0, 255]).unwrap();
assert!(Wal::new(&path).read_all().is_err());
assert_eq!(std::fs::read(path).unwrap(), [1, 0, 0, 0, 255]);
}
/// 检查点损坏与替换失败不丢旧值
#[test]
fn checkpoint_corruption_and_write_failure_are_explicit() {
let dir = tempfile::tempdir().unwrap();
let wal = Wal::new(dir.path().join("wal.log"));
for bytes in [vec![], vec![1; 7], vec![1; 9]] {
std::fs::write(dir.path().join("wal.checkpoint"), bytes).unwrap();
assert!(wal.read_checkpoint().is_err());
}
wal.write_checkpoint(7).unwrap();
std::fs::create_dir(dir.path().join("wal.checkpoint.pending")).unwrap();
assert!(wal.write_checkpoint(9).is_err());
assert_eq!(wal.read_checkpoint().unwrap(), Some(7));
}
/// 检查点写读一致
#[test]
fn checkpoint_round_trips() {