事务模型

This commit is contained in:
LIm1tZz7 committed 2026-10-03 16:49:23 +08:00
1 parent 274b7fcdcc
commit 045ccc2dc0
20 files changed
+1519 -294

No files matched your search

+2 -2
View File
@@ -28,5 +28,5 @@
- 改动说明只在交流中记录。 - 改动说明只在交流中记录。
## 当前重点 ## 当前重点
- 完善 WAL 重放幂等。 - 主线已完成:高级优化器 → 并行查询 → M1 会话隔离 → M2 快照事务 → M3 WAL 与崩溃恢复。
- 完整事务路线见 project-plan.txt 阶段 12。 - 进度与计数见 `agent_credientials/project-plan.txt`,限制与风险见 `agent_credientials/known-issues.txt`。
@@ -134,6 +134,9 @@ runStatementUncheckedM _ q@CreateDatabase{} = runStatementM q
runStatementUncheckedM _ q@DropDatabase{} = runStatementM q runStatementUncheckedM _ q@DropDatabase{} = runStatementM q
runStatementUncheckedM _ q@UseDatabase{} = runStatementM q runStatementUncheckedM _ q@UseDatabase{} = runStatementM q
runStatementUncheckedM _ ShowDatabases = runStatementM ShowDatabases runStatementUncheckedM _ ShowDatabases = runStatementM ShowDatabases
runStatementUncheckedM _ BeginTransaction = pure (Left "BEGIN is executed by the session")
runStatementUncheckedM _ CommitTransaction = pure (Left "COMMIT is executed by the session")
runStatementUncheckedM _ RollbackTransaction = pure (Left "ROLLBACK is executed by the session")
-- | 只有需要读全表才能判断约束时才扫表 -- | 只有需要读全表才能判断约束时才扫表
existingRows :: (MonadStorage m) => String -> Table -> m (Either String [Row]) existingRows :: (MonadStorage m) => String -> Table -> m (Either String [Row])
@@ -51,6 +51,8 @@ errorRules =
, ErrorRule "storage" StorageError "query_error" , ErrorRule "storage" StorageError "query_error"
, ErrorRule "storage library call failed" StorageError "query_error" , ErrorRule "storage library call failed" StorageError "query_error"
, ErrorRule "unsupported protocol" ProtocolError "query_error" , ErrorRule "unsupported protocol" ProtocolError "query_error"
, ErrorRule "executed by the session" ProtocolError "query_error"
, ErrorRule "transaction" ProtocolError "query_error"
] ]
-- | 命中的第一条规则 -- | 命中的第一条规则
@@ -38,16 +38,16 @@ scopeAt place scope c
| "ambiguous column:" `isInfixOf` err -> Left err | "ambiguous column:" `isInfixOf` err -> Left err
| otherwise -> columnAt place (scopeOuter scope) c | otherwise -> columnAt place (scopeOuter scope) c
-- | 收集 FROM 能提供的列,含外层作用域 -- | 收集 FROM 的列,列前缀由表名推导
checkFrom :: Database -> [(String, Column)] -> FromClause -> Either String [(String, Column)] checkFrom :: Database -> [(String, Column)] -> FromClause -> Either String [(String, Column)]
checkFrom _ _ FromUnit = Right [] checkFrom _ _ FromUnit = Right []
checkFrom db _ (FromTable mAlias tbl) = do checkFrom db _ (FromTable mAlias tbl) = do
t <- lookupTable db tbl (key, t) <- resolveTable db tbl
Right (prefixColumns (Just (fromMaybe tbl mAlias)) (tableCols t)) Right (prefixColumns (Just (deriveQualifier mAlias key)) (tableCols t))
checkFrom db outer (FromJoin _ left mAlias tbl cond) = do checkFrom db outer (FromJoin _ left mAlias tbl cond) = do
lcols <- checkFrom db outer left lcols <- checkFrom db outer left
t <- lookupTable db tbl (key, t) <- resolveTable db tbl
let env = lcols ++ prefixColumns (Just (fromMaybe tbl mAlias)) (tableCols t) let env = lcols ++ prefixColumns (Just (deriveQualifier mAlias key)) (tableCols t)
checkBool db "ON" (Scope env outer) cond checkBool db "ON" (Scope env outer) cond
Right env Right env
@@ -433,6 +433,10 @@ checkResolvedWith db outer q = case q of
DropDatabase name -> checkDatabaseName name DropDatabase name -> checkDatabaseName name
UseDatabase name -> checkDatabaseName name UseDatabase name -> checkDatabaseName name
ShowDatabases -> Right () ShowDatabases -> Right ()
-- | 事务控制由会话层执行,语义层不做列检查
BeginTransaction -> Right ()
CommitTransaction -> Right ()
RollbackTransaction -> Right ()
where where
tableCols' = map fst . tableCols tableCols' = map fst . tableCols
@@ -650,21 +654,21 @@ resolveStatement db q = case q of
Select{} -> resolveSelect db q Select{} -> resolveSelect db q
SelectExpr{} -> resolveSelect db q SelectExpr{} -> resolveSelect db q
Update tbl assigns mWhere -> do Update tbl assigns mWhere -> do
t <- lookupTable db tbl (key, t) <- resolveTable db tbl
let env = tableCols t let env = tableCols t
name = nameIn env "UPDATE" name = nameIn env "UPDATE"
assigns' <- mapM (\(c, e) -> (\e' -> (c, e')) <$> resolveExpr db env [] name e) assigns assigns' <- mapM (\(c, e) -> (\e' -> (c, e')) <$> resolveExpr db env [] name e) assigns
w <- mapM (resolveExpr db env [] (nameIn env "WHERE")) mWhere w <- mapM (resolveExpr db env [] (nameIn env "WHERE")) mWhere
pure (Update tbl assigns' w) pure (Update key assigns' w)
Delete tbl mWhere -> do Delete tbl mWhere -> do
t <- lookupTable db tbl (key, t) <- resolveTable db tbl
w <- mapM (resolveExpr db (tableCols t) [] (nameIn (tableCols t) "WHERE")) mWhere w <- mapM (resolveExpr db (tableCols t) [] (nameIn (tableCols t) "WHERE")) mWhere
pure (Delete tbl w) pure (Delete key w)
Insert tbl cols rows -> do Insert tbl cols rows -> do
t <- lookupTable db tbl (key, t) <- resolveTable db tbl
let env = tableCols t let env = tableCols t
rows' <- mapM (mapM (resolveExpr db env [] (nameIn env "INSERT"))) rows rows' <- mapM (mapM (resolveExpr db env [] (nameIn env "INSERT"))) rows
pure (Insert tbl cols rows') pure (Insert key cols rows')
other -> Right other other -> Right other
-- | 查一列的物理名 -- | 查一列的物理名
@@ -685,9 +689,9 @@ resolveSelect _ q = Right q
resolveQuery :: Database -> [(String, Column)] -> Statement -> [(String, Expr)] -> Either String Statement resolveQuery :: Database -> [(String, Column)] -> Statement -> [(String, Expr)] -> Either String Statement
resolveQuery db outer q items = do resolveQuery db outer q items = do
env <- checkFrom db outer (selectFrom q) env <- checkFrom db outer (selectFrom q)
-- | 单表查询里把裸列名换成物理列名 -- | 单表无别名时把裸列名换成物理列名;带别名时表头保留别名限定名
let physical c = case selectFrom q of let physical c = case selectFrom q of
FromTable Nothing tbl -> unqualify (Just tbl) c FromTable Nothing tbl -> unqualify (Just (deriveQualifier Nothing tbl)) c
_ -> c _ -> c
-- | 查一列并换算成物理名 -- | 查一列并换算成物理名
name place c = physical . fst <$> scopedColumnAt place env outer c name place c = physical . fst <$> scopedColumnAt place env outer c
@@ -707,18 +711,23 @@ resolveQuery db outer q items = do
grouping <- mapM (name "GROUP BY") (selectGroupBy q) grouping <- mapM (name "GROUP BY") (selectGroupBy q)
ordering <- mapM (\(c, d) -> (\k -> (k, d)) <$> name "ORDER BY" c) (selectOrderBy q) ordering <- mapM (\(c, d) -> (\k -> (k, d)) <$> name "ORDER BY" c) (selectOrderBy q)
source <- case selectFrom q of source <- case selectFrom q of
single@FromTable{} -> Right single FromTable mAlias tbl -> do
(key, _) <- resolveTable db tbl
Right (FromTable mAlias key)
other -> resolveFrom db outer other other -> resolveFrom db outer other
pure $ case q of pure $ case q of
Select{} -> Select (map fst projected) source condition grouping ordering (selectLimit q) Select{} -> Select (map fst projected) source condition grouping ordering (selectLimit q)
_ -> SelectExpr projected source condition grouping ordering (selectLimit q) _ -> SelectExpr projected source condition grouping ordering (selectLimit q)
-- | 连接两侧默认取表名作别名;ON 里的子查询也能引用外层列 -- | 连接两侧取表名末段作别名;ON 可引用外层列
resolveFrom :: Database -> [(String, Column)] -> FromClause -> Either String FromClause resolveFrom :: Database -> [(String, Column)] -> FromClause -> Either String FromClause
resolveFrom _ _ FromUnit = Right FromUnit resolveFrom _ _ FromUnit = Right FromUnit
resolveFrom _ _ (FromTable alias tbl) = Right (FromTable (Just (fromMaybe tbl alias)) tbl) resolveFrom db _ (FromTable alias tbl) = do
(key, _) <- resolveTable db tbl
Right (FromTable (Just (deriveQualifier alias key)) key)
resolveFrom db outer fromC@(FromJoin kind left alias tbl cond) = do resolveFrom db outer fromC@(FromJoin kind left alias tbl cond) = do
env <- checkFrom db outer fromC env <- checkFrom db outer fromC
l <- resolveFrom db outer left l <- resolveFrom db outer left
(key, _) <- resolveTable db tbl
c <- resolveExpr db env outer (fmap fst . columnAt "ON" env) cond c <- resolveExpr db env outer (fmap fst . columnAt "ON" env) cond
pure (FromJoin kind l (Just (fromMaybe tbl alias)) tbl c) pure (FromJoin kind l (Just (deriveQualifier alias key)) key c)
@@ -3,6 +3,10 @@
module ChuSQL.Core.Engine.Storage.IPC ( module ChuSQL.Core.Engine.Storage.IPC (
IPCStorage (..), IPCStorage (..),
Env (..),
envForDatabase,
defaultEnv,
runIPCStorage,
Request (..), Request (..),
Response (..), Response (..),
Account (..), Account (..),
@@ -41,21 +45,38 @@ import System.IO.Unsafe (unsafePerformIO)
-- IPC 存储实现:把 MonadStorage 的操作变成发给存储库的 JSON 请求。 -- IPC 存储实现:把 MonadStorage 的操作变成发给存储库的 JSON 请求。
newtype IPCStorage a = IPCStorage -- | 一条会话的存储环境:当前库名随会话走,不共享
{ runIPCStorage :: IO a data Env = Env
{ envDatabase :: Maybe String
} }
newtype IPCStorage a = IPCStorage
{ runIPCStorageIn :: Env -> IO a
}
-- | 按库名造一个会话环境
envForDatabase :: Maybe String -> Env
envForDatabase = Env
-- | 进程默认环境:读全局库名,给测试与 CLI 用
defaultEnv :: IO Env
defaultEnv = Env <$> readMVar databaseRef
-- | 用进程默认环境跑(兼容既有调用点)
runIPCStorage :: IPCStorage a -> IO a
runIPCStorage action = defaultEnv >>= runIPCStorageIn action
instance Functor IPCStorage where instance Functor IPCStorage where
fmap f (IPCStorage m) = IPCStorage (fmap f m) fmap f (IPCStorage m) = IPCStorage (fmap f . m)
instance Applicative IPCStorage where instance Applicative IPCStorage where
pure = IPCStorage . pure pure = IPCStorage . const . pure
IPCStorage mf <*> IPCStorage ma = IPCStorage (mf <*> ma) IPCStorage mf <*> IPCStorage ma = IPCStorage $ \env -> mf env <*> ma env
instance Monad IPCStorage where instance Monad IPCStorage where
IPCStorage m >>= k = IPCStorage $ do IPCStorage m >>= k = IPCStorage $ \env -> do
a <- m a <- m env
runIPCStorage (k a) runIPCStorageIn (k a) env
-- | 一条到存储库的链路:管请求与关闭 -- | 一条到存储库的链路:管请求与关闭
data StorageLink = StorageLink data StorageLink = StorageLink
@@ -147,10 +168,9 @@ globalRequest ReqPing = True
globalRequest _ = False globalRequest _ = False
-- | 发请求并把响应翻成 Either -- | 发请求并把响应翻成 Either
ask :: String -> Request -> (Response -> Maybe a) -> IO (Either String a) ask :: Env -> String -> Request -> (Response -> Maybe a) -> IO (Either String a)
ask what req recognize = do ask env what req recognize = do
current <- getDatabaseName resp <- sendRequest (stampDatabase (envDatabase env) req)
resp <- sendRequest (stampDatabase current req)
pure $ case resp of pure $ case resp of
RespError e -> Left e RespError e -> Left e
other -> case recognize other of other -> case recognize other of
@@ -163,99 +183,100 @@ okOnly RespOk = Just ()
okOnly _ = Nothing okOnly _ = Nothing
-- | 发 Scan -- | 发 Scan
doScan :: String -> IO (Either String [Row]) doScan :: Env -> String -> IO (Either String [Row])
doScan t = ask "scan" (ReqScan t) $ \resp -> case resp of doScan env t = ask env "scan" (ReqScan t) $ \resp -> case resp of
RespRows rows -> Just rows RespRows rows -> Just rows
_ -> Nothing _ -> Nothing
-- | 发带投影的 Scan -- | 发带投影的 Scan
doScanColumns :: String -> [String] -> IO (Either String [Row]) doScanColumns :: Env -> String -> [String] -> IO (Either String [Row])
doScanColumns t cols = ask "scan" (ReqScanColumns t cols) $ \resp -> case resp of doScanColumns env t cols = ask env "scan" (ReqScanColumns t cols) $ \resp -> case resp of
RespRows rows -> Just [[(c, v) | c <- cols, Just v <- [lookup c row]] | row <- rows] RespRows rows -> Just [[(c, v) | c <- cols, Just v <- [lookup c row]] | row <- rows]
_ -> Nothing _ -> Nothing
-- | 发分片 Scan:只取第 shard 片 -- | 发分片 Scan:只取第 shard 片
doScanShard :: String -> Maybe [String] -> Int -> Int -> IO (Either String [Row]) doScanShard :: Env -> String -> Maybe [String] -> Int -> Int -> IO (Either String [Row])
doScanShard t cols shard shards = ask "scan_shard" (ReqScanShard t cols shard shards) $ \resp -> case resp of doScanShard env t cols shard shards = ask env "scan_shard" (ReqScanShard t cols shard shards) $ \resp -> case resp of
RespRows rows -> Just rows RespRows rows -> Just rows
_ -> Nothing _ -> Nothing
-- | 发 Insert(索引由存储层维护) -- | 发 Insert(索引由存储层维护)
doInsert :: String -> Row -> IO (Either String ()) doInsert :: Env -> String -> Row -> IO (Either String ())
doInsert t r = ask "insert" (ReqInsert t r) okOnly doInsert env t r = ask env "insert" (ReqInsert t r) okOnly
-- | 发 InsertBatch:一批行一次请求 -- | 发 InsertBatch:一批行一次请求
doInsertMany :: String -> [Row] -> IO (Either String ()) doInsertMany :: Env -> String -> [Row] -> IO (Either String ())
doInsertMany t rs = ask "insert_batch" (ReqInsertBatch t rs) okOnly doInsertMany env t rs = ask env "insert_batch" (ReqInsertBatch t rs) okOnly
-- | 发 DeleteKeys -- | 发 DeleteKeys
doDeleteKeys :: String -> [Int] -> IO (Either String ()) doDeleteKeys :: Env -> String -> [Int] -> IO (Either String ())
doDeleteKeys t ks = ask "delete_keys" (ReqDeleteKeys t ks) okOnly doDeleteKeys env t ks = ask env "delete_keys" (ReqDeleteKeys t ks) okOnly
-- | 发 ReplaceAll -- | 发 ReplaceAll
doReplaceAll :: String -> [Row] -> IO (Either String ()) doReplaceAll :: Env -> String -> [Row] -> IO (Either String ())
doReplaceAll t rs = ask "replace_all" (ReqReplaceAll t rs) okOnly doReplaceAll env t rs = ask env "replace_all" (ReqReplaceAll t rs) okOnly
-- | 发 ListTables -- | 发 ListTables
doListTables :: IO (Either String [String]) doListTables :: Env -> IO (Either String [String])
doListTables = ask "list_tables" ReqListTables $ \resp -> case resp of doListTables env = ask env "list_tables" ReqListTables $ \resp -> case resp of
RespTables ts -> Just ts RespTables ts -> Just ts
_ -> Nothing _ -> Nothing
-- | 一次拿全库 schema -- | 一次拿全库 schema
doListCatalog :: IO (Either String [TableInfo]) doListCatalog :: Env -> IO (Either String [TableInfo])
doListCatalog = ask "list_catalog" ReqListCatalog $ \resp -> case resp of doListCatalog env = ask env "list_catalog" ReqListCatalog $ \resp -> case resp of
RespCatalog xs -> Just xs RespCatalog xs -> Just xs
_ -> Nothing _ -> Nothing
-- | 按某一列的索引取一行(键可以是整数或字符串) -- | 按某一列的索引取一行(键可以是整数或字符串)
doLookupByColumn :: String -> String -> Value -> IO (Either String IndexResult) doLookupByColumn :: Env -> String -> String -> Value -> IO (Either String IndexResult)
doLookupByColumn t c k = ask "lookup_by_index" (ReqLookupByColumn t c k) $ \resp -> case resp of doLookupByColumn env t c k = ask env "lookup_by_index" (ReqLookupByColumn t c k) $ \resp -> case resp of
RespRows rows -> Just (IndexRows rows) RespRows rows -> Just (IndexRows rows)
RespNoIndex -> Just NoIndex RespNoIndex -> Just NoIndex
_ -> Nothing _ -> Nothing
-- | 范围扫描:没有可用索引就回 Nothing,让上层退回扫描 -- | 范围扫描:没有可用索引就回 Nothing,让上层退回扫描
doScanRange :: doScanRange ::
Env ->
String -> String ->
String -> String ->
Maybe (Value, Bool) -> Maybe (Value, Bool) ->
Maybe (Value, Bool) -> Maybe (Value, Bool) ->
IO (Either String (Maybe [Row])) IO (Either String (Maybe [Row]))
doScanRange t c lo hi = ask "range_by_index" (ReqRangeByIndex t c lo hi) $ \resp -> case resp of doScanRange env t c lo hi = ask env "range_by_index" (ReqRangeByIndex t c lo hi) $ \resp -> case resp of
RespRows rows -> Just (Just rows) RespRows rows -> Just (Just rows)
RespNoIndex -> Just Nothing RespNoIndex -> Just Nothing
_ -> Nothing _ -> Nothing
-- | 发 DescribeTable -- | 发 DescribeTable
doDescribeTable :: String -> IO (Either String TableInfo) doDescribeTable :: Env -> String -> IO (Either String TableInfo)
doDescribeTable t = ask "describe_table" (ReqDescribeTable t) $ \resp -> case resp of doDescribeTable env t = ask env "describe_table" (ReqDescribeTable t) $ \resp -> case resp of
RespSchema info -> Just info RespSchema info -> Just info
_ -> Nothing _ -> Nothing
-- | 给某一列建索引 -- | 给某一列建索引
doCreateIndex :: String -> String -> IO (Either String ()) doCreateIndex :: Env -> String -> String -> IO (Either String ())
doCreateIndex t c = ask "create_index" (ReqCreateIndex t c) okOnly doCreateIndex env t c = ask env "create_index" (ReqCreateIndex t c) okOnly
-- | 去掉某一列的索引 -- | 去掉某一列的索引
doDropIndex :: String -> String -> IO (Either String ()) doDropIndex :: Env -> String -> String -> IO (Either String ())
doDropIndex t c = ask "drop_index" (ReqDropIndex t c) okOnly doDropIndex env t c = ask env "drop_index" (ReqDropIndex t c) okOnly
-- | 删一列(列定义、这一列上的索引、每行里的那一格一起没) -- | 删一列(列定义、这一列上的索引、每行里的那一格一起没)
doDropColumn :: String -> String -> IO (Either String ()) doDropColumn :: Env -> String -> String -> IO (Either String ())
doDropColumn t c = ask "drop_column" (ReqDropColumn t c) okOnly doDropColumn env t c = ask env "drop_column" (ReqDropColumn t c) okOnly
-- | 发 ReplaceSchema(ALTER 用) -- | 发 ReplaceSchema(ALTER 用)
doReplaceSchema :: String -> [SchemaColumn] -> [Row] -> IO (Either String ()) doReplaceSchema :: Env -> String -> [SchemaColumn] -> [Row] -> IO (Either String ())
doReplaceSchema t cols rs = ask "replace_schema" (ReqReplaceSchema t cols rs) okOnly doReplaceSchema env t cols rs = ask env "replace_schema" (ReqReplaceSchema t cols rs) okOnly
-- | 发 CreateTable -- | 发 CreateTable
doCreateTable :: String -> [SchemaColumn] -> IO (Either String ()) doCreateTable :: Env -> String -> [SchemaColumn] -> IO (Either String ())
doCreateTable t cols = ask "create_table" (ReqCreateTable t cols) okOnly doCreateTable env t cols = ask env "create_table" (ReqCreateTable t cols) okOnly
-- | 发 DropTable -- | 发 DropTable
doDropTable :: String -> IO (Either String ()) doDropTable :: Env -> String -> IO (Either String ())
doDropTable t = ask "drop_table" (ReqDropTable t) okOnly doDropTable env t = ask env "drop_table" (ReqDropTable t) okOnly
-- | 线上 schema 转本地列 -- | 线上 schema 转本地列
schemaToColumns :: [SchemaColumn] -> [(String, Column)] schemaToColumns :: [SchemaColumn] -> [(String, Column)]
@@ -292,92 +313,91 @@ toWire (n, c) =
instance MonadStorage IPCStorage where instance MonadStorage IPCStorage where
-- \| 建一个库 -- \| 建一个库
createDatabase name = IPCStorage (ask "create_database" (ReqDatabase "create_database" name) okOnly) createDatabase name = IPCStorage $ \env -> ask env "create_database" (ReqDatabase "create_database" name) okOnly
-- \| 删一个库 -- \| 删一个库
dropDatabase name = IPCStorage (ask "drop_database" (ReqDatabase "drop_database" name) okOnly) dropDatabase name = IPCStorage $ \env -> ask env "drop_database" (ReqDatabase "drop_database" name) okOnly
-- \| 切当前库 -- \| 切当前库
useDatabase name = IPCStorage (ask "use_database" (ReqDatabase "use_database" name) okOnly) useDatabase name = IPCStorage $ \env -> ask env "use_database" (ReqDatabase "use_database" name) okOnly
-- \| 列出现有库 -- \| 列出现有库
listDatabases = IPCStorage $ ask "list_databases" (ReqDatabase "list_databases" "") $ \resp -> case resp of listDatabases = IPCStorage $ \env -> ask env "list_databases" (ReqDatabase "list_databases" "") $ \resp -> case resp of
RespTables names -> Just names RespTables names -> Just names
_ -> Nothing _ -> Nothing
-- \| 发 Scan -- \| 发 Scan
scan t = IPCStorage (doScan t) scan t = IPCStorage (\env -> doScan env t)
-- \| 只取指定列 -- \| 只取指定列
scanColumns t cols = IPCStorage (doScanColumns t cols) scanColumns t cols = IPCStorage (\env -> doScanColumns env t cols)
-- \| 并行分片数:按 RTS 能力数,封顶在 parallelShardLimit -- \| 并行分片数:按 RTS 能力数,封顶在 parallelShardLimit
parallelShards = IPCStorage $ do parallelShards = IPCStorage $ \_ -> do
caps <- getNumCapabilities caps <- getNumCapabilities
pure (max 1 (min parallelShardLimit caps)) pure (max 1 (min parallelShardLimit caps))
-- \| 分片并行扫,再按分片序拼接 -- \| 分片并行扫,再按分片序拼接
scanShards width t cols scanShards width t cols
| width <= 1 = IPCStorage (maybe (doScan t) (doScanColumns t) cols) | width <= 1 = IPCStorage (\env -> maybe (doScan env t) (doScanColumns env t) cols)
| otherwise = IPCStorage $ do | otherwise = IPCStorage $ \env -> do
pool <- workerPool width pool <- workerPool width
parts <- poolRun pool [doScanShard t cols shard width | shard <- [0 .. width - 1]] parts <- poolRun pool [doScanShard env t cols shard width | shard <- [0 .. width - 1]]
pure (fmap concat (sequence parts)) pure (fmap concat (sequence parts))
-- \| 发 Insert(索引由存储层自己维护) -- \| 发 Insert(索引由存储层自己维护)
insert t r = IPCStorage (doInsert t r) insert t r = IPCStorage (\env -> doInsert env t r)
-- \| 一批行一次请求:N 行只落一次盘 -- \| 一批行一次请求:N 行只落一次盘
insertMany t rs = IPCStorage (doInsertMany t rs) insertMany t rs = IPCStorage (\env -> doInsertMany env t rs)
-- \| 按 id 批量删行 -- \| 按 id 批量删行
deleteKeys t ks = IPCStorage (doDeleteKeys t ks) deleteKeys t ks = IPCStorage (\env -> doDeleteKeys env t ks)
-- \| 发 ReplaceAll -- \| 发 ReplaceAll
replaceAll t rs = IPCStorage (doReplaceAll t rs) replaceAll t rs = IPCStorage (\env -> doReplaceAll env t rs)
-- \| 按某一列的索引取一行(没有索引就回 NoIndex) -- \| 按某一列的索引取一行(没有索引就回 NoIndex)
lookupByColumn t c k = IPCStorage (doLookupByColumn t c k) lookupByColumn t c k = IPCStorage (\env -> doLookupByColumn env t c k)
-- \| 范围扫描(没有索引就回 Nothing) -- \| 范围扫描(没有索引就回 Nothing)
scanRange t c lo hi = IPCStorage (doScanRange t c lo hi) scanRange t c lo hi = IPCStorage (\env -> doScanRange env t c lo hi)
-- \| 发 CreateTable -- \| 发 CreateTable
createTable name cols = IPCStorage (doCreateTable name (map toWire cols)) createTable name cols = IPCStorage (\env -> doCreateTable env name (map toWire cols))
-- \| ALTER:整表换列定义与全部行 -- \| ALTER:整表换列定义与全部行
replaceSchema name cols rows = IPCStorage (doReplaceSchema name (map toWire cols) rows) replaceSchema name cols rows = IPCStorage (\env -> doReplaceSchema env name (map toWire cols) rows)
-- \| 发 DropTable -- \| 发 DropTable
dropTable name = IPCStorage (doDropTable name) dropTable name = IPCStorage (\env -> doDropTable env name)
-- \| 建索引 -- \| 建索引
createIndex t c = IPCStorage (doCreateIndex t c) createIndex t c = IPCStorage (\env -> doCreateIndex env t c)
-- \| 删索引 -- \| 删索引
dropIndex t c = IPCStorage (doDropIndex t c) dropIndex t c = IPCStorage (\env -> doDropIndex env t c)
-- \| 删一列 -- \| 删一列
dropColumn t c = IPCStorage (doDropColumn t c) dropColumn t c = IPCStorage (\env -> doDropColumn env t c)
-- \| 列表 + 逐表扫描拼库 -- \| 列表 + 逐表扫描拼库
snapshot = IPCStorage $ do snapshot = IPCStorage $ \env -> do
result <- doListCatalog result <- doListCatalog env
case result of case result of
Left _ -> pure [] Left _ -> pure []
Right xs -> mapM loadEntry xs Right xs -> mapM (loadEntry env) xs
where where
loadEntry info = do loadEntry env info = do
rows <- doScan (tiTable info) rows <- doScan env (tiTable info)
pure pure
( tiTable info ( tiTable info
, Table (tiTable info) (schemaToColumns (tiColumns info)) (either (const []) id rows) (Just (metaOf info)) , Table (tiTable info) (schemaToColumns (tiColumns info)) (either (const []) id rows) (Just (metaOf info))
) )
-- \| 只问数据字典要结构,设了当前库只留它的表 -- \| 只问数据字典要结构,设了当前库只留它的表
schema = IPCStorage $ do schema = IPCStorage $ \env -> do
current <- getDatabaseName result <- ask env "all_catalogs" ReqAllCatalog $ \resp -> case resp of
result <- ask "all_catalogs" ReqAllCatalog $ \resp -> case resp of
RespCatalog infos -> Just infos RespCatalog infos -> Just infos
_ -> Nothing _ -> Nothing
pure $ case result of pure $ case result of
Left _ -> [] Left _ -> []
Right xs -> concatMap (visibleTables current) xs Right xs -> concatMap (visibleTables (envDatabase env)) xs
-- | 线上表信息转引擎侧统计 -- | 线上表信息转引擎侧统计
metaOf :: TableInfo -> TableMeta metaOf :: TableInfo -> TableMeta
@@ -53,6 +53,12 @@ data Statement
| DropDatabase String | DropDatabase String
| UseDatabase String | UseDatabase String
| ShowDatabases | ShowDatabases
-- | 开事务:BEGIN / START TRANSACTION
| BeginTransaction
-- | 提交事务
| CommitTransaction
-- | 回滚事务
| RollbackTransaction
deriving (Show, Eq) deriving (Show, Eq)
-- | 连接类型:内连接与左外连接 -- | 连接类型:内连接与左外连接
@@ -613,6 +613,14 @@ updateStatement = do
mWhere <- optional (keyword "where" *> expr) mWhere <- optional (keyword "where" *> expr)
return (Update tbl assigns mWhere) return (Update tbl assigns mWhere)
-- | 事务控制语句
transactionStatement :: Parser Statement
transactionStatement =
try (BeginTransaction <$ (keyword "start" *> keyword "transaction"))
<|> (BeginTransaction <$ (keyword "begin" *> optional (keyword "transaction")))
<|> (CommitTransaction <$ (keyword "commit" *> optional (keyword "transaction")))
<|> (RollbackTransaction <$ (keyword "rollback" *> optional (keyword "transaction")))
-- | 解析总入口 -- | 解析总入口
parseStatement :: String -> Either String Statement parseStatement :: String -> Either String Statement
parseStatement input = parseStatement input =
@@ -622,7 +630,8 @@ parseStatement input =
where where
-- | 按关键字分派到各语句解析器 -- | 按关键字分派到各语句解析器
statementP = statementP =
selectStatement transactionStatement
<|> selectStatement
<|> try (CreateDatabase <$> (keyword "create" *> keyword "database" *> identifier)) <|> try (CreateDatabase <$> (keyword "create" *> keyword "database" *> identifier))
<|> try (DropDatabase <$> (keyword "drop" *> keyword "database" *> identifier)) <|> try (DropDatabase <$> (keyword "drop" *> keyword "database" *> identifier))
<|> (UseDatabase <$> (keyword "use" *> identifier)) <|> (UseDatabase <$> (keyword "use" *> identifier))
+75 -2
View File
@@ -15,7 +15,7 @@ import ChuSQL.Core.Engine.Parallel (poolRun, workerPool)
import ChuSQL.Core.Model import ChuSQL.Core.Model
import ChuSQL.Core.Engine.Semantic (prepare) import ChuSQL.Core.Engine.Semantic (prepare)
import ChuSQL.Core.Engine.Storage (IndexResult (..), MonadStorage (..)) import ChuSQL.Core.Engine.Storage (IndexResult (..), MonadStorage (..))
import ChuSQL.Core.Engine.Storage.IPC (IPCStorage (runIPCStorage), closeConnection, doListTables, getDatabaseName, localStorageLink, setDatabaseName, setStorageLink) import ChuSQL.Core.Engine.Storage.IPC (IPCStorage (..), closeConnection, defaultEnv, doListTables, envForDatabase, getDatabaseName, localStorageLink, runIPCStorage, setDatabaseName, setStorageLink)
import ChuSQL.Core.Engine.Syntax.AST import ChuSQL.Core.Engine.Syntax.AST
import ChuSQL.Core.Engine.Syntax.Parser import ChuSQL.Core.Engine.Syntax.Parser
import Control.Concurrent (threadDelay) import Control.Concurrent (threadDelay)
@@ -113,6 +113,40 @@ main = hspec $ do
rowsOf (parseStatement "SELECT sales.users.name FROM sales.users WHERE sales.users.id = 1" >>= runStatement db) rowsOf (parseStatement "SELECT sales.users.name FROM sales.users WHERE sales.users.id = 1" >>= runStatement db)
`shouldBe` Right [[("name", VStr "Alice")]] `shouldBe` Right [[("name", VStr "Alice")]]
it "derives the column prefix when the session schema is scoped to one database" $ do
let scoped = [("users", users)]
rowsOf (parseStatement "SELECT sales.users.name FROM sales.users WHERE sales.users.id = 1" >>= runStatement scoped)
`shouldBe` Right [[("name", VStr "Alice")]]
rowsOf (parseStatement "SELECT users.name FROM users WHERE users.id = 1" >>= runStatement scoped)
`shouldBe` Right [[("name", VStr "Alice")]]
it "derives the database prefix for a bare table name" $ do
let catalog = [("sales.users", users)]
rowsOf (parseStatement "SELECT name FROM users WHERE users.id = 1" >>= runStatement catalog)
`shouldBe` Right [[("name", VStr "Alice")]]
rowsOf (parseStatement "SELECT users.name FROM sales.users WHERE users.id = 1" >>= runStatement catalog)
`shouldBe` Right [[("name", VStr "Alice")]]
it "keeps derived prefixes short in joins" $ do
let catalog = [("sales.users", users), ("sales.orders", orders)]
fmap
(map (map snd))
(parseStatement "SELECT users.name, orders.product FROM sales.users JOIN sales.orders ON users.id = orders.user_id" >>= rowsOf . runStatement catalog)
`shouldBe` Right [[VStr "Alice", VStr "Book"], [VStr "Alice", VStr "Cup"], [VStr "Bob", VStr "Pen"]]
it "writes through the derived table key" $ do
let scoped = [("users", users)]
case parseStatement "UPDATE sales.users SET age = 26 WHERE id = 1" >>= runStatement scoped of
Left err -> expectationFailure err
Right (db', _) ->
rowsOf (runStatement db' (makeSelect ["age"] "users" (Just (Eq (Col "id") (LitInt 1)))))
`shouldBe` Right [[("age", VInt 26)]]
it "rejects an ambiguous derived table name" $ do
let catalog = [("sales.users", users), ("other.users", users)]
(parseStatement "SELECT name FROM users" >>= rowsOf . runStatement catalog)
`shouldSatisfy` either (isInfixOf "ambiguous table") (const False)
it "parses database lifecycle statements" $ do it "parses database lifecycle statements" $ do
map (fmap show . parseStatement) ["CREATE DATABASE sales", "DROP DATABASE sales", "USE sales", "SHOW DATABASES"] map (fmap show . parseStatement) ["CREATE DATABASE sales", "DROP DATABASE sales", "USE sales", "SHOW DATABASES"]
`shouldBe` map Right ["CreateDatabase \"sales\"", "DropDatabase \"sales\"", "UseDatabase \"sales\"", "ShowDatabases"] `shouldBe` map Right ["CreateDatabase \"sales\"", "DropDatabase \"sales\"", "UseDatabase \"sales\"", "ShowDatabases"]
@@ -242,6 +276,33 @@ main = hspec $ do
parseStatement "SELECT name FROM" parseStatement "SELECT name FROM"
`shouldSatisfy` isLeft `shouldSatisfy` isLeft
describe "ChuSQL.Core.Engine (事务)" $ do
it "parses transaction control statements" $ do
parseStatement "BEGIN" `shouldBe` Right BeginTransaction
parseStatement "BEGIN TRANSACTION" `shouldBe` Right BeginTransaction
parseStatement "START TRANSACTION" `shouldBe` Right BeginTransaction
parseStatement "begin transaction" `shouldBe` Right BeginTransaction
parseStatement "COMMIT" `shouldBe` Right CommitTransaction
parseStatement "COMMIT TRANSACTION" `shouldBe` Right CommitTransaction
parseStatement "ROLLBACK" `shouldBe` Right RollbackTransaction
parseStatement "rollback transaction" `shouldBe` Right RollbackTransaction
it "parses the bare words only as whole words" $ do
parseStatement "BEGINNER" `shouldSatisfy` isLeft
parseStatement "COMMITTED" `shouldSatisfy` isLeft
it "reports transaction control as session work" $ do
mapM_
( \sql ->
(runStatement [("users", users)] =<< parseStatement sql)
`shouldSatisfy` either (isInfixOf "executed by the session") (const False)
)
["BEGIN", "COMMIT", "ROLLBACK"]
it "puts transaction errors under the protocol category" $ do
errorCode "BEGIN is executed by the session" `shouldBe` "query_error"
errorCategory "BEGIN is executed by the session" `shouldBe` ProtocolError
describe "ChuSQL.Core.Engine.Syntax.Parser (JOIN)" $ do describe "ChuSQL.Core.Engine.Syntax.Parser (JOIN)" $ do
it "parses a simple JOIN without aliases" $ do it "parses a simple JOIN without aliases" $ do
parseStatement "SELECT name FROM users JOIN orders ON id = user_id" parseStatement "SELECT name FROM users JOIN orders ON id = user_id"
@@ -1613,7 +1674,19 @@ main = hspec $ do
withTestSession $ \srv -> withServerEnv srv $ do withTestSession $ \srv -> withServerEnv srv $ do
runIPCStorage (insert "t1" [("id", VInt 1)]) `shouldReturn` Right () runIPCStorage (insert "t1" [("id", VInt 1)]) `shouldReturn` Right ()
runIPCStorage (insert "t2" [("id", VInt 2)]) `shouldReturn` Right () runIPCStorage (insert "t2" [("id", VInt 2)]) `shouldReturn` Right ()
doListTables `shouldReturn` Right ["t1", "t2"] env <- defaultEnv
doListTables env `shouldReturn` Right ["t1", "t2"]
it "two sessions keep their own database" $ do
withTestSession $ \_ -> do
let alpha = envForDatabase (Just "alpha")
beta = envForDatabase (Just "beta")
runIPCStorageIn (createDatabase "alpha") alpha `shouldReturn` Right ()
runIPCStorageIn (createDatabase "beta") beta `shouldReturn` Right ()
runIPCStorageIn (insert "iso" [("id", VInt 1)]) alpha `shouldReturn` Right ()
runIPCStorageIn (insert "iso" [("id", VInt 2)]) beta `shouldReturn` Right ()
runIPCStorageIn (scan "iso") alpha `shouldReturn` Right [[("id", VInt 1)]]
runIPCStorageIn (scan "iso") beta `shouldReturn` Right [[("id", VInt 2)]]
it "rows survive a restart on the same data directory" $ do it "rows survive a restart on the same data directory" $ do
withTestSession $ \srv -> do withTestSession $ \srv -> do
+31 -2
View File
@@ -15,6 +15,9 @@ module ChuSQL.Core.Model
, colNames , colNames
, colType , colType
, lookupTable , lookupTable
, resolveTable
, lastTablePart
, deriveQualifier
, allColumns , allColumns
, qualify , qualify
, unqualify , unqualify
@@ -470,6 +473,32 @@ colType t c = lookup c (tableCols t)
lookupTable :: Database -> String -> Either String Table lookupTable :: Database -> String -> Either String Table
lookupTable db t = maybe (Left ("unknown table: " ++ t)) Right (lookup t db) lookupTable db t = maybe (Left ("unknown table: " ++ t)) Right (lookup t db)
-- | 取表名引用的最后一段
lastTablePart :: String -> String
lastTablePart = reverse . takeWhile (/= '.') . reverse
-- | 推导列前缀:有别名用别名,否则取表名末段
deriveQualifier :: Maybe String -> String -> String
deriveQualifier (Just a) _ = a
deriveQualifier Nothing t = lastTablePart t
-- | 按名查表并返回 schema 键:精确、前缀、唯一后缀
resolveTable :: Database -> String -> Either String (String, Table)
resolveTable db name = case lookup name db of
Just table -> Right (name, table)
Nothing
| '.' `elem` name, Just table <- lookup bare db -> Right (bare, table)
| [(key, table)] <- suffixed -> Right (key, table)
| otherwise -> Left (miss suffixed)
where
-- | 引用里去掉库前缀的那一段
bare = lastTablePart name
-- | schema 里以这个表名收尾的键:`sales.users` 能推出裸表名引用
suffixed = [(key, table) | (key, table) <- db, ('.' : name) `isSuffixOf` key]
-- | 一个候选都没有就是查无此表,多于一个就是有歧义
miss [] = "unknown table: " ++ name
miss _ = "ambiguous table: " ++ name
-- | 通配列名 -- | 通配列名
allColumns :: String allColumns :: String
allColumns = "*" allColumns = "*"
@@ -491,5 +520,5 @@ resolveColumn name env = case [(k, v) | (k, v) <- env, matches k] of
[] -> Left ("unknown column: " ++ name) [] -> Left ("unknown column: " ++ name)
_ -> Left ("ambiguous column: " ++ name) _ -> Left ("ambiguous column: " ++ name)
where where
-- | 裸名或带后缀的名称 -- | 裸名、补充了库前缀的写法,或 schema 键自带库前缀的写法
matches k = k == name || ('.' : name) `isSuffixOf` k matches k = k == name || ('.' : name) `isSuffixOf` k || ('.' : k) `isSuffixOf` name
@@ -3,6 +3,7 @@
module ChuSQL.Core.Protocol ( module ChuSQL.Core.Protocol (
SchemaColumn (..), SchemaColumn (..),
Request (..), Request (..),
TxnOp (..),
Response (..), Response (..),
TableInfo (..), TableInfo (..),
Account (..), Account (..),
@@ -96,6 +97,35 @@ data Request
| ReqDropColumn String String | ReqDropColumn String String
| ReqReplaceSchema String [SchemaColumn] [Row] | ReqReplaceSchema String [SchemaColumn] [Row]
| ReqListCatalog | ReqListCatalog
| ReqApplyTransaction [TxnOp]
-- | 事务提交里的一条写操作
data TxnOp
= TxnUpsert String [Row]
| TxnDelete String [Int]
| TxnReplace String [Row]
deriving (Show, Eq)
-- | 事务操作编码成 JSON
instance ToJSON TxnOp where
toJSON (TxnUpsert t rs) =
object
[ "op" .= ("upsert" :: T.Text)
, "table" .= t
, "rows" .= map rowToJSON rs
]
toJSON (TxnDelete t ks) =
object
[ "op" .= ("delete" :: T.Text)
, "table" .= t
, "ids" .= ks
]
toJSON (TxnReplace t rs) =
object
[ "op" .= ("replace" :: T.Text)
, "table" .= t
, "rows" .= map rowToJSON rs
]
-- | 请求编码成 JSON -- | 请求编码成 JSON
instance ToJSON Request where instance ToJSON Request where
@@ -201,6 +231,11 @@ instance ToJSON Request where
, "rows" .= map rowToJSON rs , "rows" .= map rowToJSON rs
] ]
toJSON ReqListCatalog = object ["method" .= ("list_catalog" :: T.Text)] toJSON ReqListCatalog = object ["method" .= ("list_catalog" :: T.Text)]
toJSON (ReqApplyTransaction ops) =
object
[ "method" .= ("apply_transaction" :: T.Text)
, "ops" .= ops
]
-- | 一张表的梗概:列、行数、索引与列统计 -- | 一张表的梗概:列、行数、索引与列统计
data TableInfo = TableInfo data TableInfo = TableInfo
+23 -2
View File
@@ -171,6 +171,22 @@ pub enum Request {
columns: Vec<SchemaColumn>, columns: Vec<SchemaColumn>,
rows: Vec<Row>, rows: Vec<Row>,
}, },
/// 事务提交:把一批写操作当成一个请求原子应用
ApplyTransaction {
ops: Vec<StorageOp>,
},
}
/// 事务提交里的一条写操作
#[derive(Debug, Serialize, Deserialize)]
#[serde(tag = "op", rename_all = "snake_case")]
pub enum StorageOp {
/// 追加写入的行(提交端已先删掉要覆盖的行)
Upsert { table: String, rows: Vec<Row> },
/// 按行 id 删除
Delete { table: String, ids: Vec<i64> },
/// 整表替换(没有整数 id 的表只能这么提交)
Replace { table: String, rows: Vec<Row> },
} }
impl Request { impl Request {
@@ -202,7 +218,8 @@ impl Request {
| Self::ReplaceSchema { table, .. } => Some(table), | Self::ReplaceSchema { table, .. } => Some(table),
Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList
| Self::AccountCreate { .. } | Self::AccountReset { .. } | Self::AccountLogin { .. } | Self::AccountCreate { .. } | Self::AccountReset { .. } | Self::AccountLogin { .. }
| Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus => None, | Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus
| Self::ApplyTransaction { .. } => None,
} }
} }
} }
@@ -237,7 +254,11 @@ pub enum Response {
NoIndex, NoIndex,
Ok, Ok,
/// 系统目录状态 /// 系统目录状态
System { initialized: bool }, System {
initialized: bool,
#[serde(default)]
last_lsn: u64,
},
Error { message: String }, Error { message: String },
Catalog { schemas: Vec<TableSchemaWire> }, Catalog { schemas: Vec<TableSchemaWire> },
} }
+233 -59
View File
@@ -8,9 +8,10 @@ use crate::config::{self, Config, Loaded};
use crate::heap::{scalar_int, HeapTable}; use crate::heap::{scalar_int, HeapTable};
use crate::log; use crate::log;
use crate::protocol::{ use crate::protocol::{
reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row, SchemaColumn, TableSchemaWire, USERS_TABLE, reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row, SchemaColumn, StorageOp,
TableSchemaWire, USERS_TABLE,
}; };
use crate::wal::{Wal, WalOp}; use crate::wal::{committed_ops, Wal, WalOp};
use crate::{log_debug, log_error, log_info, log_warn}; use crate::{log_debug, log_error, log_info, log_warn};
// 进程内存储运行时:Haskell 侧经 ffi.rs 调进来。 // 进程内存储运行时:Haskell 侧经 ffi.rs 调进来。
@@ -196,6 +197,7 @@ impl Server {
/// 把一条 WAL 操作落到磁盘 /// 把一条 WAL 操作落到磁盘
fn apply_op(&self, op: &WalOp) -> std::io::Result<()> { fn apply_op(&self, op: &WalOp) -> std::io::Result<()> {
match op { match op {
WalOp::Commit => Ok(()),
WalOp::HideColumn { table, column } => { WalOp::HideColumn { table, column } => {
let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?; let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
c.remove_column(table, column)?; c.remove_column(table, column)?;
@@ -311,52 +313,119 @@ impl Server {
if self.recovery_required.load(Ordering::Acquire) { if self.recovery_required.load(Ordering::Acquire) {
return Err(std::io::Error::other("storage recovery required")); return Err(std::io::Error::other("storage recovery required"));
} }
{ self.validate_op(op)?;
let c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?; self.group_via_wal(std::slice::from_ref(op))
let validate = |table: &str, rows: &[Row]| -> std::io::Result<()> { }
let Some(schema) = c.describe(table) else { return Ok(()); };
if let Some(name) = rows.iter().flat_map(|r| r.keys()).find(|k| schema.dropped_columns.contains(*k)) { /// 校验一条操作:命中被删的列就拒绝
return Err(std::io::Error::other(format!("unknown column: {name}"))); fn validate_op(&self, op: &WalOp) -> std::io::Result<()> {
} let c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
Ok(()) let validate = |table: &str, rows: &[Row]| -> std::io::Result<()> {
}; let Some(schema) = c.describe(table) else { return Ok(()); };
match op { if let Some(name) = rows.iter().flat_map(|r| r.keys()).find(|k| schema.dropped_columns.contains(*k)) {
WalOp::Insert { table, row } => validate(table, std::slice::from_ref(row))?, return Err(std::io::Error::other(format!("unknown column: {name}")));
WalOp::InsertBatch { table, rows } | WalOp::ReplaceAll { table, rows } => validate(table, rows)?,
WalOp::HideColumn { table, column }
if !c.describe(table).is_some_and(|s| s.columns.iter().any(|col| &col.name == column)) =>
{
return Err(std::io::Error::other(format!("unknown column: {column}")));
}
_ => {}
} }
Ok(())
};
match op {
WalOp::Insert { table, row } => validate(table, std::slice::from_ref(row))?,
WalOp::InsertBatch { table, rows } | WalOp::ReplaceAll { table, rows } => validate(table, rows)?,
WalOp::HideColumn { table, column }
if !c.describe(table).is_some_and(|s| s.columns.iter().any(|col| &col.name == column)) =>
{
return Err(std::io::Error::other(format!("unknown column: {column}")));
}
_ => {}
} }
Ok(())
}
/// 事务提交:一组写操作加一个提交标记,一次落盘
///
/// 组里每条都过 `validate_op`,跟单条写一致;调用方 `dispatch` 已经持有
/// `gate.write`,因此整批对外原子可见。崩在提交标记之前,这组记录在恢复
/// 时整组丢弃;崩在标记之后,逐条按「是否已生效」补写。
fn apply_transaction(&self, ops: &[StorageOp]) -> std::io::Result<()> {
let _guard = self.write_lock.lock().unwrap();
if self.recovery_required.load(Ordering::Acquire) {
return Err(std::io::Error::other("storage recovery required"));
}
let mut wal_ops = Vec::with_capacity(ops.len());
for op in ops {
let table = match op {
StorageOp::Upsert { table, .. }
| StorageOp::Delete { table, .. }
| StorageOp::Replace { table, .. } => table,
};
if reserved_table(table) {
return Err(std::io::Error::other("reserved system table"));
}
if table.is_empty() || !table.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') {
return Err(std::io::Error::other("invalid table name"));
}
let wal_op = match op {
StorageOp::Upsert { table, rows } => WalOp::InsertBatch {
table: table.clone(),
rows: rows.clone(),
},
StorageOp::Delete { table, ids } => WalOp::DeleteKeys {
table: table.clone(),
keys: ids.clone(),
},
StorageOp::Replace { table, rows } => WalOp::ReplaceAll {
table: table.clone(),
rows: rows.clone(),
},
};
self.validate_op(&wal_op)?;
wal_ops.push(wal_op);
}
self.group_via_wal(&wal_ops)
}
/// 一组操作加提交标记落盘,成功后清空日志
fn group_via_wal(&self, ops: &[WalOp]) -> std::io::Result<()> {
std::fs::create_dir_all(&self.cfg.data_dir)?; std::fs::create_dir_all(&self.cfg.data_dir)?;
let wal = &self.wal; let wal = &self.wal;
wal.truncate()?; wal.truncate()?;
let commit_lsn = wal.append_group(ops)?;
wal.append(op)?; for op in ops {
let result = self.apply_op(op);
let result = self.apply_op(op); if result.is_err() && matches!(op, WalOp::HideColumn { .. }) {
self.recovery_required.store(true, Ordering::Release);
if result.is_err() && matches!(op, WalOp::HideColumn { .. }) { return result;
self.recovery_required.store(true, Ordering::Release); }
return result; result?;
} }
for op in ops {
if result.is_ok() {
self.flush_op_tables(op)?; self.flush_op_tables(op)?;
} }
self.flush_all_tables()?;
wal.clear()?; wal.clear()?;
result log_debug!(core, "WAL committed group at LSN {}", commit_lsn);
Ok(())
}
/// 把打开的表都 fsync 一遍
fn flush_all_tables(&self) -> std::io::Result<()> {
let handles: Vec<Arc<Mutex<HeapTable>>> = self
.tables
.lock()
.map_err(|_| std::io::Error::other("table lock poisoned"))?
.values()
.cloned()
.collect();
for handle in handles {
handle.lock().map_err(|_| std::io::Error::other("heap lock poisoned"))?.flush()?;
}
Ok(())
} }
/// 把涉及的表写回磁盘 /// 把涉及的表写回磁盘
fn flush_op_tables(&self, op: &WalOp) -> std::io::Result<()> { fn flush_op_tables(&self, op: &WalOp) -> std::io::Result<()> {
let table = match op { let table = match op {
WalOp::HideColumn { .. } => return Ok(()), WalOp::HideColumn { .. } => return Ok(()),
WalOp::Commit => return Ok(()),
WalOp::Accounts { .. } => USERS_TABLE, WalOp::Accounts { .. } => USERS_TABLE,
WalOp::Insert { table, .. } WalOp::Insert { table, .. }
| WalOp::InsertBatch { table, .. } | WalOp::InsertBatch { table, .. }
@@ -369,21 +438,97 @@ impl Server {
self.with_table(table, |t| t.flush()) self.with_table(table, |t| t.flush())
} }
/// 启动时重放残留 WAL /// 重放残留 WAL:已提交组重放,未提交组丢弃
fn recover_wal(&self) -> std::io::Result<()> { fn recover_wal(&self) -> std::io::Result<()> {
std::fs::create_dir_all(&self.cfg.data_dir)?; std::fs::create_dir_all(&self.cfg.data_dir)?;
let wal = &self.wal; let wal = &self.wal;
match wal.read()? { let scan = wal.read_all()?;
None => Ok(()), if scan.records.is_empty() {
Some(op) => { if scan.good_len > 0 {
log_warn!(core, "replaying incomplete WAL entry"); wal.truncate_to(scan.good_len)?;
self.apply_op(&op)?;
self.flush_op_tables(&op)?;
wal.clear()?;
log_info!(core, "WAL replay complete");
Ok(())
} }
return Ok(());
} }
let records = committed_ops(&scan.records);
if records.is_empty() {
log_warn!(core, "discarding uncommitted WAL records");
wal.clear()?;
return Ok(());
}
let max_lsn = scan
.records
.iter()
.rev()
.find(|(_, op)| *op == WalOp::Commit)
.map(|(lsn, _)| *lsn)
.or_else(|| records.last().map(|(lsn, _)| *lsn))
.unwrap_or(0);
log_warn!(core, "replaying {} committed WAL records", records.len());
for (_, op) in &records {
self.replay_op(op)?;
self.flush_op_tables(op)?;
}
self.flush_all_tables()?;
wal.write_checkpoint(max_lsn)?;
wal.clear()?;
log_info!(core, "WAL replay complete");
Ok(())
}
/// 重放一条操作:数据已经生效的不再重复写
fn replay_op(&self, op: &WalOp) -> std::io::Result<()> {
if self.op_already_applied(op)? {
log_debug!(core, "WAL record already applied, skipping");
return Ok(());
}
self.apply_op(op)
}
/// 这条操作对应的数据是不是已经落盘了
fn op_already_applied(&self, op: &WalOp) -> std::io::Result<bool> {
match op {
WalOp::Insert { table, row } => self.applied_rows(table, std::slice::from_ref(row)),
WalOp::InsertBatch { table, rows } => self.applied_rows(table, rows),
WalOp::ReplaceAll { table, rows } => {
if self.catalog.lock().unwrap().describe(table).is_none() {
return Ok(false);
}
let stored = self.with_table(table, |t| t.scan())?;
Ok(stored == *rows)
}
WalOp::DeleteKeys { table, keys } => {
if keys.is_empty() {
return Ok(true);
}
if self.catalog.lock().unwrap().describe(table).is_none() {
return Ok(false);
}
let rows = self.with_table(table, |t| t.scan())?;
let present = rows.iter().any(|row| {
row.get("id")
.and_then(scalar_int)
.is_some_and(|id| keys.contains(&id))
});
Ok(!present)
}
WalOp::HideColumn { table, column } => {
let c = self.catalog.lock().unwrap();
Ok(c.describe(table).is_some_and(|s| !s.columns.iter().any(|col| &col.name == column)))
}
_ => Ok(false),
}
}
/// 这些行是不是都已经在表里
fn applied_rows(&self, table: &str, rows: &[Row]) -> std::io::Result<bool> {
if rows.is_empty() {
return Ok(true);
}
if self.catalog.lock().unwrap().describe(table).is_none() {
return Ok(false);
}
let stored = self.with_table(table, |t| t.scan())?;
Ok(rows.iter().all(|row| stored.iter().any(|one| one == row)))
} }
/// 启动时重建列统计 /// 启动时重建列统计
@@ -607,6 +752,11 @@ impl Server {
.unwrap_or(false) .unwrap_or(false)
} }
/// 最近一次落盘的检查点 LSN
fn last_checkpoint(&self) -> u64 {
self.wal.read_checkpoint().ok().flatten().unwrap_or(0)
}
/// 读账号表;表没登记或数据文件缺失都算错 /// 读账号表;表没登记或数据文件缺失都算错
fn load_accounts(&self) -> std::io::Result<Vec<Account>> { fn load_accounts(&self) -> std::io::Result<Vec<Account>> {
use std::io::Error; use std::io::Error;
@@ -774,16 +924,29 @@ impl Server {
} }
Request::BootstrapSystem { user, password_hash } => { Request::BootstrapSystem { user, password_hash } => {
match self.bootstrap_system(user.as_deref(), password_hash.as_deref()) { match self.bootstrap_system(user.as_deref(), password_hash.as_deref()) {
Ok(_) => Response::System { initialized: true }, Ok(_) => Response::System { initialized: true, last_lsn: self.last_checkpoint() },
Err(e) => Response::Error { message: e.to_string() }, Err(e) => Response::Error { message: e.to_string() },
} }
} }
Request::SystemStatus => Response::System { initialized: self.system_initialized() }, Request::SystemStatus => Response::System { initialized: self.system_initialized(), last_lsn: self.last_checkpoint() },
Request::Ping => { Request::Ping => {
log_debug!(request, "ping"); log_debug!(request, "ping");
Response::Pong Response::Pong
} }
Request::ApplyTransaction { ops } => {
log_debug!(request, "apply_transaction ops={}", ops.len());
match self.apply_transaction(&ops) {
Ok(()) => Response::Ok,
Err(e) => {
log_warn!(request, "apply_transaction: {}", e);
Response::Error {
message: format!("apply_transaction: {e}"),
}
}
}
}
Request::Scan { table, columns } => { Request::Scan { table, columns } => {
log_debug!(request, "scan table={}", table); log_debug!(request, "scan table={}", table);
@@ -1347,8 +1510,8 @@ fn reserved_database(name: &str) -> bool {
/// 每个库拥有独立的目录、字典、句柄和 WAL。 /// 每个库拥有独立的目录、字典、句柄和 WAL。
struct Databases { struct Databases {
root: Config, root: Config,
system: Server, system: Arc<Server>,
named: Mutex<HashMap<String, Server>>, named: Mutex<HashMap<String, Arc<Server>>>,
} }
impl Databases { impl Databases {
@@ -1356,7 +1519,7 @@ impl Databases {
fn new(cfg: Config) -> std::io::Result<Self> { fn new(cfg: Config) -> std::io::Result<Self> {
// 系统库按需创建:目录、字典与系统表缺失时自动补齐。 // 系统库按需创建:目录、字典与系统表缺失时自动补齐。
// 其余库只在 data/databases/<name>/ 下已存在时才装载,启动不会凭空建库。 // 其余库只在 data/databases/<name>/ 下已存在时才装载,启动不会凭空建库。
let system = Self::open(&cfg, SYSTEM_DATABASE)?; let system = Arc::new(Self::open(&cfg, SYSTEM_DATABASE)?);
let root = cfg.data_dir.join("databases"); let root = cfg.data_dir.join("databases");
std::fs::create_dir_all(&root)?; std::fs::create_dir_all(&root)?;
let canonical_root = std::fs::canonicalize(&root)?; let canonical_root = std::fs::canonicalize(&root)?;
@@ -1379,7 +1542,7 @@ impl Databases {
server.rebuild_stats()?; server.rebuild_stats()?;
server.migrate_data()?; server.migrate_data()?;
server.compact_tables()?; server.compact_tables()?;
named.insert(name, server); named.insert(name, Arc::new(server));
} }
} }
Ok(Self { root: cfg, system, named: Mutex::new(named) }) Ok(Self { root: cfg, system, named: Mutex::new(named) })
@@ -1413,9 +1576,13 @@ impl Databases {
let mut named = self.named.lock().map_err(|_| Error::other("database lock poisoned"))?; let mut named = self.named.lock().map_err(|_| Error::other("database lock poisoned"))?;
match method.as_str() { match method.as_str() {
"all_catalogs" => { "all_catalogs" => {
let servers: Vec<(String, Arc<Server>)> =
std::iter::once((SYSTEM_DATABASE.to_string(), Arc::clone(&self.system)))
.chain(named.iter().map(|(name, server)| (name.clone(), Arc::clone(server))))
.collect();
drop(named);
let mut schemas = Vec::new(); let mut schemas = Vec::new();
for (database, server) in std::iter::once((SYSTEM_DATABASE, &self.system)) for (database, server) in servers {
.chain(named.iter().map(|(name, server)| (name.as_str(), server))) {
match server.handle_request(Request::ListCatalog) { match server.handle_request(Request::ListCatalog) {
Response::Catalog { schemas: entries } => { Response::Catalog { schemas: entries } => {
for mut entry in entries { for mut entry in entries {
@@ -1442,23 +1609,30 @@ impl Databases {
std::fs::create_dir(&cfg.data_dir)?; std::fs::create_dir(&cfg.data_dir)?;
let server = Server::new(cfg)?; let server = Server::new(cfg)?;
server.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.save(server.catalog_path())?; server.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.save(server.catalog_path())?;
named.insert(name, server); named.insert(name, Arc::new(server));
return Ok(Response::Ok); return Ok(Response::Ok);
} }
"drop_database" => { "drop_database" => {
if name == SYSTEM_DATABASE { return Err(Error::other("cannot drop the system database")); } if name == SYSTEM_DATABASE { return Err(Error::other("cannot drop the system database")); }
let server = named.get(&name).ok_or_else(|| Error::other("unknown database"))?; let holder = named.remove(&name).ok_or_else(|| Error::other("unknown database"))?;
let server = match Arc::try_unwrap(holder) {
Ok(server) => server,
Err(holder) => {
named.insert(name, holder);
return Err(Error::other("database is in use"));
}
};
let root = std::fs::canonicalize(self.root.data_dir.join("databases"))?; let root = std::fs::canonicalize(self.root.data_dir.join("databases"))?;
let target = std::fs::canonicalize(&server.cfg.data_dir)?; let target = std::fs::canonicalize(&server.cfg.data_dir)?;
if target.parent() != Some(root.as_path()) || !target.join("catalog.json").is_file() { if target.parent() != Some(root.as_path()) || !target.join("catalog.json").is_file() {
return Err(Error::other("invalid database directory")); return Err(Error::other("invalid database directory"));
} }
let config = server.cfg.clone(); let config = server.cfg.clone();
drop(named.remove(&name)); drop(server);
let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_err(Error::other)?.as_nanos(); let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_err(Error::other)?.as_nanos();
let tombstone = root.join(format!(".dropped-{name}-{stamp}")); let tombstone = root.join(format!(".dropped-{name}-{stamp}"));
if let Err(error) = std::fs::rename(&target, &tombstone) { if let Err(error) = std::fs::rename(&target, &tombstone) {
named.insert(name, Server::new(config)?); named.insert(name, Arc::new(Server::new(config)?));
return Err(error); return Err(error);
} }
std::fs::remove_dir_all(&tombstone).map_err(|e| Error::other(format!("database dropped; directory cleanup failed: {e}")))?; std::fs::remove_dir_all(&tombstone).map_err(|e| Error::other(format!("database dropped; directory cleanup failed: {e}")))?;
@@ -1482,10 +1656,10 @@ impl Databases {
| Request::AccountReset { .. } | Request::AccountLogin { .. } | Request::AccountReset { .. } | Request::AccountLogin { .. }
| Request::AccountDrop { .. } | Request::BootstrapSystem { .. } | Request::AccountDrop { .. } | Request::BootstrapSystem { .. }
| Request::SystemStatus | Request::Ping); | Request::SystemStatus | Request::Ping);
let server = if global { &self.system } if !global && selected.is_empty() { return Err(Error::other("no database selected")); }
else if selected.is_empty() { return Err(Error::other("no database selected")); } let server = if global || selected == SYSTEM_DATABASE { Arc::clone(&self.system) }
else if selected == SYSTEM_DATABASE { &self.system } else { Arc::clone(named.get(&selected).ok_or_else(|| Error::other("unknown database"))?) };
else { named.get(&selected).ok_or_else(|| Error::other("unknown database"))? }; drop(named);
Ok(server.handle_request(request)) Ok(server.handle_request(request))
} }
} }
+148 -28
View File
@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
use crate::protocol::{Account, Row, SchemaColumn, USERS_TABLE}; use crate::protocol::{Account, Row, SchemaColumn, USERS_TABLE};
// 预写日志:先记操作并 fsync,再改数据;重启重放残留。 // 预写日志:先记操作并 fsync,再改数据;重启按 LSN 重放已提交的组。
const OP_INSERT: u8 = 1; const OP_INSERT: u8 = 1;
const OP_REPLACE_ALL: u8 = 2; const OP_REPLACE_ALL: u8 = 2;
@@ -18,6 +18,7 @@ const OP_ACCOUNTS: u8 = 6;
const OP_REPLACE_SCHEMA: u8 = 7; const OP_REPLACE_SCHEMA: u8 = 7;
const OP_HIDE_COLUMN: u8 = 8; const OP_HIDE_COLUMN: u8 = 8;
const OP_COMPACT: u8 = 9; const OP_COMPACT: u8 = 9;
const OP_COMMIT: u8 = 10;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub enum WalOp { pub enum WalOp {
@@ -50,6 +51,8 @@ pub enum WalOp {
columns: Vec<SchemaColumn>, columns: Vec<SchemaColumn>,
rows: Vec<Row>, rows: Vec<Row>,
}, },
/// 一组写操作的提交标记;没有标记的组算未提交
Commit,
} }
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize)]
@@ -67,6 +70,13 @@ struct ReplaceSchemaPayload {
pub struct Wal { pub struct Wal {
path: PathBuf, path: PathBuf,
handle: Mutex<Option<File>>, handle: Mutex<Option<File>>,
next_lsn: Mutex<u64>,
}
/// 一次扫描的结果:记录与完好字节长度
pub struct WalScan {
pub records: Vec<(u64, WalOp)>,
pub good_len: usize,
} }
impl Wal { impl Wal {
@@ -75,6 +85,7 @@ impl Wal {
Wal { Wal {
path: path.as_ref().to_path_buf(), path: path.as_ref().to_path_buf(),
handle: Mutex::new(None), handle: Mutex::new(None),
next_lsn: Mutex::new(0),
} }
} }
@@ -83,6 +94,11 @@ impl Wal {
&self.path &self.path
} }
/// 检查点文件路径
pub fn checkpoint_path(&self) -> PathBuf {
self.path.with_extension("checkpoint")
}
/// 拿常驻句柄(没有就开一个) /// 拿常驻句柄(没有就开一个)
fn with_handle<R>(&self, f: impl FnOnce(&mut File) -> io::Result<R>) -> io::Result<R> { fn with_handle<R>(&self, f: impl FnOnce(&mut File) -> io::Result<R>) -> io::Result<R> {
let mut slot = self.handle.lock().unwrap(); let mut slot = self.handle.lock().unwrap();
@@ -99,6 +115,15 @@ impl Wal {
f(slot.as_mut().expect("just opened")) f(slot.as_mut().expect("just opened"))
} }
/// 把一段记录写到文件尾
fn write_at_end(&self, bytes: &[u8]) -> io::Result<()> {
self.with_handle(|f| {
f.seek(SeekFrom::End(0))?;
f.write_all(bytes)?;
f.sync_data()
})
}
/// 截断到空并落盘 /// 截断到空并落盘
pub fn clear(&self) -> io::Result<()> { pub fn clear(&self) -> io::Result<()> {
self.with_handle(|f| { self.with_handle(|f| {
@@ -112,34 +137,136 @@ impl Wal {
self.with_handle(truncate) self.with_handle(truncate)
} }
/// 追加一条并 fsync /// 截到指定长度(去掉撕裂的尾巴)
pub fn append(&self, op: &WalOp) -> io::Result<()> { pub fn truncate_to(&self, len: usize) -> io::Result<()> {
let bytes = encode(op)?;
self.with_handle(|f| { self.with_handle(|f| {
f.write_all(&bytes)?; f.set_len(len as u64)?;
f.seek(SeekFrom::Start(0))?;
f.sync_data() f.sync_data()
}) })
} }
/// 读当前 WAL;空则 None /// 追加一条并 fsync
pub fn append(&self, op: &WalOp) -> io::Result<()> {
let bytes = self.encode_next(op)?;
self.write_at_end(&bytes)
}
/// 追加一组操作与提交标记,返回提交 LSN
pub fn append_group(&self, ops: &[WalOp]) -> io::Result<u64> {
let mut frames = Vec::new();
for op in ops {
frames.extend_from_slice(&self.encode_next(op)?);
}
let commit_lsn = self.take_lsn()?;
frames.extend_from_slice(&encode(&WalOp::Commit, commit_lsn)?);
self.write_at_end(&frames)?;
Ok(commit_lsn)
}
/// 读当前 WAL 的第一条操作;空则 None
pub fn read(&self) -> io::Result<Option<WalOp>> { pub fn read(&self) -> io::Result<Option<WalOp>> {
let scan = self.read_all()?;
Ok(scan.records.into_iter().find_map(|(_, op)| {
if op == WalOp::Commit {
None
} else {
Some(op)
}
}))
}
/// 读全部记录,撕裂或损坏的尾巴按长度截断
pub fn read_all(&self) -> io::Result<WalScan> {
if !self.path.exists() { if !self.path.exists() {
return Ok(None); return Ok(WalScan {
records: Vec::new(),
good_len: 0,
});
} }
let mut f = File::open(&self.path)?; let mut f = File::open(&self.path)?;
let mut buf = Vec::new(); let mut buf = Vec::new();
f.read_to_end(&mut buf)?; f.read_to_end(&mut buf)?;
if buf.is_empty() { let mut records = Vec::new();
let mut offset = 0usize;
while offset + 4 <= buf.len() {
let body_len =
u32::from_le_bytes([buf[offset], buf[offset + 1], buf[offset + 2], buf[offset + 3]])
as usize;
let end = offset + 4 + body_len;
if end > buf.len() {
break;
}
match decode_body(&buf[offset + 4..end]) {
Ok(record) => {
records.push(record);
offset = end;
}
Err(_) => break,
}
}
Ok(WalScan {
records,
good_len: offset,
})
}
/// 记录检查点:到此为止的 LSN 都已落盘
pub fn write_checkpoint(&self, lsn: u64) -> io::Result<()> {
let mut f = File::create(self.checkpoint_path())?;
f.write_all(&lsn.to_le_bytes())?;
f.sync_all()
}
/// 读检查点 LSN
pub fn read_checkpoint(&self) -> io::Result<Option<u64>> {
let path = self.checkpoint_path();
if !path.exists() {
return Ok(None); return Ok(None);
} }
match decode(&buf) { let bytes = std::fs::read(&path)?;
Ok(opt) => Ok(opt), if bytes.len() < 8 {
Err(e) => Err(io::Error::new( return Ok(None);
io::ErrorKind::InvalidData, }
format!("corrupt WAL: {}", e), let mut raw = [0u8; 8];
)), raw.copy_from_slice(&bytes[..8]);
Ok(Some(u64::from_le_bytes(raw)))
}
/// 取下一条 LSN
fn take_lsn(&self) -> io::Result<u64> {
let mut slot = self.next_lsn.lock().unwrap();
if *slot == 0 {
let last = self.read_all()?.records.last().map(|(lsn, _)| *lsn).unwrap_or(0);
*slot = last + 1;
}
let lsn = *slot;
*slot += 1;
Ok(lsn)
}
/// 编一条带新 LSN 的记录
fn encode_next(&self, op: &WalOp) -> io::Result<Vec<u8>> {
let lsn = self.take_lsn()?;
encode(op, lsn)
}
}
/// 已提交的记录:提交标记前的整组;无标记的单条按旧格式算已提交
pub fn committed_ops(records: &[(u64, WalOp)]) -> Vec<(u64, WalOp)> {
let mut out = Vec::new();
let mut pending: Vec<(u64, WalOp)> = Vec::new();
for (lsn, op) in records {
if *op == WalOp::Commit {
out.append(&mut pending);
} else {
pending.push((*lsn, op.clone()));
} }
} }
if pending.len() == 1 {
out.append(&mut pending);
}
out
} }
/// 清空文件并把读写位置归零 /// 清空文件并把读写位置归零
@@ -150,7 +277,7 @@ fn truncate(f: &mut File) -> io::Result<()> {
} }
/// 把操作编成字节 /// 把操作编成字节
fn encode(op: &WalOp) -> io::Result<Vec<u8>> { fn encode(op: &WalOp, lsn: u64) -> io::Result<Vec<u8>> {
let (op_type, table, payload) = match op { let (op_type, table, payload) = match op {
WalOp::HideColumn { table, column } => (OP_HIDE_COLUMN, table.clone(), WalOp::HideColumn { table, column } => (OP_HIDE_COLUMN, table.clone(),
serde_json::to_vec(column).map_err(io::Error::other)?), serde_json::to_vec(column).map_err(io::Error::other)?),
@@ -192,11 +319,11 @@ fn encode(op: &WalOp) -> io::Result<Vec<u8>> {
.map_err(io::Error::other)?; .map_err(io::Error::other)?;
(OP_REPLACE_SCHEMA, table.clone(), p) (OP_REPLACE_SCHEMA, table.clone(), p)
} }
WalOp::Commit => (OP_COMMIT, String::new(), Vec::new()),
}; };
let table_bytes = table.as_bytes(); let table_bytes = table.as_bytes();
let lsn: u64 = 1;
let mut body = Vec::with_capacity(8 + 1 + 2 + table_bytes.len() + payload.len()); let mut body = Vec::with_capacity(8 + 1 + 2 + table_bytes.len() + payload.len());
body.extend_from_slice(&lsn.to_le_bytes()); body.extend_from_slice(&lsn.to_le_bytes());
body.push(op_type); body.push(op_type);
@@ -210,20 +337,12 @@ fn encode(op: &WalOp) -> io::Result<Vec<u8>> {
Ok(out) Ok(out)
} }
/// 从字节解回操作 /// 从一个帧体解出 LSN 与操作
fn decode(buf: &[u8]) -> Result<Option<WalOp>, String> { fn decode_body(body: &[u8]) -> Result<(u64, WalOp), String> {
if buf.len() < 4 {
return Ok(None);
}
let body_len = u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]) as usize;
if buf.len() < 4 + body_len {
return Ok(None);
}
let body = &buf[4..4 + body_len];
if body.len() < 8 + 1 + 2 { if body.len() < 8 + 1 + 2 {
return Err("body too short".into()); return Err("body too short".into());
} }
let _lsn = u64::from_le_bytes([ let lsn = u64::from_le_bytes([
body[0], body[1], body[2], body[3], body[4], body[5], body[6], body[7], body[0], body[1], body[2], body[3], body[4], body[5], body[6], body[7],
]); ]);
let op_type = body[8]; let op_type = body[8];
@@ -236,6 +355,7 @@ fn decode(buf: &[u8]) -> Result<Option<WalOp>, String> {
let payload = &body[11 + table_len..]; let payload = &body[11 + table_len..];
let op = match op_type { let op = match op_type {
OP_COMMIT => WalOp::Commit,
OP_HIDE_COLUMN => { OP_HIDE_COLUMN => {
let column = serde_json::from_slice(payload) let column = serde_json::from_slice(payload)
.map_err(|e| format!("bad hide_column payload: {e}"))?; .map_err(|e| format!("bad hide_column payload: {e}"))?;
@@ -292,5 +412,5 @@ fn decode(buf: &[u8]) -> Result<Option<WalOp>, String> {
} }
other => return Err(format!("unknown op type: {}", other)), other => return Err(format!("unknown op type: {}", other)),
}; };
Ok(Some(op)) Ok((lsn, op))
} }
+186
View File
@@ -216,6 +216,67 @@ fn concurrent_shard_reads_exclude_writers() -> Result<(), Box<dyn std::error::Er
Ok(()) Ok(())
} }
/// 跨库并发互不串数据,目录查询不被写饿死
#[test]
fn concurrent_requests_across_databases_stay_isolated() -> Result<(), Box<dyn std::error::Error>> {
let (_srv, _data) = start_server();
let mut c = connect()?;
let request = |c: &mut Conn, value: serde_json::Value| -> Result<serde_json::Value, serde_json::Error> {
serde_json::from_str(&send(c, &value.to_string()))
};
let databases = ["alpha", "beta"];
for database in databases {
assert_eq!(request(&mut c, serde_json::json!({"method":"create_database","database":database}))?["status"], "ok");
assert_eq!(
request(&mut c, serde_json::json!({"method":"create_table","database":database,"table":"items","columns":[{"name":"id","ty":"int"}]}))?["status"],
"ok"
);
}
let storage = CURRENT.with(|slot| slot.borrow().as_ref().cloned().expect("storage opened"));
let written = 40;
std::thread::scope(|scope| {
let writers: Vec<_> = databases
.iter()
.map(|database| {
let storage = Arc::clone(&storage);
let database = database.to_string();
scope.spawn(move || {
for i in 1..=written {
let line = format!(
r#"{{"method":"insert","database":"{database}","table":"items","row":{{"id":{i}}}}}"#
);
let value: serde_json::Value = serde_json::from_str(&storage.request_line(&line)).expect("json");
assert_eq!(value["status"], "ok", "{database}: {value}");
}
database
})
})
.collect();
let catalog = {
let storage = Arc::clone(&storage);
scope.spawn(move || {
for _ in 0..40 {
for line in [r#"{"method":"list_databases"}"#, r#"{"method":"all_catalogs"}"#] {
let value: serde_json::Value = serde_json::from_str(&storage.request_line(line)).expect("json");
assert_ne!(value["status"], "error", "{line}: {value}");
}
}
})
};
for writer in writers {
writer.join().expect("writer thread");
}
catalog.join().expect("catalog thread");
});
for database in databases {
let rows = request(&mut c, serde_json::json!({"method":"scan","database":database,"table":"items"}))?["rows"].clone();
assert_eq!(rows.as_array().expect("rows").len(), written, "{database} 行数不对");
}
Ok(())
}
/// 删列不改堆字节,重启后不再暴露该列 /// 删列不改堆字节,重启后不再暴露该列
#[test] #[test]
fn drop_column_keeps_heap_bytes_and_hides_values_after_restart() -> Result<(), Box<dyn std::error::Error>> { fn drop_column_keeps_heap_bytes_and_hides_values_after_restart() -> Result<(), Box<dyn std::error::Error>> {
@@ -744,6 +805,12 @@ fn account_failure_preserves_wal_and_blocks_writes() -> Result<(), Box<dyn std::
let mut c = connect()?; let mut c = connect()?;
let recovered = send(&mut c, r#"{"method":"accounts_list"}"#); let recovered = send(&mut c, r#"{"method":"accounts_list"}"#);
assert!(recovered.contains("hash-one"), "{recovered}"); assert!(recovered.contains("hash-one"), "{recovered}");
let status: serde_json::Value =
serde_json::from_str(&send(&mut c, r#"{"method":"system_status"}"#))?;
assert!(
status["last_lsn"].as_u64().unwrap_or(0) >= 1,
"重放后检查点要推进: {status}"
);
Ok(()) Ok(())
} }
@@ -1015,6 +1082,125 @@ fn delete_keys_removes_row_and_index_entry() {
assert!(r.contains(r#""status":"ok""#), "id 2 should be insertable again: {}", r); assert!(r.contains(r#""status":"ok""#), "id 2 should be insertable again: {}", r);
} }
/// 事务提交:一批写操作在同一个请求里落地
#[test]
fn apply_transaction_lands_upserts_and_deletes_together() {
let (_srv, _data) = start_server();
let mut c = connect().unwrap();
for (id, name) in [(1, "Alice"), (2, "Bob")] {
let req = format!(
r#"{{"method":"insert","table":"txn_t","row":{{"id":{},"name":"{}"}}}}"#,
id, name
);
send(&mut c, &req);
}
let batch = r#"{"method":"apply_transaction","ops":[
{"op":"delete","table":"txn_t","ids":[1]},
{"op":"upsert","table":"txn_t","rows":[{"id":3,"name":"Carol"}]}
]}"#;
let r = send(&mut c, batch);
assert!(r.contains(r#""status":"ok""#), "batch: {}", r);
let r = send(&mut c, r#"{"method":"scan","table":"txn_t"}"#);
assert!(!r.contains("Alice"), "deleted row should be gone: {}", r);
assert!(r.contains("Bob") && r.contains("Carol"), "kept and added rows: {}", r);
for line in [
r#"{"method":"apply_transaction","ops":[{"op":"upsert","table":"__system_users","rows":[{"id":1}]}]}"#,
r#"{"method":"apply_transaction","ops":[{"op":"delete","table":"../evil","ids":[1]}]}"#,
] {
let r = send(&mut c, line);
assert!(r.contains(r#""status":"error""#), "bad op rejected: {} -> {}", line, r);
}
// 没有整数 id 的表用整表替换提交
let r = send(&mut c, r#"{"method":"insert","table":"txn_no_id","row":{"name":"old"}}"#);
assert!(r.contains(r#""status":"ok""#), "insert: {}", r);
let replace = r#"{"method":"apply_transaction","ops":[{"op":"replace","table":"txn_no_id","rows":[{"name":"only"}]}]}"#;
let r = send(&mut c, replace);
assert!(r.contains(r#""status":"ok""#), "replace: {}", r);
let r = send(&mut c, r#"{"method":"scan","table":"txn_no_id"}"#);
assert!(r.contains("only") && !r.contains("old"), "replaced rows: {}", r);
}
/// 标记已落盘、数据未改的崩溃现场:重启重放并推进检查点
#[test]
fn committed_group_replays_after_restart() -> Result<(), Box<dyn std::error::Error>> {
use chusql_core_storage::wal::{Wal, WalOp};
let (server, data) = start_server();
let mut c = connect()?;
request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":1,"secret":"old"}}))?;
drop(c);
drop(server);
let mut added = std::collections::HashMap::new();
added.insert("id".to_string(), serde_json::json!(2));
added.insert("secret".to_string(), serde_json::json!("new"));
let group = vec![
WalOp::InsertBatch {
table: "items".into(),
rows: vec![added.clone()],
},
WalOp::DeleteKeys {
table: "items".into(),
keys: vec![1],
},
];
let wal = Wal::new(data.path().join("databases/main/wal.log"));
let commit_lsn = wal.append_group(&group)?;
drop(wal);
let _server = start_server_in(data.path());
let mut c = connect()?;
let rows: serde_json::Value =
serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"items"}"#))?;
assert_eq!(rows["rows"], serde_json::json!([{"id":2,"secret":"new"}]));
assert_eq!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len(), 0);
let checkpoint = std::fs::read(data.path().join("databases/main/wal.checkpoint"))?;
assert_eq!(checkpoint.len(), 8, "检查点要落盘");
assert_eq!(
u64::from_le_bytes(checkpoint[..8].try_into()?),
commit_lsn,
"检查点记到提交 LSN"
);
Ok(())
}
/// 没有提交标记的记录组在重启时整组丢掉
#[test]
fn uncommitted_group_is_discarded_on_restart() -> Result<(), Box<dyn std::error::Error>> {
use chusql_core_storage::wal::{Wal, WalOp};
let (server, data) = start_server();
let mut c = connect()?;
request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":1,"secret":"old"}}))?;
drop(c);
drop(server);
let mut lost = std::collections::HashMap::new();
lost.insert("id".to_string(), serde_json::json!(2));
lost.insert("secret".to_string(), serde_json::json!("lost"));
let wal = Wal::new(data.path().join("databases/main/wal.log"));
wal.append(&WalOp::InsertBatch {
table: "items".into(),
rows: vec![lost.clone()],
})?;
wal.append(&WalOp::InsertBatch {
table: "items".into(),
rows: vec![lost],
})?;
drop(wal);
let _server = start_server_in(data.path());
let mut c = connect()?;
let rows: serde_json::Value =
serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"items"}"#))?;
assert_eq!(rows["rows"], serde_json::json!([{"id":1,"secret":"old"}]));
assert_eq!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len(), 0);
Ok(())
}
/// 给第二列建索引后能按那一列查 /// 给第二列建索引后能按那一列查
#[test] #[test]
fn create_index_on_secondary_column() { fn create_index_on_secondary_column() {
+121 -2
View File
@@ -1,8 +1,8 @@
use chusql_core_storage::protocol::Row; use chusql_core_storage::protocol::Row;
use chusql_core_storage::wal::{Wal, WalOp}; use chusql_core_storage::wal::{committed_ops, Wal, WalOp};
use serde_json::json; use serde_json::json;
// WAL 测试:追加、读取、清空、截断尾巴。 // WAL 测试:追加、读取、清空、截断尾巴、LSN 与提交标记。
/// 用键值对构造一行 /// 用键值对构造一行
@@ -99,3 +99,122 @@ fn truncated_tail_is_ignored() {
assert!(wal.read().unwrap().is_none()); assert!(wal.read().unwrap().is_none());
} }
/// 每条记录带递增的 LSN,重开也从文件里的最大号往下走
#[test]
fn every_record_gets_an_increasing_lsn() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("wal.log");
let wal = Wal::new(&path);
for id in 1..=3 {
wal.append(&WalOp::Insert {
table: "t".into(),
row: row(&[("id", json!(id))]),
})
.unwrap();
}
let scan = wal.read_all().unwrap();
let lsns: Vec<u64> = scan.records.iter().map(|(lsn, _)| *lsn).collect();
assert_eq!(lsns, vec![1, 2, 3]);
assert_eq!(scan.good_len as u64, std::fs::metadata(&path).unwrap().len());
let reopened = Wal::new(&path);
reopened
.append(&WalOp::Insert {
table: "t".into(),
row: row(&[("id", json!(4))]),
})
.unwrap();
assert_eq!(reopened.read_all().unwrap().records.last().unwrap().0, 4);
}
/// 一组操作写完带提交标记,标记的 LSN 也在记录里
#[test]
fn append_group_writes_a_commit_marker() {
let dir = tempfile::tempdir().unwrap();
let wal = Wal::new(dir.path().join("wal.log"));
let group = [
WalOp::InsertBatch {
table: "t".into(),
rows: vec![row(&[("id", json!(2))])],
},
WalOp::DeleteKeys {
table: "t".into(),
keys: vec![1],
},
];
let commit_lsn = wal.append_group(&group).unwrap();
let scan = wal.read_all().unwrap();
assert_eq!(scan.records.len(), 3);
assert_eq!(scan.records.last().unwrap().1, WalOp::Commit);
assert_eq!(scan.records.last().unwrap().0, commit_lsn);
assert_eq!(wal.read().unwrap().unwrap(), group[0]);
assert_eq!(
committed_ops(&scan.records),
vec![(1, group[0].clone()), (2, group[1].clone())]
);
}
/// 没有提交标记的记录组算未提交,旧格式的单条仍算已提交
#[test]
fn records_without_a_commit_marker_stay_uncommitted() {
let dir = tempfile::tempdir().unwrap();
let wal = Wal::new(dir.path().join("wal.log"));
for id in 1..=2 {
wal.append(&WalOp::Insert {
table: "t".into(),
row: row(&[("id", json!(id))]),
})
.unwrap();
}
assert!(committed_ops(&wal.read_all().unwrap().records).is_empty());
wal.clear().unwrap();
wal.append(&WalOp::Insert {
table: "t".into(),
row: row(&[("id", json!(9))]),
})
.unwrap();
assert_eq!(committed_ops(&wal.read_all().unwrap().records).len(), 1);
}
/// 撕成两半的尾巴能读出完好长度并截掉
#[test]
fn trailing_torn_frame_can_be_truncated() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("wal.log");
let wal = Wal::new(&path);
for id in 1..=2 {
wal.append(&WalOp::Insert {
table: "t".into(),
row: row(&[("id", json!(id))]),
})
.unwrap();
}
let full = std::fs::read(&path).unwrap();
std::fs::write(&path, &full[..full.len() - 3]).unwrap();
let scan = wal.read_all().unwrap();
assert_eq!(scan.records.len(), 1);
assert!(scan.good_len < full.len() - 3);
wal.truncate_to(scan.good_len).unwrap();
assert_eq!(std::fs::metadata(&path).unwrap().len(), scan.good_len as u64);
assert_eq!(wal.read_all().unwrap().records.len(), 1);
}
/// 检查点写读一致
#[test]
fn checkpoint_round_trips() {
let dir = tempfile::tempdir().unwrap();
let wal = Wal::new(dir.path().join("wal.log"));
assert_eq!(wal.read_checkpoint().unwrap(), None);
wal.write_checkpoint(7).unwrap();
assert_eq!(wal.read_checkpoint().unwrap(), Some(7));
}
+62 -79
View File
@@ -6,7 +6,6 @@ module ChuSQL.Server.Backend (
columnsFromStatement, columnsFromStatement,
currentStamp, currentStamp,
exprTables, exprTables,
inDatabase,
internalTable, internalTable,
grantsTable, grantsTable,
ipcBackend, ipcBackend,
@@ -23,20 +22,23 @@ module ChuSQL.Server.Backend (
import ChuSQL.Core.Engine (runStatement, runStatementM) import ChuSQL.Core.Engine (runStatement, runStatementM)
import ChuSQL.Core.Model import ChuSQL.Core.Model
import ChuSQL.Core.Engine.Semantic (prepare) import ChuSQL.Core.Engine.Semantic (prepare)
import ChuSQL.Core.Engine.Storage (MonadStorage (schema)) import ChuSQL.Core.Engine.Storage (MonadStorage (schema, snapshot))
import ChuSQL.Core.Engine.Storage.IPC ( import ChuSQL.Core.Engine.Storage.IPC (
Account (..), Account (..),
Env,
IPCStorage (..), IPCStorage (..),
Request (..), Request (..),
Response (..), Response (..),
SchemaColumn (..), SchemaColumn (..),
TableInfo (..), TableInfo (..),
envForDatabase,
sendRawRequest, sendRawRequest,
sendRequest, sendRequest,
) )
import ChuSQL.Core.Engine.Syntax.AST import ChuSQL.Core.Engine.Syntax.AST
import ChuSQL.Core.Engine.Syntax.Parser (parseStatement) import ChuSQL.Core.Engine.Syntax.Parser (parseStatement)
import Control.Concurrent.MVar (MVar, modifyMVar, newMVar, readMVar, withMVar) import ChuSQL.Core.Protocol (TxnOp (..))
import Control.Concurrent.MVar (MVar, modifyMVar, readMVar)
import Data.List (nub) import Data.List (nub)
import Data.Text (Text) import Data.Text (Text)
import Data.Time (getCurrentTime) import Data.Time (getCurrentTime)
@@ -61,6 +63,10 @@ data Backend = Backend
, beDatabases :: IO (Either String [String]) , beDatabases :: IO (Either String [String])
-- | 原样转发一条存储请求 -- | 原样转发一条存储请求
, beStorage :: A.Value -> IO (Either String A.Value) , beStorage :: A.Value -> IO (Either String A.Value)
-- | 取一份整库快照,当事务的起点
, beSnapshot :: IO (Either String Database)
-- | 事务提交:把一批写操作交给存储层应用
, beApplyTransaction :: [TxnOp] -> IO (Either String ())
} }
-- | 按给定列名取列,固定列顺序 -- | 按给定列名取列,固定列顺序
@@ -73,36 +79,46 @@ columnsFromStatement db stmt = case prepare db stmt of
-- | 走本地存储链路的后端 -- | 走本地存储链路的后端
ipcBackend :: IO Backend ipcBackend :: IO Backend
ipcBackend = do ipcBackend = do
lock <- newMVar ()
let backend current = Backend let backend current = Backend
{ beStatement = \sql -> withMVar lock $ \_ -> runIpc current sql { beStatement = \sql -> runIpc current sql
, beCatalog = withMVar lock $ \_ -> do , beCatalog = do
response <- sendRequest (ReqInDatabase current ReqListCatalog) response <- sendRequest (ReqInDatabase current ReqListCatalog)
pure $ case response of pure $ case response of
RespCatalog infos -> Right infos RespCatalog infos -> Right infos
RespError err -> Left err RespError err -> Left err
_ -> Left "unexpected catalog response" _ -> Left "unexpected catalog response"
, bePing = withMVar lock $ \_ -> do , bePing = do
resp <- sendRequest ReqPing resp <- sendRequest ReqPing
pure (case resp of RespPong -> True; _ -> False) pure (case resp of RespPong -> True; _ -> False)
, beAccounts = \req -> withMVar lock $ \_ -> do , beAccounts = \req -> do
response <- sendRequest req response <- sendRequest req
pure $ case response of pure $ case response of
RespAccounts accounts -> Right accounts RespAccounts accounts -> Right accounts
RespError err -> Left err RespError err -> Left err
_ -> Left "unexpected account response" _ -> Left "unexpected account response"
, beWithDatabase = backend , beWithDatabase = backend
, beDatabases = withMVar lock $ \_ -> do , beDatabases = do
response <- sendRequest (ReqDatabase "list_databases" "") response <- sendRequest (ReqDatabase "list_databases" "")
pure $ case response of pure $ case response of
RespTables names -> Right names RespTables names -> Right names
RespError err -> Left err RespError err -> Left err
_ -> Left "unexpected database response" _ -> Left "unexpected database response"
, beStorage = \payload -> withMVar lock $ \_ -> runRaw payload , beStorage = \payload -> runRaw payload
, beSnapshot = Right <$> runIPCStorageIn (snapshot :: IPCStorage Database) (sessionEnv current)
, beApplyTransaction = \ops -> do
response <- sendRequest (ReqInDatabase current (ReqApplyTransaction ops))
pure $ case response of
RespOk -> Right ()
RespError err -> Left err
_ -> Left "unexpected transaction response"
} }
-- 没有默认库:没选库时 current 是空串,任何请求都得先选库 -- 没有默认库:没选库时 current 是空串,任何请求都得先选库
pure (backend "") pure (backend "")
where where
-- | 会话的存储环境:库名跟这条连接走
sessionEnv :: String -> Env
sessionEnv current = envForDatabase (if null current then Nothing else Just current)
-- | 转发一条存储请求并解析响应 -- | 转发一条存储请求并解析响应
runRaw :: A.Value -> IO (Either String A.Value) runRaw :: A.Value -> IO (Either String A.Value)
runRaw payload = do runRaw payload = do
@@ -113,18 +129,21 @@ ipcBackend = do
Left err -> Left ("storage response is not valid JSON: " ++ err) Left err -> Left ("storage response is not valid JSON: " ++ err)
Right value -> Right value Right value -> Right value
-- | 走存储链路跑一条语句 -- | 走存储链路跑一条语句:表名不再补当前库前缀
-- 引擎侧的数据字典按当前库过滤之后键是裸表名,补前缀会让查表落空;
-- 库名由 database 字段随请求带走,存储层自己按它分发。
runIpc :: String -> String -> IO (Either String StatementResult) runIpc :: String -> String -> IO (Either String StatementResult)
runIpc current sql = case fmap (inDatabase current) (parseStatement sql) of runIpc current sql = case parseStatement sql of
Left e -> pure (Left e) Left e -> pure (Left e)
Right stmt -> do Right stmt -> do
result <- runIPCStorage (runStatementM stmt) let env = sessionEnv current
result <- runIPCStorageIn (runStatementM stmt) env
case result of case result of
Left e -> pure (Left e) Left e -> pure (Left e)
Right rows -> do Right rows -> do
cols <- case rows of cols <- case rows of
[] -> do [] -> do
db <- runIPCStorage (schema :: IPCStorage Database) db <- runIPCStorageIn (schema :: IPCStorage Database) env
pure (columnsFromStatement db stmt) pure (columnsFromStatement db stmt)
(r : _) -> pure (map fst r) (r : _) -> pure (map fst r)
pure (Right (StatementResult cols rows)) pure (Right (StatementResult cols rows))
@@ -162,6 +181,10 @@ memoryBackendWith trusted name ref =
, beWithDatabase = withDatabase , beWithDatabase = withDatabase
, beDatabases = pure (Right (nub [name, systemDatabaseName])) , beDatabases = pure (Right (nub [name, systemDatabaseName]))
, beStorage = \_ -> pure (Left "storage is not available in this backend") , beStorage = \_ -> pure (Left "storage is not available in this backend")
, beSnapshot = Right . scoped <$> readMVar ref
, beApplyTransaction = \ops -> modifyMVar ref $ \db -> case applyTransactionMemory ops db of
Left err -> pure (db, Left err)
Right db' -> pure (db', Right ())
} }
where where
-- | 按视角过滤库里的表 -- | 按视角过滤库里的表
@@ -185,6 +208,8 @@ unavailableDatabase :: String -> Backend -> Backend
unavailableDatabase message backend = backend unavailableDatabase message backend = backend
{ beStatement = \_ -> pure (Left message) { beStatement = \_ -> pure (Left message)
, beCatalog = pure (Left message) , beCatalog = pure (Left message)
, beSnapshot = pure (Left message)
, beApplyTransaction = \_ -> pure (Left message)
} }
-- | 未选库时的视角:只放行不碰表的语句 -- | 未选库时的视角:只放行不碰表的语句
@@ -196,76 +221,34 @@ noDatabaseBackend backend = backend
| statementNeedsDatabase stmt -> pure (Left "no database selected") | statementNeedsDatabase stmt -> pure (Left "no database selected")
| otherwise -> beStatement backend sql | otherwise -> beStatement backend sql
, beCatalog = pure (Left "no database selected") , beCatalog = pure (Left "no database selected")
, beSnapshot = pure (Left "no database selected")
, beApplyTransaction = \_ -> pure (Left "no database selected")
} }
-- | 给语句里的裸表名补上当前库前缀,子查询里的表名也一起改 -- | 把一批事务写操作应用到内存库:先按 id 删,再按追加写
inDatabase :: String -> Statement -> Statement applyTransactionMemory :: [TxnOp] -> Database -> Either String Database
inDatabase current = walk applyTransactionMemory ops db = foldl step (Right db) ops
where where
-- | 递归改写语句里的表名 step acc op = do
walk :: Statement -> Statement current <- acc
walk stmt = case stmt of case op of
q@Select{} -> TxnDelete name keys -> do
q table <- lookupTable current name
{ selectFrom = source (selectFrom q) pure (put (name, table {tableRows = [r | r <- tableRows table, not (doomed keys r)]}) current)
, selectWhere = fmap expr (selectWhere q) TxnUpsert name rows -> do
} table <- lookupTable current name
q@SelectExpr{} -> pure (put (name, table {tableRows = tableRows table ++ rows}) current)
q TxnReplace name rows -> do
{ selectItems = [(label, expr e) | (label, e) <- selectItems q] table <- lookupTable current name
, selectFrom = source (selectFrom q) pure (put (name, table {tableRows = rows}) current)
, selectWhere = fmap expr (selectWhere q)
}
Insert t cols rows -> Insert (table t) cols (map (map expr) rows)
Delete t cond -> Delete (table t) (fmap expr cond)
Update t assigns cond -> Update (table t) [(c, expr e) | (c, e) <- assigns] (fmap expr cond)
CreateTable t cols -> CreateTable (table t) cols
DropTable t -> DropTable (table t)
CreateIndex t c -> CreateIndex (table t) c
DropIndex t c -> DropIndex (table t) c
DropColumn t c -> DropColumn (table t) c
AddColumn t def -> AddColumn (table t) def
RenameColumn t old new -> RenameColumn (table t) old new
AlterColumnType t c ty -> AlterColumnType (table t) c ty
AlterColumnDefault t c v -> AlterColumnDefault (table t) c v
AlterColumnNull t c b -> AlterColumnNull (table t) c b
other -> other
-- | 给单个表名补库前缀,已限定或没选库就原样 -- | 换掉一张表,其余保持原顺序
table t | '.' `elem` t || null current = t put entry = map (\one@(name, _) -> if name == fst entry then entry else one)
| otherwise = current ++ "." ++ t
-- | 改写 FROM 子句 -- | 这一行要不要删:按行里的 id 列比
source FromUnit = FromUnit doomed keys r = case lookup "id" r of
source (FromTable a t) = FromTable a (table t) Just (VInt k) -> k `elem` keys
source (FromJoin k l a t e) = FromJoin k (source l) a (table t) (expr e) _ -> False
-- | 改写表达式里的子查询
expr (ScalarSub sq) = ScalarSub (sub sq)
expr (ExistsSub sq neg) = ExistsSub (sub sq) neg
expr (InSub a sq neg) = InSub (expr a) (sub sq) neg
expr (InList a items neg) = InList (expr a) (map expr items) neg
expr (Add a b) = Add (expr a) (expr b)
expr (Sub a b) = Sub (expr a) (expr b)
expr (Mul a b) = Mul (expr a) (expr b)
expr (Div a b) = Div (expr a) (expr b)
expr (Neg a) = Neg (expr a)
expr (Gt a b) = Gt (expr a) (expr b)
expr (Lt a b) = Lt (expr a) (expr b)
expr (Eq a b) = Eq (expr a) (expr b)
expr (And a b) = And (expr a) (expr b)
expr (Or a b) = Or (expr a) (expr b)
expr (IsNull a) = IsNull (expr a)
expr (IsNotNull a) = IsNotNull (expr a)
expr (CountOf a) = CountOf (expr a)
expr (SumOf a) = SumOf (expr a)
expr (AvgOf a) = AvgOf (expr a)
expr (MinOf a) = MinOf (expr a)
expr (MaxOf a) = MaxOf (expr a)
expr other = other
-- | 改写一个子查询
sub sq = sq {subqueryStatement = walk (subqueryStatement sq)}
-- | 语句里出现的表 -- | 语句里出现的表
statementTables :: Statement -> [Text] statementTables :: Statement -> [Text]
+185 -14
View File
@@ -24,22 +24,24 @@ module ChuSQL.Server.Session
, queryResultJson , queryResultJson
) where ) where
import ChuSQL.Core.Model (Value (..)) import ChuSQL.Core.Model (Database, Row, Table (..), Value (..))
import ChuSQL.Core.Protocol (Account, QueryResult (..), queryResultJson) import ChuSQL.Core.Protocol (Account, QueryResult (..), TxnOp (..), queryResultJson)
import ChuSQL.Core.Engine.Storage.IPC (TableInfo (..)) import ChuSQL.Core.Engine.Storage.IPC (TableInfo (..))
import qualified ChuSQL.Core.Engine as Engine
import qualified ChuSQL.Core.Engine.Error as E import qualified ChuSQL.Core.Engine.Error as E
import ChuSQL.Core.Engine.Syntax.AST (Statement (..)) import ChuSQL.Core.Engine.Syntax.AST (Statement (..))
import ChuSQL.Core.Engine.Syntax.Parser (parseStatement) import ChuSQL.Core.Engine.Syntax.Parser (parseStatement)
import ChuSQL.Interface.AccountTable (systemTableInfo) import ChuSQL.Interface.AccountTable (systemTableInfo)
import ChuSQL.Interface.Auth (defaultSessionPolicy, newSessionStore) import ChuSQL.Interface.Auth (defaultSessionPolicy, newSessionStore)
import ChuSQL.Server.Accounts import ChuSQL.Server.Accounts
import ChuSQL.Server.Backend (Backend (..), StatementResult (..)) import ChuSQL.Server.Backend (Backend (..), StatementResult (..), columnsFromStatement)
import ChuSQL.Server.Policy (configurePasswordPolicy) import ChuSQL.Server.Policy (configurePasswordPolicy)
import ChuSQL.Server.Privileges (PrivilegeCommand, PrivilegeError (..), Privileges, RoleView, authorize, filterTables, listRoleViews, newPrivileges, privilegeCommand, runPrivilegeCommand) import ChuSQL.Server.Privileges (PrivilegeCommand, PrivilegeError (..), Privileges, RoleView, authorize, filterTables, listRoleViews, newPrivileges, privilegeCommand, runPrivilegeCommand)
import Control.Exception (IOException, try) import Control.Exception (IOException, try)
import Data.Bifunctor (first) import Data.Bifunctor (first)
import Data.Char (isAlpha, isAlphaNum) import Data.Char (isAlpha, isAlphaNum)
import Data.IORef (IORef, newIORef, readIORef, writeIORef) import Data.IORef (IORef, newIORef, readIORef, writeIORef)
import Data.Maybe (isJust)
import Data.Text (Text) import Data.Text (Text)
import qualified Data.Text as T import qualified Data.Text as T
import Data.Time.Clock (getCurrentTime) import Data.Time.Clock (getCurrentTime)
@@ -64,6 +66,13 @@ data Session = Session
, ssCurrent :: IORef Text , ssCurrent :: IORef Text
, ssPrincipal :: IORef (Maybe Principal) , ssPrincipal :: IORef (Maybe Principal)
, ssSettingsFile :: FilePath , ssSettingsFile :: FilePath
, ssTransaction :: IORef (Maybe Transaction)
}
-- | 一条会话里的显式事务:开事务时的快照与事务内改到的状态
data Transaction = Transaction
{ txBase :: Database
, txStaged :: Database
} }
-- | 开一个会话:账号与角色服务、配置策略都和 Web 端一致 -- | 开一个会话:账号与角色服务、配置策略都和 Web 端一致
@@ -76,7 +85,8 @@ newSession backend name settingsFile = do
-- 服务启动后只有系统库:没 USE 之前不预设任何工作库 -- 服务启动后只有系统库:没 USE 之前不预设任何工作库
current <- newIORef "" current <- newIORef ""
principal <- newIORef Nothing principal <- newIORef Nothing
pure (Session backend accountsService privileges current principal settingsFile) transaction <- newIORef Nothing
pure (Session backend accountsService privileges current principal settingsFile transaction)
-- | 当前生效的口令策略(只有管理员能看) -- | 当前生效的口令策略(只有管理员能看)
policyOf :: Session -> IO (Either Text PasswordPolicy) policyOf :: Session -> IO (Either Text PasswordPolicy)
@@ -168,22 +178,183 @@ runStatement session sql = fmap (first sessMessage) (runStatementCoded session s
runStatementCoded :: Session -> Text -> IO (Either SessionError QueryResult) runStatementCoded :: Session -> Text -> IO (Either SessionError QueryResult)
runStatementCoded session sql = case parseStatement (T.unpack sql) of runStatementCoded session sql = case parseStatement (T.unpack sql) of
Right (UseDatabase name) -> do Right (UseDatabase name) -> do
switched <- switchDatabaseCoded session (T.pack name) existing <- readIORef (ssTransaction session)
case switched of case existing of
Left err -> pure (Left err) Just _ -> pure (Left (SessionError "query_error" "cannot switch database inside a transaction"))
Right () -> do Nothing -> do
current <- readIORef (ssCurrent session) switched <- switchDatabaseCoded session (T.pack name)
pure (Right (emptyResult (Just current))) case switched of
Left err -> pure (Left err)
Right () -> do
current <- readIORef (ssCurrent session)
pure (Right (emptyResult (Just current)))
Right BeginTransaction -> beginTransaction session
Right CommitTransaction -> commitTransaction session
Right RollbackTransaction -> rollbackTransaction session
Right statement Right statement
| Just command <- accountCommand statement -> runAccount session command | Just command <- accountCommand statement -> runAccount session command
| Just command <- privilegeCommand statement -> runPrivilege session command | Just command <- privilegeCommand statement -> runPrivilege session command
| otherwise -> do | otherwise -> do
allowed <- authorized session statement transaction <- readIORef (ssTransaction session)
case allowed of if isJust transaction && transactionDdl statement
Left err -> pure (Left err) then pure (Left (SessionError "query_error" "DDL is not allowed in a transaction"))
Right () -> runPlain session sql else do
allowed <- authorized session statement
case allowed of
Left err -> pure (Left err)
Right ()
| isJust transaction && transactionData statement -> runInTransaction session statement
| otherwise -> runPlain session sql
Left _ -> runPlain session sql Left _ -> runPlain session sql
-- | 事务里不许改结构,会话层直接拦下
transactionDdl :: Statement -> Bool
transactionDdl statement = case statement of
CreateTable{} -> True
DropTable{} -> True
CreateIndex{} -> True
DropIndex{} -> True
DropColumn{} -> True
AddColumn{} -> True
RenameColumn{} -> True
AlterColumnType{} -> True
AlterColumnDefault{} -> True
AlterColumnNull{} -> True
CreateDatabase{} -> True
DropDatabase{} -> True
_ -> False
-- | 事务里只有数据语句走快照
transactionData :: Statement -> Bool
transactionData statement = case statement of
Select{} -> True
SelectExpr{} -> True
Insert{} -> True
Update{} -> True
Delete{} -> True
_ -> False
-- | 开事务:把当前库整库做一份快照当起点
beginTransaction :: Session -> IO (Either SessionError QueryResult)
beginTransaction session = do
current <- readIORef (ssCurrent session)
existing <- readIORef (ssTransaction session)
case existing of
Just _ -> pure (Left (SessionError "query_error" "already in a transaction"))
Nothing
| T.null current -> pure (Left (SessionError "no_database" "no database selected"))
| otherwise -> do
backend <- currentBackend session
snapshotResult <- beSnapshot backend
case snapshotResult of
Left err -> pure (Left (engineErrorCode err))
Right db -> do
writeIORef (ssTransaction session) (Just (Transaction db db))
pure (Right (emptyResult (Just current)))
-- | 提交:把快照差分交给存储层应用,成功才结束事务
commitTransaction :: Session -> IO (Either SessionError QueryResult)
commitTransaction session = do
current <- readIORef (ssCurrent session)
existing <- readIORef (ssTransaction session)
case existing of
Nothing -> pure (Left (SessionError "query_error" "no transaction in progress"))
Just transaction -> do
backend <- currentBackend session
liveResult <- beSnapshot backend
case liveResult of
Left err -> pure (Left (engineErrorCode err))
Right live -> do
applied <- beApplyTransaction backend (transactionOps (txBase transaction) (txStaged transaction) live)
case applied of
Left err -> pure (Left (engineErrorCode err))
Right () -> do
writeIORef (ssTransaction session) Nothing
pure (Right (emptyResult (Just current)))
-- | 回滚:快照丢掉,存储层没被改过
rollbackTransaction :: Session -> IO (Either SessionError QueryResult)
rollbackTransaction session = do
current <- readIORef (ssCurrent session)
existing <- readIORef (ssTransaction session)
case existing of
Nothing -> pure (Left (SessionError "query_error" "no transaction in progress"))
Just _ -> do
writeIORef (ssTransaction session) Nothing
pure (Right (emptyResult (Just current)))
-- | 事务里跑一条数据语句:只动内存快照,不碰存储层
runInTransaction :: Session -> Statement -> IO (Either SessionError QueryResult)
runInTransaction session statement = do
existing <- readIORef (ssTransaction session)
case existing of
Nothing -> pure (Left (SessionError "query_error" "no transaction in progress"))
Just transaction -> case Engine.runStatement (txStaged transaction) statement of
Left err -> pure (Left (engineErrorCode err))
Right (staged, rows) -> do
writeIORef (ssTransaction session) (Just transaction {txStaged = staged})
pure (Right (resultOfStaged (txStaged transaction) statement rows))
-- | 事务内的结果集:空结果回落到解析出来的列名
resultOfStaged :: Database -> Statement -> [Row] -> QueryResult
resultOfStaged db statement rows = resultOf (StatementResult cols rows)
where
cols = case rows of
[] -> columnsFromStatement db statement
(r : _) -> map fst r
-- | 提交差分:相对快照列出要删、要写与要替换的行
transactionOps :: Database -> Database -> Database -> [TxnOp]
transactionOps base staged live =
concatMap opsFor names
where
-- | 事务里碰得着的表:数据语句不增删表
names = nubKeys (map fst base ++ map fst staged)
-- | 一张表的差分
opsFor name
| all (isJust . rowKey) (baseRows ++ stagedRows) = keyedOps
| stagedRows == baseRows = []
| otherwise = [TxnReplace name stagedRows]
where
baseRows = rowsOf name base
stagedRows = rowsOf name staged
liveRows = rowsOf name live
liveByKey = [(k, r) | r <- liveRows, Just k <- [rowKey r]]
stagedByKey = [(k, r) | r <- stagedRows, Just k <- [rowKey r]]
baseKeys = [k | r <- baseRows, Just k <- [rowKey r]]
-- 开事务时就存在、现在没了:删;内容变了的行也先删再写
gone = [k | (k, _) <- liveByKey, k `elem` baseKeys, not (any ((== k) . fst) stagedByKey)]
rewritten = [k | (k, r) <- stagedByKey, Just liveRow <- [lookup k liveByKey], liveRow /= r]
deleted = gone ++ rewritten
-- 新增的行,以及跟库里不一样的行;并发插进来的行不动
written =
[ r
| (k, r) <- stagedByKey
, case lookup k liveByKey of
Nothing -> True
Just liveRow -> liveRow /= r
]
keyedOps =
[TxnDelete name deleted | not (null deleted)]
++ [TxnUpsert name written | not (null written)]
-- | 表名去重但保持顺序
nubKeys = foldr (\name acc -> if name `elem` acc then acc else name : acc) []
-- | 库里某张表的行
rowsOf name db = maybe [] tableRows (lookup name db)
-- | 行 id:只有整数 id 才认,跟存储层删行的口径一致
rowKey :: Row -> Maybe Int
rowKey row = case lookup "id" row of
Just (VInt k) -> Just k
_ -> Nothing
-- | 普通身份跑语句前的权限判定;管理员与未登录放行 -- | 普通身份跑语句前的权限判定;管理员与未登录放行
authorized :: Session -> Statement -> IO (Either SessionError ()) authorized :: Session -> Statement -> IO (Either SessionError ())
authorized session statement = do authorized session statement = do
+250 -7
View File
@@ -3,14 +3,20 @@
module Main (main) where module Main (main) where
import ChuSQL.Core.Engine.Storage.IPC (Account (..), Request (ReqAccountReset)) import ChuSQL.Core.Engine.Storage.IPC (
Account (..),
Request (ReqAccountReset),
closeConnection,
localStorageLink,
setStorageLink,
)
import ChuSQL.Core.Engine.Syntax.Parser (parseStatement) import ChuSQL.Core.Engine.Syntax.Parser (parseStatement)
import ChuSQL.Core.Model (Database, Table (..), Value (..), pattern TInt, pattern TStr) import ChuSQL.Core.Model (Database, Row, Table (..), Value (..), pattern TInt, pattern TStr)
import ChuSQL.Interface.Auth (hashPasswordWith) import ChuSQL.Interface.Auth (hashPasswordWith)
import ChuSQL.Interface.Protocol (ClientRequest (..), Grant (..), RoleView (..), ServerResponse (..), decodeRequest, encodeResponse) import ChuSQL.Interface.Protocol (ClientRequest (..), Grant (..), RoleView (..), ServerResponse (..), decodeRequest, encodeResponse)
import ChuSQL.Interface.RateLimit (newRateLimiter) import ChuSQL.Interface.RateLimit (newRateLimiter)
import ChuSQL.Server.Accounts (Principal (..), ensureRootAccount) import ChuSQL.Server.Accounts (Principal (..), ensureRootAccount)
import ChuSQL.Server.Backend (Backend (..), memoryBackend) import ChuSQL.Server.Backend (Backend (..), StatementResult (..), ipcBackend, memoryBackend)
import ChuSQL.Server.Privileges ( import ChuSQL.Server.Privileges (
PrivilegeCommand (..), PrivilegeCommand (..),
PrivilegeError (..), PrivilegeError (..),
@@ -20,7 +26,7 @@ import ChuSQL.Server.Privileges (
newPrivileges, newPrivileges,
runPrivilegeCommand, runPrivilegeCommand,
) )
import ChuSQL.Server.Session (QueryResult (..)) import ChuSQL.Server.Session (QueryResult (..), Session, SessionError (..), newSession, runStatementCoded)
import ChuSQL.Server.TCP ( import ChuSQL.Server.TCP (
ServerConfig (..), ServerConfig (..),
ServerEnv, ServerEnv,
@@ -30,8 +36,8 @@ import ChuSQL.Server.TCP (
newServerEnv, newServerEnv,
startServer, startServer,
) )
import Control.Concurrent (threadDelay) import Control.Concurrent (forkIO, threadDelay)
import Control.Concurrent.MVar (newMVar) import Control.Concurrent.MVar (MVar, newEmptyMVar, newMVar, putMVar, takeMVar)
import Control.Exception (IOException, bracket, finally, try) import Control.Exception (IOException, bracket, finally, try)
import Control.Monad (void) import Control.Monad (void)
import Data.Aeson (decode) import Data.Aeson (decode)
@@ -59,7 +65,7 @@ import Network.Socket (
socketToHandle, socketToHandle,
withSocketsDo, withSocketsDo,
) )
import System.Directory (doesFileExist, getTemporaryDirectory, removeFile) import System.Directory (createDirectoryIfMissing, doesFileExist, getTemporaryDirectory, removeFile, removePathForcibly)
import System.FilePath ((</>)) import System.FilePath ((</>))
import System.IO ( import System.IO (
BufferMode (LineBuffering), BufferMode (LineBuffering),
@@ -87,6 +93,8 @@ spec = do
tcpSpec tcpSpec
tcpServerSpec tcpServerSpec
privilegeSpec privilegeSpec
transactionSpec
ipcConcurrencySpec
-- 夹具:内存后端装在 MVar 里,跟 Web 测试同一套表 -- 夹具:内存后端装在 MVar 里,跟 Web 测试同一套表
@@ -545,3 +553,238 @@ privilegeSpec = describe "server privileges" $ do
Left err -> expectationFailure err Left err -> expectationFailure err
Right stmt -> authorize service root testDatabase stmt `shouldReturn` Right () Right stmt -> authorize service root testDatabase stmt `shouldReturn` Right ()
-- 真存储夹具:一个用例一份数据目录,跑完关链路删干净
-- | 本次用例的数据目录(用例串行跑,固定名够用)
ipcDataDir :: IO FilePath
ipcDataDir = do
tmp <- getTemporaryDirectory
pure (tmp </> "chusql-server-test-ipc")
-- | 写一份只属于这次用例的存储配置:数据目录与日志级别都从文件走
writeIpcConfig :: FilePath -> IO FilePath
writeIpcConfig dir = do
let cfgPath = dir ++ ".toml"
-- TOML 的普通字符串会吃反斜杠,路径统一用正斜杠
slashed = map (\c -> if c == '\\' then '/' else c) dir
writeFile
cfgPath
( unlines
[ "[storage]"
, "data_dir = \"" ++ slashed ++ "\""
, "[log]"
, "level = \"error\""
]
)
pure cfgPath
-- | 删掉这次用例的数据目录与配置,删不掉也不算错
cleanIpc :: FilePath -> IO ()
cleanIpc dir = do
_ <- try (removePathForcibly dir) :: IO (Either IOException ())
_ <- try (removePathForcibly (dir ++ ".toml")) :: IO (Either IOException ())
pure ()
-- 事务用例:内存后端跑得快,真存储的提交再在 IPC 用例里验一遍
-- | 起一个内存后端会话,当前库已切到 test
withMemorySession :: (Session -> IO a) -> IO a
withMemorySession body = do
db <- newMVar testDb
settings <- tempSettingsPath "transaction"
removeIfExists settings
session <- newSession (memoryBackend testDatabaseName db) (T.pack testDatabaseName) settings
_ <- runStatementCoded session "use test"
body session
-- | 两个会话共用一份内存库,用来验会话隔离
withTwoSessions :: (Session -> Session -> IO a) -> IO a
withTwoSessions body = do
db <- newMVar testDb
settings <- tempSettingsPath "transaction-pair"
removeIfExists settings
let backend = memoryBackend testDatabaseName db
first <- newSession backend (T.pack testDatabaseName) settings
second <- newSession backend (T.pack testDatabaseName) settings
_ <- runStatementCoded first "use test"
_ <- runStatementCoded second "use test"
body first second
-- | 跑一条语句;出错就算用例失败
mustSql :: Session -> Text -> IO QueryResult
mustSql session sql = runStatementCoded session sql >>= either failed pure
where
failed err = expectationFailure (T.unpack (sessMessage err)) >> pure (QueryResult [] [] 0 False Nothing)
-- | 取 users 的 id 列
sessionIds :: Session -> IO [Int]
sessionIds session = do
result <- mustSql session "select * from users"
pure [n | row <- qrRows result, Just (VInt n) <- [nth 0 row]]
-- | 取某个 id 的 age
sessionAge :: Session -> Int -> IO (Maybe Int)
sessionAge session wanted = do
result <- mustSql session "select * from users"
let ages = [(n, a) | row <- qrRows result, Just (VInt n) <- [nth 0 row], Just (VInt a) <- [nth 2 row]]
pure (lookup wanted ages)
-- | 取失败里的文案
failureMessage :: Either SessionError QueryResult -> Text
failureMessage outcome = either sessMessage (const "") outcome
-- | 取结果第一列的整数
intValues :: [[Value]] -> [Int]
intValues rows = [n | row <- rows, Just (VInt n) <- [nth 0 row]]
-- | 显式事务用例
transactionSpec :: Spec
transactionSpec = describe "server transactions" $ do
it "keeps uncommitted rows invisible to other sessions until COMMIT" $ withTwoSessions $ \writer reader -> do
_ <- mustSql writer "begin"
_ <- mustSql writer "insert into users (id, name, age) values (6, 'six', 26)"
sessionIds writer `shouldReturn` [1, 2, 3, 4, 5, 6]
sessionIds reader `shouldReturn` [1, 2, 3, 4, 5]
_ <- mustSql writer "commit"
ids <- sessionIds writer
ids `shouldMatchList` [1, 2, 3, 4, 5, 6]
seen <- sessionIds reader
seen `shouldMatchList` [1, 2, 3, 4, 5, 6]
it "drops every change on ROLLBACK" $ withMemorySession $ \session -> do
_ <- mustSql session "begin"
_ <- mustSql session "insert into users (id, name, age) values (6, 'six', 26)"
_ <- mustSql session "update users set age = 99 where id = 1"
_ <- mustSql session "delete from users where id = 2"
rollback <- mustSql session "rollback"
qrRowCount rollback `shouldBe` 0
kept <- sessionIds session
kept `shouldMatchList` [1, 2, 3, 4, 5]
it "commits the whole write set at once" $ withMemorySession $ \session -> do
_ <- mustSql session "begin"
_ <- mustSql session "insert into users (id, name, age) values (6, 'six', 26)"
_ <- mustSql session "update users set age = 99 where id = 1"
_ <- mustSql session "delete from users where id = 2"
_ <- mustSql session "commit"
ids <- sessionIds session
ids `shouldMatchList` [1, 3, 4, 5, 6]
sessionAge session 1 `shouldReturn` Just 99
it "rejects DDL inside a transaction and takes it after ROLLBACK" $ withMemorySession $ \session -> do
_ <- mustSql session "begin"
denied <- runStatementCoded session "create table blocked (id int)"
failureMessage denied `shouldBe` "DDL is not allowed in a transaction"
_ <- mustSql session "rollback"
_ <- mustSql session "create table fresh (id int)"
_ <- mustSql session "insert into fresh (id) values (1)"
rows <- mustSql session "select * from fresh"
qrRowCount rows `shouldBe` 1
it "refuses nested BEGIN and COMMIT outside a transaction" $ withMemorySession $ \session -> do
_ <- mustSql session "begin"
nested <- runStatementCoded session "begin"
failureMessage nested `shouldBe` "already in a transaction"
_ <- mustSql session "rollback"
late <- runStatementCoded session "commit"
failureMessage late `shouldBe` "no transaction in progress"
it "does not switch database inside a transaction" $ withMemorySession $ \session -> do
_ <- mustSql session "begin"
denied <- runStatementCoded session "use other"
failureMessage denied `shouldBe` "cannot switch database inside a transaction"
it "needs a current database to start" $ do
db <- newMVar testDb
settings <- tempSettingsPath "transaction-bare"
removeIfExists settings
session <- newSession (memoryBackend testDatabaseName db) (T.pack testDatabaseName) settings
result <- runStatementCoded session "begin"
failureMessage result `shouldBe` "no database selected"
-- | 起一份真存储并装成当前链路;动态库打不开就当用例待定
withIpcStorage :: IO () -> IO ()
withIpcStorage body = do
dir <- ipcDataDir
cleanIpc dir
createDirectoryIfMissing True dir
cfgPath <- writeIpcConfig dir
opened <- localStorageLink (Just cfgPath)
case opened of
Left err -> cleanIpc dir >> pendingWith ("cannot open the storage library: " ++ err)
Right link -> do
setStorageLink link
body `finally` (closeConnection >> cleanIpc dir)
-- | 跑一条语句,出错就算用例失败
mustRun :: IO (Either String StatementResult) -> IO ()
mustRun action = action >>= either expectationFailure (const (pure ()))
-- | 取结果行,出错就算用例失败
rowsOf :: IO (Either String StatementResult) -> IO [Row]
rowsOf action = action >>= either (\err -> expectationFailure err >> pure []) (pure . srRows)
-- | 取结果里某一列的整数
intColumn :: String -> [Row] -> [Int]
intColumn name rows = [n | row <- rows, Just (VInt n) <- [lookup name row]]
-- | 往 p 表连插一批行,结果放 MVar 交回主线程
writeRows :: Backend -> [Int] -> MVar [Either String StatementResult] -> IO ()
writeRows session ids done = do
outs <- mapM (\i -> beStatement session ("INSERT INTO p (id) VALUES (" ++ show i ++ ")")) ids
putMVar done outs
-- | 同一进程里开两个库:会话之间不串数据,也不互相挡路
ipcConcurrencySpec :: Spec
ipcConcurrencySpec = describe "server backend on the real storage" $ do
it "keeps two databases apart" $ withIpcStorage $ do
base <- ipcBackend
mustRun (beStatement base "CREATE DATABASE alpha")
mustRun (beStatement base "CREATE DATABASE beta")
let alpha = beWithDatabase base "alpha"
beta = beWithDatabase base "beta"
mustRun (beStatement alpha "CREATE TABLE t (id int)")
mustRun (beStatement beta "CREATE TABLE t (id int)")
mustRun (beStatement alpha "INSERT INTO t (id) VALUES (1)")
mustRun (beStatement beta "INSERT INTO t (id) VALUES (2)")
rowsA <- rowsOf (beStatement alpha "SELECT id FROM t")
rowsB <- rowsOf (beStatement beta "SELECT id FROM t")
intColumn "id" rowsA `shouldBe` [1]
intColumn "id" rowsB `shouldBe` [2]
it "runs two sessions at once without mixing their rows" $ withIpcStorage $ do
base <- ipcBackend
mustRun (beStatement base "CREATE DATABASE alpha")
mustRun (beStatement base "CREATE DATABASE beta")
let alpha = beWithDatabase base "alpha"
beta = beWithDatabase base "beta"
mustRun (beStatement alpha "CREATE TABLE p (id int)")
mustRun (beStatement beta "CREATE TABLE p (id int)")
doneA <- newEmptyMVar
doneB <- newEmptyMVar
_ <- forkIO (writeRows alpha [1 .. 20] doneA)
_ <- forkIO (writeRows beta [101 .. 120] doneB)
outs <- sequence [takeMVar doneA, takeMVar doneB]
[err | Left err <- concat outs] `shouldBe` []
rowsA <- rowsOf (beStatement alpha "SELECT id FROM p")
rowsB <- rowsOf (beStatement beta "SELECT id FROM p")
intColumn "id" rowsA `shouldMatchList` [1 .. 20]
intColumn "id" rowsB `shouldMatchList` [101 .. 120]
it "commits an explicit transaction to the real storage" $ withIpcStorage $ do
base <- ipcBackend
mustRun (beStatement base "CREATE DATABASE alpha")
let alpha = beWithDatabase base "alpha"
settings <- tempSettingsPath "transaction-ipc"
removeIfExists settings
session <- newSession alpha (T.pack testDatabaseName) settings
_ <- mustSql session "use alpha"
_ <- mustSql session "create table t (id int)"
_ <- mustSql session "begin"
_ <- mustSql session "insert into t (id) values (1)"
inside <- mustSql session "select * from t"
intValues (qrRows inside) `shouldBe` [1]
_ <- mustSql session "commit"
rows <- rowsOf (beStatement alpha "SELECT id FROM t")
intColumn "id" rows `shouldBe` [1]
+7 -1
View File
@@ -45,7 +45,13 @@ chusql-server
`chusql-server` 默认监听 `127.0.0.1:7777`,客户端建立连接后完成登录、数据库选择和 SQL 执行。 `chusql-server` 默认监听 `127.0.0.1:7777`,客户端建立连接后完成登录、数据库选择和 SQL 执行。
每条连接拥有独立会话状态,例如当前数据库选择不会影响其他连接。 每条连接拥有独立会话状态,例如当前数据库选择不会影响其他连接。不同连接可以同时执行语句:读请求之间可以并行,写请求之间由存储层排队。
语句里的库前缀和表限定符由引擎在语义层解析:存储层只接受精确的表名键,因此 `FROM sales.orders` 与选定 `sales` 后的 `FROM orders` 指向同一张表。
显式事务在会话层实现:`BEGIN` 把当前数据库整库读成内存快照,事务内的语句都在快照上执行;`COMMIT` 计算快照与库的差分,用一条 `apply_transaction` 请求交给存储层应用;`ROLLBACK` 直接丢弃快照。因此隔离级别是快照读加原子提交:事务里看不到别的事务未提交或后来提交的改动,提交对外一次性可见。
WAL 负责崩溃恢复:每条记录带自增 LSN,一组写操作与一条提交标记一次落盘;进程重启时按标记分组重放已提交的记录、丢弃没有标记的整组,重放前逐条做幂等检查,恢复结束把最大 LSN 写进 `wal.checkpoint` 并清空 WAL。`SystemStatus` 应答里的 `last_lsn` 报告该检查点 LSN。
系统数据库及部分管理操作仅允许管理员访问。 系统数据库及部分管理操作仅允许管理员访问。
+16
View File
@@ -38,6 +38,22 @@ csql -d sales -e "SELECT * FROM orders LIMIT 5"
csql -f json -e "SELECT count(*) FROM orders" csql -f json -e "SELECT count(*) FROM orders"
``` ```
表名与列名的写法:
- 选定数据库后语句里直接写表名即可,例如 `SELECT name FROM orders WHERE orders.id = 1`。
- 表名可以带库前缀(`sales.orders`),列名可以带表限定符(`orders.id`)或库限定符(`sales.orders.id`),这几种写法指向同一张表和同一个列。
- 只有候选唯一时才自动推导前缀;同名的表或列出现在多个候选上会以 `ambiguous table` / `ambiguous column` 报错,需要写全。
- `FROM orders AS o` 之后只能用 `o.id`,不能再写 `orders.id`。
事务:
- `BEGIN`(或 `START TRANSACTION`)开始显式事务,`COMMIT` 提交,`ROLLBACK` 回滚。
- 事务内只允许数据语句;DDL 与 `USE` 会被拒绝,`SHOW DATABASES` 这类不碰表数据的语句照常可用。
- 事务里的读都看开事务那一刻的快照:别的连接在事务期间提交的改动不会出现在事务里,本会话未提交的改动别的连接也看不见。
- 同一连接同时只能有一个事务,没有保存点与嵌套事务;开事务前要先选好数据库。
- 提交时按行 `id` 写回本事务改动过的行,事务没碰过的行不受影响;没有整数 `id` 的表按整表替换提交。
- 提交是持久的:整批改动与一条提交标记一次落盘,提交后进程被强杀也会在重启时重放;提交前崩溃则整批丢弃,不会只落一半。
### 交互模式 ### 交互模式
SQL 语句以分号结束,可跨多行输入。 SQL 语句以分号结束,可跨多行输入。