事务模型

This commit is contained in:
LIm1tZz7 committed 2026-10-03 16:49:23 +08:00
1 parent 274b7fcdcc
commit 045ccc2dc0
20 files changed
+1519 -294

No files matched your search

+2 -2
View File
@@ -28,5 +28,5 @@
- 改动说明只在交流中记录。
## 当前重点
- 完善 WAL 重放幂等。
- 完整事务路线见 project-plan.txt 阶段 12。
- 主线已完成:高级优化器 → 并行查询 → M1 会话隔离 → M2 快照事务 → M3 WAL 与崩溃恢复。
- 进度与计数见 `agent_credientials/project-plan.txt`,限制与风险见 `agent_credientials/known-issues.txt`。
@@ -134,6 +134,9 @@ runStatementUncheckedM _ q@CreateDatabase{} = runStatementM q
runStatementUncheckedM _ q@DropDatabase{} = runStatementM q
runStatementUncheckedM _ q@UseDatabase{} = runStatementM q
runStatementUncheckedM _ ShowDatabases = runStatementM ShowDatabases
runStatementUncheckedM _ BeginTransaction = pure (Left "BEGIN is executed by the session")
runStatementUncheckedM _ CommitTransaction = pure (Left "COMMIT is executed by the session")
runStatementUncheckedM _ RollbackTransaction = pure (Left "ROLLBACK is executed by the session")
-- | 只有需要读全表才能判断约束时才扫表
existingRows :: (MonadStorage m) => String -> Table -> m (Either String [Row])
@@ -51,6 +51,8 @@ errorRules =
, ErrorRule "storage" StorageError "query_error"
, ErrorRule "storage library call failed" StorageError "query_error"
, ErrorRule "unsupported protocol" ProtocolError "query_error"
, ErrorRule "executed by the session" ProtocolError "query_error"
, ErrorRule "transaction" ProtocolError "query_error"
]
-- | 命中的第一条规则
@@ -38,16 +38,16 @@ scopeAt place scope c
| "ambiguous column:" `isInfixOf` err -> Left err
| otherwise -> columnAt place (scopeOuter scope) c
-- | 收集 FROM 能提供的列,含外层作用域
-- | 收集 FROM 的列,列前缀由表名推导
checkFrom :: Database -> [(String, Column)] -> FromClause -> Either String [(String, Column)]
checkFrom _ _ FromUnit = Right []
checkFrom db _ (FromTable mAlias tbl) = do
t <- lookupTable db tbl
Right (prefixColumns (Just (fromMaybe tbl mAlias)) (tableCols t))
(key, t) <- resolveTable db tbl
Right (prefixColumns (Just (deriveQualifier mAlias key)) (tableCols t))
checkFrom db outer (FromJoin _ left mAlias tbl cond) = do
lcols <- checkFrom db outer left
t <- lookupTable db tbl
let env = lcols ++ prefixColumns (Just (fromMaybe tbl mAlias)) (tableCols t)
(key, t) <- resolveTable db tbl
let env = lcols ++ prefixColumns (Just (deriveQualifier mAlias key)) (tableCols t)
checkBool db "ON" (Scope env outer) cond
Right env
@@ -433,6 +433,10 @@ checkResolvedWith db outer q = case q of
DropDatabase name -> checkDatabaseName name
UseDatabase name -> checkDatabaseName name
ShowDatabases -> Right ()
-- | 事务控制由会话层执行,语义层不做列检查
BeginTransaction -> Right ()
CommitTransaction -> Right ()
RollbackTransaction -> Right ()
where
tableCols' = map fst . tableCols
@@ -650,21 +654,21 @@ resolveStatement db q = case q of
Select{} -> resolveSelect db q
SelectExpr{} -> resolveSelect db q
Update tbl assigns mWhere -> do
t <- lookupTable db tbl
(key, t) <- resolveTable db tbl
let env = tableCols t
name = nameIn env "UPDATE"
assigns' <- mapM (\(c, e) -> (\e' -> (c, e')) <$> resolveExpr db env [] name e) assigns
w <- mapM (resolveExpr db env [] (nameIn env "WHERE")) mWhere
pure (Update tbl assigns' w)
pure (Update key assigns' w)
Delete tbl mWhere -> do
t <- lookupTable db tbl
(key, t) <- resolveTable db tbl
w <- mapM (resolveExpr db (tableCols t) [] (nameIn (tableCols t) "WHERE")) mWhere
pure (Delete tbl w)
pure (Delete key w)
Insert tbl cols rows -> do
t <- lookupTable db tbl
(key, t) <- resolveTable db tbl
let env = tableCols t
rows' <- mapM (mapM (resolveExpr db env [] (nameIn env "INSERT"))) rows
pure (Insert tbl cols rows')
pure (Insert key cols rows')
other -> Right other
-- | 查一列的物理名
@@ -685,9 +689,9 @@ resolveSelect _ q = Right q
resolveQuery :: Database -> [(String, Column)] -> Statement -> [(String, Expr)] -> Either String Statement
resolveQuery db outer q items = do
env <- checkFrom db outer (selectFrom q)
-- | 单表查询里把裸列名换成物理列名
-- | 单表无别名时把裸列名换成物理列名;带别名时表头保留别名限定名
let physical c = case selectFrom q of
FromTable Nothing tbl -> unqualify (Just tbl) c
FromTable Nothing tbl -> unqualify (Just (deriveQualifier Nothing tbl)) c
_ -> c
-- | 查一列并换算成物理名
name place c = physical . fst <$> scopedColumnAt place env outer c
@@ -707,18 +711,23 @@ resolveQuery db outer q items = do
grouping <- mapM (name "GROUP BY") (selectGroupBy q)
ordering <- mapM (\(c, d) -> (\k -> (k, d)) <$> name "ORDER BY" c) (selectOrderBy q)
source <- case selectFrom q of
single@FromTable{} -> Right single
FromTable mAlias tbl -> do
(key, _) <- resolveTable db tbl
Right (FromTable mAlias key)
other -> resolveFrom db outer other
pure $ case q of
Select{} -> Select (map fst projected) source condition grouping ordering (selectLimit q)
_ -> SelectExpr projected source condition grouping ordering (selectLimit q)
-- | 连接两侧默认取表名作别名;ON 里的子查询也能引用外层列
-- | 连接两侧取表名末段作别名;ON 可引用外层列
resolveFrom :: Database -> [(String, Column)] -> FromClause -> Either String FromClause
resolveFrom _ _ FromUnit = Right FromUnit
resolveFrom _ _ (FromTable alias tbl) = Right (FromTable (Just (fromMaybe tbl alias)) tbl)
resolveFrom db _ (FromTable alias tbl) = do
(key, _) <- resolveTable db tbl
Right (FromTable (Just (deriveQualifier alias key)) key)
resolveFrom db outer fromC@(FromJoin kind left alias tbl cond) = do
env <- checkFrom db outer fromC
l <- resolveFrom db outer left
(key, _) <- resolveTable db tbl
c <- resolveExpr db env outer (fmap fst . columnAt "ON" env) cond
pure (FromJoin kind l (Just (fromMaybe tbl alias)) tbl c)
pure (FromJoin kind l (Just (deriveQualifier alias key)) key c)
@@ -3,6 +3,10 @@
module ChuSQL.Core.Engine.Storage.IPC (
IPCStorage (..),
Env (..),
envForDatabase,
defaultEnv,
runIPCStorage,
Request (..),
Response (..),
Account (..),
@@ -41,21 +45,38 @@ import System.IO.Unsafe (unsafePerformIO)
-- IPC 存储实现:把 MonadStorage 的操作变成发给存储库的 JSON 请求。
newtype IPCStorage a = IPCStorage
{ runIPCStorage :: IO a
-- | 一条会话的存储环境:当前库名随会话走,不共享
data Env = Env
{ envDatabase :: Maybe String
}
newtype IPCStorage a = IPCStorage
{ runIPCStorageIn :: Env -> IO a
}
-- | 按库名造一个会话环境
envForDatabase :: Maybe String -> Env
envForDatabase = Env
-- | 进程默认环境:读全局库名,给测试与 CLI 用
defaultEnv :: IO Env
defaultEnv = Env <$> readMVar databaseRef
-- | 用进程默认环境跑(兼容既有调用点)
runIPCStorage :: IPCStorage a -> IO a
runIPCStorage action = defaultEnv >>= runIPCStorageIn action
instance Functor IPCStorage where
fmap f (IPCStorage m) = IPCStorage (fmap f m)
fmap f (IPCStorage m) = IPCStorage (fmap f . m)
instance Applicative IPCStorage where
pure = IPCStorage . pure
IPCStorage mf <*> IPCStorage ma = IPCStorage (mf <*> ma)
pure = IPCStorage . const . pure
IPCStorage mf <*> IPCStorage ma = IPCStorage $ \env -> mf env <*> ma env
instance Monad IPCStorage where
IPCStorage m >>= k = IPCStorage $ do
a <- m
runIPCStorage (k a)
IPCStorage m >>= k = IPCStorage $ \env -> do
a <- m env
runIPCStorageIn (k a) env
-- | 一条到存储库的链路:管请求与关闭
data StorageLink = StorageLink
@@ -147,10 +168,9 @@ globalRequest ReqPing = True
globalRequest _ = False
-- | 发请求并把响应翻成 Either
ask :: String -> Request -> (Response -> Maybe a) -> IO (Either String a)
ask what req recognize = do
current <- getDatabaseName
resp <- sendRequest (stampDatabase current req)
ask :: Env -> String -> Request -> (Response -> Maybe a) -> IO (Either String a)
ask env what req recognize = do
resp <- sendRequest (stampDatabase (envDatabase env) req)
pure $ case resp of
RespError e -> Left e
other -> case recognize other of
@@ -163,99 +183,100 @@ okOnly RespOk = Just ()
okOnly _ = Nothing
-- | 发 Scan
doScan :: String -> IO (Either String [Row])
doScan t = ask "scan" (ReqScan t) $ \resp -> case resp of
doScan :: Env -> String -> IO (Either String [Row])
doScan env t = ask env "scan" (ReqScan t) $ \resp -> case resp of
RespRows rows -> Just rows
_ -> Nothing
-- | 发带投影的 Scan
doScanColumns :: String -> [String] -> IO (Either String [Row])
doScanColumns t cols = ask "scan" (ReqScanColumns t cols) $ \resp -> case resp of
doScanColumns :: Env -> String -> [String] -> IO (Either String [Row])
doScanColumns env t cols = ask env "scan" (ReqScanColumns t cols) $ \resp -> case resp of
RespRows rows -> Just [[(c, v) | c <- cols, Just v <- [lookup c row]] | row <- rows]
_ -> Nothing
-- | 发分片 Scan:只取第 shard 片
doScanShard :: String -> Maybe [String] -> Int -> Int -> IO (Either String [Row])
doScanShard t cols shard shards = ask "scan_shard" (ReqScanShard t cols shard shards) $ \resp -> case resp of
doScanShard :: Env -> String -> Maybe [String] -> Int -> Int -> IO (Either String [Row])
doScanShard env t cols shard shards = ask env "scan_shard" (ReqScanShard t cols shard shards) $ \resp -> case resp of
RespRows rows -> Just rows
_ -> Nothing
-- | 发 Insert(索引由存储层维护)
doInsert :: String -> Row -> IO (Either String ())
doInsert t r = ask "insert" (ReqInsert t r) okOnly
doInsert :: Env -> String -> Row -> IO (Either String ())
doInsert env t r = ask env "insert" (ReqInsert t r) okOnly
-- | 发 InsertBatch:一批行一次请求
doInsertMany :: String -> [Row] -> IO (Either String ())
doInsertMany t rs = ask "insert_batch" (ReqInsertBatch t rs) okOnly
doInsertMany :: Env -> String -> [Row] -> IO (Either String ())
doInsertMany env t rs = ask env "insert_batch" (ReqInsertBatch t rs) okOnly
-- | 发 DeleteKeys
doDeleteKeys :: String -> [Int] -> IO (Either String ())
doDeleteKeys t ks = ask "delete_keys" (ReqDeleteKeys t ks) okOnly
doDeleteKeys :: Env -> String -> [Int] -> IO (Either String ())
doDeleteKeys env t ks = ask env "delete_keys" (ReqDeleteKeys t ks) okOnly
-- | 发 ReplaceAll
doReplaceAll :: String -> [Row] -> IO (Either String ())
doReplaceAll t rs = ask "replace_all" (ReqReplaceAll t rs) okOnly
doReplaceAll :: Env -> String -> [Row] -> IO (Either String ())
doReplaceAll env t rs = ask env "replace_all" (ReqReplaceAll t rs) okOnly
-- | 发 ListTables
doListTables :: IO (Either String [String])
doListTables = ask "list_tables" ReqListTables $ \resp -> case resp of
doListTables :: Env -> IO (Either String [String])
doListTables env = ask env "list_tables" ReqListTables $ \resp -> case resp of
RespTables ts -> Just ts
_ -> Nothing
-- | 一次拿全库 schema
doListCatalog :: IO (Either String [TableInfo])
doListCatalog = ask "list_catalog" ReqListCatalog $ \resp -> case resp of
doListCatalog :: Env -> IO (Either String [TableInfo])
doListCatalog env = ask env "list_catalog" ReqListCatalog $ \resp -> case resp of
RespCatalog xs -> Just xs
_ -> Nothing
-- | 按某一列的索引取一行(键可以是整数或字符串)
doLookupByColumn :: String -> String -> Value -> IO (Either String IndexResult)
doLookupByColumn t c k = ask "lookup_by_index" (ReqLookupByColumn t c k) $ \resp -> case resp of
doLookupByColumn :: Env -> String -> String -> Value -> IO (Either String IndexResult)
doLookupByColumn env t c k = ask env "lookup_by_index" (ReqLookupByColumn t c k) $ \resp -> case resp of
RespRows rows -> Just (IndexRows rows)
RespNoIndex -> Just NoIndex
_ -> Nothing
-- | 范围扫描:没有可用索引就回 Nothing,让上层退回扫描
doScanRange ::
Env ->
String ->
String ->
Maybe (Value, Bool) ->
Maybe (Value, Bool) ->
IO (Either String (Maybe [Row]))
doScanRange t c lo hi = ask "range_by_index" (ReqRangeByIndex t c lo hi) $ \resp -> case resp of
doScanRange env t c lo hi = ask env "range_by_index" (ReqRangeByIndex t c lo hi) $ \resp -> case resp of
RespRows rows -> Just (Just rows)
RespNoIndex -> Just Nothing
_ -> Nothing
-- | 发 DescribeTable
doDescribeTable :: String -> IO (Either String TableInfo)
doDescribeTable t = ask "describe_table" (ReqDescribeTable t) $ \resp -> case resp of
doDescribeTable :: Env -> String -> IO (Either String TableInfo)
doDescribeTable env t = ask env "describe_table" (ReqDescribeTable t) $ \resp -> case resp of
RespSchema info -> Just info
_ -> Nothing
-- | 给某一列建索引
doCreateIndex :: String -> String -> IO (Either String ())
doCreateIndex t c = ask "create_index" (ReqCreateIndex t c) okOnly
doCreateIndex :: Env -> String -> String -> IO (Either String ())
doCreateIndex env t c = ask env "create_index" (ReqCreateIndex t c) okOnly
-- | 去掉某一列的索引
doDropIndex :: String -> String -> IO (Either String ())
doDropIndex t c = ask "drop_index" (ReqDropIndex t c) okOnly
doDropIndex :: Env -> String -> String -> IO (Either String ())
doDropIndex env t c = ask env "drop_index" (ReqDropIndex t c) okOnly
-- | 删一列(列定义、这一列上的索引、每行里的那一格一起没)
doDropColumn :: String -> String -> IO (Either String ())
doDropColumn t c = ask "drop_column" (ReqDropColumn t c) okOnly
doDropColumn :: Env -> String -> String -> IO (Either String ())
doDropColumn env t c = ask env "drop_column" (ReqDropColumn t c) okOnly
-- | 发 ReplaceSchema(ALTER 用)
doReplaceSchema :: String -> [SchemaColumn] -> [Row] -> IO (Either String ())
doReplaceSchema t cols rs = ask "replace_schema" (ReqReplaceSchema t cols rs) okOnly
doReplaceSchema :: Env -> String -> [SchemaColumn] -> [Row] -> IO (Either String ())
doReplaceSchema env t cols rs = ask env "replace_schema" (ReqReplaceSchema t cols rs) okOnly
-- | 发 CreateTable
doCreateTable :: String -> [SchemaColumn] -> IO (Either String ())
doCreateTable t cols = ask "create_table" (ReqCreateTable t cols) okOnly
doCreateTable :: Env -> String -> [SchemaColumn] -> IO (Either String ())
doCreateTable env t cols = ask env "create_table" (ReqCreateTable t cols) okOnly
-- | 发 DropTable
doDropTable :: String -> IO (Either String ())
doDropTable t = ask "drop_table" (ReqDropTable t) okOnly
doDropTable :: Env -> String -> IO (Either String ())
doDropTable env t = ask env "drop_table" (ReqDropTable t) okOnly
-- | 线上 schema 转本地列
schemaToColumns :: [SchemaColumn] -> [(String, Column)]
@@ -292,92 +313,91 @@ toWire (n, c) =
instance MonadStorage IPCStorage where
-- \| 建一个库
createDatabase name = IPCStorage (ask "create_database" (ReqDatabase "create_database" name) okOnly)
createDatabase name = IPCStorage $ \env -> ask env "create_database" (ReqDatabase "create_database" name) okOnly
-- \| 删一个库
dropDatabase name = IPCStorage (ask "drop_database" (ReqDatabase "drop_database" name) okOnly)
dropDatabase name = IPCStorage $ \env -> ask env "drop_database" (ReqDatabase "drop_database" name) okOnly
-- \| 切当前库
useDatabase name = IPCStorage (ask "use_database" (ReqDatabase "use_database" name) okOnly)
useDatabase name = IPCStorage $ \env -> ask env "use_database" (ReqDatabase "use_database" name) okOnly
-- \| 列出现有库
listDatabases = IPCStorage $ ask "list_databases" (ReqDatabase "list_databases" "") $ \resp -> case resp of
listDatabases = IPCStorage $ \env -> ask env "list_databases" (ReqDatabase "list_databases" "") $ \resp -> case resp of
RespTables names -> Just names
_ -> Nothing
-- \| 发 Scan
scan t = IPCStorage (doScan t)
scan t = IPCStorage (\env -> doScan env t)
-- \| 只取指定列
scanColumns t cols = IPCStorage (doScanColumns t cols)
scanColumns t cols = IPCStorage (\env -> doScanColumns env t cols)
-- \| 并行分片数:按 RTS 能力数,封顶在 parallelShardLimit
parallelShards = IPCStorage $ do
parallelShards = IPCStorage $ \_ -> do
caps <- getNumCapabilities
pure (max 1 (min parallelShardLimit caps))
-- \| 分片并行扫,再按分片序拼接
scanShards width t cols
| width <= 1 = IPCStorage (maybe (doScan t) (doScanColumns t) cols)
| otherwise = IPCStorage $ do
| width <= 1 = IPCStorage (\env -> maybe (doScan env t) (doScanColumns env t) cols)
| otherwise = IPCStorage $ \env -> do
pool <- workerPool width
parts <- poolRun pool [doScanShard t cols shard width | shard <- [0 .. width - 1]]
parts <- poolRun pool [doScanShard env t cols shard width | shard <- [0 .. width - 1]]
pure (fmap concat (sequence parts))
-- \| 发 Insert(索引由存储层自己维护)
insert t r = IPCStorage (doInsert t r)
insert t r = IPCStorage (\env -> doInsert env t r)
-- \| 一批行一次请求:N 行只落一次盘
insertMany t rs = IPCStorage (doInsertMany t rs)
insertMany t rs = IPCStorage (\env -> doInsertMany env t rs)
-- \| 按 id 批量删行
deleteKeys t ks = IPCStorage (doDeleteKeys t ks)
deleteKeys t ks = IPCStorage (\env -> doDeleteKeys env t ks)
-- \| 发 ReplaceAll
replaceAll t rs = IPCStorage (doReplaceAll t rs)
replaceAll t rs = IPCStorage (\env -> doReplaceAll env t rs)
-- \| 按某一列的索引取一行(没有索引就回 NoIndex)
lookupByColumn t c k = IPCStorage (doLookupByColumn t c k)
lookupByColumn t c k = IPCStorage (\env -> doLookupByColumn env t c k)
-- \| 范围扫描(没有索引就回 Nothing)
scanRange t c lo hi = IPCStorage (doScanRange t c lo hi)
scanRange t c lo hi = IPCStorage (\env -> doScanRange env t c lo hi)
-- \| 发 CreateTable
createTable name cols = IPCStorage (doCreateTable name (map toWire cols))
createTable name cols = IPCStorage (\env -> doCreateTable env name (map toWire cols))
-- \| ALTER:整表换列定义与全部行
replaceSchema name cols rows = IPCStorage (doReplaceSchema name (map toWire cols) rows)
replaceSchema name cols rows = IPCStorage (\env -> doReplaceSchema env name (map toWire cols) rows)
-- \| 发 DropTable
dropTable name = IPCStorage (doDropTable name)
dropTable name = IPCStorage (\env -> doDropTable env name)
-- \| 建索引
createIndex t c = IPCStorage (doCreateIndex t c)
createIndex t c = IPCStorage (\env -> doCreateIndex env t c)
-- \| 删索引
dropIndex t c = IPCStorage (doDropIndex t c)
dropIndex t c = IPCStorage (\env -> doDropIndex env t c)
-- \| 删一列
dropColumn t c = IPCStorage (doDropColumn t c)
dropColumn t c = IPCStorage (\env -> doDropColumn env t c)
-- \| 列表 + 逐表扫描拼库
snapshot = IPCStorage $ do
result <- doListCatalog
snapshot = IPCStorage $ \env -> do
result <- doListCatalog env
case result of
Left _ -> pure []
Right xs -> mapM loadEntry xs
Right xs -> mapM (loadEntry env) xs
where
loadEntry info = do
rows <- doScan (tiTable info)
loadEntry env info = do
rows <- doScan env (tiTable info)
pure
( tiTable info
, Table (tiTable info) (schemaToColumns (tiColumns info)) (either (const []) id rows) (Just (metaOf info))
)
-- \| 只问数据字典要结构,设了当前库只留它的表
schema = IPCStorage $ do
current <- getDatabaseName
result <- ask "all_catalogs" ReqAllCatalog $ \resp -> case resp of
schema = IPCStorage $ \env -> do
result <- ask env "all_catalogs" ReqAllCatalog $ \resp -> case resp of
RespCatalog infos -> Just infos
_ -> Nothing
pure $ case result of
Left _ -> []
Right xs -> concatMap (visibleTables current) xs
Right xs -> concatMap (visibleTables (envDatabase env)) xs
-- | 线上表信息转引擎侧统计
metaOf :: TableInfo -> TableMeta
@@ -53,6 +53,12 @@ data Statement
| DropDatabase String
| UseDatabase String
| ShowDatabases
-- | 开事务:BEGIN / START TRANSACTION
| BeginTransaction
-- | 提交事务
| CommitTransaction
-- | 回滚事务
| RollbackTransaction
deriving (Show, Eq)
-- | 连接类型:内连接与左外连接
@@ -613,6 +613,14 @@ updateStatement = do
mWhere <- optional (keyword "where" *> expr)
return (Update tbl assigns mWhere)
-- | 事务控制语句
transactionStatement :: Parser Statement
transactionStatement =
try (BeginTransaction <$ (keyword "start" *> keyword "transaction"))
<|> (BeginTransaction <$ (keyword "begin" *> optional (keyword "transaction")))
<|> (CommitTransaction <$ (keyword "commit" *> optional (keyword "transaction")))
<|> (RollbackTransaction <$ (keyword "rollback" *> optional (keyword "transaction")))
-- | 解析总入口
parseStatement :: String -> Either String Statement
parseStatement input =
@@ -622,7 +630,8 @@ parseStatement input =
where
-- | 按关键字分派到各语句解析器
statementP =
selectStatement
transactionStatement
<|> selectStatement
<|> try (CreateDatabase <$> (keyword "create" *> keyword "database" *> identifier))
<|> try (DropDatabase <$> (keyword "drop" *> keyword "database" *> identifier))
<|> (UseDatabase <$> (keyword "use" *> identifier))
+75 -2
View File
@@ -15,7 +15,7 @@ import ChuSQL.Core.Engine.Parallel (poolRun, workerPool)
import ChuSQL.Core.Model
import ChuSQL.Core.Engine.Semantic (prepare)
import ChuSQL.Core.Engine.Storage (IndexResult (..), MonadStorage (..))
import ChuSQL.Core.Engine.Storage.IPC (IPCStorage (runIPCStorage), closeConnection, doListTables, getDatabaseName, localStorageLink, setDatabaseName, setStorageLink)
import ChuSQL.Core.Engine.Storage.IPC (IPCStorage (..), closeConnection, defaultEnv, doListTables, envForDatabase, getDatabaseName, localStorageLink, runIPCStorage, setDatabaseName, setStorageLink)
import ChuSQL.Core.Engine.Syntax.AST
import ChuSQL.Core.Engine.Syntax.Parser
import Control.Concurrent (threadDelay)
@@ -113,6 +113,40 @@ main = hspec $ do
rowsOf (parseStatement "SELECT sales.users.name FROM sales.users WHERE sales.users.id = 1" >>= runStatement db)
`shouldBe` Right [[("name", VStr "Alice")]]
it "derives the column prefix when the session schema is scoped to one database" $ do
let scoped = [("users", users)]
rowsOf (parseStatement "SELECT sales.users.name FROM sales.users WHERE sales.users.id = 1" >>= runStatement scoped)
`shouldBe` Right [[("name", VStr "Alice")]]
rowsOf (parseStatement "SELECT users.name FROM users WHERE users.id = 1" >>= runStatement scoped)
`shouldBe` Right [[("name", VStr "Alice")]]
it "derives the database prefix for a bare table name" $ do
let catalog = [("sales.users", users)]
rowsOf (parseStatement "SELECT name FROM users WHERE users.id = 1" >>= runStatement catalog)
`shouldBe` Right [[("name", VStr "Alice")]]
rowsOf (parseStatement "SELECT users.name FROM sales.users WHERE users.id = 1" >>= runStatement catalog)
`shouldBe` Right [[("name", VStr "Alice")]]
it "keeps derived prefixes short in joins" $ do
let catalog = [("sales.users", users), ("sales.orders", orders)]
fmap
(map (map snd))
(parseStatement "SELECT users.name, orders.product FROM sales.users JOIN sales.orders ON users.id = orders.user_id" >>= rowsOf . runStatement catalog)
`shouldBe` Right [[VStr "Alice", VStr "Book"], [VStr "Alice", VStr "Cup"], [VStr "Bob", VStr "Pen"]]
it "writes through the derived table key" $ do
let scoped = [("users", users)]
case parseStatement "UPDATE sales.users SET age = 26 WHERE id = 1" >>= runStatement scoped of
Left err -> expectationFailure err
Right (db', _) ->
rowsOf (runStatement db' (makeSelect ["age"] "users" (Just (Eq (Col "id") (LitInt 1)))))
`shouldBe` Right [[("age", VInt 26)]]
it "rejects an ambiguous derived table name" $ do
let catalog = [("sales.users", users), ("other.users", users)]
(parseStatement "SELECT name FROM users" >>= rowsOf . runStatement catalog)
`shouldSatisfy` either (isInfixOf "ambiguous table") (const False)
it "parses database lifecycle statements" $ do
map (fmap show . parseStatement) ["CREATE DATABASE sales", "DROP DATABASE sales", "USE sales", "SHOW DATABASES"]
`shouldBe` map Right ["CreateDatabase \"sales\"", "DropDatabase \"sales\"", "UseDatabase \"sales\"", "ShowDatabases"]
@@ -242,6 +276,33 @@ main = hspec $ do
parseStatement "SELECT name FROM"
`shouldSatisfy` isLeft
describe "ChuSQL.Core.Engine (事务)" $ do
it "parses transaction control statements" $ do
parseStatement "BEGIN" `shouldBe` Right BeginTransaction
parseStatement "BEGIN TRANSACTION" `shouldBe` Right BeginTransaction
parseStatement "START TRANSACTION" `shouldBe` Right BeginTransaction
parseStatement "begin transaction" `shouldBe` Right BeginTransaction
parseStatement "COMMIT" `shouldBe` Right CommitTransaction
parseStatement "COMMIT TRANSACTION" `shouldBe` Right CommitTransaction
parseStatement "ROLLBACK" `shouldBe` Right RollbackTransaction
parseStatement "rollback transaction" `shouldBe` Right RollbackTransaction
it "parses the bare words only as whole words" $ do
parseStatement "BEGINNER" `shouldSatisfy` isLeft
parseStatement "COMMITTED" `shouldSatisfy` isLeft
it "reports transaction control as session work" $ do
mapM_
( \sql ->
(runStatement [("users", users)] =<< parseStatement sql)
`shouldSatisfy` either (isInfixOf "executed by the session") (const False)
)
["BEGIN", "COMMIT", "ROLLBACK"]
it "puts transaction errors under the protocol category" $ do
errorCode "BEGIN is executed by the session" `shouldBe` "query_error"
errorCategory "BEGIN is executed by the session" `shouldBe` ProtocolError
describe "ChuSQL.Core.Engine.Syntax.Parser (JOIN)" $ do
it "parses a simple JOIN without aliases" $ do
parseStatement "SELECT name FROM users JOIN orders ON id = user_id"
@@ -1613,7 +1674,19 @@ main = hspec $ do
withTestSession $ \srv -> withServerEnv srv $ do
runIPCStorage (insert "t1" [("id", VInt 1)]) `shouldReturn` Right ()
runIPCStorage (insert "t2" [("id", VInt 2)]) `shouldReturn` Right ()
doListTables `shouldReturn` Right ["t1", "t2"]
env <- defaultEnv
doListTables env `shouldReturn` Right ["t1", "t2"]
it "two sessions keep their own database" $ do
withTestSession $ \_ -> do
let alpha = envForDatabase (Just "alpha")
beta = envForDatabase (Just "beta")
runIPCStorageIn (createDatabase "alpha") alpha `shouldReturn` Right ()
runIPCStorageIn (createDatabase "beta") beta `shouldReturn` Right ()
runIPCStorageIn (insert "iso" [("id", VInt 1)]) alpha `shouldReturn` Right ()
runIPCStorageIn (insert "iso" [("id", VInt 2)]) beta `shouldReturn` Right ()
runIPCStorageIn (scan "iso") alpha `shouldReturn` Right [[("id", VInt 1)]]
runIPCStorageIn (scan "iso") beta `shouldReturn` Right [[("id", VInt 2)]]
it "rows survive a restart on the same data directory" $ do
withTestSession $ \srv -> do
+31 -2
View File
@@ -15,6 +15,9 @@ module ChuSQL.Core.Model
, colNames
, colType
, lookupTable
, resolveTable
, lastTablePart
, deriveQualifier
, allColumns
, qualify
, unqualify
@@ -470,6 +473,32 @@ colType t c = lookup c (tableCols t)
lookupTable :: Database -> String -> Either String Table
lookupTable db t = maybe (Left ("unknown table: " ++ t)) Right (lookup t db)
-- | 取表名引用的最后一段
lastTablePart :: String -> String
lastTablePart = reverse . takeWhile (/= '.') . reverse
-- | 推导列前缀:有别名用别名,否则取表名末段
deriveQualifier :: Maybe String -> String -> String
deriveQualifier (Just a) _ = a
deriveQualifier Nothing t = lastTablePart t
-- | 按名查表并返回 schema 键:精确、前缀、唯一后缀
resolveTable :: Database -> String -> Either String (String, Table)
resolveTable db name = case lookup name db of
Just table -> Right (name, table)
Nothing
| '.' `elem` name, Just table <- lookup bare db -> Right (bare, table)
| [(key, table)] <- suffixed -> Right (key, table)
| otherwise -> Left (miss suffixed)
where
-- | 引用里去掉库前缀的那一段
bare = lastTablePart name
-- | schema 里以这个表名收尾的键:`sales.users` 能推出裸表名引用
suffixed = [(key, table) | (key, table) <- db, ('.' : name) `isSuffixOf` key]
-- | 一个候选都没有就是查无此表,多于一个就是有歧义
miss [] = "unknown table: " ++ name
miss _ = "ambiguous table: " ++ name
-- | 通配列名
allColumns :: String
allColumns = "*"
@@ -491,5 +520,5 @@ resolveColumn name env = case [(k, v) | (k, v) <- env, matches k] of
[] -> Left ("unknown column: " ++ name)
_ -> Left ("ambiguous column: " ++ name)
where
-- | 裸名或带后缀的名称
matches k = k == name || ('.' : name) `isSuffixOf` k
-- | 裸名、补充了库前缀的写法,或 schema 键自带库前缀的写法
matches k = k == name || ('.' : name) `isSuffixOf` k || ('.' : k) `isSuffixOf` name
@@ -3,6 +3,7 @@
module ChuSQL.Core.Protocol (
SchemaColumn (..),
Request (..),
TxnOp (..),
Response (..),
TableInfo (..),
Account (..),
@@ -96,6 +97,35 @@ data Request
| ReqDropColumn String String
| ReqReplaceSchema String [SchemaColumn] [Row]
| ReqListCatalog
| ReqApplyTransaction [TxnOp]
-- | 事务提交里的一条写操作
data TxnOp
= TxnUpsert String [Row]
| TxnDelete String [Int]
| TxnReplace String [Row]
deriving (Show, Eq)
-- | 事务操作编码成 JSON
instance ToJSON TxnOp where
toJSON (TxnUpsert t rs) =
object
[ "op" .= ("upsert" :: T.Text)
, "table" .= t
, "rows" .= map rowToJSON rs
]
toJSON (TxnDelete t ks) =
object
[ "op" .= ("delete" :: T.Text)
, "table" .= t
, "ids" .= ks
]
toJSON (TxnReplace t rs) =
object
[ "op" .= ("replace" :: T.Text)
, "table" .= t
, "rows" .= map rowToJSON rs
]
-- | 请求编码成 JSON
instance ToJSON Request where
@@ -201,6 +231,11 @@ instance ToJSON Request where
, "rows" .= map rowToJSON rs
]
toJSON ReqListCatalog = object ["method" .= ("list_catalog" :: T.Text)]
toJSON (ReqApplyTransaction ops) =
object
[ "method" .= ("apply_transaction" :: T.Text)
, "ops" .= ops
]
-- | 一张表的梗概:列、行数、索引与列统计
data TableInfo = TableInfo
+23 -2
View File
@@ -171,6 +171,22 @@ pub enum Request {
columns: Vec<SchemaColumn>,
rows: Vec<Row>,
},
/// 事务提交:把一批写操作当成一个请求原子应用
ApplyTransaction {
ops: Vec<StorageOp>,
},
}
/// 事务提交里的一条写操作
#[derive(Debug, Serialize, Deserialize)]
#[serde(tag = "op", rename_all = "snake_case")]
pub enum StorageOp {
/// 追加写入的行(提交端已先删掉要覆盖的行)
Upsert { table: String, rows: Vec<Row> },
/// 按行 id 删除
Delete { table: String, ids: Vec<i64> },
/// 整表替换(没有整数 id 的表只能这么提交)
Replace { table: String, rows: Vec<Row> },
}
impl Request {
@@ -202,7 +218,8 @@ impl Request {
| Self::ReplaceSchema { table, .. } => Some(table),
Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList
| Self::AccountCreate { .. } | Self::AccountReset { .. } | Self::AccountLogin { .. }
| Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus => None,
| Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus
| Self::ApplyTransaction { .. } => None,
}
}
}
@@ -237,7 +254,11 @@ pub enum Response {
NoIndex,
Ok,
/// 系统目录状态
System { initialized: bool },
System {
initialized: bool,
#[serde(default)]
last_lsn: u64,
},
Error { message: String },
Catalog { schemas: Vec<TableSchemaWire> },
}
+233 -59
View File
@@ -8,9 +8,10 @@ use crate::config::{self, Config, Loaded};
use crate::heap::{scalar_int, HeapTable};
use crate::log;
use crate::protocol::{
reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row, SchemaColumn, TableSchemaWire, USERS_TABLE,
reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row, SchemaColumn, StorageOp,
TableSchemaWire, USERS_TABLE,
};
use crate::wal::{Wal, WalOp};
use crate::wal::{committed_ops, Wal, WalOp};
use crate::{log_debug, log_error, log_info, log_warn};
// 进程内存储运行时:Haskell 侧经 ffi.rs 调进来。
@@ -196,6 +197,7 @@ impl Server {
/// 把一条 WAL 操作落到磁盘
fn apply_op(&self, op: &WalOp) -> std::io::Result<()> {
match op {
WalOp::Commit => Ok(()),
WalOp::HideColumn { table, column } => {
let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
c.remove_column(table, column)?;
@@ -311,52 +313,119 @@ impl Server {
if self.recovery_required.load(Ordering::Acquire) {
return Err(std::io::Error::other("storage recovery required"));
}
{
let c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
let validate = |table: &str, rows: &[Row]| -> std::io::Result<()> {
let Some(schema) = c.describe(table) else { return Ok(()); };
if let Some(name) = rows.iter().flat_map(|r| r.keys()).find(|k| schema.dropped_columns.contains(*k)) {
return Err(std::io::Error::other(format!("unknown column: {name}")));
}
Ok(())
};
match op {
WalOp::Insert { table, row } => validate(table, std::slice::from_ref(row))?,
WalOp::InsertBatch { table, rows } | WalOp::ReplaceAll { table, rows } => validate(table, rows)?,
WalOp::HideColumn { table, column }
if !c.describe(table).is_some_and(|s| s.columns.iter().any(|col| &col.name == column)) =>
{
return Err(std::io::Error::other(format!("unknown column: {column}")));
}
_ => {}
self.validate_op(op)?;
self.group_via_wal(std::slice::from_ref(op))
}
/// 校验一条操作:命中被删的列就拒绝
fn validate_op(&self, op: &WalOp) -> std::io::Result<()> {
let c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
let validate = |table: &str, rows: &[Row]| -> std::io::Result<()> {
let Some(schema) = c.describe(table) else { return Ok(()); };
if let Some(name) = rows.iter().flat_map(|r| r.keys()).find(|k| schema.dropped_columns.contains(*k)) {
return Err(std::io::Error::other(format!("unknown column: {name}")));
}
Ok(())
};
match op {
WalOp::Insert { table, row } => validate(table, std::slice::from_ref(row))?,
WalOp::InsertBatch { table, rows } | WalOp::ReplaceAll { table, rows } => validate(table, rows)?,
WalOp::HideColumn { table, column }
if !c.describe(table).is_some_and(|s| s.columns.iter().any(|col| &col.name == column)) =>
{
return Err(std::io::Error::other(format!("unknown column: {column}")));
}
_ => {}
}
Ok(())
}
/// 事务提交:一组写操作加一个提交标记,一次落盘
///
/// 组里每条都过 `validate_op`,跟单条写一致;调用方 `dispatch` 已经持有
/// `gate.write`,因此整批对外原子可见。崩在提交标记之前,这组记录在恢复
/// 时整组丢弃;崩在标记之后,逐条按「是否已生效」补写。
fn apply_transaction(&self, ops: &[StorageOp]) -> std::io::Result<()> {
let _guard = self.write_lock.lock().unwrap();
if self.recovery_required.load(Ordering::Acquire) {
return Err(std::io::Error::other("storage recovery required"));
}
let mut wal_ops = Vec::with_capacity(ops.len());
for op in ops {
let table = match op {
StorageOp::Upsert { table, .. }
| StorageOp::Delete { table, .. }
| StorageOp::Replace { table, .. } => table,
};
if reserved_table(table) {
return Err(std::io::Error::other("reserved system table"));
}
if table.is_empty() || !table.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') {
return Err(std::io::Error::other("invalid table name"));
}
let wal_op = match op {
StorageOp::Upsert { table, rows } => WalOp::InsertBatch {
table: table.clone(),
rows: rows.clone(),
},
StorageOp::Delete { table, ids } => WalOp::DeleteKeys {
table: table.clone(),
keys: ids.clone(),
},
StorageOp::Replace { table, rows } => WalOp::ReplaceAll {
table: table.clone(),
rows: rows.clone(),
},
};
self.validate_op(&wal_op)?;
wal_ops.push(wal_op);
}
self.group_via_wal(&wal_ops)
}
/// 一组操作加提交标记落盘,成功后清空日志
fn group_via_wal(&self, ops: &[WalOp]) -> std::io::Result<()> {
std::fs::create_dir_all(&self.cfg.data_dir)?;
let wal = &self.wal;
wal.truncate()?;
wal.append(op)?;
let result = self.apply_op(op);
if result.is_err() && matches!(op, WalOp::HideColumn { .. }) {
self.recovery_required.store(true, Ordering::Release);
return result;
let commit_lsn = wal.append_group(ops)?;
for op in ops {
let result = self.apply_op(op);
if result.is_err() && matches!(op, WalOp::HideColumn { .. }) {
self.recovery_required.store(true, Ordering::Release);
return result;
}
result?;
}
if result.is_ok() {
for op in ops {
self.flush_op_tables(op)?;
}
self.flush_all_tables()?;
wal.clear()?;
result
log_debug!(core, "WAL committed group at LSN {}", commit_lsn);
Ok(())
}
/// 把打开的表都 fsync 一遍
fn flush_all_tables(&self) -> std::io::Result<()> {
let handles: Vec<Arc<Mutex<HeapTable>>> = self
.tables
.lock()
.map_err(|_| std::io::Error::other("table lock poisoned"))?
.values()
.cloned()
.collect();
for handle in handles {
handle.lock().map_err(|_| std::io::Error::other("heap lock poisoned"))?.flush()?;
}
Ok(())
}
/// 把涉及的表写回磁盘
fn flush_op_tables(&self, op: &WalOp) -> std::io::Result<()> {
let table = match op {
WalOp::HideColumn { .. } => return Ok(()),
WalOp::Commit => return Ok(()),
WalOp::Accounts { .. } => USERS_TABLE,
WalOp::Insert { table, .. }
| WalOp::InsertBatch { table, .. }
@@ -369,21 +438,97 @@ impl Server {
self.with_table(table, |t| t.flush())
}
/// 启动时重放残留 WAL
/// 重放残留 WAL:已提交组重放,未提交组丢弃
fn recover_wal(&self) -> std::io::Result<()> {
std::fs::create_dir_all(&self.cfg.data_dir)?;
let wal = &self.wal;
match wal.read()? {
None => Ok(()),
Some(op) => {
log_warn!(core, "replaying incomplete WAL entry");
self.apply_op(&op)?;
self.flush_op_tables(&op)?;
wal.clear()?;
log_info!(core, "WAL replay complete");
Ok(())
let scan = wal.read_all()?;
if scan.records.is_empty() {
if scan.good_len > 0 {
wal.truncate_to(scan.good_len)?;
}
return Ok(());
}
let records = committed_ops(&scan.records);
if records.is_empty() {
log_warn!(core, "discarding uncommitted WAL records");
wal.clear()?;
return Ok(());
}
let max_lsn = scan
.records
.iter()
.rev()
.find(|(_, op)| *op == WalOp::Commit)
.map(|(lsn, _)| *lsn)
.or_else(|| records.last().map(|(lsn, _)| *lsn))
.unwrap_or(0);
log_warn!(core, "replaying {} committed WAL records", records.len());
for (_, op) in &records {
self.replay_op(op)?;
self.flush_op_tables(op)?;
}
self.flush_all_tables()?;
wal.write_checkpoint(max_lsn)?;
wal.clear()?;
log_info!(core, "WAL replay complete");
Ok(())
}
/// 重放一条操作:数据已经生效的不再重复写
fn replay_op(&self, op: &WalOp) -> std::io::Result<()> {
if self.op_already_applied(op)? {
log_debug!(core, "WAL record already applied, skipping");
return Ok(());
}
self.apply_op(op)
}
/// 这条操作对应的数据是不是已经落盘了
fn op_already_applied(&self, op: &WalOp) -> std::io::Result<bool> {
match op {
WalOp::Insert { table, row } => self.applied_rows(table, std::slice::from_ref(row)),
WalOp::InsertBatch { table, rows } => self.applied_rows(table, rows),
WalOp::ReplaceAll { table, rows } => {
if self.catalog.lock().unwrap().describe(table).is_none() {
return Ok(false);
}
let stored = self.with_table(table, |t| t.scan())?;
Ok(stored == *rows)
}
WalOp::DeleteKeys { table, keys } => {
if keys.is_empty() {
return Ok(true);
}
if self.catalog.lock().unwrap().describe(table).is_none() {
return Ok(false);
}
let rows = self.with_table(table, |t| t.scan())?;
let present = rows.iter().any(|row| {
row.get("id")
.and_then(scalar_int)
.is_some_and(|id| keys.contains(&id))
});
Ok(!present)
}
WalOp::HideColumn { table, column } => {
let c = self.catalog.lock().unwrap();
Ok(c.describe(table).is_some_and(|s| !s.columns.iter().any(|col| &col.name == column)))
}
_ => Ok(false),
}
}
/// 这些行是不是都已经在表里
fn applied_rows(&self, table: &str, rows: &[Row]) -> std::io::Result<bool> {
if rows.is_empty() {
return Ok(true);
}
if self.catalog.lock().unwrap().describe(table).is_none() {
return Ok(false);
}
let stored = self.with_table(table, |t| t.scan())?;
Ok(rows.iter().all(|row| stored.iter().any(|one| one == row)))
}
/// 启动时重建列统计
@@ -607,6 +752,11 @@ impl Server {
.unwrap_or(false)
}
/// 最近一次落盘的检查点 LSN
fn last_checkpoint(&self) -> u64 {
self.wal.read_checkpoint().ok().flatten().unwrap_or(0)
}
/// 读账号表;表没登记或数据文件缺失都算错
fn load_accounts(&self) -> std::io::Result<Vec<Account>> {
use std::io::Error;
@@ -774,16 +924,29 @@ impl Server {
}
Request::BootstrapSystem { user, password_hash } => {
match self.bootstrap_system(user.as_deref(), password_hash.as_deref()) {
Ok(_) => Response::System { initialized: true },
Ok(_) => Response::System { initialized: true, last_lsn: self.last_checkpoint() },
Err(e) => Response::Error { message: e.to_string() },
}
}
Request::SystemStatus => Response::System { initialized: self.system_initialized() },
Request::SystemStatus => Response::System { initialized: self.system_initialized(), last_lsn: self.last_checkpoint() },
Request::Ping => {
log_debug!(request, "ping");
Response::Pong
}
Request::ApplyTransaction { ops } => {
log_debug!(request, "apply_transaction ops={}", ops.len());
match self.apply_transaction(&ops) {
Ok(()) => Response::Ok,
Err(e) => {
log_warn!(request, "apply_transaction: {}", e);
Response::Error {
message: format!("apply_transaction: {e}"),
}
}
}
}
Request::Scan { table, columns } => {
log_debug!(request, "scan table={}", table);
@@ -1347,8 +1510,8 @@ fn reserved_database(name: &str) -> bool {
/// 每个库拥有独立的目录、字典、句柄和 WAL。
struct Databases {
root: Config,
system: Server,
named: Mutex<HashMap<String, Server>>,
system: Arc<Server>,
named: Mutex<HashMap<String, Arc<Server>>>,
}
impl Databases {
@@ -1356,7 +1519,7 @@ impl Databases {
fn new(cfg: Config) -> std::io::Result<Self> {
// 系统库按需创建:目录、字典与系统表缺失时自动补齐。
// 其余库只在 data/databases/<name>/ 下已存在时才装载,启动不会凭空建库。
let system = Self::open(&cfg, SYSTEM_DATABASE)?;
let system = Arc::new(Self::open(&cfg, SYSTEM_DATABASE)?);
let root = cfg.data_dir.join("databases");
std::fs::create_dir_all(&root)?;
let canonical_root = std::fs::canonicalize(&root)?;
@@ -1379,7 +1542,7 @@ impl Databases {
server.rebuild_stats()?;
server.migrate_data()?;
server.compact_tables()?;
named.insert(name, server);
named.insert(name, Arc::new(server));
}
}
Ok(Self { root: cfg, system, named: Mutex::new(named) })
@@ -1413,9 +1576,13 @@ impl Databases {
let mut named = self.named.lock().map_err(|_| Error::other("database lock poisoned"))?;
match method.as_str() {
"all_catalogs" => {
let servers: Vec<(String, Arc<Server>)> =
std::iter::once((SYSTEM_DATABASE.to_string(), Arc::clone(&self.system)))
.chain(named.iter().map(|(name, server)| (name.clone(), Arc::clone(server))))
.collect();
drop(named);
let mut schemas = Vec::new();
for (database, server) in std::iter::once((SYSTEM_DATABASE, &self.system))
.chain(named.iter().map(|(name, server)| (name.as_str(), server))) {
for (database, server) in servers {
match server.handle_request(Request::ListCatalog) {
Response::Catalog { schemas: entries } => {
for mut entry in entries {
@@ -1442,23 +1609,30 @@ impl Databases {
std::fs::create_dir(&cfg.data_dir)?;
let server = Server::new(cfg)?;
server.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.save(server.catalog_path())?;
named.insert(name, server);
named.insert(name, Arc::new(server));
return Ok(Response::Ok);
}
"drop_database" => {
if name == SYSTEM_DATABASE { return Err(Error::other("cannot drop the system database")); }
let server = named.get(&name).ok_or_else(|| Error::other("unknown database"))?;
let holder = named.remove(&name).ok_or_else(|| Error::other("unknown database"))?;
let server = match Arc::try_unwrap(holder) {
Ok(server) => server,
Err(holder) => {
named.insert(name, holder);
return Err(Error::other("database is in use"));
}
};
let root = std::fs::canonicalize(self.root.data_dir.join("databases"))?;
let target = std::fs::canonicalize(&server.cfg.data_dir)?;
if target.parent() != Some(root.as_path()) || !target.join("catalog.json").is_file() {
return Err(Error::other("invalid database directory"));
}
let config = server.cfg.clone();
drop(named.remove(&name));
drop(server);
let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_err(Error::other)?.as_nanos();
let tombstone = root.join(format!(".dropped-{name}-{stamp}"));
if let Err(error) = std::fs::rename(&target, &tombstone) {
named.insert(name, Server::new(config)?);
named.insert(name, Arc::new(Server::new(config)?));
return Err(error);
}
std::fs::remove_dir_all(&tombstone).map_err(|e| Error::other(format!("database dropped; directory cleanup failed: {e}")))?;
@@ -1482,10 +1656,10 @@ impl Databases {
| Request::AccountReset { .. } | Request::AccountLogin { .. }
| Request::AccountDrop { .. } | Request::BootstrapSystem { .. }
| Request::SystemStatus | Request::Ping);
let server = if global { &self.system }
else if selected.is_empty() { return Err(Error::other("no database selected")); }
else if selected == SYSTEM_DATABASE { &self.system }
else { named.get(&selected).ok_or_else(|| Error::other("unknown database"))? };
if !global && selected.is_empty() { return Err(Error::other("no database selected")); }
let server = if global || selected == SYSTEM_DATABASE { Arc::clone(&self.system) }
else { Arc::clone(named.get(&selected).ok_or_else(|| Error::other("unknown database"))?) };
drop(named);
Ok(server.handle_request(request))
}
}
+148 -28
View File
@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
use crate::protocol::{Account, Row, SchemaColumn, USERS_TABLE};
// 预写日志:先记操作并 fsync,再改数据;重启重放残留。
// 预写日志:先记操作并 fsync,再改数据;重启按 LSN 重放已提交的组。
const OP_INSERT: u8 = 1;
const OP_REPLACE_ALL: u8 = 2;
@@ -18,6 +18,7 @@ const OP_ACCOUNTS: u8 = 6;
const OP_REPLACE_SCHEMA: u8 = 7;
const OP_HIDE_COLUMN: u8 = 8;
const OP_COMPACT: u8 = 9;
const OP_COMMIT: u8 = 10;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub enum WalOp {
@@ -50,6 +51,8 @@ pub enum WalOp {
columns: Vec<SchemaColumn>,
rows: Vec<Row>,
},
/// 一组写操作的提交标记;没有标记的组算未提交
Commit,
}
#[derive(Debug, Serialize, Deserialize)]
@@ -67,6 +70,13 @@ struct ReplaceSchemaPayload {
pub struct Wal {
path: PathBuf,
handle: Mutex<Option<File>>,
next_lsn: Mutex<u64>,
}
/// 一次扫描的结果:记录与完好字节长度
pub struct WalScan {
pub records: Vec<(u64, WalOp)>,
pub good_len: usize,
}
impl Wal {
@@ -75,6 +85,7 @@ impl Wal {
Wal {
path: path.as_ref().to_path_buf(),
handle: Mutex::new(None),
next_lsn: Mutex::new(0),
}
}
@@ -83,6 +94,11 @@ impl Wal {
&self.path
}
/// 检查点文件路径
pub fn checkpoint_path(&self) -> PathBuf {
self.path.with_extension("checkpoint")
}
/// 拿常驻句柄(没有就开一个)
fn with_handle<R>(&self, f: impl FnOnce(&mut File) -> io::Result<R>) -> io::Result<R> {
let mut slot = self.handle.lock().unwrap();
@@ -99,6 +115,15 @@ impl Wal {
f(slot.as_mut().expect("just opened"))
}
/// 把一段记录写到文件尾
fn write_at_end(&self, bytes: &[u8]) -> io::Result<()> {
self.with_handle(|f| {
f.seek(SeekFrom::End(0))?;
f.write_all(bytes)?;
f.sync_data()
})
}
/// 截断到空并落盘
pub fn clear(&self) -> io::Result<()> {
self.with_handle(|f| {
@@ -112,34 +137,136 @@ impl Wal {
self.with_handle(truncate)
}
/// 追加一条并 fsync
pub fn append(&self, op: &WalOp) -> io::Result<()> {
let bytes = encode(op)?;
/// 截到指定长度(去掉撕裂的尾巴)
pub fn truncate_to(&self, len: usize) -> io::Result<()> {
self.with_handle(|f| {
f.write_all(&bytes)?;
f.set_len(len as u64)?;
f.seek(SeekFrom::Start(0))?;
f.sync_data()
})
}
/// 读当前 WAL;空则 None
/// 追加一条并 fsync
pub fn append(&self, op: &WalOp) -> io::Result<()> {
let bytes = self.encode_next(op)?;
self.write_at_end(&bytes)
}
/// 追加一组操作与提交标记,返回提交 LSN
pub fn append_group(&self, ops: &[WalOp]) -> io::Result<u64> {
let mut frames = Vec::new();
for op in ops {
frames.extend_from_slice(&self.encode_next(op)?);
}
let commit_lsn = self.take_lsn()?;
frames.extend_from_slice(&encode(&WalOp::Commit, commit_lsn)?);
self.write_at_end(&frames)?;
Ok(commit_lsn)
}
/// 读当前 WAL 的第一条操作;空则 None
pub fn read(&self) -> io::Result<Option<WalOp>> {
let scan = self.read_all()?;
Ok(scan.records.into_iter().find_map(|(_, op)| {
if op == WalOp::Commit {
None
} else {
Some(op)
}
}))
}
/// 读全部记录,撕裂或损坏的尾巴按长度截断
pub fn read_all(&self) -> io::Result<WalScan> {
if !self.path.exists() {
return Ok(None);
return Ok(WalScan {
records: Vec::new(),
good_len: 0,
});
}
let mut f = File::open(&self.path)?;
let mut buf = Vec::new();
f.read_to_end(&mut buf)?;
if buf.is_empty() {
let mut records = Vec::new();
let mut offset = 0usize;
while offset + 4 <= buf.len() {
let body_len =
u32::from_le_bytes([buf[offset], buf[offset + 1], buf[offset + 2], buf[offset + 3]])
as usize;
let end = offset + 4 + body_len;
if end > buf.len() {
break;
}
match decode_body(&buf[offset + 4..end]) {
Ok(record) => {
records.push(record);
offset = end;
}
Err(_) => break,
}
}
Ok(WalScan {
records,
good_len: offset,
})
}
/// 记录检查点:到此为止的 LSN 都已落盘
pub fn write_checkpoint(&self, lsn: u64) -> io::Result<()> {
let mut f = File::create(self.checkpoint_path())?;
f.write_all(&lsn.to_le_bytes())?;
f.sync_all()
}
/// 读检查点 LSN
pub fn read_checkpoint(&self) -> io::Result<Option<u64>> {
let path = self.checkpoint_path();
if !path.exists() {
return Ok(None);
}
match decode(&buf) {
Ok(opt) => Ok(opt),
Err(e) => Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("corrupt WAL: {}", e),
)),
let bytes = std::fs::read(&path)?;
if bytes.len() < 8 {
return Ok(None);
}
let mut raw = [0u8; 8];
raw.copy_from_slice(&bytes[..8]);
Ok(Some(u64::from_le_bytes(raw)))
}
/// 取下一条 LSN
fn take_lsn(&self) -> io::Result<u64> {
let mut slot = self.next_lsn.lock().unwrap();
if *slot == 0 {
let last = self.read_all()?.records.last().map(|(lsn, _)| *lsn).unwrap_or(0);
*slot = last + 1;
}
let lsn = *slot;
*slot += 1;
Ok(lsn)
}
/// 编一条带新 LSN 的记录
fn encode_next(&self, op: &WalOp) -> io::Result<Vec<u8>> {
let lsn = self.take_lsn()?;
encode(op, lsn)
}
}
/// 已提交的记录:提交标记前的整组;无标记的单条按旧格式算已提交
pub fn committed_ops(records: &[(u64, WalOp)]) -> Vec<(u64, WalOp)> {
let mut out = Vec::new();
let mut pending: Vec<(u64, WalOp)> = Vec::new();
for (lsn, op) in records {
if *op == WalOp::Commit {
out.append(&mut pending);
} else {
pending.push((*lsn, op.clone()));
}
}
if pending.len() == 1 {
out.append(&mut pending);
}
out
}
/// 清空文件并把读写位置归零
@@ -150,7 +277,7 @@ fn truncate(f: &mut File) -> io::Result<()> {
}
/// 把操作编成字节
fn encode(op: &WalOp) -> io::Result<Vec<u8>> {
fn encode(op: &WalOp, lsn: u64) -> io::Result<Vec<u8>> {
let (op_type, table, payload) = match op {
WalOp::HideColumn { table, column } => (OP_HIDE_COLUMN, table.clone(),
serde_json::to_vec(column).map_err(io::Error::other)?),
@@ -192,11 +319,11 @@ fn encode(op: &WalOp) -> io::Result<Vec<u8>> {
.map_err(io::Error::other)?;
(OP_REPLACE_SCHEMA, table.clone(), p)
}
WalOp::Commit => (OP_COMMIT, String::new(), Vec::new()),
};
let table_bytes = table.as_bytes();
let lsn: u64 = 1;
let mut body = Vec::with_capacity(8 + 1 + 2 + table_bytes.len() + payload.len());
body.extend_from_slice(&lsn.to_le_bytes());
body.push(op_type);
@@ -210,20 +337,12 @@ fn encode(op: &WalOp) -> io::Result<Vec<u8>> {
Ok(out)
}
/// 从字节解回操作
fn decode(buf: &[u8]) -> Result<Option<WalOp>, String> {
if buf.len() < 4 {
return Ok(None);
}
let body_len = u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]) as usize;
if buf.len() < 4 + body_len {
return Ok(None);
}
let body = &buf[4..4 + body_len];
/// 从一个帧体解出 LSN 与操作
fn decode_body(body: &[u8]) -> Result<(u64, WalOp), String> {
if body.len() < 8 + 1 + 2 {
return Err("body too short".into());
}
let _lsn = u64::from_le_bytes([
let lsn = u64::from_le_bytes([
body[0], body[1], body[2], body[3], body[4], body[5], body[6], body[7],
]);
let op_type = body[8];
@@ -236,6 +355,7 @@ fn decode(buf: &[u8]) -> Result<Option<WalOp>, String> {
let payload = &body[11 + table_len..];
let op = match op_type {
OP_COMMIT => WalOp::Commit,
OP_HIDE_COLUMN => {
let column = serde_json::from_slice(payload)
.map_err(|e| format!("bad hide_column payload: {e}"))?;
@@ -292,5 +412,5 @@ fn decode(buf: &[u8]) -> Result<Option<WalOp>, String> {
}
other => return Err(format!("unknown op type: {}", other)),
};
Ok(Some(op))
Ok((lsn, op))
}
+186
View File
@@ -216,6 +216,67 @@ fn concurrent_shard_reads_exclude_writers() -> Result<(), Box<dyn std::error::Er
Ok(())
}
/// 跨库并发互不串数据,目录查询不被写饿死
#[test]
fn concurrent_requests_across_databases_stay_isolated() -> Result<(), Box<dyn std::error::Error>> {
let (_srv, _data) = start_server();
let mut c = connect()?;
let request = |c: &mut Conn, value: serde_json::Value| -> Result<serde_json::Value, serde_json::Error> {
serde_json::from_str(&send(c, &value.to_string()))
};
let databases = ["alpha", "beta"];
for database in databases {
assert_eq!(request(&mut c, serde_json::json!({"method":"create_database","database":database}))?["status"], "ok");
assert_eq!(
request(&mut c, serde_json::json!({"method":"create_table","database":database,"table":"items","columns":[{"name":"id","ty":"int"}]}))?["status"],
"ok"
);
}
let storage = CURRENT.with(|slot| slot.borrow().as_ref().cloned().expect("storage opened"));
let written = 40;
std::thread::scope(|scope| {
let writers: Vec<_> = databases
.iter()
.map(|database| {
let storage = Arc::clone(&storage);
let database = database.to_string();
scope.spawn(move || {
for i in 1..=written {
let line = format!(
r#"{{"method":"insert","database":"{database}","table":"items","row":{{"id":{i}}}}}"#
);
let value: serde_json::Value = serde_json::from_str(&storage.request_line(&line)).expect("json");
assert_eq!(value["status"], "ok", "{database}: {value}");
}
database
})
})
.collect();
let catalog = {
let storage = Arc::clone(&storage);
scope.spawn(move || {
for _ in 0..40 {
for line in [r#"{"method":"list_databases"}"#, r#"{"method":"all_catalogs"}"#] {
let value: serde_json::Value = serde_json::from_str(&storage.request_line(line)).expect("json");
assert_ne!(value["status"], "error", "{line}: {value}");
}
}
})
};
for writer in writers {
writer.join().expect("writer thread");
}
catalog.join().expect("catalog thread");
});
for database in databases {
let rows = request(&mut c, serde_json::json!({"method":"scan","database":database,"table":"items"}))?["rows"].clone();
assert_eq!(rows.as_array().expect("rows").len(), written, "{database} 行数不对");
}
Ok(())
}
/// 删列不改堆字节,重启后不再暴露该列
#[test]
fn drop_column_keeps_heap_bytes_and_hides_values_after_restart() -> Result<(), Box<dyn std::error::Error>> {
@@ -744,6 +805,12 @@ fn account_failure_preserves_wal_and_blocks_writes() -> Result<(), Box<dyn std::
let mut c = connect()?;
let recovered = send(&mut c, r#"{"method":"accounts_list"}"#);
assert!(recovered.contains("hash-one"), "{recovered}");
let status: serde_json::Value =
serde_json::from_str(&send(&mut c, r#"{"method":"system_status"}"#))?;
assert!(
status["last_lsn"].as_u64().unwrap_or(0) >= 1,
"重放后检查点要推进: {status}"
);
Ok(())
}
@@ -1015,6 +1082,125 @@ fn delete_keys_removes_row_and_index_entry() {
assert!(r.contains(r#""status":"ok""#), "id 2 should be insertable again: {}", r);
}
/// 事务提交:一批写操作在同一个请求里落地
#[test]
fn apply_transaction_lands_upserts_and_deletes_together() {
let (_srv, _data) = start_server();
let mut c = connect().unwrap();
for (id, name) in [(1, "Alice"), (2, "Bob")] {
let req = format!(
r#"{{"method":"insert","table":"txn_t","row":{{"id":{},"name":"{}"}}}}"#,
id, name
);
send(&mut c, &req);
}
let batch = r#"{"method":"apply_transaction","ops":[
{"op":"delete","table":"txn_t","ids":[1]},
{"op":"upsert","table":"txn_t","rows":[{"id":3,"name":"Carol"}]}
]}"#;
let r = send(&mut c, batch);
assert!(r.contains(r#""status":"ok""#), "batch: {}", r);
let r = send(&mut c, r#"{"method":"scan","table":"txn_t"}"#);
assert!(!r.contains("Alice"), "deleted row should be gone: {}", r);
assert!(r.contains("Bob") && r.contains("Carol"), "kept and added rows: {}", r);
for line in [
r#"{"method":"apply_transaction","ops":[{"op":"upsert","table":"__system_users","rows":[{"id":1}]}]}"#,
r#"{"method":"apply_transaction","ops":[{"op":"delete","table":"../evil","ids":[1]}]}"#,
] {
let r = send(&mut c, line);
assert!(r.contains(r#""status":"error""#), "bad op rejected: {} -> {}", line, r);
}
// 没有整数 id 的表用整表替换提交
let r = send(&mut c, r#"{"method":"insert","table":"txn_no_id","row":{"name":"old"}}"#);
assert!(r.contains(r#""status":"ok""#), "insert: {}", r);
let replace = r#"{"method":"apply_transaction","ops":[{"op":"replace","table":"txn_no_id","rows":[{"name":"only"}]}]}"#;
let r = send(&mut c, replace);
assert!(r.contains(r#""status":"ok""#), "replace: {}", r);
let r = send(&mut c, r#"{"method":"scan","table":"txn_no_id"}"#);
assert!(r.contains("only") && !r.contains("old"), "replaced rows: {}", r);
}
/// 标记已落盘、数据未改的崩溃现场:重启重放并推进检查点
#[test]
fn committed_group_replays_after_restart() -> Result<(), Box<dyn std::error::Error>> {
use chusql_core_storage::wal::{Wal, WalOp};
let (server, data) = start_server();
let mut c = connect()?;
request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":1,"secret":"old"}}))?;
drop(c);
drop(server);
let mut added = std::collections::HashMap::new();
added.insert("id".to_string(), serde_json::json!(2));
added.insert("secret".to_string(), serde_json::json!("new"));
let group = vec![
WalOp::InsertBatch {
table: "items".into(),
rows: vec![added.clone()],
},
WalOp::DeleteKeys {
table: "items".into(),
keys: vec![1],
},
];
let wal = Wal::new(data.path().join("databases/main/wal.log"));
let commit_lsn = wal.append_group(&group)?;
drop(wal);
let _server = start_server_in(data.path());
let mut c = connect()?;
let rows: serde_json::Value =
serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"items"}"#))?;
assert_eq!(rows["rows"], serde_json::json!([{"id":2,"secret":"new"}]));
assert_eq!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len(), 0);
let checkpoint = std::fs::read(data.path().join("databases/main/wal.checkpoint"))?;
assert_eq!(checkpoint.len(), 8, "检查点要落盘");
assert_eq!(
u64::from_le_bytes(checkpoint[..8].try_into()?),
commit_lsn,
"检查点记到提交 LSN"
);
Ok(())
}
/// 没有提交标记的记录组在重启时整组丢掉
#[test]
fn uncommitted_group_is_discarded_on_restart() -> Result<(), Box<dyn std::error::Error>> {
use chusql_core_storage::wal::{Wal, WalOp};
let (server, data) = start_server();
let mut c = connect()?;
request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":1,"secret":"old"}}))?;
drop(c);
drop(server);
let mut lost = std::collections::HashMap::new();
lost.insert("id".to_string(), serde_json::json!(2));
lost.insert("secret".to_string(), serde_json::json!("lost"));
let wal = Wal::new(data.path().join("databases/main/wal.log"));
wal.append(&WalOp::InsertBatch {
table: "items".into(),
rows: vec![lost.clone()],
})?;
wal.append(&WalOp::InsertBatch {
table: "items".into(),
rows: vec![lost],
})?;
drop(wal);
let _server = start_server_in(data.path());
let mut c = connect()?;
let rows: serde_json::Value =
serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"items"}"#))?;
assert_eq!(rows["rows"], serde_json::json!([{"id":1,"secret":"old"}]));
assert_eq!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len(), 0);
Ok(())
}
/// 给第二列建索引后能按那一列查
#[test]
fn create_index_on_secondary_column() {
+121 -2
View File
@@ -1,8 +1,8 @@
use chusql_core_storage::protocol::Row;
use chusql_core_storage::wal::{Wal, WalOp};
use chusql_core_storage::wal::{committed_ops, Wal, WalOp};
use serde_json::json;
// WAL 测试:追加、读取、清空、截断尾巴。
// WAL 测试:追加、读取、清空、截断尾巴、LSN 与提交标记。
/// 用键值对构造一行
@@ -99,3 +99,122 @@ fn truncated_tail_is_ignored() {
assert!(wal.read().unwrap().is_none());
}
/// 每条记录带递增的 LSN,重开也从文件里的最大号往下走
#[test]
fn every_record_gets_an_increasing_lsn() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("wal.log");
let wal = Wal::new(&path);
for id in 1..=3 {
wal.append(&WalOp::Insert {
table: "t".into(),
row: row(&[("id", json!(id))]),
})
.unwrap();
}
let scan = wal.read_all().unwrap();
let lsns: Vec<u64> = scan.records.iter().map(|(lsn, _)| *lsn).collect();
assert_eq!(lsns, vec![1, 2, 3]);
assert_eq!(scan.good_len as u64, std::fs::metadata(&path).unwrap().len());
let reopened = Wal::new(&path);
reopened
.append(&WalOp::Insert {
table: "t".into(),
row: row(&[("id", json!(4))]),
})
.unwrap();
assert_eq!(reopened.read_all().unwrap().records.last().unwrap().0, 4);
}
/// 一组操作写完带提交标记,标记的 LSN 也在记录里
#[test]
fn append_group_writes_a_commit_marker() {
let dir = tempfile::tempdir().unwrap();
let wal = Wal::new(dir.path().join("wal.log"));
let group = [
WalOp::InsertBatch {
table: "t".into(),
rows: vec![row(&[("id", json!(2))])],
},
WalOp::DeleteKeys {
table: "t".into(),
keys: vec![1],
},
];
let commit_lsn = wal.append_group(&group).unwrap();
let scan = wal.read_all().unwrap();
assert_eq!(scan.records.len(), 3);
assert_eq!(scan.records.last().unwrap().1, WalOp::Commit);
assert_eq!(scan.records.last().unwrap().0, commit_lsn);
assert_eq!(wal.read().unwrap().unwrap(), group[0]);
assert_eq!(
committed_ops(&scan.records),
vec![(1, group[0].clone()), (2, group[1].clone())]
);
}
/// 没有提交标记的记录组算未提交,旧格式的单条仍算已提交
#[test]
fn records_without_a_commit_marker_stay_uncommitted() {
let dir = tempfile::tempdir().unwrap();
let wal = Wal::new(dir.path().join("wal.log"));
for id in 1..=2 {
wal.append(&WalOp::Insert {
table: "t".into(),
row: row(&[("id", json!(id))]),
})
.unwrap();
}
assert!(committed_ops(&wal.read_all().unwrap().records).is_empty());
wal.clear().unwrap();
wal.append(&WalOp::Insert {
table: "t".into(),
row: row(&[("id", json!(9))]),
})
.unwrap();
assert_eq!(committed_ops(&wal.read_all().unwrap().records).len(), 1);
}
/// 撕成两半的尾巴能读出完好长度并截掉
#[test]
fn trailing_torn_frame_can_be_truncated() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("wal.log");
let wal = Wal::new(&path);
for id in 1..=2 {
wal.append(&WalOp::Insert {
table: "t".into(),
row: row(&[("id", json!(id))]),
})
.unwrap();
}
let full = std::fs::read(&path).unwrap();
std::fs::write(&path, &full[..full.len() - 3]).unwrap();
let scan = wal.read_all().unwrap();
assert_eq!(scan.records.len(), 1);
assert!(scan.good_len < full.len() - 3);
wal.truncate_to(scan.good_len).unwrap();
assert_eq!(std::fs::metadata(&path).unwrap().len(), scan.good_len as u64);
assert_eq!(wal.read_all().unwrap().records.len(), 1);
}
/// 检查点写读一致
#[test]
fn checkpoint_round_trips() {
let dir = tempfile::tempdir().unwrap();
let wal = Wal::new(dir.path().join("wal.log"));
assert_eq!(wal.read_checkpoint().unwrap(), None);
wal.write_checkpoint(7).unwrap();
assert_eq!(wal.read_checkpoint().unwrap(), Some(7));
}
+62 -79
View File
@@ -6,7 +6,6 @@ module ChuSQL.Server.Backend (
columnsFromStatement,
currentStamp,
exprTables,
inDatabase,
internalTable,
grantsTable,
ipcBackend,
@@ -23,20 +22,23 @@ module ChuSQL.Server.Backend (
import ChuSQL.Core.Engine (runStatement, runStatementM)
import ChuSQL.Core.Model
import ChuSQL.Core.Engine.Semantic (prepare)
import ChuSQL.Core.Engine.Storage (MonadStorage (schema))
import ChuSQL.Core.Engine.Storage (MonadStorage (schema, snapshot))
import ChuSQL.Core.Engine.Storage.IPC (
Account (..),
Env,
IPCStorage (..),
Request (..),
Response (..),
SchemaColumn (..),
TableInfo (..),
envForDatabase,
sendRawRequest,
sendRequest,
)
import ChuSQL.Core.Engine.Syntax.AST
import ChuSQL.Core.Engine.Syntax.Parser (parseStatement)
import Control.Concurrent.MVar (MVar, modifyMVar, newMVar, readMVar, withMVar)
import ChuSQL.Core.Protocol (TxnOp (..))
import Control.Concurrent.MVar (MVar, modifyMVar, readMVar)
import Data.List (nub)
import Data.Text (Text)
import Data.Time (getCurrentTime)
@@ -61,6 +63,10 @@ data Backend = Backend
, beDatabases :: IO (Either String [String])
-- | 原样转发一条存储请求
, beStorage :: A.Value -> IO (Either String A.Value)
-- | 取一份整库快照,当事务的起点
, beSnapshot :: IO (Either String Database)
-- | 事务提交:把一批写操作交给存储层应用
, beApplyTransaction :: [TxnOp] -> IO (Either String ())
}
-- | 按给定列名取列,固定列顺序
@@ -73,36 +79,46 @@ columnsFromStatement db stmt = case prepare db stmt of
-- | 走本地存储链路的后端
ipcBackend :: IO Backend
ipcBackend = do
lock <- newMVar ()
let backend current = Backend
{ beStatement = \sql -> withMVar lock $ \_ -> runIpc current sql
, beCatalog = withMVar lock $ \_ -> do
{ beStatement = \sql -> runIpc current sql
, beCatalog = do
response <- sendRequest (ReqInDatabase current ReqListCatalog)
pure $ case response of
RespCatalog infos -> Right infos
RespError err -> Left err
_ -> Left "unexpected catalog response"
, bePing = withMVar lock $ \_ -> do
, bePing = do
resp <- sendRequest ReqPing
pure (case resp of RespPong -> True; _ -> False)
, beAccounts = \req -> withMVar lock $ \_ -> do
, beAccounts = \req -> do
response <- sendRequest req
pure $ case response of
RespAccounts accounts -> Right accounts
RespError err -> Left err
_ -> Left "unexpected account response"
, beWithDatabase = backend
, beDatabases = withMVar lock $ \_ -> do
, beDatabases = do
response <- sendRequest (ReqDatabase "list_databases" "")
pure $ case response of
RespTables names -> Right names
RespError err -> Left err
_ -> Left "unexpected database response"
, beStorage = \payload -> withMVar lock $ \_ -> runRaw payload
, beStorage = \payload -> runRaw payload
, beSnapshot = Right <$> runIPCStorageIn (snapshot :: IPCStorage Database) (sessionEnv current)
, beApplyTransaction = \ops -> do
response <- sendRequest (ReqInDatabase current (ReqApplyTransaction ops))
pure $ case response of
RespOk -> Right ()
RespError err -> Left err
_ -> Left "unexpected transaction response"
}
-- 没有默认库:没选库时 current 是空串,任何请求都得先选库
pure (backend "")
where
-- | 会话的存储环境:库名跟这条连接走
sessionEnv :: String -> Env
sessionEnv current = envForDatabase (if null current then Nothing else Just current)
-- | 转发一条存储请求并解析响应
runRaw :: A.Value -> IO (Either String A.Value)
runRaw payload = do
@@ -113,18 +129,21 @@ ipcBackend = do
Left err -> Left ("storage response is not valid JSON: " ++ err)
Right value -> Right value
-- | 走存储链路跑一条语句
-- | 走存储链路跑一条语句:表名不再补当前库前缀
-- 引擎侧的数据字典按当前库过滤之后键是裸表名,补前缀会让查表落空;
-- 库名由 database 字段随请求带走,存储层自己按它分发。
runIpc :: String -> String -> IO (Either String StatementResult)
runIpc current sql = case fmap (inDatabase current) (parseStatement sql) of
runIpc current sql = case parseStatement sql of
Left e -> pure (Left e)
Right stmt -> do
result <- runIPCStorage (runStatementM stmt)
let env = sessionEnv current
result <- runIPCStorageIn (runStatementM stmt) env
case result of
Left e -> pure (Left e)
Right rows -> do
cols <- case rows of
[] -> do
db <- runIPCStorage (schema :: IPCStorage Database)
db <- runIPCStorageIn (schema :: IPCStorage Database) env
pure (columnsFromStatement db stmt)
(r : _) -> pure (map fst r)
pure (Right (StatementResult cols rows))
@@ -162,6 +181,10 @@ memoryBackendWith trusted name ref =
, beWithDatabase = withDatabase
, beDatabases = pure (Right (nub [name, systemDatabaseName]))
, beStorage = \_ -> pure (Left "storage is not available in this backend")
, beSnapshot = Right . scoped <$> readMVar ref
, beApplyTransaction = \ops -> modifyMVar ref $ \db -> case applyTransactionMemory ops db of
Left err -> pure (db, Left err)
Right db' -> pure (db', Right ())
}
where
-- | 按视角过滤库里的表
@@ -185,6 +208,8 @@ unavailableDatabase :: String -> Backend -> Backend
unavailableDatabase message backend = backend
{ beStatement = \_ -> pure (Left message)
, beCatalog = pure (Left message)
, beSnapshot = pure (Left message)
, beApplyTransaction = \_ -> pure (Left message)
}
-- | 未选库时的视角:只放行不碰表的语句
@@ -196,76 +221,34 @@ noDatabaseBackend backend = backend
| statementNeedsDatabase stmt -> pure (Left "no database selected")
| otherwise -> beStatement backend sql
, beCatalog = pure (Left "no database selected")
, beSnapshot = pure (Left "no database selected")
, beApplyTransaction = \_ -> pure (Left "no database selected")
}
-- | 给语句里的裸表名补上当前库前缀,子查询里的表名也一起改
inDatabase :: String -> Statement -> Statement
inDatabase current = walk
-- | 把一批事务写操作应用到内存库:先按 id 删,再按追加写
applyTransactionMemory :: [TxnOp] -> Database -> Either String Database
applyTransactionMemory ops db = foldl step (Right db) ops
where
-- | 递归改写语句里的表名
walk :: Statement -> Statement
walk stmt = case stmt of
q@Select{} ->
q
{ selectFrom = source (selectFrom q)
, selectWhere = fmap expr (selectWhere q)
}
q@SelectExpr{} ->
q
{ selectItems = [(label, expr e) | (label, e) <- selectItems q]
, selectFrom = source (selectFrom q)
, selectWhere = fmap expr (selectWhere q)
}
Insert t cols rows -> Insert (table t) cols (map (map expr) rows)
Delete t cond -> Delete (table t) (fmap expr cond)
Update t assigns cond -> Update (table t) [(c, expr e) | (c, e) <- assigns] (fmap expr cond)
CreateTable t cols -> CreateTable (table t) cols
DropTable t -> DropTable (table t)
CreateIndex t c -> CreateIndex (table t) c
DropIndex t c -> DropIndex (table t) c
DropColumn t c -> DropColumn (table t) c
AddColumn t def -> AddColumn (table t) def
RenameColumn t old new -> RenameColumn (table t) old new
AlterColumnType t c ty -> AlterColumnType (table t) c ty
AlterColumnDefault t c v -> AlterColumnDefault (table t) c v
AlterColumnNull t c b -> AlterColumnNull (table t) c b
other -> other
step acc op = do
current <- acc
case op of
TxnDelete name keys -> do
table <- lookupTable current name
pure (put (name, table {tableRows = [r | r <- tableRows table, not (doomed keys r)]}) current)
TxnUpsert name rows -> do
table <- lookupTable current name
pure (put (name, table {tableRows = tableRows table ++ rows}) current)
TxnReplace name rows -> do
table <- lookupTable current name
pure (put (name, table {tableRows = rows}) current)
-- | 给单个表名补库前缀,已限定或没选库就原样
table t | '.' `elem` t || null current = t
| otherwise = current ++ "." ++ t
-- | 换掉一张表,其余保持原顺序
put entry = map (\one@(name, _) -> if name == fst entry then entry else one)
-- | 改写 FROM 子句
source FromUnit = FromUnit
source (FromTable a t) = FromTable a (table t)
source (FromJoin k l a t e) = FromJoin k (source l) a (table t) (expr e)
-- | 改写表达式里的子查询
expr (ScalarSub sq) = ScalarSub (sub sq)
expr (ExistsSub sq neg) = ExistsSub (sub sq) neg
expr (InSub a sq neg) = InSub (expr a) (sub sq) neg
expr (InList a items neg) = InList (expr a) (map expr items) neg
expr (Add a b) = Add (expr a) (expr b)
expr (Sub a b) = Sub (expr a) (expr b)
expr (Mul a b) = Mul (expr a) (expr b)
expr (Div a b) = Div (expr a) (expr b)
expr (Neg a) = Neg (expr a)
expr (Gt a b) = Gt (expr a) (expr b)
expr (Lt a b) = Lt (expr a) (expr b)
expr (Eq a b) = Eq (expr a) (expr b)
expr (And a b) = And (expr a) (expr b)
expr (Or a b) = Or (expr a) (expr b)
expr (IsNull a) = IsNull (expr a)
expr (IsNotNull a) = IsNotNull (expr a)
expr (CountOf a) = CountOf (expr a)
expr (SumOf a) = SumOf (expr a)
expr (AvgOf a) = AvgOf (expr a)
expr (MinOf a) = MinOf (expr a)
expr (MaxOf a) = MaxOf (expr a)
expr other = other
-- | 改写一个子查询
sub sq = sq {subqueryStatement = walk (subqueryStatement sq)}
-- | 这一行要不要删:按行里的 id 列比
doomed keys r = case lookup "id" r of
Just (VInt k) -> k `elem` keys
_ -> False
-- | 语句里出现的表
statementTables :: Statement -> [Text]
+185 -14
View File
@@ -24,22 +24,24 @@ module ChuSQL.Server.Session
, queryResultJson
) where
import ChuSQL.Core.Model (Value (..))
import ChuSQL.Core.Protocol (Account, QueryResult (..), queryResultJson)
import ChuSQL.Core.Model (Database, Row, Table (..), Value (..))
import ChuSQL.Core.Protocol (Account, QueryResult (..), TxnOp (..), queryResultJson)
import ChuSQL.Core.Engine.Storage.IPC (TableInfo (..))
import qualified ChuSQL.Core.Engine as Engine
import qualified ChuSQL.Core.Engine.Error as E
import ChuSQL.Core.Engine.Syntax.AST (Statement (..))
import ChuSQL.Core.Engine.Syntax.Parser (parseStatement)
import ChuSQL.Interface.AccountTable (systemTableInfo)
import ChuSQL.Interface.Auth (defaultSessionPolicy, newSessionStore)
import ChuSQL.Server.Accounts
import ChuSQL.Server.Backend (Backend (..), StatementResult (..))
import ChuSQL.Server.Backend (Backend (..), StatementResult (..), columnsFromStatement)
import ChuSQL.Server.Policy (configurePasswordPolicy)
import ChuSQL.Server.Privileges (PrivilegeCommand, PrivilegeError (..), Privileges, RoleView, authorize, filterTables, listRoleViews, newPrivileges, privilegeCommand, runPrivilegeCommand)
import Control.Exception (IOException, try)
import Data.Bifunctor (first)
import Data.Char (isAlpha, isAlphaNum)
import Data.IORef (IORef, newIORef, readIORef, writeIORef)
import Data.Maybe (isJust)
import Data.Text (Text)
import qualified Data.Text as T
import Data.Time.Clock (getCurrentTime)
@@ -64,6 +66,13 @@ data Session = Session
, ssCurrent :: IORef Text
, ssPrincipal :: IORef (Maybe Principal)
, ssSettingsFile :: FilePath
, ssTransaction :: IORef (Maybe Transaction)
}
-- | 一条会话里的显式事务:开事务时的快照与事务内改到的状态
data Transaction = Transaction
{ txBase :: Database
, txStaged :: Database
}
-- | 开一个会话:账号与角色服务、配置策略都和 Web 端一致
@@ -76,7 +85,8 @@ newSession backend name settingsFile = do
-- 服务启动后只有系统库:没 USE 之前不预设任何工作库
current <- newIORef ""
principal <- newIORef Nothing
pure (Session backend accountsService privileges current principal settingsFile)
transaction <- newIORef Nothing
pure (Session backend accountsService privileges current principal settingsFile transaction)
-- | 当前生效的口令策略(只有管理员能看)
policyOf :: Session -> IO (Either Text PasswordPolicy)
@@ -168,22 +178,183 @@ runStatement session sql = fmap (first sessMessage) (runStatementCoded session s
runStatementCoded :: Session -> Text -> IO (Either SessionError QueryResult)
runStatementCoded session sql = case parseStatement (T.unpack sql) of
Right (UseDatabase name) -> do
switched <- switchDatabaseCoded session (T.pack name)
case switched of
Left err -> pure (Left err)
Right () -> do
current <- readIORef (ssCurrent session)
pure (Right (emptyResult (Just current)))
existing <- readIORef (ssTransaction session)
case existing of
Just _ -> pure (Left (SessionError "query_error" "cannot switch database inside a transaction"))
Nothing -> do
switched <- switchDatabaseCoded session (T.pack name)
case switched of
Left err -> pure (Left err)
Right () -> do
current <- readIORef (ssCurrent session)
pure (Right (emptyResult (Just current)))
Right BeginTransaction -> beginTransaction session
Right CommitTransaction -> commitTransaction session
Right RollbackTransaction -> rollbackTransaction session
Right statement
| Just command <- accountCommand statement -> runAccount session command
| Just command <- privilegeCommand statement -> runPrivilege session command
| otherwise -> do
allowed <- authorized session statement
case allowed of
Left err -> pure (Left err)
Right () -> runPlain session sql
transaction <- readIORef (ssTransaction session)
if isJust transaction && transactionDdl statement
then pure (Left (SessionError "query_error" "DDL is not allowed in a transaction"))
else do
allowed <- authorized session statement
case allowed of
Left err -> pure (Left err)
Right ()
| isJust transaction && transactionData statement -> runInTransaction session statement
| otherwise -> runPlain session sql
Left _ -> runPlain session sql
-- | 事务里不许改结构,会话层直接拦下
transactionDdl :: Statement -> Bool
transactionDdl statement = case statement of
CreateTable{} -> True
DropTable{} -> True
CreateIndex{} -> True
DropIndex{} -> True
DropColumn{} -> True
AddColumn{} -> True
RenameColumn{} -> True
AlterColumnType{} -> True
AlterColumnDefault{} -> True
AlterColumnNull{} -> True
CreateDatabase{} -> True
DropDatabase{} -> True
_ -> False
-- | 事务里只有数据语句走快照
transactionData :: Statement -> Bool
transactionData statement = case statement of
Select{} -> True
SelectExpr{} -> True
Insert{} -> True
Update{} -> True
Delete{} -> True
_ -> False
-- | 开事务:把当前库整库做一份快照当起点
beginTransaction :: Session -> IO (Either SessionError QueryResult)
beginTransaction session = do
current <- readIORef (ssCurrent session)
existing <- readIORef (ssTransaction session)
case existing of
Just _ -> pure (Left (SessionError "query_error" "already in a transaction"))
Nothing
| T.null current -> pure (Left (SessionError "no_database" "no database selected"))
| otherwise -> do
backend <- currentBackend session
snapshotResult <- beSnapshot backend
case snapshotResult of
Left err -> pure (Left (engineErrorCode err))
Right db -> do
writeIORef (ssTransaction session) (Just (Transaction db db))
pure (Right (emptyResult (Just current)))
-- | 提交:把快照差分交给存储层应用,成功才结束事务
commitTransaction :: Session -> IO (Either SessionError QueryResult)
commitTransaction session = do
current <- readIORef (ssCurrent session)
existing <- readIORef (ssTransaction session)
case existing of
Nothing -> pure (Left (SessionError "query_error" "no transaction in progress"))
Just transaction -> do
backend <- currentBackend session
liveResult <- beSnapshot backend
case liveResult of
Left err -> pure (Left (engineErrorCode err))
Right live -> do
applied <- beApplyTransaction backend (transactionOps (txBase transaction) (txStaged transaction) live)
case applied of
Left err -> pure (Left (engineErrorCode err))
Right () -> do
writeIORef (ssTransaction session) Nothing
pure (Right (emptyResult (Just current)))
-- | 回滚:快照丢掉,存储层没被改过
rollbackTransaction :: Session -> IO (Either SessionError QueryResult)
rollbackTransaction session = do
current <- readIORef (ssCurrent session)
existing <- readIORef (ssTransaction session)
case existing of
Nothing -> pure (Left (SessionError "query_error" "no transaction in progress"))
Just _ -> do
writeIORef (ssTransaction session) Nothing
pure (Right (emptyResult (Just current)))
-- | 事务里跑一条数据语句:只动内存快照,不碰存储层
runInTransaction :: Session -> Statement -> IO (Either SessionError QueryResult)
runInTransaction session statement = do
existing <- readIORef (ssTransaction session)
case existing of
Nothing -> pure (Left (SessionError "query_error" "no transaction in progress"))
Just transaction -> case Engine.runStatement (txStaged transaction) statement of
Left err -> pure (Left (engineErrorCode err))
Right (staged, rows) -> do
writeIORef (ssTransaction session) (Just transaction {txStaged = staged})
pure (Right (resultOfStaged (txStaged transaction) statement rows))
-- | 事务内的结果集:空结果回落到解析出来的列名
resultOfStaged :: Database -> Statement -> [Row] -> QueryResult
resultOfStaged db statement rows = resultOf (StatementResult cols rows)
where
cols = case rows of
[] -> columnsFromStatement db statement
(r : _) -> map fst r
-- | 提交差分:相对快照列出要删、要写与要替换的行
transactionOps :: Database -> Database -> Database -> [TxnOp]
transactionOps base staged live =
concatMap opsFor names
where
-- | 事务里碰得着的表:数据语句不增删表
names = nubKeys (map fst base ++ map fst staged)
-- | 一张表的差分
opsFor name
| all (isJust . rowKey) (baseRows ++ stagedRows) = keyedOps
| stagedRows == baseRows = []
| otherwise = [TxnReplace name stagedRows]
where
baseRows = rowsOf name base
stagedRows = rowsOf name staged
liveRows = rowsOf name live
liveByKey = [(k, r) | r <- liveRows, Just k <- [rowKey r]]
stagedByKey = [(k, r) | r <- stagedRows, Just k <- [rowKey r]]
baseKeys = [k | r <- baseRows, Just k <- [rowKey r]]
-- 开事务时就存在、现在没了:删;内容变了的行也先删再写
gone = [k | (k, _) <- liveByKey, k `elem` baseKeys, not (any ((== k) . fst) stagedByKey)]
rewritten = [k | (k, r) <- stagedByKey, Just liveRow <- [lookup k liveByKey], liveRow /= r]
deleted = gone ++ rewritten
-- 新增的行,以及跟库里不一样的行;并发插进来的行不动
written =
[ r
| (k, r) <- stagedByKey
, case lookup k liveByKey of
Nothing -> True
Just liveRow -> liveRow /= r
]
keyedOps =
[TxnDelete name deleted | not (null deleted)]
++ [TxnUpsert name written | not (null written)]
-- | 表名去重但保持顺序
nubKeys = foldr (\name acc -> if name `elem` acc then acc else name : acc) []
-- | 库里某张表的行
rowsOf name db = maybe [] tableRows (lookup name db)
-- | 行 id:只有整数 id 才认,跟存储层删行的口径一致
rowKey :: Row -> Maybe Int
rowKey row = case lookup "id" row of
Just (VInt k) -> Just k
_ -> Nothing
-- | 普通身份跑语句前的权限判定;管理员与未登录放行
authorized :: Session -> Statement -> IO (Either SessionError ())
authorized session statement = do
+250 -7
View File
@@ -3,14 +3,20 @@
module Main (main) where
import ChuSQL.Core.Engine.Storage.IPC (Account (..), Request (ReqAccountReset))
import ChuSQL.Core.Engine.Storage.IPC (
Account (..),
Request (ReqAccountReset),
closeConnection,
localStorageLink,
setStorageLink,
)
import ChuSQL.Core.Engine.Syntax.Parser (parseStatement)
import ChuSQL.Core.Model (Database, Table (..), Value (..), pattern TInt, pattern TStr)
import ChuSQL.Core.Model (Database, Row, Table (..), Value (..), pattern TInt, pattern TStr)
import ChuSQL.Interface.Auth (hashPasswordWith)
import ChuSQL.Interface.Protocol (ClientRequest (..), Grant (..), RoleView (..), ServerResponse (..), decodeRequest, encodeResponse)
import ChuSQL.Interface.RateLimit (newRateLimiter)
import ChuSQL.Server.Accounts (Principal (..), ensureRootAccount)
import ChuSQL.Server.Backend (Backend (..), memoryBackend)
import ChuSQL.Server.Backend (Backend (..), StatementResult (..), ipcBackend, memoryBackend)
import ChuSQL.Server.Privileges (
PrivilegeCommand (..),
PrivilegeError (..),
@@ -20,7 +26,7 @@ import ChuSQL.Server.Privileges (
newPrivileges,
runPrivilegeCommand,
)
import ChuSQL.Server.Session (QueryResult (..))
import ChuSQL.Server.Session (QueryResult (..), Session, SessionError (..), newSession, runStatementCoded)
import ChuSQL.Server.TCP (
ServerConfig (..),
ServerEnv,
@@ -30,8 +36,8 @@ import ChuSQL.Server.TCP (
newServerEnv,
startServer,
)
import Control.Concurrent (threadDelay)
import Control.Concurrent.MVar (newMVar)
import Control.Concurrent (forkIO, threadDelay)
import Control.Concurrent.MVar (MVar, newEmptyMVar, newMVar, putMVar, takeMVar)
import Control.Exception (IOException, bracket, finally, try)
import Control.Monad (void)
import Data.Aeson (decode)
@@ -59,7 +65,7 @@ import Network.Socket (
socketToHandle,
withSocketsDo,
)
import System.Directory (doesFileExist, getTemporaryDirectory, removeFile)
import System.Directory (createDirectoryIfMissing, doesFileExist, getTemporaryDirectory, removeFile, removePathForcibly)
import System.FilePath ((</>))
import System.IO (
BufferMode (LineBuffering),
@@ -87,6 +93,8 @@ spec = do
tcpSpec
tcpServerSpec
privilegeSpec
transactionSpec
ipcConcurrencySpec
-- 夹具:内存后端装在 MVar 里,跟 Web 测试同一套表
@@ -545,3 +553,238 @@ privilegeSpec = describe "server privileges" $ do
Left err -> expectationFailure err
Right stmt -> authorize service root testDatabase stmt `shouldReturn` Right ()
-- 真存储夹具:一个用例一份数据目录,跑完关链路删干净
-- | 本次用例的数据目录(用例串行跑,固定名够用)
ipcDataDir :: IO FilePath
ipcDataDir = do
tmp <- getTemporaryDirectory
pure (tmp </> "chusql-server-test-ipc")
-- | 写一份只属于这次用例的存储配置:数据目录与日志级别都从文件走
writeIpcConfig :: FilePath -> IO FilePath
writeIpcConfig dir = do
let cfgPath = dir ++ ".toml"
-- TOML 的普通字符串会吃反斜杠,路径统一用正斜杠
slashed = map (\c -> if c == '\\' then '/' else c) dir
writeFile
cfgPath
( unlines
[ "[storage]"
, "data_dir = \"" ++ slashed ++ "\""
, "[log]"
, "level = \"error\""
]
)
pure cfgPath
-- | 删掉这次用例的数据目录与配置,删不掉也不算错
cleanIpc :: FilePath -> IO ()
cleanIpc dir = do
_ <- try (removePathForcibly dir) :: IO (Either IOException ())
_ <- try (removePathForcibly (dir ++ ".toml")) :: IO (Either IOException ())
pure ()
-- 事务用例:内存后端跑得快,真存储的提交再在 IPC 用例里验一遍
-- | 起一个内存后端会话,当前库已切到 test
withMemorySession :: (Session -> IO a) -> IO a
withMemorySession body = do
db <- newMVar testDb
settings <- tempSettingsPath "transaction"
removeIfExists settings
session <- newSession (memoryBackend testDatabaseName db) (T.pack testDatabaseName) settings
_ <- runStatementCoded session "use test"
body session
-- | 两个会话共用一份内存库,用来验会话隔离
withTwoSessions :: (Session -> Session -> IO a) -> IO a
withTwoSessions body = do
db <- newMVar testDb
settings <- tempSettingsPath "transaction-pair"
removeIfExists settings
let backend = memoryBackend testDatabaseName db
first <- newSession backend (T.pack testDatabaseName) settings
second <- newSession backend (T.pack testDatabaseName) settings
_ <- runStatementCoded first "use test"
_ <- runStatementCoded second "use test"
body first second
-- | 跑一条语句;出错就算用例失败
mustSql :: Session -> Text -> IO QueryResult
mustSql session sql = runStatementCoded session sql >>= either failed pure
where
failed err = expectationFailure (T.unpack (sessMessage err)) >> pure (QueryResult [] [] 0 False Nothing)
-- | 取 users 的 id 列
sessionIds :: Session -> IO [Int]
sessionIds session = do
result <- mustSql session "select * from users"
pure [n | row <- qrRows result, Just (VInt n) <- [nth 0 row]]
-- | 取某个 id 的 age
sessionAge :: Session -> Int -> IO (Maybe Int)
sessionAge session wanted = do
result <- mustSql session "select * from users"
let ages = [(n, a) | row <- qrRows result, Just (VInt n) <- [nth 0 row], Just (VInt a) <- [nth 2 row]]
pure (lookup wanted ages)
-- | 取失败里的文案
failureMessage :: Either SessionError QueryResult -> Text
failureMessage outcome = either sessMessage (const "") outcome
-- | 取结果第一列的整数
intValues :: [[Value]] -> [Int]
intValues rows = [n | row <- rows, Just (VInt n) <- [nth 0 row]]
-- | 显式事务用例
transactionSpec :: Spec
transactionSpec = describe "server transactions" $ do
it "keeps uncommitted rows invisible to other sessions until COMMIT" $ withTwoSessions $ \writer reader -> do
_ <- mustSql writer "begin"
_ <- mustSql writer "insert into users (id, name, age) values (6, 'six', 26)"
sessionIds writer `shouldReturn` [1, 2, 3, 4, 5, 6]
sessionIds reader `shouldReturn` [1, 2, 3, 4, 5]
_ <- mustSql writer "commit"
ids <- sessionIds writer
ids `shouldMatchList` [1, 2, 3, 4, 5, 6]
seen <- sessionIds reader
seen `shouldMatchList` [1, 2, 3, 4, 5, 6]
it "drops every change on ROLLBACK" $ withMemorySession $ \session -> do
_ <- mustSql session "begin"
_ <- mustSql session "insert into users (id, name, age) values (6, 'six', 26)"
_ <- mustSql session "update users set age = 99 where id = 1"
_ <- mustSql session "delete from users where id = 2"
rollback <- mustSql session "rollback"
qrRowCount rollback `shouldBe` 0
kept <- sessionIds session
kept `shouldMatchList` [1, 2, 3, 4, 5]
it "commits the whole write set at once" $ withMemorySession $ \session -> do
_ <- mustSql session "begin"
_ <- mustSql session "insert into users (id, name, age) values (6, 'six', 26)"
_ <- mustSql session "update users set age = 99 where id = 1"
_ <- mustSql session "delete from users where id = 2"
_ <- mustSql session "commit"
ids <- sessionIds session
ids `shouldMatchList` [1, 3, 4, 5, 6]
sessionAge session 1 `shouldReturn` Just 99
it "rejects DDL inside a transaction and takes it after ROLLBACK" $ withMemorySession $ \session -> do
_ <- mustSql session "begin"
denied <- runStatementCoded session "create table blocked (id int)"
failureMessage denied `shouldBe` "DDL is not allowed in a transaction"
_ <- mustSql session "rollback"
_ <- mustSql session "create table fresh (id int)"
_ <- mustSql session "insert into fresh (id) values (1)"
rows <- mustSql session "select * from fresh"
qrRowCount rows `shouldBe` 1
it "refuses nested BEGIN and COMMIT outside a transaction" $ withMemorySession $ \session -> do
_ <- mustSql session "begin"
nested <- runStatementCoded session "begin"
failureMessage nested `shouldBe` "already in a transaction"
_ <- mustSql session "rollback"
late <- runStatementCoded session "commit"
failureMessage late `shouldBe` "no transaction in progress"
it "does not switch database inside a transaction" $ withMemorySession $ \session -> do
_ <- mustSql session "begin"
denied <- runStatementCoded session "use other"
failureMessage denied `shouldBe` "cannot switch database inside a transaction"
it "needs a current database to start" $ do
db <- newMVar testDb
settings <- tempSettingsPath "transaction-bare"
removeIfExists settings
session <- newSession (memoryBackend testDatabaseName db) (T.pack testDatabaseName) settings
result <- runStatementCoded session "begin"
failureMessage result `shouldBe` "no database selected"
-- | 起一份真存储并装成当前链路;动态库打不开就当用例待定
withIpcStorage :: IO () -> IO ()
withIpcStorage body = do
dir <- ipcDataDir
cleanIpc dir
createDirectoryIfMissing True dir
cfgPath <- writeIpcConfig dir
opened <- localStorageLink (Just cfgPath)
case opened of
Left err -> cleanIpc dir >> pendingWith ("cannot open the storage library: " ++ err)
Right link -> do
setStorageLink link
body `finally` (closeConnection >> cleanIpc dir)
-- | 跑一条语句,出错就算用例失败
mustRun :: IO (Either String StatementResult) -> IO ()
mustRun action = action >>= either expectationFailure (const (pure ()))
-- | 取结果行,出错就算用例失败
rowsOf :: IO (Either String StatementResult) -> IO [Row]
rowsOf action = action >>= either (\err -> expectationFailure err >> pure []) (pure . srRows)
-- | 取结果里某一列的整数
intColumn :: String -> [Row] -> [Int]
intColumn name rows = [n | row <- rows, Just (VInt n) <- [lookup name row]]
-- | 往 p 表连插一批行,结果放 MVar 交回主线程
writeRows :: Backend -> [Int] -> MVar [Either String StatementResult] -> IO ()
writeRows session ids done = do
outs <- mapM (\i -> beStatement session ("INSERT INTO p (id) VALUES (" ++ show i ++ ")")) ids
putMVar done outs
-- | 同一进程里开两个库:会话之间不串数据,也不互相挡路
ipcConcurrencySpec :: Spec
ipcConcurrencySpec = describe "server backend on the real storage" $ do
it "keeps two databases apart" $ withIpcStorage $ do
base <- ipcBackend
mustRun (beStatement base "CREATE DATABASE alpha")
mustRun (beStatement base "CREATE DATABASE beta")
let alpha = beWithDatabase base "alpha"
beta = beWithDatabase base "beta"
mustRun (beStatement alpha "CREATE TABLE t (id int)")
mustRun (beStatement beta "CREATE TABLE t (id int)")
mustRun (beStatement alpha "INSERT INTO t (id) VALUES (1)")
mustRun (beStatement beta "INSERT INTO t (id) VALUES (2)")
rowsA <- rowsOf (beStatement alpha "SELECT id FROM t")
rowsB <- rowsOf (beStatement beta "SELECT id FROM t")
intColumn "id" rowsA `shouldBe` [1]
intColumn "id" rowsB `shouldBe` [2]
it "runs two sessions at once without mixing their rows" $ withIpcStorage $ do
base <- ipcBackend
mustRun (beStatement base "CREATE DATABASE alpha")
mustRun (beStatement base "CREATE DATABASE beta")
let alpha = beWithDatabase base "alpha"
beta = beWithDatabase base "beta"
mustRun (beStatement alpha "CREATE TABLE p (id int)")
mustRun (beStatement beta "CREATE TABLE p (id int)")
doneA <- newEmptyMVar
doneB <- newEmptyMVar
_ <- forkIO (writeRows alpha [1 .. 20] doneA)
_ <- forkIO (writeRows beta [101 .. 120] doneB)
outs <- sequence [takeMVar doneA, takeMVar doneB]
[err | Left err <- concat outs] `shouldBe` []
rowsA <- rowsOf (beStatement alpha "SELECT id FROM p")
rowsB <- rowsOf (beStatement beta "SELECT id FROM p")
intColumn "id" rowsA `shouldMatchList` [1 .. 20]
intColumn "id" rowsB `shouldMatchList` [101 .. 120]
it "commits an explicit transaction to the real storage" $ withIpcStorage $ do
base <- ipcBackend
mustRun (beStatement base "CREATE DATABASE alpha")
let alpha = beWithDatabase base "alpha"
settings <- tempSettingsPath "transaction-ipc"
removeIfExists settings
session <- newSession alpha (T.pack testDatabaseName) settings
_ <- mustSql session "use alpha"
_ <- mustSql session "create table t (id int)"
_ <- mustSql session "begin"
_ <- mustSql session "insert into t (id) values (1)"
inside <- mustSql session "select * from t"
intValues (qrRows inside) `shouldBe` [1]
_ <- mustSql session "commit"
rows <- rowsOf (beStatement alpha "SELECT id FROM t")
intColumn "id" rows `shouldBe` [1]
+7 -1
View File
@@ -45,7 +45,13 @@ chusql-server
`chusql-server` 默认监听 `127.0.0.1:7777`,客户端建立连接后完成登录、数据库选择和 SQL 执行。
每条连接拥有独立会话状态,例如当前数据库选择不会影响其他连接。
每条连接拥有独立会话状态,例如当前数据库选择不会影响其他连接。不同连接可以同时执行语句:读请求之间可以并行,写请求之间由存储层排队。
语句里的库前缀和表限定符由引擎在语义层解析:存储层只接受精确的表名键,因此 `FROM sales.orders` 与选定 `sales` 后的 `FROM orders` 指向同一张表。
显式事务在会话层实现:`BEGIN` 把当前数据库整库读成内存快照,事务内的语句都在快照上执行;`COMMIT` 计算快照与库的差分,用一条 `apply_transaction` 请求交给存储层应用;`ROLLBACK` 直接丢弃快照。因此隔离级别是快照读加原子提交:事务里看不到别的事务未提交或后来提交的改动,提交对外一次性可见。
WAL 负责崩溃恢复:每条记录带自增 LSN,一组写操作与一条提交标记一次落盘;进程重启时按标记分组重放已提交的记录、丢弃没有标记的整组,重放前逐条做幂等检查,恢复结束把最大 LSN 写进 `wal.checkpoint` 并清空 WAL。`SystemStatus` 应答里的 `last_lsn` 报告该检查点 LSN。
系统数据库及部分管理操作仅允许管理员访问。
+16
View File
@@ -38,6 +38,22 @@ csql -d sales -e "SELECT * FROM orders LIMIT 5"
csql -f json -e "SELECT count(*) FROM orders"
```
表名与列名的写法:
- 选定数据库后语句里直接写表名即可,例如 `SELECT name FROM orders WHERE orders.id = 1`。
- 表名可以带库前缀(`sales.orders`),列名可以带表限定符(`orders.id`)或库限定符(`sales.orders.id`),这几种写法指向同一张表和同一个列。
- 只有候选唯一时才自动推导前缀;同名的表或列出现在多个候选上会以 `ambiguous table` / `ambiguous column` 报错,需要写全。
- `FROM orders AS o` 之后只能用 `o.id`,不能再写 `orders.id`。
事务:
- `BEGIN`(或 `START TRANSACTION`)开始显式事务,`COMMIT` 提交,`ROLLBACK` 回滚。
- 事务内只允许数据语句;DDL 与 `USE` 会被拒绝,`SHOW DATABASES` 这类不碰表数据的语句照常可用。
- 事务里的读都看开事务那一刻的快照:别的连接在事务期间提交的改动不会出现在事务里,本会话未提交的改动别的连接也看不见。
- 同一连接同时只能有一个事务,没有保存点与嵌套事务;开事务前要先选好数据库。
- 提交时按行 `id` 写回本事务改动过的行,事务没碰过的行不受影响;没有整数 `id` 的表按整表替换提交。
- 提交是持久的:整批改动与一条提交标记一次落盘,提交后进程被强杀也会在重启时重放;提交前崩溃则整批丢弃,不会只落一半。
### 交互模式
SQL 语句以分号结束,可跨多行输入。