事务模型
This commit is contained in:
1 parent
274b7fcdcc
commit
045ccc2dc0
20 files changed
+1486
-261
No files matched your search
@@ -28,5 +28,5 @@
|
||||
- 改动说明只在交流中记录。
|
||||
|
||||
## 当前重点
|
||||
- 完善 WAL 重放幂等。
|
||||
- 完整事务路线见 project-plan.txt 阶段 12。
|
||||
- 主线已完成:高级优化器 → 并行查询 → M1 会话隔离 → M2 快照事务 → M3 WAL 与崩溃恢复。
|
||||
- 进度与计数见 `agent_credientials/project-plan.txt`,限制与风险见 `agent_credientials/known-issues.txt`。
|
||||
@@ -134,6 +134,9 @@ runStatementUncheckedM _ q@CreateDatabase{} = runStatementM q
|
||||
runStatementUncheckedM _ q@DropDatabase{} = runStatementM q
|
||||
runStatementUncheckedM _ q@UseDatabase{} = runStatementM q
|
||||
runStatementUncheckedM _ ShowDatabases = runStatementM ShowDatabases
|
||||
runStatementUncheckedM _ BeginTransaction = pure (Left "BEGIN is executed by the session")
|
||||
runStatementUncheckedM _ CommitTransaction = pure (Left "COMMIT is executed by the session")
|
||||
runStatementUncheckedM _ RollbackTransaction = pure (Left "ROLLBACK is executed by the session")
|
||||
|
||||
-- | 只有需要读全表才能判断约束时才扫表
|
||||
existingRows :: (MonadStorage m) => String -> Table -> m (Either String [Row])
|
||||
|
||||
@@ -51,6 +51,8 @@ errorRules =
|
||||
, ErrorRule "storage" StorageError "query_error"
|
||||
, ErrorRule "storage library call failed" StorageError "query_error"
|
||||
, ErrorRule "unsupported protocol" ProtocolError "query_error"
|
||||
, ErrorRule "executed by the session" ProtocolError "query_error"
|
||||
, ErrorRule "transaction" ProtocolError "query_error"
|
||||
]
|
||||
|
||||
-- | 命中的第一条规则
|
||||
|
||||
@@ -38,16 +38,16 @@ scopeAt place scope c
|
||||
| "ambiguous column:" `isInfixOf` err -> Left err
|
||||
| otherwise -> columnAt place (scopeOuter scope) c
|
||||
|
||||
-- | 收集 FROM 能提供的列,含外层作用域
|
||||
-- | 收集 FROM 的列,列前缀由表名推导
|
||||
checkFrom :: Database -> [(String, Column)] -> FromClause -> Either String [(String, Column)]
|
||||
checkFrom _ _ FromUnit = Right []
|
||||
checkFrom db _ (FromTable mAlias tbl) = do
|
||||
t <- lookupTable db tbl
|
||||
Right (prefixColumns (Just (fromMaybe tbl mAlias)) (tableCols t))
|
||||
(key, t) <- resolveTable db tbl
|
||||
Right (prefixColumns (Just (deriveQualifier mAlias key)) (tableCols t))
|
||||
checkFrom db outer (FromJoin _ left mAlias tbl cond) = do
|
||||
lcols <- checkFrom db outer left
|
||||
t <- lookupTable db tbl
|
||||
let env = lcols ++ prefixColumns (Just (fromMaybe tbl mAlias)) (tableCols t)
|
||||
(key, t) <- resolveTable db tbl
|
||||
let env = lcols ++ prefixColumns (Just (deriveQualifier mAlias key)) (tableCols t)
|
||||
checkBool db "ON" (Scope env outer) cond
|
||||
Right env
|
||||
|
||||
@@ -433,6 +433,10 @@ checkResolvedWith db outer q = case q of
|
||||
DropDatabase name -> checkDatabaseName name
|
||||
UseDatabase name -> checkDatabaseName name
|
||||
ShowDatabases -> Right ()
|
||||
-- | 事务控制由会话层执行,语义层不做列检查
|
||||
BeginTransaction -> Right ()
|
||||
CommitTransaction -> Right ()
|
||||
RollbackTransaction -> Right ()
|
||||
where
|
||||
tableCols' = map fst . tableCols
|
||||
|
||||
@@ -650,21 +654,21 @@ resolveStatement db q = case q of
|
||||
Select{} -> resolveSelect db q
|
||||
SelectExpr{} -> resolveSelect db q
|
||||
Update tbl assigns mWhere -> do
|
||||
t <- lookupTable db tbl
|
||||
(key, t) <- resolveTable db tbl
|
||||
let env = tableCols t
|
||||
name = nameIn env "UPDATE"
|
||||
assigns' <- mapM (\(c, e) -> (\e' -> (c, e')) <$> resolveExpr db env [] name e) assigns
|
||||
w <- mapM (resolveExpr db env [] (nameIn env "WHERE")) mWhere
|
||||
pure (Update tbl assigns' w)
|
||||
pure (Update key assigns' w)
|
||||
Delete tbl mWhere -> do
|
||||
t <- lookupTable db tbl
|
||||
(key, t) <- resolveTable db tbl
|
||||
w <- mapM (resolveExpr db (tableCols t) [] (nameIn (tableCols t) "WHERE")) mWhere
|
||||
pure (Delete tbl w)
|
||||
pure (Delete key w)
|
||||
Insert tbl cols rows -> do
|
||||
t <- lookupTable db tbl
|
||||
(key, t) <- resolveTable db tbl
|
||||
let env = tableCols t
|
||||
rows' <- mapM (mapM (resolveExpr db env [] (nameIn env "INSERT"))) rows
|
||||
pure (Insert tbl cols rows')
|
||||
pure (Insert key cols rows')
|
||||
other -> Right other
|
||||
|
||||
-- | 查一列的物理名
|
||||
@@ -685,9 +689,9 @@ resolveSelect _ q = Right q
|
||||
resolveQuery :: Database -> [(String, Column)] -> Statement -> [(String, Expr)] -> Either String Statement
|
||||
resolveQuery db outer q items = do
|
||||
env <- checkFrom db outer (selectFrom q)
|
||||
-- | 单表查询里把裸列名换成物理列名
|
||||
-- | 单表无别名时把裸列名换成物理列名;带别名时表头保留别名限定名
|
||||
let physical c = case selectFrom q of
|
||||
FromTable Nothing tbl -> unqualify (Just tbl) c
|
||||
FromTable Nothing tbl -> unqualify (Just (deriveQualifier Nothing tbl)) c
|
||||
_ -> c
|
||||
-- | 查一列并换算成物理名
|
||||
name place c = physical . fst <$> scopedColumnAt place env outer c
|
||||
@@ -707,18 +711,23 @@ resolveQuery db outer q items = do
|
||||
grouping <- mapM (name "GROUP BY") (selectGroupBy q)
|
||||
ordering <- mapM (\(c, d) -> (\k -> (k, d)) <$> name "ORDER BY" c) (selectOrderBy q)
|
||||
source <- case selectFrom q of
|
||||
single@FromTable{} -> Right single
|
||||
FromTable mAlias tbl -> do
|
||||
(key, _) <- resolveTable db tbl
|
||||
Right (FromTable mAlias key)
|
||||
other -> resolveFrom db outer other
|
||||
pure $ case q of
|
||||
Select{} -> Select (map fst projected) source condition grouping ordering (selectLimit q)
|
||||
_ -> SelectExpr projected source condition grouping ordering (selectLimit q)
|
||||
|
||||
-- | 连接两侧默认取表名作别名;ON 里的子查询也能引用外层列
|
||||
-- | 连接两侧取表名末段作别名;ON 可引用外层列
|
||||
resolveFrom :: Database -> [(String, Column)] -> FromClause -> Either String FromClause
|
||||
resolveFrom _ _ FromUnit = Right FromUnit
|
||||
resolveFrom _ _ (FromTable alias tbl) = Right (FromTable (Just (fromMaybe tbl alias)) tbl)
|
||||
resolveFrom db _ (FromTable alias tbl) = do
|
||||
(key, _) <- resolveTable db tbl
|
||||
Right (FromTable (Just (deriveQualifier alias key)) key)
|
||||
resolveFrom db outer fromC@(FromJoin kind left alias tbl cond) = do
|
||||
env <- checkFrom db outer fromC
|
||||
l <- resolveFrom db outer left
|
||||
(key, _) <- resolveTable db tbl
|
||||
c <- resolveExpr db env outer (fmap fst . columnAt "ON" env) cond
|
||||
pure (FromJoin kind l (Just (fromMaybe tbl alias)) tbl c)
|
||||
pure (FromJoin kind l (Just (deriveQualifier alias key)) key c)
|
||||
@@ -3,6 +3,10 @@
|
||||
|
||||
module ChuSQL.Core.Engine.Storage.IPC (
|
||||
IPCStorage (..),
|
||||
Env (..),
|
||||
envForDatabase,
|
||||
defaultEnv,
|
||||
runIPCStorage,
|
||||
Request (..),
|
||||
Response (..),
|
||||
Account (..),
|
||||
@@ -41,21 +45,38 @@ import System.IO.Unsafe (unsafePerformIO)
|
||||
|
||||
-- IPC 存储实现:把 MonadStorage 的操作变成发给存储库的 JSON 请求。
|
||||
|
||||
newtype IPCStorage a = IPCStorage
|
||||
{ runIPCStorage :: IO a
|
||||
-- | 一条会话的存储环境:当前库名随会话走,不共享
|
||||
data Env = Env
|
||||
{ envDatabase :: Maybe String
|
||||
}
|
||||
|
||||
newtype IPCStorage a = IPCStorage
|
||||
{ runIPCStorageIn :: Env -> IO a
|
||||
}
|
||||
|
||||
-- | 按库名造一个会话环境
|
||||
envForDatabase :: Maybe String -> Env
|
||||
envForDatabase = Env
|
||||
|
||||
-- | 进程默认环境:读全局库名,给测试与 CLI 用
|
||||
defaultEnv :: IO Env
|
||||
defaultEnv = Env <$> readMVar databaseRef
|
||||
|
||||
-- | 用进程默认环境跑(兼容既有调用点)
|
||||
runIPCStorage :: IPCStorage a -> IO a
|
||||
runIPCStorage action = defaultEnv >>= runIPCStorageIn action
|
||||
|
||||
instance Functor IPCStorage where
|
||||
fmap f (IPCStorage m) = IPCStorage (fmap f m)
|
||||
fmap f (IPCStorage m) = IPCStorage (fmap f . m)
|
||||
|
||||
instance Applicative IPCStorage where
|
||||
pure = IPCStorage . pure
|
||||
IPCStorage mf <*> IPCStorage ma = IPCStorage (mf <*> ma)
|
||||
pure = IPCStorage . const . pure
|
||||
IPCStorage mf <*> IPCStorage ma = IPCStorage $ \env -> mf env <*> ma env
|
||||
|
||||
instance Monad IPCStorage where
|
||||
IPCStorage m >>= k = IPCStorage $ do
|
||||
a <- m
|
||||
runIPCStorage (k a)
|
||||
IPCStorage m >>= k = IPCStorage $ \env -> do
|
||||
a <- m env
|
||||
runIPCStorageIn (k a) env
|
||||
|
||||
-- | 一条到存储库的链路:管请求与关闭
|
||||
data StorageLink = StorageLink
|
||||
@@ -147,10 +168,9 @@ globalRequest ReqPing = True
|
||||
globalRequest _ = False
|
||||
|
||||
-- | 发请求并把响应翻成 Either
|
||||
ask :: String -> Request -> (Response -> Maybe a) -> IO (Either String a)
|
||||
ask what req recognize = do
|
||||
current <- getDatabaseName
|
||||
resp <- sendRequest (stampDatabase current req)
|
||||
ask :: Env -> String -> Request -> (Response -> Maybe a) -> IO (Either String a)
|
||||
ask env what req recognize = do
|
||||
resp <- sendRequest (stampDatabase (envDatabase env) req)
|
||||
pure $ case resp of
|
||||
RespError e -> Left e
|
||||
other -> case recognize other of
|
||||
@@ -163,99 +183,100 @@ okOnly RespOk = Just ()
|
||||
okOnly _ = Nothing
|
||||
|
||||
-- | 发 Scan
|
||||
doScan :: String -> IO (Either String [Row])
|
||||
doScan t = ask "scan" (ReqScan t) $ \resp -> case resp of
|
||||
doScan :: Env -> String -> IO (Either String [Row])
|
||||
doScan env t = ask env "scan" (ReqScan t) $ \resp -> case resp of
|
||||
RespRows rows -> Just rows
|
||||
_ -> Nothing
|
||||
|
||||
-- | 发带投影的 Scan
|
||||
doScanColumns :: String -> [String] -> IO (Either String [Row])
|
||||
doScanColumns t cols = ask "scan" (ReqScanColumns t cols) $ \resp -> case resp of
|
||||
doScanColumns :: Env -> String -> [String] -> IO (Either String [Row])
|
||||
doScanColumns env t cols = ask env "scan" (ReqScanColumns t cols) $ \resp -> case resp of
|
||||
RespRows rows -> Just [[(c, v) | c <- cols, Just v <- [lookup c row]] | row <- rows]
|
||||
_ -> Nothing
|
||||
|
||||
-- | 发分片 Scan:只取第 shard 片
|
||||
doScanShard :: String -> Maybe [String] -> Int -> Int -> IO (Either String [Row])
|
||||
doScanShard t cols shard shards = ask "scan_shard" (ReqScanShard t cols shard shards) $ \resp -> case resp of
|
||||
doScanShard :: Env -> String -> Maybe [String] -> Int -> Int -> IO (Either String [Row])
|
||||
doScanShard env t cols shard shards = ask env "scan_shard" (ReqScanShard t cols shard shards) $ \resp -> case resp of
|
||||
RespRows rows -> Just rows
|
||||
_ -> Nothing
|
||||
|
||||
-- | 发 Insert(索引由存储层维护)
|
||||
doInsert :: String -> Row -> IO (Either String ())
|
||||
doInsert t r = ask "insert" (ReqInsert t r) okOnly
|
||||
doInsert :: Env -> String -> Row -> IO (Either String ())
|
||||
doInsert env t r = ask env "insert" (ReqInsert t r) okOnly
|
||||
|
||||
-- | 发 InsertBatch:一批行一次请求
|
||||
doInsertMany :: String -> [Row] -> IO (Either String ())
|
||||
doInsertMany t rs = ask "insert_batch" (ReqInsertBatch t rs) okOnly
|
||||
doInsertMany :: Env -> String -> [Row] -> IO (Either String ())
|
||||
doInsertMany env t rs = ask env "insert_batch" (ReqInsertBatch t rs) okOnly
|
||||
|
||||
-- | 发 DeleteKeys
|
||||
doDeleteKeys :: String -> [Int] -> IO (Either String ())
|
||||
doDeleteKeys t ks = ask "delete_keys" (ReqDeleteKeys t ks) okOnly
|
||||
doDeleteKeys :: Env -> String -> [Int] -> IO (Either String ())
|
||||
doDeleteKeys env t ks = ask env "delete_keys" (ReqDeleteKeys t ks) okOnly
|
||||
|
||||
-- | 发 ReplaceAll
|
||||
doReplaceAll :: String -> [Row] -> IO (Either String ())
|
||||
doReplaceAll t rs = ask "replace_all" (ReqReplaceAll t rs) okOnly
|
||||
doReplaceAll :: Env -> String -> [Row] -> IO (Either String ())
|
||||
doReplaceAll env t rs = ask env "replace_all" (ReqReplaceAll t rs) okOnly
|
||||
|
||||
-- | 发 ListTables
|
||||
doListTables :: IO (Either String [String])
|
||||
doListTables = ask "list_tables" ReqListTables $ \resp -> case resp of
|
||||
doListTables :: Env -> IO (Either String [String])
|
||||
doListTables env = ask env "list_tables" ReqListTables $ \resp -> case resp of
|
||||
RespTables ts -> Just ts
|
||||
_ -> Nothing
|
||||
|
||||
-- | 一次拿全库 schema
|
||||
doListCatalog :: IO (Either String [TableInfo])
|
||||
doListCatalog = ask "list_catalog" ReqListCatalog $ \resp -> case resp of
|
||||
doListCatalog :: Env -> IO (Either String [TableInfo])
|
||||
doListCatalog env = ask env "list_catalog" ReqListCatalog $ \resp -> case resp of
|
||||
RespCatalog xs -> Just xs
|
||||
_ -> Nothing
|
||||
|
||||
-- | 按某一列的索引取一行(键可以是整数或字符串)
|
||||
doLookupByColumn :: String -> String -> Value -> IO (Either String IndexResult)
|
||||
doLookupByColumn t c k = ask "lookup_by_index" (ReqLookupByColumn t c k) $ \resp -> case resp of
|
||||
doLookupByColumn :: Env -> String -> String -> Value -> IO (Either String IndexResult)
|
||||
doLookupByColumn env t c k = ask env "lookup_by_index" (ReqLookupByColumn t c k) $ \resp -> case resp of
|
||||
RespRows rows -> Just (IndexRows rows)
|
||||
RespNoIndex -> Just NoIndex
|
||||
_ -> Nothing
|
||||
|
||||
-- | 范围扫描:没有可用索引就回 Nothing,让上层退回扫描
|
||||
doScanRange ::
|
||||
Env ->
|
||||
String ->
|
||||
String ->
|
||||
Maybe (Value, Bool) ->
|
||||
Maybe (Value, Bool) ->
|
||||
IO (Either String (Maybe [Row]))
|
||||
doScanRange t c lo hi = ask "range_by_index" (ReqRangeByIndex t c lo hi) $ \resp -> case resp of
|
||||
doScanRange env t c lo hi = ask env "range_by_index" (ReqRangeByIndex t c lo hi) $ \resp -> case resp of
|
||||
RespRows rows -> Just (Just rows)
|
||||
RespNoIndex -> Just Nothing
|
||||
_ -> Nothing
|
||||
|
||||
-- | 发 DescribeTable
|
||||
doDescribeTable :: String -> IO (Either String TableInfo)
|
||||
doDescribeTable t = ask "describe_table" (ReqDescribeTable t) $ \resp -> case resp of
|
||||
doDescribeTable :: Env -> String -> IO (Either String TableInfo)
|
||||
doDescribeTable env t = ask env "describe_table" (ReqDescribeTable t) $ \resp -> case resp of
|
||||
RespSchema info -> Just info
|
||||
_ -> Nothing
|
||||
|
||||
-- | 给某一列建索引
|
||||
doCreateIndex :: String -> String -> IO (Either String ())
|
||||
doCreateIndex t c = ask "create_index" (ReqCreateIndex t c) okOnly
|
||||
doCreateIndex :: Env -> String -> String -> IO (Either String ())
|
||||
doCreateIndex env t c = ask env "create_index" (ReqCreateIndex t c) okOnly
|
||||
|
||||
-- | 去掉某一列的索引
|
||||
doDropIndex :: String -> String -> IO (Either String ())
|
||||
doDropIndex t c = ask "drop_index" (ReqDropIndex t c) okOnly
|
||||
doDropIndex :: Env -> String -> String -> IO (Either String ())
|
||||
doDropIndex env t c = ask env "drop_index" (ReqDropIndex t c) okOnly
|
||||
|
||||
-- | 删一列(列定义、这一列上的索引、每行里的那一格一起没)
|
||||
doDropColumn :: String -> String -> IO (Either String ())
|
||||
doDropColumn t c = ask "drop_column" (ReqDropColumn t c) okOnly
|
||||
doDropColumn :: Env -> String -> String -> IO (Either String ())
|
||||
doDropColumn env t c = ask env "drop_column" (ReqDropColumn t c) okOnly
|
||||
|
||||
-- | 发 ReplaceSchema(ALTER 用)
|
||||
doReplaceSchema :: String -> [SchemaColumn] -> [Row] -> IO (Either String ())
|
||||
doReplaceSchema t cols rs = ask "replace_schema" (ReqReplaceSchema t cols rs) okOnly
|
||||
doReplaceSchema :: Env -> String -> [SchemaColumn] -> [Row] -> IO (Either String ())
|
||||
doReplaceSchema env t cols rs = ask env "replace_schema" (ReqReplaceSchema t cols rs) okOnly
|
||||
|
||||
-- | 发 CreateTable
|
||||
doCreateTable :: String -> [SchemaColumn] -> IO (Either String ())
|
||||
doCreateTable t cols = ask "create_table" (ReqCreateTable t cols) okOnly
|
||||
doCreateTable :: Env -> String -> [SchemaColumn] -> IO (Either String ())
|
||||
doCreateTable env t cols = ask env "create_table" (ReqCreateTable t cols) okOnly
|
||||
|
||||
-- | 发 DropTable
|
||||
doDropTable :: String -> IO (Either String ())
|
||||
doDropTable t = ask "drop_table" (ReqDropTable t) okOnly
|
||||
doDropTable :: Env -> String -> IO (Either String ())
|
||||
doDropTable env t = ask env "drop_table" (ReqDropTable t) okOnly
|
||||
|
||||
-- | 线上 schema 转本地列
|
||||
schemaToColumns :: [SchemaColumn] -> [(String, Column)]
|
||||
@@ -292,92 +313,91 @@ toWire (n, c) =
|
||||
|
||||
instance MonadStorage IPCStorage where
|
||||
-- \| 建一个库
|
||||
createDatabase name = IPCStorage (ask "create_database" (ReqDatabase "create_database" name) okOnly)
|
||||
createDatabase name = IPCStorage $ \env -> ask env "create_database" (ReqDatabase "create_database" name) okOnly
|
||||
-- \| 删一个库
|
||||
dropDatabase name = IPCStorage (ask "drop_database" (ReqDatabase "drop_database" name) okOnly)
|
||||
dropDatabase name = IPCStorage $ \env -> ask env "drop_database" (ReqDatabase "drop_database" name) okOnly
|
||||
-- \| 切当前库
|
||||
useDatabase name = IPCStorage (ask "use_database" (ReqDatabase "use_database" name) okOnly)
|
||||
useDatabase name = IPCStorage $ \env -> ask env "use_database" (ReqDatabase "use_database" name) okOnly
|
||||
-- \| 列出现有库
|
||||
listDatabases = IPCStorage $ ask "list_databases" (ReqDatabase "list_databases" "") $ \resp -> case resp of
|
||||
listDatabases = IPCStorage $ \env -> ask env "list_databases" (ReqDatabase "list_databases" "") $ \resp -> case resp of
|
||||
RespTables names -> Just names
|
||||
_ -> Nothing
|
||||
-- \| 发 Scan
|
||||
scan t = IPCStorage (doScan t)
|
||||
scan t = IPCStorage (\env -> doScan env t)
|
||||
-- \| 只取指定列
|
||||
scanColumns t cols = IPCStorage (doScanColumns t cols)
|
||||
scanColumns t cols = IPCStorage (\env -> doScanColumns env t cols)
|
||||
|
||||
-- \| 并行分片数:按 RTS 能力数,封顶在 parallelShardLimit
|
||||
parallelShards = IPCStorage $ do
|
||||
parallelShards = IPCStorage $ \_ -> do
|
||||
caps <- getNumCapabilities
|
||||
pure (max 1 (min parallelShardLimit caps))
|
||||
|
||||
-- \| 分片并行扫,再按分片序拼接
|
||||
scanShards width t cols
|
||||
| width <= 1 = IPCStorage (maybe (doScan t) (doScanColumns t) cols)
|
||||
| otherwise = IPCStorage $ do
|
||||
| width <= 1 = IPCStorage (\env -> maybe (doScan env t) (doScanColumns env t) cols)
|
||||
| otherwise = IPCStorage $ \env -> do
|
||||
pool <- workerPool width
|
||||
parts <- poolRun pool [doScanShard t cols shard width | shard <- [0 .. width - 1]]
|
||||
parts <- poolRun pool [doScanShard env t cols shard width | shard <- [0 .. width - 1]]
|
||||
pure (fmap concat (sequence parts))
|
||||
|
||||
-- \| 发 Insert(索引由存储层自己维护)
|
||||
insert t r = IPCStorage (doInsert t r)
|
||||
insert t r = IPCStorage (\env -> doInsert env t r)
|
||||
|
||||
-- \| 一批行一次请求:N 行只落一次盘
|
||||
insertMany t rs = IPCStorage (doInsertMany t rs)
|
||||
insertMany t rs = IPCStorage (\env -> doInsertMany env t rs)
|
||||
|
||||
-- \| 按 id 批量删行
|
||||
deleteKeys t ks = IPCStorage (doDeleteKeys t ks)
|
||||
deleteKeys t ks = IPCStorage (\env -> doDeleteKeys env t ks)
|
||||
|
||||
-- \| 发 ReplaceAll
|
||||
replaceAll t rs = IPCStorage (doReplaceAll t rs)
|
||||
replaceAll t rs = IPCStorage (\env -> doReplaceAll env t rs)
|
||||
|
||||
-- \| 按某一列的索引取一行(没有索引就回 NoIndex)
|
||||
lookupByColumn t c k = IPCStorage (doLookupByColumn t c k)
|
||||
lookupByColumn t c k = IPCStorage (\env -> doLookupByColumn env t c k)
|
||||
|
||||
-- \| 范围扫描(没有索引就回 Nothing)
|
||||
scanRange t c lo hi = IPCStorage (doScanRange t c lo hi)
|
||||
scanRange t c lo hi = IPCStorage (\env -> doScanRange env t c lo hi)
|
||||
|
||||
-- \| 发 CreateTable
|
||||
createTable name cols = IPCStorage (doCreateTable name (map toWire cols))
|
||||
createTable name cols = IPCStorage (\env -> doCreateTable env name (map toWire cols))
|
||||
|
||||
-- \| ALTER:整表换列定义与全部行
|
||||
replaceSchema name cols rows = IPCStorage (doReplaceSchema name (map toWire cols) rows)
|
||||
replaceSchema name cols rows = IPCStorage (\env -> doReplaceSchema env name (map toWire cols) rows)
|
||||
|
||||
-- \| 发 DropTable
|
||||
dropTable name = IPCStorage (doDropTable name)
|
||||
dropTable name = IPCStorage (\env -> doDropTable env name)
|
||||
|
||||
-- \| 建索引
|
||||
createIndex t c = IPCStorage (doCreateIndex t c)
|
||||
createIndex t c = IPCStorage (\env -> doCreateIndex env t c)
|
||||
|
||||
-- \| 删索引
|
||||
dropIndex t c = IPCStorage (doDropIndex t c)
|
||||
dropIndex t c = IPCStorage (\env -> doDropIndex env t c)
|
||||
|
||||
-- \| 删一列
|
||||
dropColumn t c = IPCStorage (doDropColumn t c)
|
||||
dropColumn t c = IPCStorage (\env -> doDropColumn env t c)
|
||||
|
||||
-- \| 列表 + 逐表扫描拼库
|
||||
snapshot = IPCStorage $ do
|
||||
result <- doListCatalog
|
||||
snapshot = IPCStorage $ \env -> do
|
||||
result <- doListCatalog env
|
||||
case result of
|
||||
Left _ -> pure []
|
||||
Right xs -> mapM loadEntry xs
|
||||
Right xs -> mapM (loadEntry env) xs
|
||||
where
|
||||
loadEntry info = do
|
||||
rows <- doScan (tiTable info)
|
||||
loadEntry env info = do
|
||||
rows <- doScan env (tiTable info)
|
||||
pure
|
||||
( tiTable info
|
||||
, Table (tiTable info) (schemaToColumns (tiColumns info)) (either (const []) id rows) (Just (metaOf info))
|
||||
)
|
||||
|
||||
-- \| 只问数据字典要结构,设了当前库只留它的表
|
||||
schema = IPCStorage $ do
|
||||
current <- getDatabaseName
|
||||
result <- ask "all_catalogs" ReqAllCatalog $ \resp -> case resp of
|
||||
schema = IPCStorage $ \env -> do
|
||||
result <- ask env "all_catalogs" ReqAllCatalog $ \resp -> case resp of
|
||||
RespCatalog infos -> Just infos
|
||||
_ -> Nothing
|
||||
pure $ case result of
|
||||
Left _ -> []
|
||||
Right xs -> concatMap (visibleTables current) xs
|
||||
Right xs -> concatMap (visibleTables (envDatabase env)) xs
|
||||
|
||||
-- | 线上表信息转引擎侧统计
|
||||
metaOf :: TableInfo -> TableMeta
|
||||
|
||||
@@ -53,6 +53,12 @@ data Statement
|
||||
| DropDatabase String
|
||||
| UseDatabase String
|
||||
| ShowDatabases
|
||||
-- | 开事务:BEGIN / START TRANSACTION
|
||||
| BeginTransaction
|
||||
-- | 提交事务
|
||||
| CommitTransaction
|
||||
-- | 回滚事务
|
||||
| RollbackTransaction
|
||||
deriving (Show, Eq)
|
||||
|
||||
-- | 连接类型:内连接与左外连接
|
||||
|
||||
@@ -613,6 +613,14 @@ updateStatement = do
|
||||
mWhere <- optional (keyword "where" *> expr)
|
||||
return (Update tbl assigns mWhere)
|
||||
|
||||
-- | 事务控制语句
|
||||
transactionStatement :: Parser Statement
|
||||
transactionStatement =
|
||||
try (BeginTransaction <$ (keyword "start" *> keyword "transaction"))
|
||||
<|> (BeginTransaction <$ (keyword "begin" *> optional (keyword "transaction")))
|
||||
<|> (CommitTransaction <$ (keyword "commit" *> optional (keyword "transaction")))
|
||||
<|> (RollbackTransaction <$ (keyword "rollback" *> optional (keyword "transaction")))
|
||||
|
||||
-- | 解析总入口
|
||||
parseStatement :: String -> Either String Statement
|
||||
parseStatement input =
|
||||
@@ -622,7 +630,8 @@ parseStatement input =
|
||||
where
|
||||
-- | 按关键字分派到各语句解析器
|
||||
statementP =
|
||||
selectStatement
|
||||
transactionStatement
|
||||
<|> selectStatement
|
||||
<|> try (CreateDatabase <$> (keyword "create" *> keyword "database" *> identifier))
|
||||
<|> try (DropDatabase <$> (keyword "drop" *> keyword "database" *> identifier))
|
||||
<|> (UseDatabase <$> (keyword "use" *> identifier))
|
||||
|
||||
@@ -15,7 +15,7 @@ import ChuSQL.Core.Engine.Parallel (poolRun, workerPool)
|
||||
import ChuSQL.Core.Model
|
||||
import ChuSQL.Core.Engine.Semantic (prepare)
|
||||
import ChuSQL.Core.Engine.Storage (IndexResult (..), MonadStorage (..))
|
||||
import ChuSQL.Core.Engine.Storage.IPC (IPCStorage (runIPCStorage), closeConnection, doListTables, getDatabaseName, localStorageLink, setDatabaseName, setStorageLink)
|
||||
import ChuSQL.Core.Engine.Storage.IPC (IPCStorage (..), closeConnection, defaultEnv, doListTables, envForDatabase, getDatabaseName, localStorageLink, runIPCStorage, setDatabaseName, setStorageLink)
|
||||
import ChuSQL.Core.Engine.Syntax.AST
|
||||
import ChuSQL.Core.Engine.Syntax.Parser
|
||||
import Control.Concurrent (threadDelay)
|
||||
@@ -113,6 +113,40 @@ main = hspec $ do
|
||||
rowsOf (parseStatement "SELECT sales.users.name FROM sales.users WHERE sales.users.id = 1" >>= runStatement db)
|
||||
`shouldBe` Right [[("name", VStr "Alice")]]
|
||||
|
||||
it "derives the column prefix when the session schema is scoped to one database" $ do
|
||||
let scoped = [("users", users)]
|
||||
rowsOf (parseStatement "SELECT sales.users.name FROM sales.users WHERE sales.users.id = 1" >>= runStatement scoped)
|
||||
`shouldBe` Right [[("name", VStr "Alice")]]
|
||||
rowsOf (parseStatement "SELECT users.name FROM users WHERE users.id = 1" >>= runStatement scoped)
|
||||
`shouldBe` Right [[("name", VStr "Alice")]]
|
||||
|
||||
it "derives the database prefix for a bare table name" $ do
|
||||
let catalog = [("sales.users", users)]
|
||||
rowsOf (parseStatement "SELECT name FROM users WHERE users.id = 1" >>= runStatement catalog)
|
||||
`shouldBe` Right [[("name", VStr "Alice")]]
|
||||
rowsOf (parseStatement "SELECT users.name FROM sales.users WHERE users.id = 1" >>= runStatement catalog)
|
||||
`shouldBe` Right [[("name", VStr "Alice")]]
|
||||
|
||||
it "keeps derived prefixes short in joins" $ do
|
||||
let catalog = [("sales.users", users), ("sales.orders", orders)]
|
||||
fmap
|
||||
(map (map snd))
|
||||
(parseStatement "SELECT users.name, orders.product FROM sales.users JOIN sales.orders ON users.id = orders.user_id" >>= rowsOf . runStatement catalog)
|
||||
`shouldBe` Right [[VStr "Alice", VStr "Book"], [VStr "Alice", VStr "Cup"], [VStr "Bob", VStr "Pen"]]
|
||||
|
||||
it "writes through the derived table key" $ do
|
||||
let scoped = [("users", users)]
|
||||
case parseStatement "UPDATE sales.users SET age = 26 WHERE id = 1" >>= runStatement scoped of
|
||||
Left err -> expectationFailure err
|
||||
Right (db', _) ->
|
||||
rowsOf (runStatement db' (makeSelect ["age"] "users" (Just (Eq (Col "id") (LitInt 1)))))
|
||||
`shouldBe` Right [[("age", VInt 26)]]
|
||||
|
||||
it "rejects an ambiguous derived table name" $ do
|
||||
let catalog = [("sales.users", users), ("other.users", users)]
|
||||
(parseStatement "SELECT name FROM users" >>= rowsOf . runStatement catalog)
|
||||
`shouldSatisfy` either (isInfixOf "ambiguous table") (const False)
|
||||
|
||||
it "parses database lifecycle statements" $ do
|
||||
map (fmap show . parseStatement) ["CREATE DATABASE sales", "DROP DATABASE sales", "USE sales", "SHOW DATABASES"]
|
||||
`shouldBe` map Right ["CreateDatabase \"sales\"", "DropDatabase \"sales\"", "UseDatabase \"sales\"", "ShowDatabases"]
|
||||
@@ -242,6 +276,33 @@ main = hspec $ do
|
||||
parseStatement "SELECT name FROM"
|
||||
`shouldSatisfy` isLeft
|
||||
|
||||
describe "ChuSQL.Core.Engine (事务)" $ do
|
||||
it "parses transaction control statements" $ do
|
||||
parseStatement "BEGIN" `shouldBe` Right BeginTransaction
|
||||
parseStatement "BEGIN TRANSACTION" `shouldBe` Right BeginTransaction
|
||||
parseStatement "START TRANSACTION" `shouldBe` Right BeginTransaction
|
||||
parseStatement "begin transaction" `shouldBe` Right BeginTransaction
|
||||
parseStatement "COMMIT" `shouldBe` Right CommitTransaction
|
||||
parseStatement "COMMIT TRANSACTION" `shouldBe` Right CommitTransaction
|
||||
parseStatement "ROLLBACK" `shouldBe` Right RollbackTransaction
|
||||
parseStatement "rollback transaction" `shouldBe` Right RollbackTransaction
|
||||
|
||||
it "parses the bare words only as whole words" $ do
|
||||
parseStatement "BEGINNER" `shouldSatisfy` isLeft
|
||||
parseStatement "COMMITTED" `shouldSatisfy` isLeft
|
||||
|
||||
it "reports transaction control as session work" $ do
|
||||
mapM_
|
||||
( \sql ->
|
||||
(runStatement [("users", users)] =<< parseStatement sql)
|
||||
`shouldSatisfy` either (isInfixOf "executed by the session") (const False)
|
||||
)
|
||||
["BEGIN", "COMMIT", "ROLLBACK"]
|
||||
|
||||
it "puts transaction errors under the protocol category" $ do
|
||||
errorCode "BEGIN is executed by the session" `shouldBe` "query_error"
|
||||
errorCategory "BEGIN is executed by the session" `shouldBe` ProtocolError
|
||||
|
||||
describe "ChuSQL.Core.Engine.Syntax.Parser (JOIN)" $ do
|
||||
it "parses a simple JOIN without aliases" $ do
|
||||
parseStatement "SELECT name FROM users JOIN orders ON id = user_id"
|
||||
@@ -1613,7 +1674,19 @@ main = hspec $ do
|
||||
withTestSession $ \srv -> withServerEnv srv $ do
|
||||
runIPCStorage (insert "t1" [("id", VInt 1)]) `shouldReturn` Right ()
|
||||
runIPCStorage (insert "t2" [("id", VInt 2)]) `shouldReturn` Right ()
|
||||
doListTables `shouldReturn` Right ["t1", "t2"]
|
||||
env <- defaultEnv
|
||||
doListTables env `shouldReturn` Right ["t1", "t2"]
|
||||
|
||||
it "two sessions keep their own database" $ do
|
||||
withTestSession $ \_ -> do
|
||||
let alpha = envForDatabase (Just "alpha")
|
||||
beta = envForDatabase (Just "beta")
|
||||
runIPCStorageIn (createDatabase "alpha") alpha `shouldReturn` Right ()
|
||||
runIPCStorageIn (createDatabase "beta") beta `shouldReturn` Right ()
|
||||
runIPCStorageIn (insert "iso" [("id", VInt 1)]) alpha `shouldReturn` Right ()
|
||||
runIPCStorageIn (insert "iso" [("id", VInt 2)]) beta `shouldReturn` Right ()
|
||||
runIPCStorageIn (scan "iso") alpha `shouldReturn` Right [[("id", VInt 1)]]
|
||||
runIPCStorageIn (scan "iso") beta `shouldReturn` Right [[("id", VInt 2)]]
|
||||
|
||||
it "rows survive a restart on the same data directory" $ do
|
||||
withTestSession $ \srv -> do
|
||||
|
||||
@@ -15,6 +15,9 @@ module ChuSQL.Core.Model
|
||||
, colNames
|
||||
, colType
|
||||
, lookupTable
|
||||
, resolveTable
|
||||
, lastTablePart
|
||||
, deriveQualifier
|
||||
, allColumns
|
||||
, qualify
|
||||
, unqualify
|
||||
@@ -470,6 +473,32 @@ colType t c = lookup c (tableCols t)
|
||||
lookupTable :: Database -> String -> Either String Table
|
||||
lookupTable db t = maybe (Left ("unknown table: " ++ t)) Right (lookup t db)
|
||||
|
||||
-- | 取表名引用的最后一段
|
||||
lastTablePart :: String -> String
|
||||
lastTablePart = reverse . takeWhile (/= '.') . reverse
|
||||
|
||||
-- | 推导列前缀:有别名用别名,否则取表名末段
|
||||
deriveQualifier :: Maybe String -> String -> String
|
||||
deriveQualifier (Just a) _ = a
|
||||
deriveQualifier Nothing t = lastTablePart t
|
||||
|
||||
-- | 按名查表并返回 schema 键:精确、前缀、唯一后缀
|
||||
resolveTable :: Database -> String -> Either String (String, Table)
|
||||
resolveTable db name = case lookup name db of
|
||||
Just table -> Right (name, table)
|
||||
Nothing
|
||||
| '.' `elem` name, Just table <- lookup bare db -> Right (bare, table)
|
||||
| [(key, table)] <- suffixed -> Right (key, table)
|
||||
| otherwise -> Left (miss suffixed)
|
||||
where
|
||||
-- | 引用里去掉库前缀的那一段
|
||||
bare = lastTablePart name
|
||||
-- | schema 里以这个表名收尾的键:`sales.users` 能推出裸表名引用
|
||||
suffixed = [(key, table) | (key, table) <- db, ('.' : name) `isSuffixOf` key]
|
||||
-- | 一个候选都没有就是查无此表,多于一个就是有歧义
|
||||
miss [] = "unknown table: " ++ name
|
||||
miss _ = "ambiguous table: " ++ name
|
||||
|
||||
-- | 通配列名
|
||||
allColumns :: String
|
||||
allColumns = "*"
|
||||
@@ -491,5 +520,5 @@ resolveColumn name env = case [(k, v) | (k, v) <- env, matches k] of
|
||||
[] -> Left ("unknown column: " ++ name)
|
||||
_ -> Left ("ambiguous column: " ++ name)
|
||||
where
|
||||
-- | 裸名或带后缀的名称
|
||||
matches k = k == name || ('.' : name) `isSuffixOf` k
|
||||
-- | 裸名、补充了库前缀的写法,或 schema 键自带库前缀的写法
|
||||
matches k = k == name || ('.' : name) `isSuffixOf` k || ('.' : k) `isSuffixOf` name
|
||||
@@ -3,6 +3,7 @@
|
||||
module ChuSQL.Core.Protocol (
|
||||
SchemaColumn (..),
|
||||
Request (..),
|
||||
TxnOp (..),
|
||||
Response (..),
|
||||
TableInfo (..),
|
||||
Account (..),
|
||||
@@ -96,6 +97,35 @@ data Request
|
||||
| ReqDropColumn String String
|
||||
| ReqReplaceSchema String [SchemaColumn] [Row]
|
||||
| ReqListCatalog
|
||||
| ReqApplyTransaction [TxnOp]
|
||||
|
||||
-- | 事务提交里的一条写操作
|
||||
data TxnOp
|
||||
= TxnUpsert String [Row]
|
||||
| TxnDelete String [Int]
|
||||
| TxnReplace String [Row]
|
||||
deriving (Show, Eq)
|
||||
|
||||
-- | 事务操作编码成 JSON
|
||||
instance ToJSON TxnOp where
|
||||
toJSON (TxnUpsert t rs) =
|
||||
object
|
||||
[ "op" .= ("upsert" :: T.Text)
|
||||
, "table" .= t
|
||||
, "rows" .= map rowToJSON rs
|
||||
]
|
||||
toJSON (TxnDelete t ks) =
|
||||
object
|
||||
[ "op" .= ("delete" :: T.Text)
|
||||
, "table" .= t
|
||||
, "ids" .= ks
|
||||
]
|
||||
toJSON (TxnReplace t rs) =
|
||||
object
|
||||
[ "op" .= ("replace" :: T.Text)
|
||||
, "table" .= t
|
||||
, "rows" .= map rowToJSON rs
|
||||
]
|
||||
|
||||
-- | 请求编码成 JSON
|
||||
instance ToJSON Request where
|
||||
@@ -201,6 +231,11 @@ instance ToJSON Request where
|
||||
, "rows" .= map rowToJSON rs
|
||||
]
|
||||
toJSON ReqListCatalog = object ["method" .= ("list_catalog" :: T.Text)]
|
||||
toJSON (ReqApplyTransaction ops) =
|
||||
object
|
||||
[ "method" .= ("apply_transaction" :: T.Text)
|
||||
, "ops" .= ops
|
||||
]
|
||||
|
||||
-- | 一张表的梗概:列、行数、索引与列统计
|
||||
data TableInfo = TableInfo
|
||||
|
||||
@@ -171,6 +171,22 @@ pub enum Request {
|
||||
columns: Vec<SchemaColumn>,
|
||||
rows: Vec<Row>,
|
||||
},
|
||||
/// 事务提交:把一批写操作当成一个请求原子应用
|
||||
ApplyTransaction {
|
||||
ops: Vec<StorageOp>,
|
||||
},
|
||||
}
|
||||
|
||||
/// 事务提交里的一条写操作
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "op", rename_all = "snake_case")]
|
||||
pub enum StorageOp {
|
||||
/// 追加写入的行(提交端已先删掉要覆盖的行)
|
||||
Upsert { table: String, rows: Vec<Row> },
|
||||
/// 按行 id 删除
|
||||
Delete { table: String, ids: Vec<i64> },
|
||||
/// 整表替换(没有整数 id 的表只能这么提交)
|
||||
Replace { table: String, rows: Vec<Row> },
|
||||
}
|
||||
|
||||
impl Request {
|
||||
@@ -202,7 +218,8 @@ impl Request {
|
||||
| Self::ReplaceSchema { table, .. } => Some(table),
|
||||
Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList
|
||||
| Self::AccountCreate { .. } | Self::AccountReset { .. } | Self::AccountLogin { .. }
|
||||
| Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus => None,
|
||||
| Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus
|
||||
| Self::ApplyTransaction { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -237,7 +254,11 @@ pub enum Response {
|
||||
NoIndex,
|
||||
Ok,
|
||||
/// 系统目录状态
|
||||
System { initialized: bool },
|
||||
System {
|
||||
initialized: bool,
|
||||
#[serde(default)]
|
||||
last_lsn: u64,
|
||||
},
|
||||
Error { message: String },
|
||||
Catalog { schemas: Vec<TableSchemaWire> },
|
||||
}
|
||||
|
||||
@@ -8,9 +8,10 @@ use crate::config::{self, Config, Loaded};
|
||||
use crate::heap::{scalar_int, HeapTable};
|
||||
use crate::log;
|
||||
use crate::protocol::{
|
||||
reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row, SchemaColumn, TableSchemaWire, USERS_TABLE,
|
||||
reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row, SchemaColumn, StorageOp,
|
||||
TableSchemaWire, USERS_TABLE,
|
||||
};
|
||||
use crate::wal::{Wal, WalOp};
|
||||
use crate::wal::{committed_ops, Wal, WalOp};
|
||||
use crate::{log_debug, log_error, log_info, log_warn};
|
||||
|
||||
// 进程内存储运行时:Haskell 侧经 ffi.rs 调进来。
|
||||
@@ -196,6 +197,7 @@ impl Server {
|
||||
/// 把一条 WAL 操作落到磁盘
|
||||
fn apply_op(&self, op: &WalOp) -> std::io::Result<()> {
|
||||
match op {
|
||||
WalOp::Commit => Ok(()),
|
||||
WalOp::HideColumn { table, column } => {
|
||||
let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||
c.remove_column(table, column)?;
|
||||
@@ -311,7 +313,12 @@ impl Server {
|
||||
if self.recovery_required.load(Ordering::Acquire) {
|
||||
return Err(std::io::Error::other("storage recovery required"));
|
||||
}
|
||||
{
|
||||
self.validate_op(op)?;
|
||||
self.group_via_wal(std::slice::from_ref(op))
|
||||
}
|
||||
|
||||
/// 校验一条操作:命中被删的列就拒绝
|
||||
fn validate_op(&self, op: &WalOp) -> std::io::Result<()> {
|
||||
let c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||
let validate = |table: &str, rows: &[Row]| -> std::io::Result<()> {
|
||||
let Some(schema) = c.describe(table) else { return Ok(()); };
|
||||
@@ -330,33 +337,95 @@ impl Server {
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 事务提交:一组写操作加一个提交标记,一次落盘
|
||||
///
|
||||
/// 组里每条都过 `validate_op`,跟单条写一致;调用方 `dispatch` 已经持有
|
||||
/// `gate.write`,因此整批对外原子可见。崩在提交标记之前,这组记录在恢复
|
||||
/// 时整组丢弃;崩在标记之后,逐条按「是否已生效」补写。
|
||||
fn apply_transaction(&self, ops: &[StorageOp]) -> std::io::Result<()> {
|
||||
let _guard = self.write_lock.lock().unwrap();
|
||||
if self.recovery_required.load(Ordering::Acquire) {
|
||||
return Err(std::io::Error::other("storage recovery required"));
|
||||
}
|
||||
let mut wal_ops = Vec::with_capacity(ops.len());
|
||||
for op in ops {
|
||||
let table = match op {
|
||||
StorageOp::Upsert { table, .. }
|
||||
| StorageOp::Delete { table, .. }
|
||||
| StorageOp::Replace { table, .. } => table,
|
||||
};
|
||||
if reserved_table(table) {
|
||||
return Err(std::io::Error::other("reserved system table"));
|
||||
}
|
||||
if table.is_empty() || !table.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') {
|
||||
return Err(std::io::Error::other("invalid table name"));
|
||||
}
|
||||
let wal_op = match op {
|
||||
StorageOp::Upsert { table, rows } => WalOp::InsertBatch {
|
||||
table: table.clone(),
|
||||
rows: rows.clone(),
|
||||
},
|
||||
StorageOp::Delete { table, ids } => WalOp::DeleteKeys {
|
||||
table: table.clone(),
|
||||
keys: ids.clone(),
|
||||
},
|
||||
StorageOp::Replace { table, rows } => WalOp::ReplaceAll {
|
||||
table: table.clone(),
|
||||
rows: rows.clone(),
|
||||
},
|
||||
};
|
||||
self.validate_op(&wal_op)?;
|
||||
wal_ops.push(wal_op);
|
||||
}
|
||||
self.group_via_wal(&wal_ops)
|
||||
}
|
||||
|
||||
/// 一组操作加提交标记落盘,成功后清空日志
|
||||
fn group_via_wal(&self, ops: &[WalOp]) -> std::io::Result<()> {
|
||||
std::fs::create_dir_all(&self.cfg.data_dir)?;
|
||||
let wal = &self.wal;
|
||||
|
||||
wal.truncate()?;
|
||||
|
||||
wal.append(op)?;
|
||||
|
||||
let commit_lsn = wal.append_group(ops)?;
|
||||
for op in ops {
|
||||
let result = self.apply_op(op);
|
||||
|
||||
if result.is_err() && matches!(op, WalOp::HideColumn { .. }) {
|
||||
self.recovery_required.store(true, Ordering::Release);
|
||||
return result;
|
||||
}
|
||||
|
||||
if result.is_ok() {
|
||||
result?;
|
||||
}
|
||||
for op in ops {
|
||||
self.flush_op_tables(op)?;
|
||||
}
|
||||
|
||||
self.flush_all_tables()?;
|
||||
wal.clear()?;
|
||||
result
|
||||
log_debug!(core, "WAL committed group at LSN {}", commit_lsn);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 把打开的表都 fsync 一遍
|
||||
fn flush_all_tables(&self) -> std::io::Result<()> {
|
||||
let handles: Vec<Arc<Mutex<HeapTable>>> = self
|
||||
.tables
|
||||
.lock()
|
||||
.map_err(|_| std::io::Error::other("table lock poisoned"))?
|
||||
.values()
|
||||
.cloned()
|
||||
.collect();
|
||||
for handle in handles {
|
||||
handle.lock().map_err(|_| std::io::Error::other("heap lock poisoned"))?.flush()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 把涉及的表写回磁盘
|
||||
fn flush_op_tables(&self, op: &WalOp) -> std::io::Result<()> {
|
||||
let table = match op {
|
||||
WalOp::HideColumn { .. } => return Ok(()),
|
||||
WalOp::Commit => return Ok(()),
|
||||
WalOp::Accounts { .. } => USERS_TABLE,
|
||||
WalOp::Insert { table, .. }
|
||||
| WalOp::InsertBatch { table, .. }
|
||||
@@ -369,21 +438,97 @@ impl Server {
|
||||
self.with_table(table, |t| t.flush())
|
||||
}
|
||||
|
||||
/// 启动时重放残留 WAL
|
||||
/// 重放残留 WAL:已提交组重放,未提交组丢弃
|
||||
fn recover_wal(&self) -> std::io::Result<()> {
|
||||
std::fs::create_dir_all(&self.cfg.data_dir)?;
|
||||
let wal = &self.wal;
|
||||
match wal.read()? {
|
||||
None => Ok(()),
|
||||
Some(op) => {
|
||||
log_warn!(core, "replaying incomplete WAL entry");
|
||||
self.apply_op(&op)?;
|
||||
self.flush_op_tables(&op)?;
|
||||
let scan = wal.read_all()?;
|
||||
if scan.records.is_empty() {
|
||||
if scan.good_len > 0 {
|
||||
wal.truncate_to(scan.good_len)?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
let records = committed_ops(&scan.records);
|
||||
if records.is_empty() {
|
||||
log_warn!(core, "discarding uncommitted WAL records");
|
||||
wal.clear()?;
|
||||
return Ok(());
|
||||
}
|
||||
let max_lsn = scan
|
||||
.records
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|(_, op)| *op == WalOp::Commit)
|
||||
.map(|(lsn, _)| *lsn)
|
||||
.or_else(|| records.last().map(|(lsn, _)| *lsn))
|
||||
.unwrap_or(0);
|
||||
log_warn!(core, "replaying {} committed WAL records", records.len());
|
||||
for (_, op) in &records {
|
||||
self.replay_op(op)?;
|
||||
self.flush_op_tables(op)?;
|
||||
}
|
||||
self.flush_all_tables()?;
|
||||
wal.write_checkpoint(max_lsn)?;
|
||||
wal.clear()?;
|
||||
log_info!(core, "WAL replay complete");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 重放一条操作:数据已经生效的不再重复写
|
||||
fn replay_op(&self, op: &WalOp) -> std::io::Result<()> {
|
||||
if self.op_already_applied(op)? {
|
||||
log_debug!(core, "WAL record already applied, skipping");
|
||||
return Ok(());
|
||||
}
|
||||
self.apply_op(op)
|
||||
}
|
||||
|
||||
/// 这条操作对应的数据是不是已经落盘了
|
||||
fn op_already_applied(&self, op: &WalOp) -> std::io::Result<bool> {
|
||||
match op {
|
||||
WalOp::Insert { table, row } => self.applied_rows(table, std::slice::from_ref(row)),
|
||||
WalOp::InsertBatch { table, rows } => self.applied_rows(table, rows),
|
||||
WalOp::ReplaceAll { table, rows } => {
|
||||
if self.catalog.lock().unwrap().describe(table).is_none() {
|
||||
return Ok(false);
|
||||
}
|
||||
let stored = self.with_table(table, |t| t.scan())?;
|
||||
Ok(stored == *rows)
|
||||
}
|
||||
WalOp::DeleteKeys { table, keys } => {
|
||||
if keys.is_empty() {
|
||||
return Ok(true);
|
||||
}
|
||||
if self.catalog.lock().unwrap().describe(table).is_none() {
|
||||
return Ok(false);
|
||||
}
|
||||
let rows = self.with_table(table, |t| t.scan())?;
|
||||
let present = rows.iter().any(|row| {
|
||||
row.get("id")
|
||||
.and_then(scalar_int)
|
||||
.is_some_and(|id| keys.contains(&id))
|
||||
});
|
||||
Ok(!present)
|
||||
}
|
||||
WalOp::HideColumn { table, column } => {
|
||||
let c = self.catalog.lock().unwrap();
|
||||
Ok(c.describe(table).is_some_and(|s| !s.columns.iter().any(|col| &col.name == column)))
|
||||
}
|
||||
_ => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
/// 这些行是不是都已经在表里
|
||||
fn applied_rows(&self, table: &str, rows: &[Row]) -> std::io::Result<bool> {
|
||||
if rows.is_empty() {
|
||||
return Ok(true);
|
||||
}
|
||||
if self.catalog.lock().unwrap().describe(table).is_none() {
|
||||
return Ok(false);
|
||||
}
|
||||
let stored = self.with_table(table, |t| t.scan())?;
|
||||
Ok(rows.iter().all(|row| stored.iter().any(|one| one == row)))
|
||||
}
|
||||
|
||||
/// 启动时重建列统计
|
||||
@@ -607,6 +752,11 @@ impl Server {
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// 最近一次落盘的检查点 LSN
|
||||
fn last_checkpoint(&self) -> u64 {
|
||||
self.wal.read_checkpoint().ok().flatten().unwrap_or(0)
|
||||
}
|
||||
|
||||
/// 读账号表;表没登记或数据文件缺失都算错
|
||||
fn load_accounts(&self) -> std::io::Result<Vec<Account>> {
|
||||
use std::io::Error;
|
||||
@@ -774,16 +924,29 @@ impl Server {
|
||||
}
|
||||
Request::BootstrapSystem { user, password_hash } => {
|
||||
match self.bootstrap_system(user.as_deref(), password_hash.as_deref()) {
|
||||
Ok(_) => Response::System { initialized: true },
|
||||
Ok(_) => Response::System { initialized: true, last_lsn: self.last_checkpoint() },
|
||||
Err(e) => Response::Error { message: e.to_string() },
|
||||
}
|
||||
}
|
||||
Request::SystemStatus => Response::System { initialized: self.system_initialized() },
|
||||
Request::SystemStatus => Response::System { initialized: self.system_initialized(), last_lsn: self.last_checkpoint() },
|
||||
Request::Ping => {
|
||||
log_debug!(request, "ping");
|
||||
Response::Pong
|
||||
}
|
||||
|
||||
Request::ApplyTransaction { ops } => {
|
||||
log_debug!(request, "apply_transaction ops={}", ops.len());
|
||||
match self.apply_transaction(&ops) {
|
||||
Ok(()) => Response::Ok,
|
||||
Err(e) => {
|
||||
log_warn!(request, "apply_transaction: {}", e);
|
||||
Response::Error {
|
||||
message: format!("apply_transaction: {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Request::Scan { table, columns } => {
|
||||
log_debug!(request, "scan table={}", table);
|
||||
|
||||
@@ -1347,8 +1510,8 @@ fn reserved_database(name: &str) -> bool {
|
||||
/// 每个库拥有独立的目录、字典、句柄和 WAL。
|
||||
struct Databases {
|
||||
root: Config,
|
||||
system: Server,
|
||||
named: Mutex<HashMap<String, Server>>,
|
||||
system: Arc<Server>,
|
||||
named: Mutex<HashMap<String, Arc<Server>>>,
|
||||
}
|
||||
|
||||
impl Databases {
|
||||
@@ -1356,7 +1519,7 @@ impl Databases {
|
||||
fn new(cfg: Config) -> std::io::Result<Self> {
|
||||
// 系统库按需创建:目录、字典与系统表缺失时自动补齐。
|
||||
// 其余库只在 data/databases/<name>/ 下已存在时才装载,启动不会凭空建库。
|
||||
let system = Self::open(&cfg, SYSTEM_DATABASE)?;
|
||||
let system = Arc::new(Self::open(&cfg, SYSTEM_DATABASE)?);
|
||||
let root = cfg.data_dir.join("databases");
|
||||
std::fs::create_dir_all(&root)?;
|
||||
let canonical_root = std::fs::canonicalize(&root)?;
|
||||
@@ -1379,7 +1542,7 @@ impl Databases {
|
||||
server.rebuild_stats()?;
|
||||
server.migrate_data()?;
|
||||
server.compact_tables()?;
|
||||
named.insert(name, server);
|
||||
named.insert(name, Arc::new(server));
|
||||
}
|
||||
}
|
||||
Ok(Self { root: cfg, system, named: Mutex::new(named) })
|
||||
@@ -1413,9 +1576,13 @@ impl Databases {
|
||||
let mut named = self.named.lock().map_err(|_| Error::other("database lock poisoned"))?;
|
||||
match method.as_str() {
|
||||
"all_catalogs" => {
|
||||
let servers: Vec<(String, Arc<Server>)> =
|
||||
std::iter::once((SYSTEM_DATABASE.to_string(), Arc::clone(&self.system)))
|
||||
.chain(named.iter().map(|(name, server)| (name.clone(), Arc::clone(server))))
|
||||
.collect();
|
||||
drop(named);
|
||||
let mut schemas = Vec::new();
|
||||
for (database, server) in std::iter::once((SYSTEM_DATABASE, &self.system))
|
||||
.chain(named.iter().map(|(name, server)| (name.as_str(), server))) {
|
||||
for (database, server) in servers {
|
||||
match server.handle_request(Request::ListCatalog) {
|
||||
Response::Catalog { schemas: entries } => {
|
||||
for mut entry in entries {
|
||||
@@ -1442,23 +1609,30 @@ impl Databases {
|
||||
std::fs::create_dir(&cfg.data_dir)?;
|
||||
let server = Server::new(cfg)?;
|
||||
server.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.save(server.catalog_path())?;
|
||||
named.insert(name, server);
|
||||
named.insert(name, Arc::new(server));
|
||||
return Ok(Response::Ok);
|
||||
}
|
||||
"drop_database" => {
|
||||
if name == SYSTEM_DATABASE { return Err(Error::other("cannot drop the system database")); }
|
||||
let server = named.get(&name).ok_or_else(|| Error::other("unknown database"))?;
|
||||
let holder = named.remove(&name).ok_or_else(|| Error::other("unknown database"))?;
|
||||
let server = match Arc::try_unwrap(holder) {
|
||||
Ok(server) => server,
|
||||
Err(holder) => {
|
||||
named.insert(name, holder);
|
||||
return Err(Error::other("database is in use"));
|
||||
}
|
||||
};
|
||||
let root = std::fs::canonicalize(self.root.data_dir.join("databases"))?;
|
||||
let target = std::fs::canonicalize(&server.cfg.data_dir)?;
|
||||
if target.parent() != Some(root.as_path()) || !target.join("catalog.json").is_file() {
|
||||
return Err(Error::other("invalid database directory"));
|
||||
}
|
||||
let config = server.cfg.clone();
|
||||
drop(named.remove(&name));
|
||||
drop(server);
|
||||
let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_err(Error::other)?.as_nanos();
|
||||
let tombstone = root.join(format!(".dropped-{name}-{stamp}"));
|
||||
if let Err(error) = std::fs::rename(&target, &tombstone) {
|
||||
named.insert(name, Server::new(config)?);
|
||||
named.insert(name, Arc::new(Server::new(config)?));
|
||||
return Err(error);
|
||||
}
|
||||
std::fs::remove_dir_all(&tombstone).map_err(|e| Error::other(format!("database dropped; directory cleanup failed: {e}")))?;
|
||||
@@ -1482,10 +1656,10 @@ impl Databases {
|
||||
| Request::AccountReset { .. } | Request::AccountLogin { .. }
|
||||
| Request::AccountDrop { .. } | Request::BootstrapSystem { .. }
|
||||
| Request::SystemStatus | Request::Ping);
|
||||
let server = if global { &self.system }
|
||||
else if selected.is_empty() { return Err(Error::other("no database selected")); }
|
||||
else if selected == SYSTEM_DATABASE { &self.system }
|
||||
else { named.get(&selected).ok_or_else(|| Error::other("unknown database"))? };
|
||||
if !global && selected.is_empty() { return Err(Error::other("no database selected")); }
|
||||
let server = if global || selected == SYSTEM_DATABASE { Arc::clone(&self.system) }
|
||||
else { Arc::clone(named.get(&selected).ok_or_else(|| Error::other("unknown database"))?) };
|
||||
drop(named);
|
||||
Ok(server.handle_request(request))
|
||||
}
|
||||
}
|
||||
|
||||
+148
-28
@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::protocol::{Account, Row, SchemaColumn, USERS_TABLE};
|
||||
|
||||
// 预写日志:先记操作并 fsync,再改数据;重启重放残留。
|
||||
// 预写日志:先记操作并 fsync,再改数据;重启按 LSN 重放已提交的组。
|
||||
|
||||
const OP_INSERT: u8 = 1;
|
||||
const OP_REPLACE_ALL: u8 = 2;
|
||||
@@ -18,6 +18,7 @@ const OP_ACCOUNTS: u8 = 6;
|
||||
const OP_REPLACE_SCHEMA: u8 = 7;
|
||||
const OP_HIDE_COLUMN: u8 = 8;
|
||||
const OP_COMPACT: u8 = 9;
|
||||
const OP_COMMIT: u8 = 10;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub enum WalOp {
|
||||
@@ -50,6 +51,8 @@ pub enum WalOp {
|
||||
columns: Vec<SchemaColumn>,
|
||||
rows: Vec<Row>,
|
||||
},
|
||||
/// 一组写操作的提交标记;没有标记的组算未提交
|
||||
Commit,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
@@ -67,6 +70,13 @@ struct ReplaceSchemaPayload {
|
||||
pub struct Wal {
|
||||
path: PathBuf,
|
||||
handle: Mutex<Option<File>>,
|
||||
next_lsn: Mutex<u64>,
|
||||
}
|
||||
|
||||
/// 一次扫描的结果:记录与完好字节长度
|
||||
pub struct WalScan {
|
||||
pub records: Vec<(u64, WalOp)>,
|
||||
pub good_len: usize,
|
||||
}
|
||||
|
||||
impl Wal {
|
||||
@@ -75,6 +85,7 @@ impl Wal {
|
||||
Wal {
|
||||
path: path.as_ref().to_path_buf(),
|
||||
handle: Mutex::new(None),
|
||||
next_lsn: Mutex::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,6 +94,11 @@ impl Wal {
|
||||
&self.path
|
||||
}
|
||||
|
||||
/// 检查点文件路径
|
||||
pub fn checkpoint_path(&self) -> PathBuf {
|
||||
self.path.with_extension("checkpoint")
|
||||
}
|
||||
|
||||
/// 拿常驻句柄(没有就开一个)
|
||||
fn with_handle<R>(&self, f: impl FnOnce(&mut File) -> io::Result<R>) -> io::Result<R> {
|
||||
let mut slot = self.handle.lock().unwrap();
|
||||
@@ -99,6 +115,15 @@ impl Wal {
|
||||
f(slot.as_mut().expect("just opened"))
|
||||
}
|
||||
|
||||
/// 把一段记录写到文件尾
|
||||
fn write_at_end(&self, bytes: &[u8]) -> io::Result<()> {
|
||||
self.with_handle(|f| {
|
||||
f.seek(SeekFrom::End(0))?;
|
||||
f.write_all(bytes)?;
|
||||
f.sync_data()
|
||||
})
|
||||
}
|
||||
|
||||
/// 截断到空并落盘
|
||||
pub fn clear(&self) -> io::Result<()> {
|
||||
self.with_handle(|f| {
|
||||
@@ -112,34 +137,136 @@ impl Wal {
|
||||
self.with_handle(truncate)
|
||||
}
|
||||
|
||||
/// 追加一条并 fsync
|
||||
pub fn append(&self, op: &WalOp) -> io::Result<()> {
|
||||
let bytes = encode(op)?;
|
||||
/// 截到指定长度(去掉撕裂的尾巴)
|
||||
pub fn truncate_to(&self, len: usize) -> io::Result<()> {
|
||||
self.with_handle(|f| {
|
||||
f.write_all(&bytes)?;
|
||||
f.set_len(len as u64)?;
|
||||
f.seek(SeekFrom::Start(0))?;
|
||||
f.sync_data()
|
||||
})
|
||||
}
|
||||
|
||||
/// 读当前 WAL;空则 None
|
||||
/// 追加一条并 fsync
|
||||
pub fn append(&self, op: &WalOp) -> io::Result<()> {
|
||||
let bytes = self.encode_next(op)?;
|
||||
self.write_at_end(&bytes)
|
||||
}
|
||||
|
||||
/// 追加一组操作与提交标记,返回提交 LSN
|
||||
pub fn append_group(&self, ops: &[WalOp]) -> io::Result<u64> {
|
||||
let mut frames = Vec::new();
|
||||
for op in ops {
|
||||
frames.extend_from_slice(&self.encode_next(op)?);
|
||||
}
|
||||
let commit_lsn = self.take_lsn()?;
|
||||
frames.extend_from_slice(&encode(&WalOp::Commit, commit_lsn)?);
|
||||
self.write_at_end(&frames)?;
|
||||
Ok(commit_lsn)
|
||||
}
|
||||
|
||||
/// 读当前 WAL 的第一条操作;空则 None
|
||||
pub fn read(&self) -> io::Result<Option<WalOp>> {
|
||||
let scan = self.read_all()?;
|
||||
Ok(scan.records.into_iter().find_map(|(_, op)| {
|
||||
if op == WalOp::Commit {
|
||||
None
|
||||
} else {
|
||||
Some(op)
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
/// 读全部记录,撕裂或损坏的尾巴按长度截断
|
||||
pub fn read_all(&self) -> io::Result<WalScan> {
|
||||
if !self.path.exists() {
|
||||
return Ok(None);
|
||||
return Ok(WalScan {
|
||||
records: Vec::new(),
|
||||
good_len: 0,
|
||||
});
|
||||
}
|
||||
let mut f = File::open(&self.path)?;
|
||||
let mut buf = Vec::new();
|
||||
f.read_to_end(&mut buf)?;
|
||||
if buf.is_empty() {
|
||||
let mut records = Vec::new();
|
||||
let mut offset = 0usize;
|
||||
while offset + 4 <= buf.len() {
|
||||
let body_len =
|
||||
u32::from_le_bytes([buf[offset], buf[offset + 1], buf[offset + 2], buf[offset + 3]])
|
||||
as usize;
|
||||
let end = offset + 4 + body_len;
|
||||
if end > buf.len() {
|
||||
break;
|
||||
}
|
||||
match decode_body(&buf[offset + 4..end]) {
|
||||
Ok(record) => {
|
||||
records.push(record);
|
||||
offset = end;
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
Ok(WalScan {
|
||||
records,
|
||||
good_len: offset,
|
||||
})
|
||||
}
|
||||
|
||||
/// 记录检查点:到此为止的 LSN 都已落盘
|
||||
pub fn write_checkpoint(&self, lsn: u64) -> io::Result<()> {
|
||||
let mut f = File::create(self.checkpoint_path())?;
|
||||
f.write_all(&lsn.to_le_bytes())?;
|
||||
f.sync_all()
|
||||
}
|
||||
|
||||
/// 读检查点 LSN
|
||||
pub fn read_checkpoint(&self) -> io::Result<Option<u64>> {
|
||||
let path = self.checkpoint_path();
|
||||
if !path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
match decode(&buf) {
|
||||
Ok(opt) => Ok(opt),
|
||||
Err(e) => Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("corrupt WAL: {}", e),
|
||||
)),
|
||||
let bytes = std::fs::read(&path)?;
|
||||
if bytes.len() < 8 {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut raw = [0u8; 8];
|
||||
raw.copy_from_slice(&bytes[..8]);
|
||||
Ok(Some(u64::from_le_bytes(raw)))
|
||||
}
|
||||
|
||||
/// 取下一条 LSN
|
||||
fn take_lsn(&self) -> io::Result<u64> {
|
||||
let mut slot = self.next_lsn.lock().unwrap();
|
||||
if *slot == 0 {
|
||||
let last = self.read_all()?.records.last().map(|(lsn, _)| *lsn).unwrap_or(0);
|
||||
*slot = last + 1;
|
||||
}
|
||||
let lsn = *slot;
|
||||
*slot += 1;
|
||||
Ok(lsn)
|
||||
}
|
||||
|
||||
/// 编一条带新 LSN 的记录
|
||||
fn encode_next(&self, op: &WalOp) -> io::Result<Vec<u8>> {
|
||||
let lsn = self.take_lsn()?;
|
||||
encode(op, lsn)
|
||||
}
|
||||
}
|
||||
|
||||
/// 已提交的记录:提交标记前的整组;无标记的单条按旧格式算已提交
|
||||
pub fn committed_ops(records: &[(u64, WalOp)]) -> Vec<(u64, WalOp)> {
|
||||
let mut out = Vec::new();
|
||||
let mut pending: Vec<(u64, WalOp)> = Vec::new();
|
||||
for (lsn, op) in records {
|
||||
if *op == WalOp::Commit {
|
||||
out.append(&mut pending);
|
||||
} else {
|
||||
pending.push((*lsn, op.clone()));
|
||||
}
|
||||
}
|
||||
if pending.len() == 1 {
|
||||
out.append(&mut pending);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// 清空文件并把读写位置归零
|
||||
@@ -150,7 +277,7 @@ fn truncate(f: &mut File) -> io::Result<()> {
|
||||
}
|
||||
|
||||
/// 把操作编成字节
|
||||
fn encode(op: &WalOp) -> io::Result<Vec<u8>> {
|
||||
fn encode(op: &WalOp, lsn: u64) -> io::Result<Vec<u8>> {
|
||||
let (op_type, table, payload) = match op {
|
||||
WalOp::HideColumn { table, column } => (OP_HIDE_COLUMN, table.clone(),
|
||||
serde_json::to_vec(column).map_err(io::Error::other)?),
|
||||
@@ -192,11 +319,11 @@ fn encode(op: &WalOp) -> io::Result<Vec<u8>> {
|
||||
.map_err(io::Error::other)?;
|
||||
(OP_REPLACE_SCHEMA, table.clone(), p)
|
||||
}
|
||||
WalOp::Commit => (OP_COMMIT, String::new(), Vec::new()),
|
||||
};
|
||||
|
||||
let table_bytes = table.as_bytes();
|
||||
|
||||
let lsn: u64 = 1;
|
||||
let mut body = Vec::with_capacity(8 + 1 + 2 + table_bytes.len() + payload.len());
|
||||
body.extend_from_slice(&lsn.to_le_bytes());
|
||||
body.push(op_type);
|
||||
@@ -210,20 +337,12 @@ fn encode(op: &WalOp) -> io::Result<Vec<u8>> {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// 从字节解回操作
|
||||
fn decode(buf: &[u8]) -> Result<Option<WalOp>, String> {
|
||||
if buf.len() < 4 {
|
||||
return Ok(None);
|
||||
}
|
||||
let body_len = u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]) as usize;
|
||||
if buf.len() < 4 + body_len {
|
||||
return Ok(None);
|
||||
}
|
||||
let body = &buf[4..4 + body_len];
|
||||
/// 从一个帧体解出 LSN 与操作
|
||||
fn decode_body(body: &[u8]) -> Result<(u64, WalOp), String> {
|
||||
if body.len() < 8 + 1 + 2 {
|
||||
return Err("body too short".into());
|
||||
}
|
||||
let _lsn = u64::from_le_bytes([
|
||||
let lsn = u64::from_le_bytes([
|
||||
body[0], body[1], body[2], body[3], body[4], body[5], body[6], body[7],
|
||||
]);
|
||||
let op_type = body[8];
|
||||
@@ -236,6 +355,7 @@ fn decode(buf: &[u8]) -> Result<Option<WalOp>, String> {
|
||||
let payload = &body[11 + table_len..];
|
||||
|
||||
let op = match op_type {
|
||||
OP_COMMIT => WalOp::Commit,
|
||||
OP_HIDE_COLUMN => {
|
||||
let column = serde_json::from_slice(payload)
|
||||
.map_err(|e| format!("bad hide_column payload: {e}"))?;
|
||||
@@ -292,5 +412,5 @@ fn decode(buf: &[u8]) -> Result<Option<WalOp>, String> {
|
||||
}
|
||||
other => return Err(format!("unknown op type: {}", other)),
|
||||
};
|
||||
Ok(Some(op))
|
||||
Ok((lsn, op))
|
||||
}
|
||||
@@ -216,6 +216,67 @@ fn concurrent_shard_reads_exclude_writers() -> Result<(), Box<dyn std::error::Er
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 跨库并发互不串数据,目录查询不被写饿死
|
||||
#[test]
|
||||
fn concurrent_requests_across_databases_stay_isolated() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let (_srv, _data) = start_server();
|
||||
let mut c = connect()?;
|
||||
let request = |c: &mut Conn, value: serde_json::Value| -> Result<serde_json::Value, serde_json::Error> {
|
||||
serde_json::from_str(&send(c, &value.to_string()))
|
||||
};
|
||||
let databases = ["alpha", "beta"];
|
||||
for database in databases {
|
||||
assert_eq!(request(&mut c, serde_json::json!({"method":"create_database","database":database}))?["status"], "ok");
|
||||
assert_eq!(
|
||||
request(&mut c, serde_json::json!({"method":"create_table","database":database,"table":"items","columns":[{"name":"id","ty":"int"}]}))?["status"],
|
||||
"ok"
|
||||
);
|
||||
}
|
||||
|
||||
let storage = CURRENT.with(|slot| slot.borrow().as_ref().cloned().expect("storage opened"));
|
||||
let written = 40;
|
||||
std::thread::scope(|scope| {
|
||||
let writers: Vec<_> = databases
|
||||
.iter()
|
||||
.map(|database| {
|
||||
let storage = Arc::clone(&storage);
|
||||
let database = database.to_string();
|
||||
scope.spawn(move || {
|
||||
for i in 1..=written {
|
||||
let line = format!(
|
||||
r#"{{"method":"insert","database":"{database}","table":"items","row":{{"id":{i}}}}}"#
|
||||
);
|
||||
let value: serde_json::Value = serde_json::from_str(&storage.request_line(&line)).expect("json");
|
||||
assert_eq!(value["status"], "ok", "{database}: {value}");
|
||||
}
|
||||
database
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let catalog = {
|
||||
let storage = Arc::clone(&storage);
|
||||
scope.spawn(move || {
|
||||
for _ in 0..40 {
|
||||
for line in [r#"{"method":"list_databases"}"#, r#"{"method":"all_catalogs"}"#] {
|
||||
let value: serde_json::Value = serde_json::from_str(&storage.request_line(line)).expect("json");
|
||||
assert_ne!(value["status"], "error", "{line}: {value}");
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
for writer in writers {
|
||||
writer.join().expect("writer thread");
|
||||
}
|
||||
catalog.join().expect("catalog thread");
|
||||
});
|
||||
|
||||
for database in databases {
|
||||
let rows = request(&mut c, serde_json::json!({"method":"scan","database":database,"table":"items"}))?["rows"].clone();
|
||||
assert_eq!(rows.as_array().expect("rows").len(), written, "{database} 行数不对");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 删列不改堆字节,重启后不再暴露该列
|
||||
#[test]
|
||||
fn drop_column_keeps_heap_bytes_and_hides_values_after_restart() -> Result<(), Box<dyn std::error::Error>> {
|
||||
@@ -744,6 +805,12 @@ fn account_failure_preserves_wal_and_blocks_writes() -> Result<(), Box<dyn std::
|
||||
let mut c = connect()?;
|
||||
let recovered = send(&mut c, r#"{"method":"accounts_list"}"#);
|
||||
assert!(recovered.contains("hash-one"), "{recovered}");
|
||||
let status: serde_json::Value =
|
||||
serde_json::from_str(&send(&mut c, r#"{"method":"system_status"}"#))?;
|
||||
assert!(
|
||||
status["last_lsn"].as_u64().unwrap_or(0) >= 1,
|
||||
"重放后检查点要推进: {status}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1015,6 +1082,125 @@ fn delete_keys_removes_row_and_index_entry() {
|
||||
assert!(r.contains(r#""status":"ok""#), "id 2 should be insertable again: {}", r);
|
||||
}
|
||||
|
||||
/// 事务提交:一批写操作在同一个请求里落地
|
||||
#[test]
|
||||
fn apply_transaction_lands_upserts_and_deletes_together() {
|
||||
let (_srv, _data) = start_server();
|
||||
let mut c = connect().unwrap();
|
||||
|
||||
for (id, name) in [(1, "Alice"), (2, "Bob")] {
|
||||
let req = format!(
|
||||
r#"{{"method":"insert","table":"txn_t","row":{{"id":{},"name":"{}"}}}}"#,
|
||||
id, name
|
||||
);
|
||||
send(&mut c, &req);
|
||||
}
|
||||
|
||||
let batch = r#"{"method":"apply_transaction","ops":[
|
||||
{"op":"delete","table":"txn_t","ids":[1]},
|
||||
{"op":"upsert","table":"txn_t","rows":[{"id":3,"name":"Carol"}]}
|
||||
]}"#;
|
||||
let r = send(&mut c, batch);
|
||||
assert!(r.contains(r#""status":"ok""#), "batch: {}", r);
|
||||
|
||||
let r = send(&mut c, r#"{"method":"scan","table":"txn_t"}"#);
|
||||
assert!(!r.contains("Alice"), "deleted row should be gone: {}", r);
|
||||
assert!(r.contains("Bob") && r.contains("Carol"), "kept and added rows: {}", r);
|
||||
|
||||
for line in [
|
||||
r#"{"method":"apply_transaction","ops":[{"op":"upsert","table":"__system_users","rows":[{"id":1}]}]}"#,
|
||||
r#"{"method":"apply_transaction","ops":[{"op":"delete","table":"../evil","ids":[1]}]}"#,
|
||||
] {
|
||||
let r = send(&mut c, line);
|
||||
assert!(r.contains(r#""status":"error""#), "bad op rejected: {} -> {}", line, r);
|
||||
}
|
||||
|
||||
// 没有整数 id 的表用整表替换提交
|
||||
let r = send(&mut c, r#"{"method":"insert","table":"txn_no_id","row":{"name":"old"}}"#);
|
||||
assert!(r.contains(r#""status":"ok""#), "insert: {}", r);
|
||||
let replace = r#"{"method":"apply_transaction","ops":[{"op":"replace","table":"txn_no_id","rows":[{"name":"only"}]}]}"#;
|
||||
let r = send(&mut c, replace);
|
||||
assert!(r.contains(r#""status":"ok""#), "replace: {}", r);
|
||||
let r = send(&mut c, r#"{"method":"scan","table":"txn_no_id"}"#);
|
||||
assert!(r.contains("only") && !r.contains("old"), "replaced rows: {}", r);
|
||||
}
|
||||
|
||||
/// 标记已落盘、数据未改的崩溃现场:重启重放并推进检查点
|
||||
#[test]
|
||||
fn committed_group_replays_after_restart() -> Result<(), Box<dyn std::error::Error>> {
|
||||
use chusql_core_storage::wal::{Wal, WalOp};
|
||||
let (server, data) = start_server();
|
||||
let mut c = connect()?;
|
||||
request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":1,"secret":"old"}}))?;
|
||||
drop(c);
|
||||
drop(server);
|
||||
|
||||
let mut added = std::collections::HashMap::new();
|
||||
added.insert("id".to_string(), serde_json::json!(2));
|
||||
added.insert("secret".to_string(), serde_json::json!("new"));
|
||||
let group = vec![
|
||||
WalOp::InsertBatch {
|
||||
table: "items".into(),
|
||||
rows: vec![added.clone()],
|
||||
},
|
||||
WalOp::DeleteKeys {
|
||||
table: "items".into(),
|
||||
keys: vec![1],
|
||||
},
|
||||
];
|
||||
let wal = Wal::new(data.path().join("databases/main/wal.log"));
|
||||
let commit_lsn = wal.append_group(&group)?;
|
||||
drop(wal);
|
||||
|
||||
let _server = start_server_in(data.path());
|
||||
let mut c = connect()?;
|
||||
let rows: serde_json::Value =
|
||||
serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"items"}"#))?;
|
||||
assert_eq!(rows["rows"], serde_json::json!([{"id":2,"secret":"new"}]));
|
||||
assert_eq!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len(), 0);
|
||||
let checkpoint = std::fs::read(data.path().join("databases/main/wal.checkpoint"))?;
|
||||
assert_eq!(checkpoint.len(), 8, "检查点要落盘");
|
||||
assert_eq!(
|
||||
u64::from_le_bytes(checkpoint[..8].try_into()?),
|
||||
commit_lsn,
|
||||
"检查点记到提交 LSN"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 没有提交标记的记录组在重启时整组丢掉
|
||||
#[test]
|
||||
fn uncommitted_group_is_discarded_on_restart() -> Result<(), Box<dyn std::error::Error>> {
|
||||
use chusql_core_storage::wal::{Wal, WalOp};
|
||||
let (server, data) = start_server();
|
||||
let mut c = connect()?;
|
||||
request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":1,"secret":"old"}}))?;
|
||||
drop(c);
|
||||
drop(server);
|
||||
|
||||
let mut lost = std::collections::HashMap::new();
|
||||
lost.insert("id".to_string(), serde_json::json!(2));
|
||||
lost.insert("secret".to_string(), serde_json::json!("lost"));
|
||||
let wal = Wal::new(data.path().join("databases/main/wal.log"));
|
||||
wal.append(&WalOp::InsertBatch {
|
||||
table: "items".into(),
|
||||
rows: vec![lost.clone()],
|
||||
})?;
|
||||
wal.append(&WalOp::InsertBatch {
|
||||
table: "items".into(),
|
||||
rows: vec![lost],
|
||||
})?;
|
||||
drop(wal);
|
||||
|
||||
let _server = start_server_in(data.path());
|
||||
let mut c = connect()?;
|
||||
let rows: serde_json::Value =
|
||||
serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"items"}"#))?;
|
||||
assert_eq!(rows["rows"], serde_json::json!([{"id":1,"secret":"old"}]));
|
||||
assert_eq!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len(), 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 给第二列建索引后能按那一列查
|
||||
#[test]
|
||||
fn create_index_on_secondary_column() {
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use chusql_core_storage::protocol::Row;
|
||||
use chusql_core_storage::wal::{Wal, WalOp};
|
||||
use chusql_core_storage::wal::{committed_ops, Wal, WalOp};
|
||||
use serde_json::json;
|
||||
|
||||
// WAL 测试:追加、读取、清空、截断尾巴。
|
||||
// WAL 测试:追加、读取、清空、截断尾巴、LSN 与提交标记。
|
||||
|
||||
|
||||
/// 用键值对构造一行
|
||||
@@ -99,3 +99,122 @@ fn truncated_tail_is_ignored() {
|
||||
assert!(wal.read().unwrap().is_none());
|
||||
}
|
||||
|
||||
/// 每条记录带递增的 LSN,重开也从文件里的最大号往下走
|
||||
#[test]
|
||||
fn every_record_gets_an_increasing_lsn() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("wal.log");
|
||||
let wal = Wal::new(&path);
|
||||
|
||||
for id in 1..=3 {
|
||||
wal.append(&WalOp::Insert {
|
||||
table: "t".into(),
|
||||
row: row(&[("id", json!(id))]),
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
let scan = wal.read_all().unwrap();
|
||||
let lsns: Vec<u64> = scan.records.iter().map(|(lsn, _)| *lsn).collect();
|
||||
assert_eq!(lsns, vec![1, 2, 3]);
|
||||
assert_eq!(scan.good_len as u64, std::fs::metadata(&path).unwrap().len());
|
||||
|
||||
let reopened = Wal::new(&path);
|
||||
reopened
|
||||
.append(&WalOp::Insert {
|
||||
table: "t".into(),
|
||||
row: row(&[("id", json!(4))]),
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(reopened.read_all().unwrap().records.last().unwrap().0, 4);
|
||||
}
|
||||
|
||||
/// 一组操作写完带提交标记,标记的 LSN 也在记录里
|
||||
#[test]
|
||||
fn append_group_writes_a_commit_marker() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let wal = Wal::new(dir.path().join("wal.log"));
|
||||
|
||||
let group = [
|
||||
WalOp::InsertBatch {
|
||||
table: "t".into(),
|
||||
rows: vec![row(&[("id", json!(2))])],
|
||||
},
|
||||
WalOp::DeleteKeys {
|
||||
table: "t".into(),
|
||||
keys: vec![1],
|
||||
},
|
||||
];
|
||||
let commit_lsn = wal.append_group(&group).unwrap();
|
||||
|
||||
let scan = wal.read_all().unwrap();
|
||||
assert_eq!(scan.records.len(), 3);
|
||||
assert_eq!(scan.records.last().unwrap().1, WalOp::Commit);
|
||||
assert_eq!(scan.records.last().unwrap().0, commit_lsn);
|
||||
assert_eq!(wal.read().unwrap().unwrap(), group[0]);
|
||||
assert_eq!(
|
||||
committed_ops(&scan.records),
|
||||
vec![(1, group[0].clone()), (2, group[1].clone())]
|
||||
);
|
||||
}
|
||||
|
||||
/// 没有提交标记的记录组算未提交,旧格式的单条仍算已提交
|
||||
#[test]
|
||||
fn records_without_a_commit_marker_stay_uncommitted() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let wal = Wal::new(dir.path().join("wal.log"));
|
||||
|
||||
for id in 1..=2 {
|
||||
wal.append(&WalOp::Insert {
|
||||
table: "t".into(),
|
||||
row: row(&[("id", json!(id))]),
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
assert!(committed_ops(&wal.read_all().unwrap().records).is_empty());
|
||||
|
||||
wal.clear().unwrap();
|
||||
wal.append(&WalOp::Insert {
|
||||
table: "t".into(),
|
||||
row: row(&[("id", json!(9))]),
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(committed_ops(&wal.read_all().unwrap().records).len(), 1);
|
||||
}
|
||||
|
||||
/// 撕成两半的尾巴能读出完好长度并截掉
|
||||
#[test]
|
||||
fn trailing_torn_frame_can_be_truncated() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("wal.log");
|
||||
let wal = Wal::new(&path);
|
||||
|
||||
for id in 1..=2 {
|
||||
wal.append(&WalOp::Insert {
|
||||
table: "t".into(),
|
||||
row: row(&[("id", json!(id))]),
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
let full = std::fs::read(&path).unwrap();
|
||||
std::fs::write(&path, &full[..full.len() - 3]).unwrap();
|
||||
|
||||
let scan = wal.read_all().unwrap();
|
||||
assert_eq!(scan.records.len(), 1);
|
||||
assert!(scan.good_len < full.len() - 3);
|
||||
|
||||
wal.truncate_to(scan.good_len).unwrap();
|
||||
assert_eq!(std::fs::metadata(&path).unwrap().len(), scan.good_len as u64);
|
||||
assert_eq!(wal.read_all().unwrap().records.len(), 1);
|
||||
}
|
||||
|
||||
/// 检查点写读一致
|
||||
#[test]
|
||||
fn checkpoint_round_trips() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let wal = Wal::new(dir.path().join("wal.log"));
|
||||
|
||||
assert_eq!(wal.read_checkpoint().unwrap(), None);
|
||||
wal.write_checkpoint(7).unwrap();
|
||||
assert_eq!(wal.read_checkpoint().unwrap(), Some(7));
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ module ChuSQL.Server.Backend (
|
||||
columnsFromStatement,
|
||||
currentStamp,
|
||||
exprTables,
|
||||
inDatabase,
|
||||
internalTable,
|
||||
grantsTable,
|
||||
ipcBackend,
|
||||
@@ -23,20 +22,23 @@ module ChuSQL.Server.Backend (
|
||||
import ChuSQL.Core.Engine (runStatement, runStatementM)
|
||||
import ChuSQL.Core.Model
|
||||
import ChuSQL.Core.Engine.Semantic (prepare)
|
||||
import ChuSQL.Core.Engine.Storage (MonadStorage (schema))
|
||||
import ChuSQL.Core.Engine.Storage (MonadStorage (schema, snapshot))
|
||||
import ChuSQL.Core.Engine.Storage.IPC (
|
||||
Account (..),
|
||||
Env,
|
||||
IPCStorage (..),
|
||||
Request (..),
|
||||
Response (..),
|
||||
SchemaColumn (..),
|
||||
TableInfo (..),
|
||||
envForDatabase,
|
||||
sendRawRequest,
|
||||
sendRequest,
|
||||
)
|
||||
import ChuSQL.Core.Engine.Syntax.AST
|
||||
import ChuSQL.Core.Engine.Syntax.Parser (parseStatement)
|
||||
import Control.Concurrent.MVar (MVar, modifyMVar, newMVar, readMVar, withMVar)
|
||||
import ChuSQL.Core.Protocol (TxnOp (..))
|
||||
import Control.Concurrent.MVar (MVar, modifyMVar, readMVar)
|
||||
import Data.List (nub)
|
||||
import Data.Text (Text)
|
||||
import Data.Time (getCurrentTime)
|
||||
@@ -61,6 +63,10 @@ data Backend = Backend
|
||||
, beDatabases :: IO (Either String [String])
|
||||
-- | 原样转发一条存储请求
|
||||
, beStorage :: A.Value -> IO (Either String A.Value)
|
||||
-- | 取一份整库快照,当事务的起点
|
||||
, beSnapshot :: IO (Either String Database)
|
||||
-- | 事务提交:把一批写操作交给存储层应用
|
||||
, beApplyTransaction :: [TxnOp] -> IO (Either String ())
|
||||
}
|
||||
|
||||
-- | 按给定列名取列,固定列顺序
|
||||
@@ -73,36 +79,46 @@ columnsFromStatement db stmt = case prepare db stmt of
|
||||
-- | 走本地存储链路的后端
|
||||
ipcBackend :: IO Backend
|
||||
ipcBackend = do
|
||||
lock <- newMVar ()
|
||||
let backend current = Backend
|
||||
{ beStatement = \sql -> withMVar lock $ \_ -> runIpc current sql
|
||||
, beCatalog = withMVar lock $ \_ -> do
|
||||
{ beStatement = \sql -> runIpc current sql
|
||||
, beCatalog = do
|
||||
response <- sendRequest (ReqInDatabase current ReqListCatalog)
|
||||
pure $ case response of
|
||||
RespCatalog infos -> Right infos
|
||||
RespError err -> Left err
|
||||
_ -> Left "unexpected catalog response"
|
||||
, bePing = withMVar lock $ \_ -> do
|
||||
, bePing = do
|
||||
resp <- sendRequest ReqPing
|
||||
pure (case resp of RespPong -> True; _ -> False)
|
||||
, beAccounts = \req -> withMVar lock $ \_ -> do
|
||||
, beAccounts = \req -> do
|
||||
response <- sendRequest req
|
||||
pure $ case response of
|
||||
RespAccounts accounts -> Right accounts
|
||||
RespError err -> Left err
|
||||
_ -> Left "unexpected account response"
|
||||
, beWithDatabase = backend
|
||||
, beDatabases = withMVar lock $ \_ -> do
|
||||
, beDatabases = do
|
||||
response <- sendRequest (ReqDatabase "list_databases" "")
|
||||
pure $ case response of
|
||||
RespTables names -> Right names
|
||||
RespError err -> Left err
|
||||
_ -> Left "unexpected database response"
|
||||
, beStorage = \payload -> withMVar lock $ \_ -> runRaw payload
|
||||
, beStorage = \payload -> runRaw payload
|
||||
, beSnapshot = Right <$> runIPCStorageIn (snapshot :: IPCStorage Database) (sessionEnv current)
|
||||
, beApplyTransaction = \ops -> do
|
||||
response <- sendRequest (ReqInDatabase current (ReqApplyTransaction ops))
|
||||
pure $ case response of
|
||||
RespOk -> Right ()
|
||||
RespError err -> Left err
|
||||
_ -> Left "unexpected transaction response"
|
||||
}
|
||||
-- 没有默认库:没选库时 current 是空串,任何请求都得先选库
|
||||
pure (backend "")
|
||||
where
|
||||
-- | 会话的存储环境:库名跟这条连接走
|
||||
sessionEnv :: String -> Env
|
||||
sessionEnv current = envForDatabase (if null current then Nothing else Just current)
|
||||
|
||||
-- | 转发一条存储请求并解析响应
|
||||
runRaw :: A.Value -> IO (Either String A.Value)
|
||||
runRaw payload = do
|
||||
@@ -113,18 +129,21 @@ ipcBackend = do
|
||||
Left err -> Left ("storage response is not valid JSON: " ++ err)
|
||||
Right value -> Right value
|
||||
|
||||
-- | 走存储链路跑一条语句
|
||||
-- | 走存储链路跑一条语句:表名不再补当前库前缀
|
||||
-- 引擎侧的数据字典按当前库过滤之后键是裸表名,补前缀会让查表落空;
|
||||
-- 库名由 database 字段随请求带走,存储层自己按它分发。
|
||||
runIpc :: String -> String -> IO (Either String StatementResult)
|
||||
runIpc current sql = case fmap (inDatabase current) (parseStatement sql) of
|
||||
runIpc current sql = case parseStatement sql of
|
||||
Left e -> pure (Left e)
|
||||
Right stmt -> do
|
||||
result <- runIPCStorage (runStatementM stmt)
|
||||
let env = sessionEnv current
|
||||
result <- runIPCStorageIn (runStatementM stmt) env
|
||||
case result of
|
||||
Left e -> pure (Left e)
|
||||
Right rows -> do
|
||||
cols <- case rows of
|
||||
[] -> do
|
||||
db <- runIPCStorage (schema :: IPCStorage Database)
|
||||
db <- runIPCStorageIn (schema :: IPCStorage Database) env
|
||||
pure (columnsFromStatement db stmt)
|
||||
(r : _) -> pure (map fst r)
|
||||
pure (Right (StatementResult cols rows))
|
||||
@@ -162,6 +181,10 @@ memoryBackendWith trusted name ref =
|
||||
, beWithDatabase = withDatabase
|
||||
, beDatabases = pure (Right (nub [name, systemDatabaseName]))
|
||||
, beStorage = \_ -> pure (Left "storage is not available in this backend")
|
||||
, beSnapshot = Right . scoped <$> readMVar ref
|
||||
, beApplyTransaction = \ops -> modifyMVar ref $ \db -> case applyTransactionMemory ops db of
|
||||
Left err -> pure (db, Left err)
|
||||
Right db' -> pure (db', Right ())
|
||||
}
|
||||
where
|
||||
-- | 按视角过滤库里的表
|
||||
@@ -185,6 +208,8 @@ unavailableDatabase :: String -> Backend -> Backend
|
||||
unavailableDatabase message backend = backend
|
||||
{ beStatement = \_ -> pure (Left message)
|
||||
, beCatalog = pure (Left message)
|
||||
, beSnapshot = pure (Left message)
|
||||
, beApplyTransaction = \_ -> pure (Left message)
|
||||
}
|
||||
|
||||
-- | 未选库时的视角:只放行不碰表的语句
|
||||
@@ -196,76 +221,34 @@ noDatabaseBackend backend = backend
|
||||
| statementNeedsDatabase stmt -> pure (Left "no database selected")
|
||||
| otherwise -> beStatement backend sql
|
||||
, beCatalog = pure (Left "no database selected")
|
||||
, beSnapshot = pure (Left "no database selected")
|
||||
, beApplyTransaction = \_ -> pure (Left "no database selected")
|
||||
}
|
||||
|
||||
-- | 给语句里的裸表名补上当前库前缀,子查询里的表名也一起改
|
||||
inDatabase :: String -> Statement -> Statement
|
||||
inDatabase current = walk
|
||||
-- | 把一批事务写操作应用到内存库:先按 id 删,再按追加写
|
||||
applyTransactionMemory :: [TxnOp] -> Database -> Either String Database
|
||||
applyTransactionMemory ops db = foldl step (Right db) ops
|
||||
where
|
||||
-- | 递归改写语句里的表名
|
||||
walk :: Statement -> Statement
|
||||
walk stmt = case stmt of
|
||||
q@Select{} ->
|
||||
q
|
||||
{ selectFrom = source (selectFrom q)
|
||||
, selectWhere = fmap expr (selectWhere q)
|
||||
}
|
||||
q@SelectExpr{} ->
|
||||
q
|
||||
{ selectItems = [(label, expr e) | (label, e) <- selectItems q]
|
||||
, selectFrom = source (selectFrom q)
|
||||
, selectWhere = fmap expr (selectWhere q)
|
||||
}
|
||||
Insert t cols rows -> Insert (table t) cols (map (map expr) rows)
|
||||
Delete t cond -> Delete (table t) (fmap expr cond)
|
||||
Update t assigns cond -> Update (table t) [(c, expr e) | (c, e) <- assigns] (fmap expr cond)
|
||||
CreateTable t cols -> CreateTable (table t) cols
|
||||
DropTable t -> DropTable (table t)
|
||||
CreateIndex t c -> CreateIndex (table t) c
|
||||
DropIndex t c -> DropIndex (table t) c
|
||||
DropColumn t c -> DropColumn (table t) c
|
||||
AddColumn t def -> AddColumn (table t) def
|
||||
RenameColumn t old new -> RenameColumn (table t) old new
|
||||
AlterColumnType t c ty -> AlterColumnType (table t) c ty
|
||||
AlterColumnDefault t c v -> AlterColumnDefault (table t) c v
|
||||
AlterColumnNull t c b -> AlterColumnNull (table t) c b
|
||||
other -> other
|
||||
step acc op = do
|
||||
current <- acc
|
||||
case op of
|
||||
TxnDelete name keys -> do
|
||||
table <- lookupTable current name
|
||||
pure (put (name, table {tableRows = [r | r <- tableRows table, not (doomed keys r)]}) current)
|
||||
TxnUpsert name rows -> do
|
||||
table <- lookupTable current name
|
||||
pure (put (name, table {tableRows = tableRows table ++ rows}) current)
|
||||
TxnReplace name rows -> do
|
||||
table <- lookupTable current name
|
||||
pure (put (name, table {tableRows = rows}) current)
|
||||
|
||||
-- | 给单个表名补库前缀,已限定或没选库就原样
|
||||
table t | '.' `elem` t || null current = t
|
||||
| otherwise = current ++ "." ++ t
|
||||
-- | 换掉一张表,其余保持原顺序
|
||||
put entry = map (\one@(name, _) -> if name == fst entry then entry else one)
|
||||
|
||||
-- | 改写 FROM 子句
|
||||
source FromUnit = FromUnit
|
||||
source (FromTable a t) = FromTable a (table t)
|
||||
source (FromJoin k l a t e) = FromJoin k (source l) a (table t) (expr e)
|
||||
|
||||
-- | 改写表达式里的子查询
|
||||
expr (ScalarSub sq) = ScalarSub (sub sq)
|
||||
expr (ExistsSub sq neg) = ExistsSub (sub sq) neg
|
||||
expr (InSub a sq neg) = InSub (expr a) (sub sq) neg
|
||||
expr (InList a items neg) = InList (expr a) (map expr items) neg
|
||||
expr (Add a b) = Add (expr a) (expr b)
|
||||
expr (Sub a b) = Sub (expr a) (expr b)
|
||||
expr (Mul a b) = Mul (expr a) (expr b)
|
||||
expr (Div a b) = Div (expr a) (expr b)
|
||||
expr (Neg a) = Neg (expr a)
|
||||
expr (Gt a b) = Gt (expr a) (expr b)
|
||||
expr (Lt a b) = Lt (expr a) (expr b)
|
||||
expr (Eq a b) = Eq (expr a) (expr b)
|
||||
expr (And a b) = And (expr a) (expr b)
|
||||
expr (Or a b) = Or (expr a) (expr b)
|
||||
expr (IsNull a) = IsNull (expr a)
|
||||
expr (IsNotNull a) = IsNotNull (expr a)
|
||||
expr (CountOf a) = CountOf (expr a)
|
||||
expr (SumOf a) = SumOf (expr a)
|
||||
expr (AvgOf a) = AvgOf (expr a)
|
||||
expr (MinOf a) = MinOf (expr a)
|
||||
expr (MaxOf a) = MaxOf (expr a)
|
||||
expr other = other
|
||||
|
||||
-- | 改写一个子查询
|
||||
sub sq = sq {subqueryStatement = walk (subqueryStatement sq)}
|
||||
-- | 这一行要不要删:按行里的 id 列比
|
||||
doomed keys r = case lookup "id" r of
|
||||
Just (VInt k) -> k `elem` keys
|
||||
_ -> False
|
||||
|
||||
-- | 语句里出现的表
|
||||
statementTables :: Statement -> [Text]
|
||||
|
||||
@@ -24,22 +24,24 @@ module ChuSQL.Server.Session
|
||||
, queryResultJson
|
||||
) where
|
||||
|
||||
import ChuSQL.Core.Model (Value (..))
|
||||
import ChuSQL.Core.Protocol (Account, QueryResult (..), queryResultJson)
|
||||
import ChuSQL.Core.Model (Database, Row, Table (..), Value (..))
|
||||
import ChuSQL.Core.Protocol (Account, QueryResult (..), TxnOp (..), queryResultJson)
|
||||
import ChuSQL.Core.Engine.Storage.IPC (TableInfo (..))
|
||||
import qualified ChuSQL.Core.Engine as Engine
|
||||
import qualified ChuSQL.Core.Engine.Error as E
|
||||
import ChuSQL.Core.Engine.Syntax.AST (Statement (..))
|
||||
import ChuSQL.Core.Engine.Syntax.Parser (parseStatement)
|
||||
import ChuSQL.Interface.AccountTable (systemTableInfo)
|
||||
import ChuSQL.Interface.Auth (defaultSessionPolicy, newSessionStore)
|
||||
import ChuSQL.Server.Accounts
|
||||
import ChuSQL.Server.Backend (Backend (..), StatementResult (..))
|
||||
import ChuSQL.Server.Backend (Backend (..), StatementResult (..), columnsFromStatement)
|
||||
import ChuSQL.Server.Policy (configurePasswordPolicy)
|
||||
import ChuSQL.Server.Privileges (PrivilegeCommand, PrivilegeError (..), Privileges, RoleView, authorize, filterTables, listRoleViews, newPrivileges, privilegeCommand, runPrivilegeCommand)
|
||||
import Control.Exception (IOException, try)
|
||||
import Data.Bifunctor (first)
|
||||
import Data.Char (isAlpha, isAlphaNum)
|
||||
import Data.IORef (IORef, newIORef, readIORef, writeIORef)
|
||||
import Data.Maybe (isJust)
|
||||
import Data.Text (Text)
|
||||
import qualified Data.Text as T
|
||||
import Data.Time.Clock (getCurrentTime)
|
||||
@@ -64,6 +66,13 @@ data Session = Session
|
||||
, ssCurrent :: IORef Text
|
||||
, ssPrincipal :: IORef (Maybe Principal)
|
||||
, ssSettingsFile :: FilePath
|
||||
, ssTransaction :: IORef (Maybe Transaction)
|
||||
}
|
||||
|
||||
-- | 一条会话里的显式事务:开事务时的快照与事务内改到的状态
|
||||
data Transaction = Transaction
|
||||
{ txBase :: Database
|
||||
, txStaged :: Database
|
||||
}
|
||||
|
||||
-- | 开一个会话:账号与角色服务、配置策略都和 Web 端一致
|
||||
@@ -76,7 +85,8 @@ newSession backend name settingsFile = do
|
||||
-- 服务启动后只有系统库:没 USE 之前不预设任何工作库
|
||||
current <- newIORef ""
|
||||
principal <- newIORef Nothing
|
||||
pure (Session backend accountsService privileges current principal settingsFile)
|
||||
transaction <- newIORef Nothing
|
||||
pure (Session backend accountsService privileges current principal settingsFile transaction)
|
||||
|
||||
-- | 当前生效的口令策略(只有管理员能看)
|
||||
policyOf :: Session -> IO (Either Text PasswordPolicy)
|
||||
@@ -168,22 +178,183 @@ runStatement session sql = fmap (first sessMessage) (runStatementCoded session s
|
||||
runStatementCoded :: Session -> Text -> IO (Either SessionError QueryResult)
|
||||
runStatementCoded session sql = case parseStatement (T.unpack sql) of
|
||||
Right (UseDatabase name) -> do
|
||||
existing <- readIORef (ssTransaction session)
|
||||
case existing of
|
||||
Just _ -> pure (Left (SessionError "query_error" "cannot switch database inside a transaction"))
|
||||
Nothing -> do
|
||||
switched <- switchDatabaseCoded session (T.pack name)
|
||||
case switched of
|
||||
Left err -> pure (Left err)
|
||||
Right () -> do
|
||||
current <- readIORef (ssCurrent session)
|
||||
pure (Right (emptyResult (Just current)))
|
||||
Right BeginTransaction -> beginTransaction session
|
||||
Right CommitTransaction -> commitTransaction session
|
||||
Right RollbackTransaction -> rollbackTransaction session
|
||||
Right statement
|
||||
| Just command <- accountCommand statement -> runAccount session command
|
||||
| Just command <- privilegeCommand statement -> runPrivilege session command
|
||||
| otherwise -> do
|
||||
transaction <- readIORef (ssTransaction session)
|
||||
if isJust transaction && transactionDdl statement
|
||||
then pure (Left (SessionError "query_error" "DDL is not allowed in a transaction"))
|
||||
else do
|
||||
allowed <- authorized session statement
|
||||
case allowed of
|
||||
Left err -> pure (Left err)
|
||||
Right () -> runPlain session sql
|
||||
Right ()
|
||||
| isJust transaction && transactionData statement -> runInTransaction session statement
|
||||
| otherwise -> runPlain session sql
|
||||
Left _ -> runPlain session sql
|
||||
|
||||
-- | 事务里不许改结构,会话层直接拦下
|
||||
transactionDdl :: Statement -> Bool
|
||||
transactionDdl statement = case statement of
|
||||
CreateTable{} -> True
|
||||
DropTable{} -> True
|
||||
CreateIndex{} -> True
|
||||
DropIndex{} -> True
|
||||
DropColumn{} -> True
|
||||
AddColumn{} -> True
|
||||
RenameColumn{} -> True
|
||||
AlterColumnType{} -> True
|
||||
AlterColumnDefault{} -> True
|
||||
AlterColumnNull{} -> True
|
||||
CreateDatabase{} -> True
|
||||
DropDatabase{} -> True
|
||||
_ -> False
|
||||
|
||||
-- | 事务里只有数据语句走快照
|
||||
transactionData :: Statement -> Bool
|
||||
transactionData statement = case statement of
|
||||
Select{} -> True
|
||||
SelectExpr{} -> True
|
||||
Insert{} -> True
|
||||
Update{} -> True
|
||||
Delete{} -> True
|
||||
_ -> False
|
||||
|
||||
-- | 开事务:把当前库整库做一份快照当起点
|
||||
beginTransaction :: Session -> IO (Either SessionError QueryResult)
|
||||
beginTransaction session = do
|
||||
current <- readIORef (ssCurrent session)
|
||||
existing <- readIORef (ssTransaction session)
|
||||
case existing of
|
||||
Just _ -> pure (Left (SessionError "query_error" "already in a transaction"))
|
||||
Nothing
|
||||
| T.null current -> pure (Left (SessionError "no_database" "no database selected"))
|
||||
| otherwise -> do
|
||||
backend <- currentBackend session
|
||||
snapshotResult <- beSnapshot backend
|
||||
case snapshotResult of
|
||||
Left err -> pure (Left (engineErrorCode err))
|
||||
Right db -> do
|
||||
writeIORef (ssTransaction session) (Just (Transaction db db))
|
||||
pure (Right (emptyResult (Just current)))
|
||||
|
||||
-- | 提交:把快照差分交给存储层应用,成功才结束事务
|
||||
commitTransaction :: Session -> IO (Either SessionError QueryResult)
|
||||
commitTransaction session = do
|
||||
current <- readIORef (ssCurrent session)
|
||||
existing <- readIORef (ssTransaction session)
|
||||
case existing of
|
||||
Nothing -> pure (Left (SessionError "query_error" "no transaction in progress"))
|
||||
Just transaction -> do
|
||||
backend <- currentBackend session
|
||||
liveResult <- beSnapshot backend
|
||||
case liveResult of
|
||||
Left err -> pure (Left (engineErrorCode err))
|
||||
Right live -> do
|
||||
applied <- beApplyTransaction backend (transactionOps (txBase transaction) (txStaged transaction) live)
|
||||
case applied of
|
||||
Left err -> pure (Left (engineErrorCode err))
|
||||
Right () -> do
|
||||
writeIORef (ssTransaction session) Nothing
|
||||
pure (Right (emptyResult (Just current)))
|
||||
|
||||
-- | 回滚:快照丢掉,存储层没被改过
|
||||
rollbackTransaction :: Session -> IO (Either SessionError QueryResult)
|
||||
rollbackTransaction session = do
|
||||
current <- readIORef (ssCurrent session)
|
||||
existing <- readIORef (ssTransaction session)
|
||||
case existing of
|
||||
Nothing -> pure (Left (SessionError "query_error" "no transaction in progress"))
|
||||
Just _ -> do
|
||||
writeIORef (ssTransaction session) Nothing
|
||||
pure (Right (emptyResult (Just current)))
|
||||
|
||||
-- | 事务里跑一条数据语句:只动内存快照,不碰存储层
|
||||
runInTransaction :: Session -> Statement -> IO (Either SessionError QueryResult)
|
||||
runInTransaction session statement = do
|
||||
existing <- readIORef (ssTransaction session)
|
||||
case existing of
|
||||
Nothing -> pure (Left (SessionError "query_error" "no transaction in progress"))
|
||||
Just transaction -> case Engine.runStatement (txStaged transaction) statement of
|
||||
Left err -> pure (Left (engineErrorCode err))
|
||||
Right (staged, rows) -> do
|
||||
writeIORef (ssTransaction session) (Just transaction {txStaged = staged})
|
||||
pure (Right (resultOfStaged (txStaged transaction) statement rows))
|
||||
|
||||
-- | 事务内的结果集:空结果回落到解析出来的列名
|
||||
resultOfStaged :: Database -> Statement -> [Row] -> QueryResult
|
||||
resultOfStaged db statement rows = resultOf (StatementResult cols rows)
|
||||
where
|
||||
cols = case rows of
|
||||
[] -> columnsFromStatement db statement
|
||||
(r : _) -> map fst r
|
||||
|
||||
-- | 提交差分:相对快照列出要删、要写与要替换的行
|
||||
transactionOps :: Database -> Database -> Database -> [TxnOp]
|
||||
transactionOps base staged live =
|
||||
concatMap opsFor names
|
||||
where
|
||||
-- | 事务里碰得着的表:数据语句不增删表
|
||||
names = nubKeys (map fst base ++ map fst staged)
|
||||
|
||||
-- | 一张表的差分
|
||||
opsFor name
|
||||
| all (isJust . rowKey) (baseRows ++ stagedRows) = keyedOps
|
||||
| stagedRows == baseRows = []
|
||||
| otherwise = [TxnReplace name stagedRows]
|
||||
where
|
||||
baseRows = rowsOf name base
|
||||
stagedRows = rowsOf name staged
|
||||
liveRows = rowsOf name live
|
||||
|
||||
liveByKey = [(k, r) | r <- liveRows, Just k <- [rowKey r]]
|
||||
stagedByKey = [(k, r) | r <- stagedRows, Just k <- [rowKey r]]
|
||||
baseKeys = [k | r <- baseRows, Just k <- [rowKey r]]
|
||||
|
||||
-- 开事务时就存在、现在没了:删;内容变了的行也先删再写
|
||||
gone = [k | (k, _) <- liveByKey, k `elem` baseKeys, not (any ((== k) . fst) stagedByKey)]
|
||||
rewritten = [k | (k, r) <- stagedByKey, Just liveRow <- [lookup k liveByKey], liveRow /= r]
|
||||
deleted = gone ++ rewritten
|
||||
|
||||
-- 新增的行,以及跟库里不一样的行;并发插进来的行不动
|
||||
written =
|
||||
[ r
|
||||
| (k, r) <- stagedByKey
|
||||
, case lookup k liveByKey of
|
||||
Nothing -> True
|
||||
Just liveRow -> liveRow /= r
|
||||
]
|
||||
|
||||
keyedOps =
|
||||
[TxnDelete name deleted | not (null deleted)]
|
||||
++ [TxnUpsert name written | not (null written)]
|
||||
|
||||
-- | 表名去重但保持顺序
|
||||
nubKeys = foldr (\name acc -> if name `elem` acc then acc else name : acc) []
|
||||
|
||||
-- | 库里某张表的行
|
||||
rowsOf name db = maybe [] tableRows (lookup name db)
|
||||
|
||||
-- | 行 id:只有整数 id 才认,跟存储层删行的口径一致
|
||||
rowKey :: Row -> Maybe Int
|
||||
rowKey row = case lookup "id" row of
|
||||
Just (VInt k) -> Just k
|
||||
_ -> Nothing
|
||||
|
||||
-- | 普通身份跑语句前的权限判定;管理员与未登录放行
|
||||
authorized :: Session -> Statement -> IO (Either SessionError ())
|
||||
authorized session statement = do
|
||||
|
||||
+250
-7
@@ -3,14 +3,20 @@
|
||||
|
||||
module Main (main) where
|
||||
|
||||
import ChuSQL.Core.Engine.Storage.IPC (Account (..), Request (ReqAccountReset))
|
||||
import ChuSQL.Core.Engine.Storage.IPC (
|
||||
Account (..),
|
||||
Request (ReqAccountReset),
|
||||
closeConnection,
|
||||
localStorageLink,
|
||||
setStorageLink,
|
||||
)
|
||||
import ChuSQL.Core.Engine.Syntax.Parser (parseStatement)
|
||||
import ChuSQL.Core.Model (Database, Table (..), Value (..), pattern TInt, pattern TStr)
|
||||
import ChuSQL.Core.Model (Database, Row, Table (..), Value (..), pattern TInt, pattern TStr)
|
||||
import ChuSQL.Interface.Auth (hashPasswordWith)
|
||||
import ChuSQL.Interface.Protocol (ClientRequest (..), Grant (..), RoleView (..), ServerResponse (..), decodeRequest, encodeResponse)
|
||||
import ChuSQL.Interface.RateLimit (newRateLimiter)
|
||||
import ChuSQL.Server.Accounts (Principal (..), ensureRootAccount)
|
||||
import ChuSQL.Server.Backend (Backend (..), memoryBackend)
|
||||
import ChuSQL.Server.Backend (Backend (..), StatementResult (..), ipcBackend, memoryBackend)
|
||||
import ChuSQL.Server.Privileges (
|
||||
PrivilegeCommand (..),
|
||||
PrivilegeError (..),
|
||||
@@ -20,7 +26,7 @@ import ChuSQL.Server.Privileges (
|
||||
newPrivileges,
|
||||
runPrivilegeCommand,
|
||||
)
|
||||
import ChuSQL.Server.Session (QueryResult (..))
|
||||
import ChuSQL.Server.Session (QueryResult (..), Session, SessionError (..), newSession, runStatementCoded)
|
||||
import ChuSQL.Server.TCP (
|
||||
ServerConfig (..),
|
||||
ServerEnv,
|
||||
@@ -30,8 +36,8 @@ import ChuSQL.Server.TCP (
|
||||
newServerEnv,
|
||||
startServer,
|
||||
)
|
||||
import Control.Concurrent (threadDelay)
|
||||
import Control.Concurrent.MVar (newMVar)
|
||||
import Control.Concurrent (forkIO, threadDelay)
|
||||
import Control.Concurrent.MVar (MVar, newEmptyMVar, newMVar, putMVar, takeMVar)
|
||||
import Control.Exception (IOException, bracket, finally, try)
|
||||
import Control.Monad (void)
|
||||
import Data.Aeson (decode)
|
||||
@@ -59,7 +65,7 @@ import Network.Socket (
|
||||
socketToHandle,
|
||||
withSocketsDo,
|
||||
)
|
||||
import System.Directory (doesFileExist, getTemporaryDirectory, removeFile)
|
||||
import System.Directory (createDirectoryIfMissing, doesFileExist, getTemporaryDirectory, removeFile, removePathForcibly)
|
||||
import System.FilePath ((</>))
|
||||
import System.IO (
|
||||
BufferMode (LineBuffering),
|
||||
@@ -87,6 +93,8 @@ spec = do
|
||||
tcpSpec
|
||||
tcpServerSpec
|
||||
privilegeSpec
|
||||
transactionSpec
|
||||
ipcConcurrencySpec
|
||||
|
||||
-- 夹具:内存后端装在 MVar 里,跟 Web 测试同一套表
|
||||
|
||||
@@ -545,3 +553,238 @@ privilegeSpec = describe "server privileges" $ do
|
||||
Left err -> expectationFailure err
|
||||
Right stmt -> authorize service root testDatabase stmt `shouldReturn` Right ()
|
||||
|
||||
-- 真存储夹具:一个用例一份数据目录,跑完关链路删干净
|
||||
|
||||
-- | 本次用例的数据目录(用例串行跑,固定名够用)
|
||||
ipcDataDir :: IO FilePath
|
||||
ipcDataDir = do
|
||||
tmp <- getTemporaryDirectory
|
||||
pure (tmp </> "chusql-server-test-ipc")
|
||||
|
||||
-- | 写一份只属于这次用例的存储配置:数据目录与日志级别都从文件走
|
||||
writeIpcConfig :: FilePath -> IO FilePath
|
||||
writeIpcConfig dir = do
|
||||
let cfgPath = dir ++ ".toml"
|
||||
-- TOML 的普通字符串会吃反斜杠,路径统一用正斜杠
|
||||
slashed = map (\c -> if c == '\\' then '/' else c) dir
|
||||
writeFile
|
||||
cfgPath
|
||||
( unlines
|
||||
[ "[storage]"
|
||||
, "data_dir = \"" ++ slashed ++ "\""
|
||||
, "[log]"
|
||||
, "level = \"error\""
|
||||
]
|
||||
)
|
||||
pure cfgPath
|
||||
|
||||
-- | 删掉这次用例的数据目录与配置,删不掉也不算错
|
||||
cleanIpc :: FilePath -> IO ()
|
||||
cleanIpc dir = do
|
||||
_ <- try (removePathForcibly dir) :: IO (Either IOException ())
|
||||
_ <- try (removePathForcibly (dir ++ ".toml")) :: IO (Either IOException ())
|
||||
pure ()
|
||||
|
||||
-- 事务用例:内存后端跑得快,真存储的提交再在 IPC 用例里验一遍
|
||||
|
||||
-- | 起一个内存后端会话,当前库已切到 test
|
||||
withMemorySession :: (Session -> IO a) -> IO a
|
||||
withMemorySession body = do
|
||||
db <- newMVar testDb
|
||||
settings <- tempSettingsPath "transaction"
|
||||
removeIfExists settings
|
||||
session <- newSession (memoryBackend testDatabaseName db) (T.pack testDatabaseName) settings
|
||||
_ <- runStatementCoded session "use test"
|
||||
body session
|
||||
|
||||
-- | 两个会话共用一份内存库,用来验会话隔离
|
||||
withTwoSessions :: (Session -> Session -> IO a) -> IO a
|
||||
withTwoSessions body = do
|
||||
db <- newMVar testDb
|
||||
settings <- tempSettingsPath "transaction-pair"
|
||||
removeIfExists settings
|
||||
let backend = memoryBackend testDatabaseName db
|
||||
first <- newSession backend (T.pack testDatabaseName) settings
|
||||
second <- newSession backend (T.pack testDatabaseName) settings
|
||||
_ <- runStatementCoded first "use test"
|
||||
_ <- runStatementCoded second "use test"
|
||||
body first second
|
||||
|
||||
-- | 跑一条语句;出错就算用例失败
|
||||
mustSql :: Session -> Text -> IO QueryResult
|
||||
mustSql session sql = runStatementCoded session sql >>= either failed pure
|
||||
where
|
||||
failed err = expectationFailure (T.unpack (sessMessage err)) >> pure (QueryResult [] [] 0 False Nothing)
|
||||
|
||||
-- | 取 users 的 id 列
|
||||
sessionIds :: Session -> IO [Int]
|
||||
sessionIds session = do
|
||||
result <- mustSql session "select * from users"
|
||||
pure [n | row <- qrRows result, Just (VInt n) <- [nth 0 row]]
|
||||
|
||||
-- | 取某个 id 的 age
|
||||
sessionAge :: Session -> Int -> IO (Maybe Int)
|
||||
sessionAge session wanted = do
|
||||
result <- mustSql session "select * from users"
|
||||
let ages = [(n, a) | row <- qrRows result, Just (VInt n) <- [nth 0 row], Just (VInt a) <- [nth 2 row]]
|
||||
pure (lookup wanted ages)
|
||||
|
||||
-- | 取失败里的文案
|
||||
failureMessage :: Either SessionError QueryResult -> Text
|
||||
failureMessage outcome = either sessMessage (const "") outcome
|
||||
|
||||
-- | 取结果第一列的整数
|
||||
intValues :: [[Value]] -> [Int]
|
||||
intValues rows = [n | row <- rows, Just (VInt n) <- [nth 0 row]]
|
||||
|
||||
-- | 显式事务用例
|
||||
transactionSpec :: Spec
|
||||
transactionSpec = describe "server transactions" $ do
|
||||
it "keeps uncommitted rows invisible to other sessions until COMMIT" $ withTwoSessions $ \writer reader -> do
|
||||
_ <- mustSql writer "begin"
|
||||
_ <- mustSql writer "insert into users (id, name, age) values (6, 'six', 26)"
|
||||
sessionIds writer `shouldReturn` [1, 2, 3, 4, 5, 6]
|
||||
sessionIds reader `shouldReturn` [1, 2, 3, 4, 5]
|
||||
_ <- mustSql writer "commit"
|
||||
ids <- sessionIds writer
|
||||
ids `shouldMatchList` [1, 2, 3, 4, 5, 6]
|
||||
seen <- sessionIds reader
|
||||
seen `shouldMatchList` [1, 2, 3, 4, 5, 6]
|
||||
|
||||
it "drops every change on ROLLBACK" $ withMemorySession $ \session -> do
|
||||
_ <- mustSql session "begin"
|
||||
_ <- mustSql session "insert into users (id, name, age) values (6, 'six', 26)"
|
||||
_ <- mustSql session "update users set age = 99 where id = 1"
|
||||
_ <- mustSql session "delete from users where id = 2"
|
||||
rollback <- mustSql session "rollback"
|
||||
qrRowCount rollback `shouldBe` 0
|
||||
kept <- sessionIds session
|
||||
kept `shouldMatchList` [1, 2, 3, 4, 5]
|
||||
|
||||
it "commits the whole write set at once" $ withMemorySession $ \session -> do
|
||||
_ <- mustSql session "begin"
|
||||
_ <- mustSql session "insert into users (id, name, age) values (6, 'six', 26)"
|
||||
_ <- mustSql session "update users set age = 99 where id = 1"
|
||||
_ <- mustSql session "delete from users where id = 2"
|
||||
_ <- mustSql session "commit"
|
||||
ids <- sessionIds session
|
||||
ids `shouldMatchList` [1, 3, 4, 5, 6]
|
||||
sessionAge session 1 `shouldReturn` Just 99
|
||||
|
||||
it "rejects DDL inside a transaction and takes it after ROLLBACK" $ withMemorySession $ \session -> do
|
||||
_ <- mustSql session "begin"
|
||||
denied <- runStatementCoded session "create table blocked (id int)"
|
||||
failureMessage denied `shouldBe` "DDL is not allowed in a transaction"
|
||||
_ <- mustSql session "rollback"
|
||||
_ <- mustSql session "create table fresh (id int)"
|
||||
_ <- mustSql session "insert into fresh (id) values (1)"
|
||||
rows <- mustSql session "select * from fresh"
|
||||
qrRowCount rows `shouldBe` 1
|
||||
|
||||
it "refuses nested BEGIN and COMMIT outside a transaction" $ withMemorySession $ \session -> do
|
||||
_ <- mustSql session "begin"
|
||||
nested <- runStatementCoded session "begin"
|
||||
failureMessage nested `shouldBe` "already in a transaction"
|
||||
_ <- mustSql session "rollback"
|
||||
late <- runStatementCoded session "commit"
|
||||
failureMessage late `shouldBe` "no transaction in progress"
|
||||
|
||||
it "does not switch database inside a transaction" $ withMemorySession $ \session -> do
|
||||
_ <- mustSql session "begin"
|
||||
denied <- runStatementCoded session "use other"
|
||||
failureMessage denied `shouldBe` "cannot switch database inside a transaction"
|
||||
|
||||
it "needs a current database to start" $ do
|
||||
db <- newMVar testDb
|
||||
settings <- tempSettingsPath "transaction-bare"
|
||||
removeIfExists settings
|
||||
session <- newSession (memoryBackend testDatabaseName db) (T.pack testDatabaseName) settings
|
||||
result <- runStatementCoded session "begin"
|
||||
failureMessage result `shouldBe` "no database selected"
|
||||
|
||||
-- | 起一份真存储并装成当前链路;动态库打不开就当用例待定
|
||||
withIpcStorage :: IO () -> IO ()
|
||||
withIpcStorage body = do
|
||||
dir <- ipcDataDir
|
||||
cleanIpc dir
|
||||
createDirectoryIfMissing True dir
|
||||
cfgPath <- writeIpcConfig dir
|
||||
opened <- localStorageLink (Just cfgPath)
|
||||
case opened of
|
||||
Left err -> cleanIpc dir >> pendingWith ("cannot open the storage library: " ++ err)
|
||||
Right link -> do
|
||||
setStorageLink link
|
||||
body `finally` (closeConnection >> cleanIpc dir)
|
||||
|
||||
-- | 跑一条语句,出错就算用例失败
|
||||
mustRun :: IO (Either String StatementResult) -> IO ()
|
||||
mustRun action = action >>= either expectationFailure (const (pure ()))
|
||||
|
||||
-- | 取结果行,出错就算用例失败
|
||||
rowsOf :: IO (Either String StatementResult) -> IO [Row]
|
||||
rowsOf action = action >>= either (\err -> expectationFailure err >> pure []) (pure . srRows)
|
||||
|
||||
-- | 取结果里某一列的整数
|
||||
intColumn :: String -> [Row] -> [Int]
|
||||
intColumn name rows = [n | row <- rows, Just (VInt n) <- [lookup name row]]
|
||||
|
||||
-- | 往 p 表连插一批行,结果放 MVar 交回主线程
|
||||
writeRows :: Backend -> [Int] -> MVar [Either String StatementResult] -> IO ()
|
||||
writeRows session ids done = do
|
||||
outs <- mapM (\i -> beStatement session ("INSERT INTO p (id) VALUES (" ++ show i ++ ")")) ids
|
||||
putMVar done outs
|
||||
|
||||
-- | 同一进程里开两个库:会话之间不串数据,也不互相挡路
|
||||
ipcConcurrencySpec :: Spec
|
||||
ipcConcurrencySpec = describe "server backend on the real storage" $ do
|
||||
it "keeps two databases apart" $ withIpcStorage $ do
|
||||
base <- ipcBackend
|
||||
mustRun (beStatement base "CREATE DATABASE alpha")
|
||||
mustRun (beStatement base "CREATE DATABASE beta")
|
||||
let alpha = beWithDatabase base "alpha"
|
||||
beta = beWithDatabase base "beta"
|
||||
mustRun (beStatement alpha "CREATE TABLE t (id int)")
|
||||
mustRun (beStatement beta "CREATE TABLE t (id int)")
|
||||
mustRun (beStatement alpha "INSERT INTO t (id) VALUES (1)")
|
||||
mustRun (beStatement beta "INSERT INTO t (id) VALUES (2)")
|
||||
rowsA <- rowsOf (beStatement alpha "SELECT id FROM t")
|
||||
rowsB <- rowsOf (beStatement beta "SELECT id FROM t")
|
||||
intColumn "id" rowsA `shouldBe` [1]
|
||||
intColumn "id" rowsB `shouldBe` [2]
|
||||
|
||||
it "runs two sessions at once without mixing their rows" $ withIpcStorage $ do
|
||||
base <- ipcBackend
|
||||
mustRun (beStatement base "CREATE DATABASE alpha")
|
||||
mustRun (beStatement base "CREATE DATABASE beta")
|
||||
let alpha = beWithDatabase base "alpha"
|
||||
beta = beWithDatabase base "beta"
|
||||
mustRun (beStatement alpha "CREATE TABLE p (id int)")
|
||||
mustRun (beStatement beta "CREATE TABLE p (id int)")
|
||||
doneA <- newEmptyMVar
|
||||
doneB <- newEmptyMVar
|
||||
_ <- forkIO (writeRows alpha [1 .. 20] doneA)
|
||||
_ <- forkIO (writeRows beta [101 .. 120] doneB)
|
||||
outs <- sequence [takeMVar doneA, takeMVar doneB]
|
||||
[err | Left err <- concat outs] `shouldBe` []
|
||||
rowsA <- rowsOf (beStatement alpha "SELECT id FROM p")
|
||||
rowsB <- rowsOf (beStatement beta "SELECT id FROM p")
|
||||
intColumn "id" rowsA `shouldMatchList` [1 .. 20]
|
||||
intColumn "id" rowsB `shouldMatchList` [101 .. 120]
|
||||
|
||||
it "commits an explicit transaction to the real storage" $ withIpcStorage $ do
|
||||
base <- ipcBackend
|
||||
mustRun (beStatement base "CREATE DATABASE alpha")
|
||||
let alpha = beWithDatabase base "alpha"
|
||||
settings <- tempSettingsPath "transaction-ipc"
|
||||
removeIfExists settings
|
||||
session <- newSession alpha (T.pack testDatabaseName) settings
|
||||
_ <- mustSql session "use alpha"
|
||||
_ <- mustSql session "create table t (id int)"
|
||||
_ <- mustSql session "begin"
|
||||
_ <- mustSql session "insert into t (id) values (1)"
|
||||
inside <- mustSql session "select * from t"
|
||||
intValues (qrRows inside) `shouldBe` [1]
|
||||
_ <- mustSql session "commit"
|
||||
rows <- rowsOf (beStatement alpha "SELECT id FROM t")
|
||||
intColumn "id" rows `shouldBe` [1]
|
||||
|
||||
@@ -45,7 +45,13 @@ chusql-server
|
||||
|
||||
`chusql-server` 默认监听 `127.0.0.1:7777`,客户端建立连接后完成登录、数据库选择和 SQL 执行。
|
||||
|
||||
每条连接拥有独立会话状态,例如当前数据库选择不会影响其他连接。
|
||||
每条连接拥有独立会话状态,例如当前数据库选择不会影响其他连接。不同连接可以同时执行语句:读请求之间可以并行,写请求之间由存储层排队。
|
||||
|
||||
语句里的库前缀和表限定符由引擎在语义层解析:存储层只接受精确的表名键,因此 `FROM sales.orders` 与选定 `sales` 后的 `FROM orders` 指向同一张表。
|
||||
|
||||
显式事务在会话层实现:`BEGIN` 把当前数据库整库读成内存快照,事务内的语句都在快照上执行;`COMMIT` 计算快照与库的差分,用一条 `apply_transaction` 请求交给存储层应用;`ROLLBACK` 直接丢弃快照。因此隔离级别是快照读加原子提交:事务里看不到别的事务未提交或后来提交的改动,提交对外一次性可见。
|
||||
|
||||
WAL 负责崩溃恢复:每条记录带自增 LSN,一组写操作与一条提交标记一次落盘;进程重启时按标记分组重放已提交的记录、丢弃没有标记的整组,重放前逐条做幂等检查,恢复结束把最大 LSN 写进 `wal.checkpoint` 并清空 WAL。`SystemStatus` 应答里的 `last_lsn` 报告该检查点 LSN。
|
||||
|
||||
系统数据库及部分管理操作仅允许管理员访问。
|
||||
|
||||
|
||||
@@ -38,6 +38,22 @@ csql -d sales -e "SELECT * FROM orders LIMIT 5"
|
||||
csql -f json -e "SELECT count(*) FROM orders"
|
||||
```
|
||||
|
||||
表名与列名的写法:
|
||||
|
||||
- 选定数据库后语句里直接写表名即可,例如 `SELECT name FROM orders WHERE orders.id = 1`。
|
||||
- 表名可以带库前缀(`sales.orders`),列名可以带表限定符(`orders.id`)或库限定符(`sales.orders.id`),这几种写法指向同一张表和同一个列。
|
||||
- 只有候选唯一时才自动推导前缀;同名的表或列出现在多个候选上会以 `ambiguous table` / `ambiguous column` 报错,需要写全。
|
||||
- `FROM orders AS o` 之后只能用 `o.id`,不能再写 `orders.id`。
|
||||
|
||||
事务:
|
||||
|
||||
- `BEGIN`(或 `START TRANSACTION`)开始显式事务,`COMMIT` 提交,`ROLLBACK` 回滚。
|
||||
- 事务内只允许数据语句;DDL 与 `USE` 会被拒绝,`SHOW DATABASES` 这类不碰表数据的语句照常可用。
|
||||
- 事务里的读都看开事务那一刻的快照:别的连接在事务期间提交的改动不会出现在事务里,本会话未提交的改动别的连接也看不见。
|
||||
- 同一连接同时只能有一个事务,没有保存点与嵌套事务;开事务前要先选好数据库。
|
||||
- 提交时按行 `id` 写回本事务改动过的行,事务没碰过的行不受影响;没有整数 `id` 的表按整表替换提交。
|
||||
- 提交是持久的:整批改动与一条提交标记一次落盘,提交后进程被强杀也会在重启时重放;提交前崩溃则整批丢弃,不会只落一半。
|
||||
|
||||
### 交互模式
|
||||
|
||||
SQL 语句以分号结束,可跨多行输入。
|
||||
|
||||
Reference in new issue
Block a user