diff --git a/AGENTS.md b/AGENTS.md index 5948870..ee090e6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,5 +28,5 @@ - 改动说明只在交流中记录。 ## 当前重点 -- 完善 WAL 重放幂等。 -- 完整事务路线见 project-plan.txt 阶段 12。 +- 主线已完成:高级优化器 → 并行查询 → M1 会话隔离 → M2 快照事务 → M3 WAL 与崩溃恢复。 +- 进度与计数见 `agent_credientials/project-plan.txt`,限制与风险见 `agent_credientials/known-issues.txt`。 diff --git a/chusql-core/engine/src/ChuSQL/Core/Engine.hs b/chusql-core/engine/src/ChuSQL/Core/Engine.hs index 588114a..57a4902 100644 --- a/chusql-core/engine/src/ChuSQL/Core/Engine.hs +++ b/chusql-core/engine/src/ChuSQL/Core/Engine.hs @@ -134,6 +134,9 @@ runStatementUncheckedM _ q@CreateDatabase{} = runStatementM q runStatementUncheckedM _ q@DropDatabase{} = runStatementM q runStatementUncheckedM _ q@UseDatabase{} = runStatementM q runStatementUncheckedM _ ShowDatabases = runStatementM ShowDatabases +runStatementUncheckedM _ BeginTransaction = pure (Left "BEGIN is executed by the session") +runStatementUncheckedM _ CommitTransaction = pure (Left "COMMIT is executed by the session") +runStatementUncheckedM _ RollbackTransaction = pure (Left "ROLLBACK is executed by the session") -- | 只有需要读全表才能判断约束时才扫表 existingRows :: (MonadStorage m) => String -> Table -> m (Either String [Row]) diff --git a/chusql-core/engine/src/ChuSQL/Core/Engine/Error.hs b/chusql-core/engine/src/ChuSQL/Core/Engine/Error.hs index e33a6bc..964e75d 100644 --- a/chusql-core/engine/src/ChuSQL/Core/Engine/Error.hs +++ b/chusql-core/engine/src/ChuSQL/Core/Engine/Error.hs @@ -51,6 +51,8 @@ errorRules = , ErrorRule "storage" StorageError "query_error" , ErrorRule "storage library call failed" StorageError "query_error" , ErrorRule "unsupported protocol" ProtocolError "query_error" + , ErrorRule "executed by the session" ProtocolError "query_error" + , ErrorRule "transaction" ProtocolError "query_error" ] -- | 命中的第一条规则 diff --git a/chusql-core/engine/src/ChuSQL/Core/Engine/Semantic.hs b/chusql-core/engine/src/ChuSQL/Core/Engine/Semantic.hs index b897d49..7f9fed4 100644 --- a/chusql-core/engine/src/ChuSQL/Core/Engine/Semantic.hs +++ b/chusql-core/engine/src/ChuSQL/Core/Engine/Semantic.hs @@ -38,16 +38,16 @@ scopeAt place scope c | "ambiguous column:" `isInfixOf` err -> Left err | otherwise -> columnAt place (scopeOuter scope) c --- | 收集 FROM 能提供的列,含外层作用域 +-- | 收集 FROM 的列,列前缀由表名推导 checkFrom :: Database -> [(String, Column)] -> FromClause -> Either String [(String, Column)] checkFrom _ _ FromUnit = Right [] checkFrom db _ (FromTable mAlias tbl) = do - t <- lookupTable db tbl - Right (prefixColumns (Just (fromMaybe tbl mAlias)) (tableCols t)) + (key, t) <- resolveTable db tbl + Right (prefixColumns (Just (deriveQualifier mAlias key)) (tableCols t)) checkFrom db outer (FromJoin _ left mAlias tbl cond) = do lcols <- checkFrom db outer left - t <- lookupTable db tbl - let env = lcols ++ prefixColumns (Just (fromMaybe tbl mAlias)) (tableCols t) + (key, t) <- resolveTable db tbl + let env = lcols ++ prefixColumns (Just (deriveQualifier mAlias key)) (tableCols t) checkBool db "ON" (Scope env outer) cond Right env @@ -433,6 +433,10 @@ checkResolvedWith db outer q = case q of DropDatabase name -> checkDatabaseName name UseDatabase name -> checkDatabaseName name ShowDatabases -> Right () + -- | 事务控制由会话层执行,语义层不做列检查 + BeginTransaction -> Right () + CommitTransaction -> Right () + RollbackTransaction -> Right () where tableCols' = map fst . tableCols @@ -650,21 +654,21 @@ resolveStatement db q = case q of Select{} -> resolveSelect db q SelectExpr{} -> resolveSelect db q Update tbl assigns mWhere -> do - t <- lookupTable db tbl + (key, t) <- resolveTable db tbl let env = tableCols t name = nameIn env "UPDATE" assigns' <- mapM (\(c, e) -> (\e' -> (c, e')) <$> resolveExpr db env [] name e) assigns w <- mapM (resolveExpr db env [] (nameIn env "WHERE")) mWhere - pure (Update tbl assigns' w) + pure (Update key assigns' w) Delete tbl mWhere -> do - t <- lookupTable db tbl + (key, t) <- resolveTable db tbl w <- mapM (resolveExpr db (tableCols t) [] (nameIn (tableCols t) "WHERE")) mWhere - pure (Delete tbl w) + pure (Delete key w) Insert tbl cols rows -> do - t <- lookupTable db tbl + (key, t) <- resolveTable db tbl let env = tableCols t rows' <- mapM (mapM (resolveExpr db env [] (nameIn env "INSERT"))) rows - pure (Insert tbl cols rows') + pure (Insert key cols rows') other -> Right other -- | 查一列的物理名 @@ -685,9 +689,9 @@ resolveSelect _ q = Right q resolveQuery :: Database -> [(String, Column)] -> Statement -> [(String, Expr)] -> Either String Statement resolveQuery db outer q items = do env <- checkFrom db outer (selectFrom q) - -- | 单表查询里把裸列名换成物理列名 + -- | 单表无别名时把裸列名换成物理列名;带别名时表头保留别名限定名 let physical c = case selectFrom q of - FromTable Nothing tbl -> unqualify (Just tbl) c + FromTable Nothing tbl -> unqualify (Just (deriveQualifier Nothing tbl)) c _ -> c -- | 查一列并换算成物理名 name place c = physical . fst <$> scopedColumnAt place env outer c @@ -707,18 +711,23 @@ resolveQuery db outer q items = do grouping <- mapM (name "GROUP BY") (selectGroupBy q) ordering <- mapM (\(c, d) -> (\k -> (k, d)) <$> name "ORDER BY" c) (selectOrderBy q) source <- case selectFrom q of - single@FromTable{} -> Right single + FromTable mAlias tbl -> do + (key, _) <- resolveTable db tbl + Right (FromTable mAlias key) other -> resolveFrom db outer other pure $ case q of Select{} -> Select (map fst projected) source condition grouping ordering (selectLimit q) _ -> SelectExpr projected source condition grouping ordering (selectLimit q) --- | 连接两侧默认取表名作别名;ON 里的子查询也能引用外层列 +-- | 连接两侧取表名末段作别名;ON 可引用外层列 resolveFrom :: Database -> [(String, Column)] -> FromClause -> Either String FromClause resolveFrom _ _ FromUnit = Right FromUnit -resolveFrom _ _ (FromTable alias tbl) = Right (FromTable (Just (fromMaybe tbl alias)) tbl) +resolveFrom db _ (FromTable alias tbl) = do + (key, _) <- resolveTable db tbl + Right (FromTable (Just (deriveQualifier alias key)) key) resolveFrom db outer fromC@(FromJoin kind left alias tbl cond) = do env <- checkFrom db outer fromC l <- resolveFrom db outer left + (key, _) <- resolveTable db tbl c <- resolveExpr db env outer (fmap fst . columnAt "ON" env) cond - pure (FromJoin kind l (Just (fromMaybe tbl alias)) tbl c) + pure (FromJoin kind l (Just (deriveQualifier alias key)) key c) diff --git a/chusql-core/engine/src/ChuSQL/Core/Engine/Storage/IPC.hs b/chusql-core/engine/src/ChuSQL/Core/Engine/Storage/IPC.hs index ad0f81b..ef73166 100644 --- a/chusql-core/engine/src/ChuSQL/Core/Engine/Storage/IPC.hs +++ b/chusql-core/engine/src/ChuSQL/Core/Engine/Storage/IPC.hs @@ -3,6 +3,10 @@ module ChuSQL.Core.Engine.Storage.IPC ( IPCStorage (..), + Env (..), + envForDatabase, + defaultEnv, + runIPCStorage, Request (..), Response (..), Account (..), @@ -41,21 +45,38 @@ import System.IO.Unsafe (unsafePerformIO) -- IPC 存储实现:把 MonadStorage 的操作变成发给存储库的 JSON 请求。 -newtype IPCStorage a = IPCStorage - { runIPCStorage :: IO a +-- | 一条会话的存储环境:当前库名随会话走,不共享 +data Env = Env + { envDatabase :: Maybe String } +newtype IPCStorage a = IPCStorage + { runIPCStorageIn :: Env -> IO a + } + +-- | 按库名造一个会话环境 +envForDatabase :: Maybe String -> Env +envForDatabase = Env + +-- | 进程默认环境:读全局库名,给测试与 CLI 用 +defaultEnv :: IO Env +defaultEnv = Env <$> readMVar databaseRef + +-- | 用进程默认环境跑(兼容既有调用点) +runIPCStorage :: IPCStorage a -> IO a +runIPCStorage action = defaultEnv >>= runIPCStorageIn action + instance Functor IPCStorage where - fmap f (IPCStorage m) = IPCStorage (fmap f m) + fmap f (IPCStorage m) = IPCStorage (fmap f . m) instance Applicative IPCStorage where - pure = IPCStorage . pure - IPCStorage mf <*> IPCStorage ma = IPCStorage (mf <*> ma) + pure = IPCStorage . const . pure + IPCStorage mf <*> IPCStorage ma = IPCStorage $ \env -> mf env <*> ma env instance Monad IPCStorage where - IPCStorage m >>= k = IPCStorage $ do - a <- m - runIPCStorage (k a) + IPCStorage m >>= k = IPCStorage $ \env -> do + a <- m env + runIPCStorageIn (k a) env -- | 一条到存储库的链路:管请求与关闭 data StorageLink = StorageLink @@ -147,10 +168,9 @@ globalRequest ReqPing = True globalRequest _ = False -- | 发请求并把响应翻成 Either -ask :: String -> Request -> (Response -> Maybe a) -> IO (Either String a) -ask what req recognize = do - current <- getDatabaseName - resp <- sendRequest (stampDatabase current req) +ask :: Env -> String -> Request -> (Response -> Maybe a) -> IO (Either String a) +ask env what req recognize = do + resp <- sendRequest (stampDatabase (envDatabase env) req) pure $ case resp of RespError e -> Left e other -> case recognize other of @@ -163,99 +183,100 @@ okOnly RespOk = Just () okOnly _ = Nothing -- | 发 Scan -doScan :: String -> IO (Either String [Row]) -doScan t = ask "scan" (ReqScan t) $ \resp -> case resp of +doScan :: Env -> String -> IO (Either String [Row]) +doScan env t = ask env "scan" (ReqScan t) $ \resp -> case resp of RespRows rows -> Just rows _ -> Nothing -- | 发带投影的 Scan -doScanColumns :: String -> [String] -> IO (Either String [Row]) -doScanColumns t cols = ask "scan" (ReqScanColumns t cols) $ \resp -> case resp of +doScanColumns :: Env -> String -> [String] -> IO (Either String [Row]) +doScanColumns env t cols = ask env "scan" (ReqScanColumns t cols) $ \resp -> case resp of RespRows rows -> Just [[(c, v) | c <- cols, Just v <- [lookup c row]] | row <- rows] _ -> Nothing -- | 发分片 Scan:只取第 shard 片 -doScanShard :: String -> Maybe [String] -> Int -> Int -> IO (Either String [Row]) -doScanShard t cols shard shards = ask "scan_shard" (ReqScanShard t cols shard shards) $ \resp -> case resp of +doScanShard :: Env -> String -> Maybe [String] -> Int -> Int -> IO (Either String [Row]) +doScanShard env t cols shard shards = ask env "scan_shard" (ReqScanShard t cols shard shards) $ \resp -> case resp of RespRows rows -> Just rows _ -> Nothing -- | 发 Insert(索引由存储层维护) -doInsert :: String -> Row -> IO (Either String ()) -doInsert t r = ask "insert" (ReqInsert t r) okOnly +doInsert :: Env -> String -> Row -> IO (Either String ()) +doInsert env t r = ask env "insert" (ReqInsert t r) okOnly -- | 发 InsertBatch:一批行一次请求 -doInsertMany :: String -> [Row] -> IO (Either String ()) -doInsertMany t rs = ask "insert_batch" (ReqInsertBatch t rs) okOnly +doInsertMany :: Env -> String -> [Row] -> IO (Either String ()) +doInsertMany env t rs = ask env "insert_batch" (ReqInsertBatch t rs) okOnly -- | 发 DeleteKeys -doDeleteKeys :: String -> [Int] -> IO (Either String ()) -doDeleteKeys t ks = ask "delete_keys" (ReqDeleteKeys t ks) okOnly +doDeleteKeys :: Env -> String -> [Int] -> IO (Either String ()) +doDeleteKeys env t ks = ask env "delete_keys" (ReqDeleteKeys t ks) okOnly -- | 发 ReplaceAll -doReplaceAll :: String -> [Row] -> IO (Either String ()) -doReplaceAll t rs = ask "replace_all" (ReqReplaceAll t rs) okOnly +doReplaceAll :: Env -> String -> [Row] -> IO (Either String ()) +doReplaceAll env t rs = ask env "replace_all" (ReqReplaceAll t rs) okOnly -- | 发 ListTables -doListTables :: IO (Either String [String]) -doListTables = ask "list_tables" ReqListTables $ \resp -> case resp of +doListTables :: Env -> IO (Either String [String]) +doListTables env = ask env "list_tables" ReqListTables $ \resp -> case resp of RespTables ts -> Just ts _ -> Nothing -- | 一次拿全库 schema -doListCatalog :: IO (Either String [TableInfo]) -doListCatalog = ask "list_catalog" ReqListCatalog $ \resp -> case resp of +doListCatalog :: Env -> IO (Either String [TableInfo]) +doListCatalog env = ask env "list_catalog" ReqListCatalog $ \resp -> case resp of RespCatalog xs -> Just xs _ -> Nothing -- | 按某一列的索引取一行(键可以是整数或字符串) -doLookupByColumn :: String -> String -> Value -> IO (Either String IndexResult) -doLookupByColumn t c k = ask "lookup_by_index" (ReqLookupByColumn t c k) $ \resp -> case resp of +doLookupByColumn :: Env -> String -> String -> Value -> IO (Either String IndexResult) +doLookupByColumn env t c k = ask env "lookup_by_index" (ReqLookupByColumn t c k) $ \resp -> case resp of RespRows rows -> Just (IndexRows rows) RespNoIndex -> Just NoIndex _ -> Nothing -- | 范围扫描:没有可用索引就回 Nothing,让上层退回扫描 doScanRange :: + Env -> String -> String -> Maybe (Value, Bool) -> Maybe (Value, Bool) -> IO (Either String (Maybe [Row])) -doScanRange t c lo hi = ask "range_by_index" (ReqRangeByIndex t c lo hi) $ \resp -> case resp of +doScanRange env t c lo hi = ask env "range_by_index" (ReqRangeByIndex t c lo hi) $ \resp -> case resp of RespRows rows -> Just (Just rows) RespNoIndex -> Just Nothing _ -> Nothing -- | 发 DescribeTable -doDescribeTable :: String -> IO (Either String TableInfo) -doDescribeTable t = ask "describe_table" (ReqDescribeTable t) $ \resp -> case resp of +doDescribeTable :: Env -> String -> IO (Either String TableInfo) +doDescribeTable env t = ask env "describe_table" (ReqDescribeTable t) $ \resp -> case resp of RespSchema info -> Just info _ -> Nothing -- | 给某一列建索引 -doCreateIndex :: String -> String -> IO (Either String ()) -doCreateIndex t c = ask "create_index" (ReqCreateIndex t c) okOnly +doCreateIndex :: Env -> String -> String -> IO (Either String ()) +doCreateIndex env t c = ask env "create_index" (ReqCreateIndex t c) okOnly -- | 去掉某一列的索引 -doDropIndex :: String -> String -> IO (Either String ()) -doDropIndex t c = ask "drop_index" (ReqDropIndex t c) okOnly +doDropIndex :: Env -> String -> String -> IO (Either String ()) +doDropIndex env t c = ask env "drop_index" (ReqDropIndex t c) okOnly -- | 删一列(列定义、这一列上的索引、每行里的那一格一起没) -doDropColumn :: String -> String -> IO (Either String ()) -doDropColumn t c = ask "drop_column" (ReqDropColumn t c) okOnly +doDropColumn :: Env -> String -> String -> IO (Either String ()) +doDropColumn env t c = ask env "drop_column" (ReqDropColumn t c) okOnly -- | 发 ReplaceSchema(ALTER 用) -doReplaceSchema :: String -> [SchemaColumn] -> [Row] -> IO (Either String ()) -doReplaceSchema t cols rs = ask "replace_schema" (ReqReplaceSchema t cols rs) okOnly +doReplaceSchema :: Env -> String -> [SchemaColumn] -> [Row] -> IO (Either String ()) +doReplaceSchema env t cols rs = ask env "replace_schema" (ReqReplaceSchema t cols rs) okOnly -- | 发 CreateTable -doCreateTable :: String -> [SchemaColumn] -> IO (Either String ()) -doCreateTable t cols = ask "create_table" (ReqCreateTable t cols) okOnly +doCreateTable :: Env -> String -> [SchemaColumn] -> IO (Either String ()) +doCreateTable env t cols = ask env "create_table" (ReqCreateTable t cols) okOnly -- | 发 DropTable -doDropTable :: String -> IO (Either String ()) -doDropTable t = ask "drop_table" (ReqDropTable t) okOnly +doDropTable :: Env -> String -> IO (Either String ()) +doDropTable env t = ask env "drop_table" (ReqDropTable t) okOnly -- | 线上 schema 转本地列 schemaToColumns :: [SchemaColumn] -> [(String, Column)] @@ -292,92 +313,91 @@ toWire (n, c) = instance MonadStorage IPCStorage where -- \| 建一个库 - createDatabase name = IPCStorage (ask "create_database" (ReqDatabase "create_database" name) okOnly) + createDatabase name = IPCStorage $ \env -> ask env "create_database" (ReqDatabase "create_database" name) okOnly -- \| 删一个库 - dropDatabase name = IPCStorage (ask "drop_database" (ReqDatabase "drop_database" name) okOnly) + dropDatabase name = IPCStorage $ \env -> ask env "drop_database" (ReqDatabase "drop_database" name) okOnly -- \| 切当前库 - useDatabase name = IPCStorage (ask "use_database" (ReqDatabase "use_database" name) okOnly) + useDatabase name = IPCStorage $ \env -> ask env "use_database" (ReqDatabase "use_database" name) okOnly -- \| 列出现有库 - listDatabases = IPCStorage $ ask "list_databases" (ReqDatabase "list_databases" "") $ \resp -> case resp of + listDatabases = IPCStorage $ \env -> ask env "list_databases" (ReqDatabase "list_databases" "") $ \resp -> case resp of RespTables names -> Just names _ -> Nothing -- \| 发 Scan - scan t = IPCStorage (doScan t) + scan t = IPCStorage (\env -> doScan env t) -- \| 只取指定列 - scanColumns t cols = IPCStorage (doScanColumns t cols) + scanColumns t cols = IPCStorage (\env -> doScanColumns env t cols) -- \| 并行分片数:按 RTS 能力数,封顶在 parallelShardLimit - parallelShards = IPCStorage $ do + parallelShards = IPCStorage $ \_ -> do caps <- getNumCapabilities pure (max 1 (min parallelShardLimit caps)) -- \| 分片并行扫,再按分片序拼接 scanShards width t cols - | width <= 1 = IPCStorage (maybe (doScan t) (doScanColumns t) cols) - | otherwise = IPCStorage $ do + | width <= 1 = IPCStorage (\env -> maybe (doScan env t) (doScanColumns env t) cols) + | otherwise = IPCStorage $ \env -> do pool <- workerPool width - parts <- poolRun pool [doScanShard t cols shard width | shard <- [0 .. width - 1]] + parts <- poolRun pool [doScanShard env t cols shard width | shard <- [0 .. width - 1]] pure (fmap concat (sequence parts)) -- \| 发 Insert(索引由存储层自己维护) - insert t r = IPCStorage (doInsert t r) + insert t r = IPCStorage (\env -> doInsert env t r) -- \| 一批行一次请求:N 行只落一次盘 - insertMany t rs = IPCStorage (doInsertMany t rs) + insertMany t rs = IPCStorage (\env -> doInsertMany env t rs) -- \| 按 id 批量删行 - deleteKeys t ks = IPCStorage (doDeleteKeys t ks) + deleteKeys t ks = IPCStorage (\env -> doDeleteKeys env t ks) -- \| 发 ReplaceAll - replaceAll t rs = IPCStorage (doReplaceAll t rs) + replaceAll t rs = IPCStorage (\env -> doReplaceAll env t rs) -- \| 按某一列的索引取一行(没有索引就回 NoIndex) - lookupByColumn t c k = IPCStorage (doLookupByColumn t c k) + lookupByColumn t c k = IPCStorage (\env -> doLookupByColumn env t c k) -- \| 范围扫描(没有索引就回 Nothing) - scanRange t c lo hi = IPCStorage (doScanRange t c lo hi) + scanRange t c lo hi = IPCStorage (\env -> doScanRange env t c lo hi) -- \| 发 CreateTable - createTable name cols = IPCStorage (doCreateTable name (map toWire cols)) + createTable name cols = IPCStorage (\env -> doCreateTable env name (map toWire cols)) -- \| ALTER:整表换列定义与全部行 - replaceSchema name cols rows = IPCStorage (doReplaceSchema name (map toWire cols) rows) + replaceSchema name cols rows = IPCStorage (\env -> doReplaceSchema env name (map toWire cols) rows) -- \| 发 DropTable - dropTable name = IPCStorage (doDropTable name) + dropTable name = IPCStorage (\env -> doDropTable env name) -- \| 建索引 - createIndex t c = IPCStorage (doCreateIndex t c) + createIndex t c = IPCStorage (\env -> doCreateIndex env t c) -- \| 删索引 - dropIndex t c = IPCStorage (doDropIndex t c) + dropIndex t c = IPCStorage (\env -> doDropIndex env t c) -- \| 删一列 - dropColumn t c = IPCStorage (doDropColumn t c) + dropColumn t c = IPCStorage (\env -> doDropColumn env t c) -- \| 列表 + 逐表扫描拼库 - snapshot = IPCStorage $ do - result <- doListCatalog + snapshot = IPCStorage $ \env -> do + result <- doListCatalog env case result of Left _ -> pure [] - Right xs -> mapM loadEntry xs + Right xs -> mapM (loadEntry env) xs where - loadEntry info = do - rows <- doScan (tiTable info) + loadEntry env info = do + rows <- doScan env (tiTable info) pure ( tiTable info , Table (tiTable info) (schemaToColumns (tiColumns info)) (either (const []) id rows) (Just (metaOf info)) ) -- \| 只问数据字典要结构,设了当前库只留它的表 - schema = IPCStorage $ do - current <- getDatabaseName - result <- ask "all_catalogs" ReqAllCatalog $ \resp -> case resp of + schema = IPCStorage $ \env -> do + result <- ask env "all_catalogs" ReqAllCatalog $ \resp -> case resp of RespCatalog infos -> Just infos _ -> Nothing pure $ case result of Left _ -> [] - Right xs -> concatMap (visibleTables current) xs + Right xs -> concatMap (visibleTables (envDatabase env)) xs -- | 线上表信息转引擎侧统计 metaOf :: TableInfo -> TableMeta diff --git a/chusql-core/engine/src/ChuSQL/Core/Engine/Syntax/AST.hs b/chusql-core/engine/src/ChuSQL/Core/Engine/Syntax/AST.hs index 01500ae..fa6f53b 100644 --- a/chusql-core/engine/src/ChuSQL/Core/Engine/Syntax/AST.hs +++ b/chusql-core/engine/src/ChuSQL/Core/Engine/Syntax/AST.hs @@ -53,6 +53,12 @@ data Statement | DropDatabase String | UseDatabase String | ShowDatabases + -- | 开事务:BEGIN / START TRANSACTION + | BeginTransaction + -- | 提交事务 + | CommitTransaction + -- | 回滚事务 + | RollbackTransaction deriving (Show, Eq) -- | 连接类型:内连接与左外连接 diff --git a/chusql-core/engine/src/ChuSQL/Core/Engine/Syntax/Parser.hs b/chusql-core/engine/src/ChuSQL/Core/Engine/Syntax/Parser.hs index 6bb3ac2..d028d2e 100644 --- a/chusql-core/engine/src/ChuSQL/Core/Engine/Syntax/Parser.hs +++ b/chusql-core/engine/src/ChuSQL/Core/Engine/Syntax/Parser.hs @@ -613,6 +613,14 @@ updateStatement = do mWhere <- optional (keyword "where" *> expr) return (Update tbl assigns mWhere) +-- | 事务控制语句 +transactionStatement :: Parser Statement +transactionStatement = + try (BeginTransaction <$ (keyword "start" *> keyword "transaction")) + <|> (BeginTransaction <$ (keyword "begin" *> optional (keyword "transaction"))) + <|> (CommitTransaction <$ (keyword "commit" *> optional (keyword "transaction"))) + <|> (RollbackTransaction <$ (keyword "rollback" *> optional (keyword "transaction"))) + -- | 解析总入口 parseStatement :: String -> Either String Statement parseStatement input = @@ -622,7 +630,8 @@ parseStatement input = where -- | 按关键字分派到各语句解析器 statementP = - selectStatement + transactionStatement + <|> selectStatement <|> try (CreateDatabase <$> (keyword "create" *> keyword "database" *> identifier)) <|> try (DropDatabase <$> (keyword "drop" *> keyword "database" *> identifier)) <|> (UseDatabase <$> (keyword "use" *> identifier)) diff --git a/chusql-core/engine/test/Spec.hs b/chusql-core/engine/test/Spec.hs index c03c5bc..490fe56 100644 --- a/chusql-core/engine/test/Spec.hs +++ b/chusql-core/engine/test/Spec.hs @@ -15,7 +15,7 @@ import ChuSQL.Core.Engine.Parallel (poolRun, workerPool) import ChuSQL.Core.Model import ChuSQL.Core.Engine.Semantic (prepare) import ChuSQL.Core.Engine.Storage (IndexResult (..), MonadStorage (..)) -import ChuSQL.Core.Engine.Storage.IPC (IPCStorage (runIPCStorage), closeConnection, doListTables, getDatabaseName, localStorageLink, setDatabaseName, setStorageLink) +import ChuSQL.Core.Engine.Storage.IPC (IPCStorage (..), closeConnection, defaultEnv, doListTables, envForDatabase, getDatabaseName, localStorageLink, runIPCStorage, setDatabaseName, setStorageLink) import ChuSQL.Core.Engine.Syntax.AST import ChuSQL.Core.Engine.Syntax.Parser import Control.Concurrent (threadDelay) @@ -113,6 +113,40 @@ main = hspec $ do rowsOf (parseStatement "SELECT sales.users.name FROM sales.users WHERE sales.users.id = 1" >>= runStatement db) `shouldBe` Right [[("name", VStr "Alice")]] + it "derives the column prefix when the session schema is scoped to one database" $ do + let scoped = [("users", users)] + rowsOf (parseStatement "SELECT sales.users.name FROM sales.users WHERE sales.users.id = 1" >>= runStatement scoped) + `shouldBe` Right [[("name", VStr "Alice")]] + rowsOf (parseStatement "SELECT users.name FROM users WHERE users.id = 1" >>= runStatement scoped) + `shouldBe` Right [[("name", VStr "Alice")]] + + it "derives the database prefix for a bare table name" $ do + let catalog = [("sales.users", users)] + rowsOf (parseStatement "SELECT name FROM users WHERE users.id = 1" >>= runStatement catalog) + `shouldBe` Right [[("name", VStr "Alice")]] + rowsOf (parseStatement "SELECT users.name FROM sales.users WHERE users.id = 1" >>= runStatement catalog) + `shouldBe` Right [[("name", VStr "Alice")]] + + it "keeps derived prefixes short in joins" $ do + let catalog = [("sales.users", users), ("sales.orders", orders)] + fmap + (map (map snd)) + (parseStatement "SELECT users.name, orders.product FROM sales.users JOIN sales.orders ON users.id = orders.user_id" >>= rowsOf . runStatement catalog) + `shouldBe` Right [[VStr "Alice", VStr "Book"], [VStr "Alice", VStr "Cup"], [VStr "Bob", VStr "Pen"]] + + it "writes through the derived table key" $ do + let scoped = [("users", users)] + case parseStatement "UPDATE sales.users SET age = 26 WHERE id = 1" >>= runStatement scoped of + Left err -> expectationFailure err + Right (db', _) -> + rowsOf (runStatement db' (makeSelect ["age"] "users" (Just (Eq (Col "id") (LitInt 1))))) + `shouldBe` Right [[("age", VInt 26)]] + + it "rejects an ambiguous derived table name" $ do + let catalog = [("sales.users", users), ("other.users", users)] + (parseStatement "SELECT name FROM users" >>= rowsOf . runStatement catalog) + `shouldSatisfy` either (isInfixOf "ambiguous table") (const False) + it "parses database lifecycle statements" $ do map (fmap show . parseStatement) ["CREATE DATABASE sales", "DROP DATABASE sales", "USE sales", "SHOW DATABASES"] `shouldBe` map Right ["CreateDatabase \"sales\"", "DropDatabase \"sales\"", "UseDatabase \"sales\"", "ShowDatabases"] @@ -242,6 +276,33 @@ main = hspec $ do parseStatement "SELECT name FROM" `shouldSatisfy` isLeft + describe "ChuSQL.Core.Engine (事务)" $ do + it "parses transaction control statements" $ do + parseStatement "BEGIN" `shouldBe` Right BeginTransaction + parseStatement "BEGIN TRANSACTION" `shouldBe` Right BeginTransaction + parseStatement "START TRANSACTION" `shouldBe` Right BeginTransaction + parseStatement "begin transaction" `shouldBe` Right BeginTransaction + parseStatement "COMMIT" `shouldBe` Right CommitTransaction + parseStatement "COMMIT TRANSACTION" `shouldBe` Right CommitTransaction + parseStatement "ROLLBACK" `shouldBe` Right RollbackTransaction + parseStatement "rollback transaction" `shouldBe` Right RollbackTransaction + + it "parses the bare words only as whole words" $ do + parseStatement "BEGINNER" `shouldSatisfy` isLeft + parseStatement "COMMITTED" `shouldSatisfy` isLeft + + it "reports transaction control as session work" $ do + mapM_ + ( \sql -> + (runStatement [("users", users)] =<< parseStatement sql) + `shouldSatisfy` either (isInfixOf "executed by the session") (const False) + ) + ["BEGIN", "COMMIT", "ROLLBACK"] + + it "puts transaction errors under the protocol category" $ do + errorCode "BEGIN is executed by the session" `shouldBe` "query_error" + errorCategory "BEGIN is executed by the session" `shouldBe` ProtocolError + describe "ChuSQL.Core.Engine.Syntax.Parser (JOIN)" $ do it "parses a simple JOIN without aliases" $ do parseStatement "SELECT name FROM users JOIN orders ON id = user_id" @@ -1613,7 +1674,19 @@ main = hspec $ do withTestSession $ \srv -> withServerEnv srv $ do runIPCStorage (insert "t1" [("id", VInt 1)]) `shouldReturn` Right () runIPCStorage (insert "t2" [("id", VInt 2)]) `shouldReturn` Right () - doListTables `shouldReturn` Right ["t1", "t2"] + env <- defaultEnv + doListTables env `shouldReturn` Right ["t1", "t2"] + + it "two sessions keep their own database" $ do + withTestSession $ \_ -> do + let alpha = envForDatabase (Just "alpha") + beta = envForDatabase (Just "beta") + runIPCStorageIn (createDatabase "alpha") alpha `shouldReturn` Right () + runIPCStorageIn (createDatabase "beta") beta `shouldReturn` Right () + runIPCStorageIn (insert "iso" [("id", VInt 1)]) alpha `shouldReturn` Right () + runIPCStorageIn (insert "iso" [("id", VInt 2)]) beta `shouldReturn` Right () + runIPCStorageIn (scan "iso") alpha `shouldReturn` Right [[("id", VInt 1)]] + runIPCStorageIn (scan "iso") beta `shouldReturn` Right [[("id", VInt 2)]] it "rows survive a restart on the same data directory" $ do withTestSession $ \srv -> do diff --git a/chusql-core/model/src/ChuSQL/Core/Model.hs b/chusql-core/model/src/ChuSQL/Core/Model.hs index 4998ca7..2849bec 100644 --- a/chusql-core/model/src/ChuSQL/Core/Model.hs +++ b/chusql-core/model/src/ChuSQL/Core/Model.hs @@ -15,6 +15,9 @@ module ChuSQL.Core.Model , colNames , colType , lookupTable + , resolveTable + , lastTablePart + , deriveQualifier , allColumns , qualify , unqualify @@ -470,6 +473,32 @@ colType t c = lookup c (tableCols t) lookupTable :: Database -> String -> Either String Table lookupTable db t = maybe (Left ("unknown table: " ++ t)) Right (lookup t db) +-- | 取表名引用的最后一段 +lastTablePart :: String -> String +lastTablePart = reverse . takeWhile (/= '.') . reverse + +-- | 推导列前缀:有别名用别名,否则取表名末段 +deriveQualifier :: Maybe String -> String -> String +deriveQualifier (Just a) _ = a +deriveQualifier Nothing t = lastTablePart t + +-- | 按名查表并返回 schema 键:精确、前缀、唯一后缀 +resolveTable :: Database -> String -> Either String (String, Table) +resolveTable db name = case lookup name db of + Just table -> Right (name, table) + Nothing + | '.' `elem` name, Just table <- lookup bare db -> Right (bare, table) + | [(key, table)] <- suffixed -> Right (key, table) + | otherwise -> Left (miss suffixed) + where + -- | 引用里去掉库前缀的那一段 + bare = lastTablePart name + -- | schema 里以这个表名收尾的键:`sales.users` 能推出裸表名引用 + suffixed = [(key, table) | (key, table) <- db, ('.' : name) `isSuffixOf` key] + -- | 一个候选都没有就是查无此表,多于一个就是有歧义 + miss [] = "unknown table: " ++ name + miss _ = "ambiguous table: " ++ name + -- | 通配列名 allColumns :: String allColumns = "*" @@ -491,5 +520,5 @@ resolveColumn name env = case [(k, v) | (k, v) <- env, matches k] of [] -> Left ("unknown column: " ++ name) _ -> Left ("ambiguous column: " ++ name) where - -- | 裸名或带后缀的名称 - matches k = k == name || ('.' : name) `isSuffixOf` k + -- | 裸名、补充了库前缀的写法,或 schema 键自带库前缀的写法 + matches k = k == name || ('.' : name) `isSuffixOf` k || ('.' : k) `isSuffixOf` name diff --git a/chusql-core/model/src/ChuSQL/Core/Protocol.hs b/chusql-core/model/src/ChuSQL/Core/Protocol.hs index 6f812f1..027ca57 100644 --- a/chusql-core/model/src/ChuSQL/Core/Protocol.hs +++ b/chusql-core/model/src/ChuSQL/Core/Protocol.hs @@ -3,6 +3,7 @@ module ChuSQL.Core.Protocol ( SchemaColumn (..), Request (..), + TxnOp (..), Response (..), TableInfo (..), Account (..), @@ -96,6 +97,35 @@ data Request | ReqDropColumn String String | ReqReplaceSchema String [SchemaColumn] [Row] | ReqListCatalog + | ReqApplyTransaction [TxnOp] + +-- | 事务提交里的一条写操作 +data TxnOp + = TxnUpsert String [Row] + | TxnDelete String [Int] + | TxnReplace String [Row] + deriving (Show, Eq) + +-- | 事务操作编码成 JSON +instance ToJSON TxnOp where + toJSON (TxnUpsert t rs) = + object + [ "op" .= ("upsert" :: T.Text) + , "table" .= t + , "rows" .= map rowToJSON rs + ] + toJSON (TxnDelete t ks) = + object + [ "op" .= ("delete" :: T.Text) + , "table" .= t + , "ids" .= ks + ] + toJSON (TxnReplace t rs) = + object + [ "op" .= ("replace" :: T.Text) + , "table" .= t + , "rows" .= map rowToJSON rs + ] -- | 请求编码成 JSON instance ToJSON Request where @@ -201,6 +231,11 @@ instance ToJSON Request where , "rows" .= map rowToJSON rs ] toJSON ReqListCatalog = object ["method" .= ("list_catalog" :: T.Text)] + toJSON (ReqApplyTransaction ops) = + object + [ "method" .= ("apply_transaction" :: T.Text) + , "ops" .= ops + ] -- | 一张表的梗概:列、行数、索引与列统计 data TableInfo = TableInfo diff --git a/chusql-core/storage/src/protocol.rs b/chusql-core/storage/src/protocol.rs index cb6be70..39276f5 100644 --- a/chusql-core/storage/src/protocol.rs +++ b/chusql-core/storage/src/protocol.rs @@ -171,6 +171,22 @@ pub enum Request { columns: Vec, rows: Vec, }, + /// 事务提交:把一批写操作当成一个请求原子应用 + ApplyTransaction { + ops: Vec, + }, +} + +/// 事务提交里的一条写操作 +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum StorageOp { + /// 追加写入的行(提交端已先删掉要覆盖的行) + Upsert { table: String, rows: Vec }, + /// 按行 id 删除 + Delete { table: String, ids: Vec }, + /// 整表替换(没有整数 id 的表只能这么提交) + Replace { table: String, rows: Vec }, } impl Request { @@ -202,7 +218,8 @@ impl Request { | Self::ReplaceSchema { table, .. } => Some(table), Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList | Self::AccountCreate { .. } | Self::AccountReset { .. } | Self::AccountLogin { .. } - | Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus => None, + | Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus + | Self::ApplyTransaction { .. } => None, } } } @@ -237,7 +254,11 @@ pub enum Response { NoIndex, Ok, /// 系统目录状态 - System { initialized: bool }, + System { + initialized: bool, + #[serde(default)] + last_lsn: u64, + }, Error { message: String }, Catalog { schemas: Vec }, } diff --git a/chusql-core/storage/src/runtime.rs b/chusql-core/storage/src/runtime.rs index 808a194..f8531b9 100644 --- a/chusql-core/storage/src/runtime.rs +++ b/chusql-core/storage/src/runtime.rs @@ -8,9 +8,10 @@ use crate::config::{self, Config, Loaded}; use crate::heap::{scalar_int, HeapTable}; use crate::log; use crate::protocol::{ - reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row, SchemaColumn, TableSchemaWire, USERS_TABLE, + reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row, SchemaColumn, StorageOp, + TableSchemaWire, USERS_TABLE, }; -use crate::wal::{Wal, WalOp}; +use crate::wal::{committed_ops, Wal, WalOp}; use crate::{log_debug, log_error, log_info, log_warn}; // 进程内存储运行时:Haskell 侧经 ffi.rs 调进来。 @@ -196,6 +197,7 @@ impl Server { /// 把一条 WAL 操作落到磁盘 fn apply_op(&self, op: &WalOp) -> std::io::Result<()> { match op { + WalOp::Commit => Ok(()), WalOp::HideColumn { table, column } => { let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?; c.remove_column(table, column)?; @@ -311,52 +313,119 @@ impl Server { if self.recovery_required.load(Ordering::Acquire) { return Err(std::io::Error::other("storage recovery required")); } - { - let c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?; - let validate = |table: &str, rows: &[Row]| -> std::io::Result<()> { - let Some(schema) = c.describe(table) else { return Ok(()); }; - if let Some(name) = rows.iter().flat_map(|r| r.keys()).find(|k| schema.dropped_columns.contains(*k)) { - return Err(std::io::Error::other(format!("unknown column: {name}"))); - } - Ok(()) - }; - match op { - WalOp::Insert { table, row } => validate(table, std::slice::from_ref(row))?, - WalOp::InsertBatch { table, rows } | WalOp::ReplaceAll { table, rows } => validate(table, rows)?, - WalOp::HideColumn { table, column } - if !c.describe(table).is_some_and(|s| s.columns.iter().any(|col| &col.name == column)) => - { - return Err(std::io::Error::other(format!("unknown column: {column}"))); - } - _ => {} + self.validate_op(op)?; + self.group_via_wal(std::slice::from_ref(op)) + } + + /// 校验一条操作:命中被删的列就拒绝 + fn validate_op(&self, op: &WalOp) -> std::io::Result<()> { + let c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?; + let validate = |table: &str, rows: &[Row]| -> std::io::Result<()> { + let Some(schema) = c.describe(table) else { return Ok(()); }; + if let Some(name) = rows.iter().flat_map(|r| r.keys()).find(|k| schema.dropped_columns.contains(*k)) { + return Err(std::io::Error::other(format!("unknown column: {name}"))); } + Ok(()) + }; + match op { + WalOp::Insert { table, row } => validate(table, std::slice::from_ref(row))?, + WalOp::InsertBatch { table, rows } | WalOp::ReplaceAll { table, rows } => validate(table, rows)?, + WalOp::HideColumn { table, column } + if !c.describe(table).is_some_and(|s| s.columns.iter().any(|col| &col.name == column)) => + { + return Err(std::io::Error::other(format!("unknown column: {column}"))); + } + _ => {} } + Ok(()) + } + + /// 事务提交:一组写操作加一个提交标记,一次落盘 + /// + /// 组里每条都过 `validate_op`,跟单条写一致;调用方 `dispatch` 已经持有 + /// `gate.write`,因此整批对外原子可见。崩在提交标记之前,这组记录在恢复 + /// 时整组丢弃;崩在标记之后,逐条按「是否已生效」补写。 + fn apply_transaction(&self, ops: &[StorageOp]) -> std::io::Result<()> { + let _guard = self.write_lock.lock().unwrap(); + if self.recovery_required.load(Ordering::Acquire) { + return Err(std::io::Error::other("storage recovery required")); + } + let mut wal_ops = Vec::with_capacity(ops.len()); + for op in ops { + let table = match op { + StorageOp::Upsert { table, .. } + | StorageOp::Delete { table, .. } + | StorageOp::Replace { table, .. } => table, + }; + if reserved_table(table) { + return Err(std::io::Error::other("reserved system table")); + } + if table.is_empty() || !table.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + return Err(std::io::Error::other("invalid table name")); + } + let wal_op = match op { + StorageOp::Upsert { table, rows } => WalOp::InsertBatch { + table: table.clone(), + rows: rows.clone(), + }, + StorageOp::Delete { table, ids } => WalOp::DeleteKeys { + table: table.clone(), + keys: ids.clone(), + }, + StorageOp::Replace { table, rows } => WalOp::ReplaceAll { + table: table.clone(), + rows: rows.clone(), + }, + }; + self.validate_op(&wal_op)?; + wal_ops.push(wal_op); + } + self.group_via_wal(&wal_ops) + } + + /// 一组操作加提交标记落盘,成功后清空日志 + fn group_via_wal(&self, ops: &[WalOp]) -> std::io::Result<()> { std::fs::create_dir_all(&self.cfg.data_dir)?; let wal = &self.wal; - wal.truncate()?; - - wal.append(op)?; - - let result = self.apply_op(op); - - if result.is_err() && matches!(op, WalOp::HideColumn { .. }) { - self.recovery_required.store(true, Ordering::Release); - return result; + let commit_lsn = wal.append_group(ops)?; + for op in ops { + let result = self.apply_op(op); + if result.is_err() && matches!(op, WalOp::HideColumn { .. }) { + self.recovery_required.store(true, Ordering::Release); + return result; + } + result?; } - - if result.is_ok() { + for op in ops { self.flush_op_tables(op)?; } - + self.flush_all_tables()?; wal.clear()?; - result + log_debug!(core, "WAL committed group at LSN {}", commit_lsn); + Ok(()) + } + + /// 把打开的表都 fsync 一遍 + fn flush_all_tables(&self) -> std::io::Result<()> { + let handles: Vec>> = self + .tables + .lock() + .map_err(|_| std::io::Error::other("table lock poisoned"))? + .values() + .cloned() + .collect(); + for handle in handles { + handle.lock().map_err(|_| std::io::Error::other("heap lock poisoned"))?.flush()?; + } + Ok(()) } /// 把涉及的表写回磁盘 fn flush_op_tables(&self, op: &WalOp) -> std::io::Result<()> { let table = match op { WalOp::HideColumn { .. } => return Ok(()), + WalOp::Commit => return Ok(()), WalOp::Accounts { .. } => USERS_TABLE, WalOp::Insert { table, .. } | WalOp::InsertBatch { table, .. } @@ -369,21 +438,97 @@ impl Server { self.with_table(table, |t| t.flush()) } - /// 启动时重放残留 WAL + /// 重放残留 WAL:已提交组重放,未提交组丢弃 fn recover_wal(&self) -> std::io::Result<()> { std::fs::create_dir_all(&self.cfg.data_dir)?; let wal = &self.wal; - match wal.read()? { - None => Ok(()), - Some(op) => { - log_warn!(core, "replaying incomplete WAL entry"); - self.apply_op(&op)?; - self.flush_op_tables(&op)?; - wal.clear()?; - log_info!(core, "WAL replay complete"); - Ok(()) + let scan = wal.read_all()?; + if scan.records.is_empty() { + if scan.good_len > 0 { + wal.truncate_to(scan.good_len)?; } + return Ok(()); } + let records = committed_ops(&scan.records); + if records.is_empty() { + log_warn!(core, "discarding uncommitted WAL records"); + wal.clear()?; + return Ok(()); + } + let max_lsn = scan + .records + .iter() + .rev() + .find(|(_, op)| *op == WalOp::Commit) + .map(|(lsn, _)| *lsn) + .or_else(|| records.last().map(|(lsn, _)| *lsn)) + .unwrap_or(0); + log_warn!(core, "replaying {} committed WAL records", records.len()); + for (_, op) in &records { + self.replay_op(op)?; + self.flush_op_tables(op)?; + } + self.flush_all_tables()?; + wal.write_checkpoint(max_lsn)?; + wal.clear()?; + log_info!(core, "WAL replay complete"); + Ok(()) + } + + /// 重放一条操作:数据已经生效的不再重复写 + fn replay_op(&self, op: &WalOp) -> std::io::Result<()> { + if self.op_already_applied(op)? { + log_debug!(core, "WAL record already applied, skipping"); + return Ok(()); + } + self.apply_op(op) + } + + /// 这条操作对应的数据是不是已经落盘了 + fn op_already_applied(&self, op: &WalOp) -> std::io::Result { + match op { + WalOp::Insert { table, row } => self.applied_rows(table, std::slice::from_ref(row)), + WalOp::InsertBatch { table, rows } => self.applied_rows(table, rows), + WalOp::ReplaceAll { table, rows } => { + if self.catalog.lock().unwrap().describe(table).is_none() { + return Ok(false); + } + let stored = self.with_table(table, |t| t.scan())?; + Ok(stored == *rows) + } + WalOp::DeleteKeys { table, keys } => { + if keys.is_empty() { + return Ok(true); + } + if self.catalog.lock().unwrap().describe(table).is_none() { + return Ok(false); + } + let rows = self.with_table(table, |t| t.scan())?; + let present = rows.iter().any(|row| { + row.get("id") + .and_then(scalar_int) + .is_some_and(|id| keys.contains(&id)) + }); + Ok(!present) + } + WalOp::HideColumn { table, column } => { + let c = self.catalog.lock().unwrap(); + Ok(c.describe(table).is_some_and(|s| !s.columns.iter().any(|col| &col.name == column))) + } + _ => Ok(false), + } + } + + /// 这些行是不是都已经在表里 + fn applied_rows(&self, table: &str, rows: &[Row]) -> std::io::Result { + if rows.is_empty() { + return Ok(true); + } + if self.catalog.lock().unwrap().describe(table).is_none() { + return Ok(false); + } + let stored = self.with_table(table, |t| t.scan())?; + Ok(rows.iter().all(|row| stored.iter().any(|one| one == row))) } /// 启动时重建列统计 @@ -607,6 +752,11 @@ impl Server { .unwrap_or(false) } + /// 最近一次落盘的检查点 LSN + fn last_checkpoint(&self) -> u64 { + self.wal.read_checkpoint().ok().flatten().unwrap_or(0) + } + /// 读账号表;表没登记或数据文件缺失都算错 fn load_accounts(&self) -> std::io::Result> { use std::io::Error; @@ -774,16 +924,29 @@ impl Server { } Request::BootstrapSystem { user, password_hash } => { match self.bootstrap_system(user.as_deref(), password_hash.as_deref()) { - Ok(_) => Response::System { initialized: true }, + Ok(_) => Response::System { initialized: true, last_lsn: self.last_checkpoint() }, Err(e) => Response::Error { message: e.to_string() }, } } - Request::SystemStatus => Response::System { initialized: self.system_initialized() }, + Request::SystemStatus => Response::System { initialized: self.system_initialized(), last_lsn: self.last_checkpoint() }, Request::Ping => { log_debug!(request, "ping"); Response::Pong } + Request::ApplyTransaction { ops } => { + log_debug!(request, "apply_transaction ops={}", ops.len()); + match self.apply_transaction(&ops) { + Ok(()) => Response::Ok, + Err(e) => { + log_warn!(request, "apply_transaction: {}", e); + Response::Error { + message: format!("apply_transaction: {e}"), + } + } + } + } + Request::Scan { table, columns } => { log_debug!(request, "scan table={}", table); @@ -1347,8 +1510,8 @@ fn reserved_database(name: &str) -> bool { /// 每个库拥有独立的目录、字典、句柄和 WAL。 struct Databases { root: Config, - system: Server, - named: Mutex>, + system: Arc, + named: Mutex>>, } impl Databases { @@ -1356,7 +1519,7 @@ impl Databases { fn new(cfg: Config) -> std::io::Result { // 系统库按需创建:目录、字典与系统表缺失时自动补齐。 // 其余库只在 data/databases// 下已存在时才装载,启动不会凭空建库。 - let system = Self::open(&cfg, SYSTEM_DATABASE)?; + let system = Arc::new(Self::open(&cfg, SYSTEM_DATABASE)?); let root = cfg.data_dir.join("databases"); std::fs::create_dir_all(&root)?; let canonical_root = std::fs::canonicalize(&root)?; @@ -1379,7 +1542,7 @@ impl Databases { server.rebuild_stats()?; server.migrate_data()?; server.compact_tables()?; - named.insert(name, server); + named.insert(name, Arc::new(server)); } } Ok(Self { root: cfg, system, named: Mutex::new(named) }) @@ -1413,9 +1576,13 @@ impl Databases { let mut named = self.named.lock().map_err(|_| Error::other("database lock poisoned"))?; match method.as_str() { "all_catalogs" => { + let servers: Vec<(String, Arc)> = + std::iter::once((SYSTEM_DATABASE.to_string(), Arc::clone(&self.system))) + .chain(named.iter().map(|(name, server)| (name.clone(), Arc::clone(server)))) + .collect(); + drop(named); let mut schemas = Vec::new(); - for (database, server) in std::iter::once((SYSTEM_DATABASE, &self.system)) - .chain(named.iter().map(|(name, server)| (name.as_str(), server))) { + for (database, server) in servers { match server.handle_request(Request::ListCatalog) { Response::Catalog { schemas: entries } => { for mut entry in entries { @@ -1442,23 +1609,30 @@ impl Databases { std::fs::create_dir(&cfg.data_dir)?; let server = Server::new(cfg)?; server.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.save(server.catalog_path())?; - named.insert(name, server); + named.insert(name, Arc::new(server)); return Ok(Response::Ok); } "drop_database" => { if name == SYSTEM_DATABASE { return Err(Error::other("cannot drop the system database")); } - let server = named.get(&name).ok_or_else(|| Error::other("unknown database"))?; + let holder = named.remove(&name).ok_or_else(|| Error::other("unknown database"))?; + let server = match Arc::try_unwrap(holder) { + Ok(server) => server, + Err(holder) => { + named.insert(name, holder); + return Err(Error::other("database is in use")); + } + }; let root = std::fs::canonicalize(self.root.data_dir.join("databases"))?; let target = std::fs::canonicalize(&server.cfg.data_dir)?; if target.parent() != Some(root.as_path()) || !target.join("catalog.json").is_file() { return Err(Error::other("invalid database directory")); } let config = server.cfg.clone(); - drop(named.remove(&name)); + drop(server); let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_err(Error::other)?.as_nanos(); let tombstone = root.join(format!(".dropped-{name}-{stamp}")); if let Err(error) = std::fs::rename(&target, &tombstone) { - named.insert(name, Server::new(config)?); + named.insert(name, Arc::new(Server::new(config)?)); return Err(error); } std::fs::remove_dir_all(&tombstone).map_err(|e| Error::other(format!("database dropped; directory cleanup failed: {e}")))?; @@ -1482,10 +1656,10 @@ impl Databases { | Request::AccountReset { .. } | Request::AccountLogin { .. } | Request::AccountDrop { .. } | Request::BootstrapSystem { .. } | Request::SystemStatus | Request::Ping); - let server = if global { &self.system } - else if selected.is_empty() { return Err(Error::other("no database selected")); } - else if selected == SYSTEM_DATABASE { &self.system } - else { named.get(&selected).ok_or_else(|| Error::other("unknown database"))? }; + if !global && selected.is_empty() { return Err(Error::other("no database selected")); } + let server = if global || selected == SYSTEM_DATABASE { Arc::clone(&self.system) } + else { Arc::clone(named.get(&selected).ok_or_else(|| Error::other("unknown database"))?) }; + drop(named); Ok(server.handle_request(request)) } } diff --git a/chusql-core/storage/src/wal.rs b/chusql-core/storage/src/wal.rs index ecae033..ca403da 100644 --- a/chusql-core/storage/src/wal.rs +++ b/chusql-core/storage/src/wal.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use crate::protocol::{Account, Row, SchemaColumn, USERS_TABLE}; -// 预写日志:先记操作并 fsync,再改数据;重启重放残留。 +// 预写日志:先记操作并 fsync,再改数据;重启按 LSN 重放已提交的组。 const OP_INSERT: u8 = 1; const OP_REPLACE_ALL: u8 = 2; @@ -18,6 +18,7 @@ const OP_ACCOUNTS: u8 = 6; const OP_REPLACE_SCHEMA: u8 = 7; const OP_HIDE_COLUMN: u8 = 8; const OP_COMPACT: u8 = 9; +const OP_COMMIT: u8 = 10; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub enum WalOp { @@ -50,6 +51,8 @@ pub enum WalOp { columns: Vec, rows: Vec, }, + /// 一组写操作的提交标记;没有标记的组算未提交 + Commit, } #[derive(Debug, Serialize, Deserialize)] @@ -67,6 +70,13 @@ struct ReplaceSchemaPayload { pub struct Wal { path: PathBuf, handle: Mutex>, + next_lsn: Mutex, +} + +/// 一次扫描的结果:记录与完好字节长度 +pub struct WalScan { + pub records: Vec<(u64, WalOp)>, + pub good_len: usize, } impl Wal { @@ -75,6 +85,7 @@ impl Wal { Wal { path: path.as_ref().to_path_buf(), handle: Mutex::new(None), + next_lsn: Mutex::new(0), } } @@ -83,6 +94,11 @@ impl Wal { &self.path } + /// 检查点文件路径 + pub fn checkpoint_path(&self) -> PathBuf { + self.path.with_extension("checkpoint") + } + /// 拿常驻句柄(没有就开一个) fn with_handle(&self, f: impl FnOnce(&mut File) -> io::Result) -> io::Result { let mut slot = self.handle.lock().unwrap(); @@ -99,6 +115,15 @@ impl Wal { f(slot.as_mut().expect("just opened")) } + /// 把一段记录写到文件尾 + fn write_at_end(&self, bytes: &[u8]) -> io::Result<()> { + self.with_handle(|f| { + f.seek(SeekFrom::End(0))?; + f.write_all(bytes)?; + f.sync_data() + }) + } + /// 截断到空并落盘 pub fn clear(&self) -> io::Result<()> { self.with_handle(|f| { @@ -112,34 +137,136 @@ impl Wal { self.with_handle(truncate) } - /// 追加一条并 fsync - pub fn append(&self, op: &WalOp) -> io::Result<()> { - let bytes = encode(op)?; + /// 截到指定长度(去掉撕裂的尾巴) + pub fn truncate_to(&self, len: usize) -> io::Result<()> { self.with_handle(|f| { - f.write_all(&bytes)?; + f.set_len(len as u64)?; + f.seek(SeekFrom::Start(0))?; f.sync_data() }) } - /// 读当前 WAL;空则 None + /// 追加一条并 fsync + pub fn append(&self, op: &WalOp) -> io::Result<()> { + let bytes = self.encode_next(op)?; + self.write_at_end(&bytes) + } + + /// 追加一组操作与提交标记,返回提交 LSN + pub fn append_group(&self, ops: &[WalOp]) -> io::Result { + let mut frames = Vec::new(); + for op in ops { + frames.extend_from_slice(&self.encode_next(op)?); + } + let commit_lsn = self.take_lsn()?; + frames.extend_from_slice(&encode(&WalOp::Commit, commit_lsn)?); + self.write_at_end(&frames)?; + Ok(commit_lsn) + } + + /// 读当前 WAL 的第一条操作;空则 None pub fn read(&self) -> io::Result> { + let scan = self.read_all()?; + Ok(scan.records.into_iter().find_map(|(_, op)| { + if op == WalOp::Commit { + None + } else { + Some(op) + } + })) + } + + /// 读全部记录,撕裂或损坏的尾巴按长度截断 + pub fn read_all(&self) -> io::Result { if !self.path.exists() { - return Ok(None); + return Ok(WalScan { + records: Vec::new(), + good_len: 0, + }); } let mut f = File::open(&self.path)?; let mut buf = Vec::new(); f.read_to_end(&mut buf)?; - if buf.is_empty() { + let mut records = Vec::new(); + let mut offset = 0usize; + while offset + 4 <= buf.len() { + let body_len = + u32::from_le_bytes([buf[offset], buf[offset + 1], buf[offset + 2], buf[offset + 3]]) + as usize; + let end = offset + 4 + body_len; + if end > buf.len() { + break; + } + match decode_body(&buf[offset + 4..end]) { + Ok(record) => { + records.push(record); + offset = end; + } + Err(_) => break, + } + } + Ok(WalScan { + records, + good_len: offset, + }) + } + + /// 记录检查点:到此为止的 LSN 都已落盘 + pub fn write_checkpoint(&self, lsn: u64) -> io::Result<()> { + let mut f = File::create(self.checkpoint_path())?; + f.write_all(&lsn.to_le_bytes())?; + f.sync_all() + } + + /// 读检查点 LSN + pub fn read_checkpoint(&self) -> io::Result> { + let path = self.checkpoint_path(); + if !path.exists() { return Ok(None); } - match decode(&buf) { - Ok(opt) => Ok(opt), - Err(e) => Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("corrupt WAL: {}", e), - )), + let bytes = std::fs::read(&path)?; + if bytes.len() < 8 { + return Ok(None); + } + let mut raw = [0u8; 8]; + raw.copy_from_slice(&bytes[..8]); + Ok(Some(u64::from_le_bytes(raw))) + } + + /// 取下一条 LSN + fn take_lsn(&self) -> io::Result { + let mut slot = self.next_lsn.lock().unwrap(); + if *slot == 0 { + let last = self.read_all()?.records.last().map(|(lsn, _)| *lsn).unwrap_or(0); + *slot = last + 1; + } + let lsn = *slot; + *slot += 1; + Ok(lsn) + } + + /// 编一条带新 LSN 的记录 + fn encode_next(&self, op: &WalOp) -> io::Result> { + let lsn = self.take_lsn()?; + encode(op, lsn) + } +} + +/// 已提交的记录:提交标记前的整组;无标记的单条按旧格式算已提交 +pub fn committed_ops(records: &[(u64, WalOp)]) -> Vec<(u64, WalOp)> { + let mut out = Vec::new(); + let mut pending: Vec<(u64, WalOp)> = Vec::new(); + for (lsn, op) in records { + if *op == WalOp::Commit { + out.append(&mut pending); + } else { + pending.push((*lsn, op.clone())); } } + if pending.len() == 1 { + out.append(&mut pending); + } + out } /// 清空文件并把读写位置归零 @@ -150,7 +277,7 @@ fn truncate(f: &mut File) -> io::Result<()> { } /// 把操作编成字节 -fn encode(op: &WalOp) -> io::Result> { +fn encode(op: &WalOp, lsn: u64) -> io::Result> { let (op_type, table, payload) = match op { WalOp::HideColumn { table, column } => (OP_HIDE_COLUMN, table.clone(), serde_json::to_vec(column).map_err(io::Error::other)?), @@ -192,11 +319,11 @@ fn encode(op: &WalOp) -> io::Result> { .map_err(io::Error::other)?; (OP_REPLACE_SCHEMA, table.clone(), p) } + WalOp::Commit => (OP_COMMIT, String::new(), Vec::new()), }; let table_bytes = table.as_bytes(); - let lsn: u64 = 1; let mut body = Vec::with_capacity(8 + 1 + 2 + table_bytes.len() + payload.len()); body.extend_from_slice(&lsn.to_le_bytes()); body.push(op_type); @@ -210,20 +337,12 @@ fn encode(op: &WalOp) -> io::Result> { Ok(out) } -/// 从字节解回操作 -fn decode(buf: &[u8]) -> Result, String> { - if buf.len() < 4 { - return Ok(None); - } - let body_len = u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]) as usize; - if buf.len() < 4 + body_len { - return Ok(None); - } - let body = &buf[4..4 + body_len]; +/// 从一个帧体解出 LSN 与操作 +fn decode_body(body: &[u8]) -> Result<(u64, WalOp), String> { if body.len() < 8 + 1 + 2 { return Err("body too short".into()); } - let _lsn = u64::from_le_bytes([ + let lsn = u64::from_le_bytes([ body[0], body[1], body[2], body[3], body[4], body[5], body[6], body[7], ]); let op_type = body[8]; @@ -236,6 +355,7 @@ fn decode(buf: &[u8]) -> Result, String> { let payload = &body[11 + table_len..]; let op = match op_type { + OP_COMMIT => WalOp::Commit, OP_HIDE_COLUMN => { let column = serde_json::from_slice(payload) .map_err(|e| format!("bad hide_column payload: {e}"))?; @@ -292,5 +412,5 @@ fn decode(buf: &[u8]) -> Result, String> { } other => return Err(format!("unknown op type: {}", other)), }; - Ok(Some(op)) + Ok((lsn, op)) } diff --git a/chusql-core/storage/tests/runtime_test.rs b/chusql-core/storage/tests/runtime_test.rs index 45b9fe4..639315b 100644 --- a/chusql-core/storage/tests/runtime_test.rs +++ b/chusql-core/storage/tests/runtime_test.rs @@ -216,6 +216,67 @@ fn concurrent_shard_reads_exclude_writers() -> Result<(), Box Result<(), Box> { + let (_srv, _data) = start_server(); + let mut c = connect()?; + let request = |c: &mut Conn, value: serde_json::Value| -> Result { + serde_json::from_str(&send(c, &value.to_string())) + }; + let databases = ["alpha", "beta"]; + for database in databases { + assert_eq!(request(&mut c, serde_json::json!({"method":"create_database","database":database}))?["status"], "ok"); + assert_eq!( + request(&mut c, serde_json::json!({"method":"create_table","database":database,"table":"items","columns":[{"name":"id","ty":"int"}]}))?["status"], + "ok" + ); + } + + let storage = CURRENT.with(|slot| slot.borrow().as_ref().cloned().expect("storage opened")); + let written = 40; + std::thread::scope(|scope| { + let writers: Vec<_> = databases + .iter() + .map(|database| { + let storage = Arc::clone(&storage); + let database = database.to_string(); + scope.spawn(move || { + for i in 1..=written { + let line = format!( + r#"{{"method":"insert","database":"{database}","table":"items","row":{{"id":{i}}}}}"# + ); + let value: serde_json::Value = serde_json::from_str(&storage.request_line(&line)).expect("json"); + assert_eq!(value["status"], "ok", "{database}: {value}"); + } + database + }) + }) + .collect(); + let catalog = { + let storage = Arc::clone(&storage); + scope.spawn(move || { + for _ in 0..40 { + for line in [r#"{"method":"list_databases"}"#, r#"{"method":"all_catalogs"}"#] { + let value: serde_json::Value = serde_json::from_str(&storage.request_line(line)).expect("json"); + assert_ne!(value["status"], "error", "{line}: {value}"); + } + } + }) + }; + for writer in writers { + writer.join().expect("writer thread"); + } + catalog.join().expect("catalog thread"); + }); + + for database in databases { + let rows = request(&mut c, serde_json::json!({"method":"scan","database":database,"table":"items"}))?["rows"].clone(); + assert_eq!(rows.as_array().expect("rows").len(), written, "{database} 行数不对"); + } + Ok(()) +} + /// 删列不改堆字节,重启后不再暴露该列 #[test] fn drop_column_keeps_heap_bytes_and_hides_values_after_restart() -> Result<(), Box> { @@ -744,6 +805,12 @@ fn account_failure_preserves_wal_and_blocks_writes() -> Result<(), Box= 1, + "重放后检查点要推进: {status}" + ); Ok(()) } @@ -1015,6 +1082,125 @@ fn delete_keys_removes_row_and_index_entry() { assert!(r.contains(r#""status":"ok""#), "id 2 should be insertable again: {}", r); } +/// 事务提交:一批写操作在同一个请求里落地 +#[test] +fn apply_transaction_lands_upserts_and_deletes_together() { + let (_srv, _data) = start_server(); + let mut c = connect().unwrap(); + + for (id, name) in [(1, "Alice"), (2, "Bob")] { + let req = format!( + r#"{{"method":"insert","table":"txn_t","row":{{"id":{},"name":"{}"}}}}"#, + id, name + ); + send(&mut c, &req); + } + + let batch = r#"{"method":"apply_transaction","ops":[ + {"op":"delete","table":"txn_t","ids":[1]}, + {"op":"upsert","table":"txn_t","rows":[{"id":3,"name":"Carol"}]} + ]}"#; + let r = send(&mut c, batch); + assert!(r.contains(r#""status":"ok""#), "batch: {}", r); + + let r = send(&mut c, r#"{"method":"scan","table":"txn_t"}"#); + assert!(!r.contains("Alice"), "deleted row should be gone: {}", r); + assert!(r.contains("Bob") && r.contains("Carol"), "kept and added rows: {}", r); + + for line in [ + r#"{"method":"apply_transaction","ops":[{"op":"upsert","table":"__system_users","rows":[{"id":1}]}]}"#, + r#"{"method":"apply_transaction","ops":[{"op":"delete","table":"../evil","ids":[1]}]}"#, + ] { + let r = send(&mut c, line); + assert!(r.contains(r#""status":"error""#), "bad op rejected: {} -> {}", line, r); + } + + // 没有整数 id 的表用整表替换提交 + let r = send(&mut c, r#"{"method":"insert","table":"txn_no_id","row":{"name":"old"}}"#); + assert!(r.contains(r#""status":"ok""#), "insert: {}", r); + let replace = r#"{"method":"apply_transaction","ops":[{"op":"replace","table":"txn_no_id","rows":[{"name":"only"}]}]}"#; + let r = send(&mut c, replace); + assert!(r.contains(r#""status":"ok""#), "replace: {}", r); + let r = send(&mut c, r#"{"method":"scan","table":"txn_no_id"}"#); + assert!(r.contains("only") && !r.contains("old"), "replaced rows: {}", r); +} + +/// 标记已落盘、数据未改的崩溃现场:重启重放并推进检查点 +#[test] +fn committed_group_replays_after_restart() -> Result<(), Box> { + use chusql_core_storage::wal::{Wal, WalOp}; + let (server, data) = start_server(); + let mut c = connect()?; + request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":1,"secret":"old"}}))?; + drop(c); + drop(server); + + let mut added = std::collections::HashMap::new(); + added.insert("id".to_string(), serde_json::json!(2)); + added.insert("secret".to_string(), serde_json::json!("new")); + let group = vec![ + WalOp::InsertBatch { + table: "items".into(), + rows: vec![added.clone()], + }, + WalOp::DeleteKeys { + table: "items".into(), + keys: vec![1], + }, + ]; + let wal = Wal::new(data.path().join("databases/main/wal.log")); + let commit_lsn = wal.append_group(&group)?; + drop(wal); + + let _server = start_server_in(data.path()); + let mut c = connect()?; + let rows: serde_json::Value = + serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"items"}"#))?; + assert_eq!(rows["rows"], serde_json::json!([{"id":2,"secret":"new"}])); + assert_eq!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len(), 0); + let checkpoint = std::fs::read(data.path().join("databases/main/wal.checkpoint"))?; + assert_eq!(checkpoint.len(), 8, "检查点要落盘"); + assert_eq!( + u64::from_le_bytes(checkpoint[..8].try_into()?), + commit_lsn, + "检查点记到提交 LSN" + ); + Ok(()) +} + +/// 没有提交标记的记录组在重启时整组丢掉 +#[test] +fn uncommitted_group_is_discarded_on_restart() -> Result<(), Box> { + use chusql_core_storage::wal::{Wal, WalOp}; + let (server, data) = start_server(); + let mut c = connect()?; + request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":1,"secret":"old"}}))?; + drop(c); + drop(server); + + let mut lost = std::collections::HashMap::new(); + lost.insert("id".to_string(), serde_json::json!(2)); + lost.insert("secret".to_string(), serde_json::json!("lost")); + let wal = Wal::new(data.path().join("databases/main/wal.log")); + wal.append(&WalOp::InsertBatch { + table: "items".into(), + rows: vec![lost.clone()], + })?; + wal.append(&WalOp::InsertBatch { + table: "items".into(), + rows: vec![lost], + })?; + drop(wal); + + let _server = start_server_in(data.path()); + let mut c = connect()?; + let rows: serde_json::Value = + serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"items"}"#))?; + assert_eq!(rows["rows"], serde_json::json!([{"id":1,"secret":"old"}])); + assert_eq!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len(), 0); + Ok(()) +} + /// 给第二列建索引后能按那一列查 #[test] fn create_index_on_secondary_column() { diff --git a/chusql-core/storage/tests/wal_test.rs b/chusql-core/storage/tests/wal_test.rs index 38990f1..930f562 100644 --- a/chusql-core/storage/tests/wal_test.rs +++ b/chusql-core/storage/tests/wal_test.rs @@ -1,8 +1,8 @@ use chusql_core_storage::protocol::Row; -use chusql_core_storage::wal::{Wal, WalOp}; +use chusql_core_storage::wal::{committed_ops, Wal, WalOp}; use serde_json::json; -// WAL 测试:追加、读取、清空、截断尾巴。 +// WAL 测试:追加、读取、清空、截断尾巴、LSN 与提交标记。 /// 用键值对构造一行 @@ -99,3 +99,122 @@ fn truncated_tail_is_ignored() { assert!(wal.read().unwrap().is_none()); } +/// 每条记录带递增的 LSN,重开也从文件里的最大号往下走 +#[test] +fn every_record_gets_an_increasing_lsn() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("wal.log"); + let wal = Wal::new(&path); + + for id in 1..=3 { + wal.append(&WalOp::Insert { + table: "t".into(), + row: row(&[("id", json!(id))]), + }) + .unwrap(); + } + let scan = wal.read_all().unwrap(); + let lsns: Vec = scan.records.iter().map(|(lsn, _)| *lsn).collect(); + assert_eq!(lsns, vec![1, 2, 3]); + assert_eq!(scan.good_len as u64, std::fs::metadata(&path).unwrap().len()); + + let reopened = Wal::new(&path); + reopened + .append(&WalOp::Insert { + table: "t".into(), + row: row(&[("id", json!(4))]), + }) + .unwrap(); + assert_eq!(reopened.read_all().unwrap().records.last().unwrap().0, 4); +} + +/// 一组操作写完带提交标记,标记的 LSN 也在记录里 +#[test] +fn append_group_writes_a_commit_marker() { + let dir = tempfile::tempdir().unwrap(); + let wal = Wal::new(dir.path().join("wal.log")); + + let group = [ + WalOp::InsertBatch { + table: "t".into(), + rows: vec![row(&[("id", json!(2))])], + }, + WalOp::DeleteKeys { + table: "t".into(), + keys: vec![1], + }, + ]; + let commit_lsn = wal.append_group(&group).unwrap(); + + let scan = wal.read_all().unwrap(); + assert_eq!(scan.records.len(), 3); + assert_eq!(scan.records.last().unwrap().1, WalOp::Commit); + assert_eq!(scan.records.last().unwrap().0, commit_lsn); + assert_eq!(wal.read().unwrap().unwrap(), group[0]); + assert_eq!( + committed_ops(&scan.records), + vec![(1, group[0].clone()), (2, group[1].clone())] + ); +} + +/// 没有提交标记的记录组算未提交,旧格式的单条仍算已提交 +#[test] +fn records_without_a_commit_marker_stay_uncommitted() { + let dir = tempfile::tempdir().unwrap(); + let wal = Wal::new(dir.path().join("wal.log")); + + for id in 1..=2 { + wal.append(&WalOp::Insert { + table: "t".into(), + row: row(&[("id", json!(id))]), + }) + .unwrap(); + } + assert!(committed_ops(&wal.read_all().unwrap().records).is_empty()); + + wal.clear().unwrap(); + wal.append(&WalOp::Insert { + table: "t".into(), + row: row(&[("id", json!(9))]), + }) + .unwrap(); + assert_eq!(committed_ops(&wal.read_all().unwrap().records).len(), 1); +} + +/// 撕成两半的尾巴能读出完好长度并截掉 +#[test] +fn trailing_torn_frame_can_be_truncated() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("wal.log"); + let wal = Wal::new(&path); + + for id in 1..=2 { + wal.append(&WalOp::Insert { + table: "t".into(), + row: row(&[("id", json!(id))]), + }) + .unwrap(); + } + let full = std::fs::read(&path).unwrap(); + std::fs::write(&path, &full[..full.len() - 3]).unwrap(); + + let scan = wal.read_all().unwrap(); + assert_eq!(scan.records.len(), 1); + assert!(scan.good_len < full.len() - 3); + + wal.truncate_to(scan.good_len).unwrap(); + assert_eq!(std::fs::metadata(&path).unwrap().len(), scan.good_len as u64); + assert_eq!(wal.read_all().unwrap().records.len(), 1); +} + +/// 检查点写读一致 +#[test] +fn checkpoint_round_trips() { + let dir = tempfile::tempdir().unwrap(); + let wal = Wal::new(dir.path().join("wal.log")); + + assert_eq!(wal.read_checkpoint().unwrap(), None); + wal.write_checkpoint(7).unwrap(); + assert_eq!(wal.read_checkpoint().unwrap(), Some(7)); +} + diff --git a/chusql-server/src/ChuSQL/Server/Backend.hs b/chusql-server/src/ChuSQL/Server/Backend.hs index 728a8c4..5624cd2 100644 --- a/chusql-server/src/ChuSQL/Server/Backend.hs +++ b/chusql-server/src/ChuSQL/Server/Backend.hs @@ -6,7 +6,6 @@ module ChuSQL.Server.Backend ( columnsFromStatement, currentStamp, exprTables, - inDatabase, internalTable, grantsTable, ipcBackend, @@ -23,20 +22,23 @@ module ChuSQL.Server.Backend ( import ChuSQL.Core.Engine (runStatement, runStatementM) import ChuSQL.Core.Model import ChuSQL.Core.Engine.Semantic (prepare) -import ChuSQL.Core.Engine.Storage (MonadStorage (schema)) +import ChuSQL.Core.Engine.Storage (MonadStorage (schema, snapshot)) import ChuSQL.Core.Engine.Storage.IPC ( Account (..), + Env, IPCStorage (..), Request (..), Response (..), SchemaColumn (..), TableInfo (..), + envForDatabase, sendRawRequest, sendRequest, ) import ChuSQL.Core.Engine.Syntax.AST import ChuSQL.Core.Engine.Syntax.Parser (parseStatement) -import Control.Concurrent.MVar (MVar, modifyMVar, newMVar, readMVar, withMVar) +import ChuSQL.Core.Protocol (TxnOp (..)) +import Control.Concurrent.MVar (MVar, modifyMVar, readMVar) import Data.List (nub) import Data.Text (Text) import Data.Time (getCurrentTime) @@ -61,6 +63,10 @@ data Backend = Backend , beDatabases :: IO (Either String [String]) -- | 原样转发一条存储请求 , beStorage :: A.Value -> IO (Either String A.Value) + -- | 取一份整库快照,当事务的起点 + , beSnapshot :: IO (Either String Database) + -- | 事务提交:把一批写操作交给存储层应用 + , beApplyTransaction :: [TxnOp] -> IO (Either String ()) } -- | 按给定列名取列,固定列顺序 @@ -73,36 +79,46 @@ columnsFromStatement db stmt = case prepare db stmt of -- | 走本地存储链路的后端 ipcBackend :: IO Backend ipcBackend = do - lock <- newMVar () let backend current = Backend - { beStatement = \sql -> withMVar lock $ \_ -> runIpc current sql - , beCatalog = withMVar lock $ \_ -> do + { beStatement = \sql -> runIpc current sql + , beCatalog = do response <- sendRequest (ReqInDatabase current ReqListCatalog) pure $ case response of RespCatalog infos -> Right infos RespError err -> Left err _ -> Left "unexpected catalog response" - , bePing = withMVar lock $ \_ -> do + , bePing = do resp <- sendRequest ReqPing pure (case resp of RespPong -> True; _ -> False) - , beAccounts = \req -> withMVar lock $ \_ -> do + , beAccounts = \req -> do response <- sendRequest req pure $ case response of RespAccounts accounts -> Right accounts RespError err -> Left err _ -> Left "unexpected account response" , beWithDatabase = backend - , beDatabases = withMVar lock $ \_ -> do + , beDatabases = do response <- sendRequest (ReqDatabase "list_databases" "") pure $ case response of RespTables names -> Right names RespError err -> Left err _ -> Left "unexpected database response" - , beStorage = \payload -> withMVar lock $ \_ -> runRaw payload + , beStorage = \payload -> runRaw payload + , beSnapshot = Right <$> runIPCStorageIn (snapshot :: IPCStorage Database) (sessionEnv current) + , beApplyTransaction = \ops -> do + response <- sendRequest (ReqInDatabase current (ReqApplyTransaction ops)) + pure $ case response of + RespOk -> Right () + RespError err -> Left err + _ -> Left "unexpected transaction response" } -- 没有默认库:没选库时 current 是空串,任何请求都得先选库 pure (backend "") where + -- | 会话的存储环境:库名跟这条连接走 + sessionEnv :: String -> Env + sessionEnv current = envForDatabase (if null current then Nothing else Just current) + -- | 转发一条存储请求并解析响应 runRaw :: A.Value -> IO (Either String A.Value) runRaw payload = do @@ -113,18 +129,21 @@ ipcBackend = do Left err -> Left ("storage response is not valid JSON: " ++ err) Right value -> Right value - -- | 走存储链路跑一条语句 + -- | 走存储链路跑一条语句:表名不再补当前库前缀 + -- 引擎侧的数据字典按当前库过滤之后键是裸表名,补前缀会让查表落空; + -- 库名由 database 字段随请求带走,存储层自己按它分发。 runIpc :: String -> String -> IO (Either String StatementResult) - runIpc current sql = case fmap (inDatabase current) (parseStatement sql) of + runIpc current sql = case parseStatement sql of Left e -> pure (Left e) Right stmt -> do - result <- runIPCStorage (runStatementM stmt) + let env = sessionEnv current + result <- runIPCStorageIn (runStatementM stmt) env case result of Left e -> pure (Left e) Right rows -> do cols <- case rows of [] -> do - db <- runIPCStorage (schema :: IPCStorage Database) + db <- runIPCStorageIn (schema :: IPCStorage Database) env pure (columnsFromStatement db stmt) (r : _) -> pure (map fst r) pure (Right (StatementResult cols rows)) @@ -162,6 +181,10 @@ memoryBackendWith trusted name ref = , beWithDatabase = withDatabase , beDatabases = pure (Right (nub [name, systemDatabaseName])) , beStorage = \_ -> pure (Left "storage is not available in this backend") + , beSnapshot = Right . scoped <$> readMVar ref + , beApplyTransaction = \ops -> modifyMVar ref $ \db -> case applyTransactionMemory ops db of + Left err -> pure (db, Left err) + Right db' -> pure (db', Right ()) } where -- | 按视角过滤库里的表 @@ -185,6 +208,8 @@ unavailableDatabase :: String -> Backend -> Backend unavailableDatabase message backend = backend { beStatement = \_ -> pure (Left message) , beCatalog = pure (Left message) + , beSnapshot = pure (Left message) + , beApplyTransaction = \_ -> pure (Left message) } -- | 未选库时的视角:只放行不碰表的语句 @@ -196,76 +221,34 @@ noDatabaseBackend backend = backend | statementNeedsDatabase stmt -> pure (Left "no database selected") | otherwise -> beStatement backend sql , beCatalog = pure (Left "no database selected") + , beSnapshot = pure (Left "no database selected") + , beApplyTransaction = \_ -> pure (Left "no database selected") } --- | 给语句里的裸表名补上当前库前缀,子查询里的表名也一起改 -inDatabase :: String -> Statement -> Statement -inDatabase current = walk +-- | 把一批事务写操作应用到内存库:先按 id 删,再按追加写 +applyTransactionMemory :: [TxnOp] -> Database -> Either String Database +applyTransactionMemory ops db = foldl step (Right db) ops where - -- | 递归改写语句里的表名 - walk :: Statement -> Statement - walk stmt = case stmt of - q@Select{} -> - q - { selectFrom = source (selectFrom q) - , selectWhere = fmap expr (selectWhere q) - } - q@SelectExpr{} -> - q - { selectItems = [(label, expr e) | (label, e) <- selectItems q] - , selectFrom = source (selectFrom q) - , selectWhere = fmap expr (selectWhere q) - } - Insert t cols rows -> Insert (table t) cols (map (map expr) rows) - Delete t cond -> Delete (table t) (fmap expr cond) - Update t assigns cond -> Update (table t) [(c, expr e) | (c, e) <- assigns] (fmap expr cond) - CreateTable t cols -> CreateTable (table t) cols - DropTable t -> DropTable (table t) - CreateIndex t c -> CreateIndex (table t) c - DropIndex t c -> DropIndex (table t) c - DropColumn t c -> DropColumn (table t) c - AddColumn t def -> AddColumn (table t) def - RenameColumn t old new -> RenameColumn (table t) old new - AlterColumnType t c ty -> AlterColumnType (table t) c ty - AlterColumnDefault t c v -> AlterColumnDefault (table t) c v - AlterColumnNull t c b -> AlterColumnNull (table t) c b - other -> other + step acc op = do + current <- acc + case op of + TxnDelete name keys -> do + table <- lookupTable current name + pure (put (name, table {tableRows = [r | r <- tableRows table, not (doomed keys r)]}) current) + TxnUpsert name rows -> do + table <- lookupTable current name + pure (put (name, table {tableRows = tableRows table ++ rows}) current) + TxnReplace name rows -> do + table <- lookupTable current name + pure (put (name, table {tableRows = rows}) current) - -- | 给单个表名补库前缀,已限定或没选库就原样 - table t | '.' `elem` t || null current = t - | otherwise = current ++ "." ++ t + -- | 换掉一张表,其余保持原顺序 + put entry = map (\one@(name, _) -> if name == fst entry then entry else one) - -- | 改写 FROM 子句 - source FromUnit = FromUnit - source (FromTable a t) = FromTable a (table t) - source (FromJoin k l a t e) = FromJoin k (source l) a (table t) (expr e) - - -- | 改写表达式里的子查询 - expr (ScalarSub sq) = ScalarSub (sub sq) - expr (ExistsSub sq neg) = ExistsSub (sub sq) neg - expr (InSub a sq neg) = InSub (expr a) (sub sq) neg - expr (InList a items neg) = InList (expr a) (map expr items) neg - expr (Add a b) = Add (expr a) (expr b) - expr (Sub a b) = Sub (expr a) (expr b) - expr (Mul a b) = Mul (expr a) (expr b) - expr (Div a b) = Div (expr a) (expr b) - expr (Neg a) = Neg (expr a) - expr (Gt a b) = Gt (expr a) (expr b) - expr (Lt a b) = Lt (expr a) (expr b) - expr (Eq a b) = Eq (expr a) (expr b) - expr (And a b) = And (expr a) (expr b) - expr (Or a b) = Or (expr a) (expr b) - expr (IsNull a) = IsNull (expr a) - expr (IsNotNull a) = IsNotNull (expr a) - expr (CountOf a) = CountOf (expr a) - expr (SumOf a) = SumOf (expr a) - expr (AvgOf a) = AvgOf (expr a) - expr (MinOf a) = MinOf (expr a) - expr (MaxOf a) = MaxOf (expr a) - expr other = other - - -- | 改写一个子查询 - sub sq = sq {subqueryStatement = walk (subqueryStatement sq)} + -- | 这一行要不要删:按行里的 id 列比 + doomed keys r = case lookup "id" r of + Just (VInt k) -> k `elem` keys + _ -> False -- | 语句里出现的表 statementTables :: Statement -> [Text] diff --git a/chusql-server/src/ChuSQL/Server/Session.hs b/chusql-server/src/ChuSQL/Server/Session.hs index 6c1103c..8d13334 100644 --- a/chusql-server/src/ChuSQL/Server/Session.hs +++ b/chusql-server/src/ChuSQL/Server/Session.hs @@ -24,22 +24,24 @@ module ChuSQL.Server.Session , queryResultJson ) where -import ChuSQL.Core.Model (Value (..)) -import ChuSQL.Core.Protocol (Account, QueryResult (..), queryResultJson) +import ChuSQL.Core.Model (Database, Row, Table (..), Value (..)) +import ChuSQL.Core.Protocol (Account, QueryResult (..), TxnOp (..), queryResultJson) import ChuSQL.Core.Engine.Storage.IPC (TableInfo (..)) +import qualified ChuSQL.Core.Engine as Engine import qualified ChuSQL.Core.Engine.Error as E import ChuSQL.Core.Engine.Syntax.AST (Statement (..)) import ChuSQL.Core.Engine.Syntax.Parser (parseStatement) import ChuSQL.Interface.AccountTable (systemTableInfo) import ChuSQL.Interface.Auth (defaultSessionPolicy, newSessionStore) import ChuSQL.Server.Accounts -import ChuSQL.Server.Backend (Backend (..), StatementResult (..)) +import ChuSQL.Server.Backend (Backend (..), StatementResult (..), columnsFromStatement) import ChuSQL.Server.Policy (configurePasswordPolicy) import ChuSQL.Server.Privileges (PrivilegeCommand, PrivilegeError (..), Privileges, RoleView, authorize, filterTables, listRoleViews, newPrivileges, privilegeCommand, runPrivilegeCommand) import Control.Exception (IOException, try) import Data.Bifunctor (first) import Data.Char (isAlpha, isAlphaNum) import Data.IORef (IORef, newIORef, readIORef, writeIORef) +import Data.Maybe (isJust) import Data.Text (Text) import qualified Data.Text as T import Data.Time.Clock (getCurrentTime) @@ -64,6 +66,13 @@ data Session = Session , ssCurrent :: IORef Text , ssPrincipal :: IORef (Maybe Principal) , ssSettingsFile :: FilePath + , ssTransaction :: IORef (Maybe Transaction) + } + +-- | 一条会话里的显式事务:开事务时的快照与事务内改到的状态 +data Transaction = Transaction + { txBase :: Database + , txStaged :: Database } -- | 开一个会话:账号与角色服务、配置策略都和 Web 端一致 @@ -76,7 +85,8 @@ newSession backend name settingsFile = do -- 服务启动后只有系统库:没 USE 之前不预设任何工作库 current <- newIORef "" principal <- newIORef Nothing - pure (Session backend accountsService privileges current principal settingsFile) + transaction <- newIORef Nothing + pure (Session backend accountsService privileges current principal settingsFile transaction) -- | 当前生效的口令策略(只有管理员能看) policyOf :: Session -> IO (Either Text PasswordPolicy) @@ -168,22 +178,183 @@ runStatement session sql = fmap (first sessMessage) (runStatementCoded session s runStatementCoded :: Session -> Text -> IO (Either SessionError QueryResult) runStatementCoded session sql = case parseStatement (T.unpack sql) of Right (UseDatabase name) -> do - switched <- switchDatabaseCoded session (T.pack name) - case switched of - Left err -> pure (Left err) - Right () -> do - current <- readIORef (ssCurrent session) - pure (Right (emptyResult (Just current))) + existing <- readIORef (ssTransaction session) + case existing of + Just _ -> pure (Left (SessionError "query_error" "cannot switch database inside a transaction")) + Nothing -> do + switched <- switchDatabaseCoded session (T.pack name) + case switched of + Left err -> pure (Left err) + Right () -> do + current <- readIORef (ssCurrent session) + pure (Right (emptyResult (Just current))) + Right BeginTransaction -> beginTransaction session + Right CommitTransaction -> commitTransaction session + Right RollbackTransaction -> rollbackTransaction session Right statement | Just command <- accountCommand statement -> runAccount session command | Just command <- privilegeCommand statement -> runPrivilege session command | otherwise -> do - allowed <- authorized session statement - case allowed of - Left err -> pure (Left err) - Right () -> runPlain session sql + transaction <- readIORef (ssTransaction session) + if isJust transaction && transactionDdl statement + then pure (Left (SessionError "query_error" "DDL is not allowed in a transaction")) + else do + allowed <- authorized session statement + case allowed of + Left err -> pure (Left err) + Right () + | isJust transaction && transactionData statement -> runInTransaction session statement + | otherwise -> runPlain session sql Left _ -> runPlain session sql +-- | 事务里不许改结构,会话层直接拦下 +transactionDdl :: Statement -> Bool +transactionDdl statement = case statement of + CreateTable{} -> True + DropTable{} -> True + CreateIndex{} -> True + DropIndex{} -> True + DropColumn{} -> True + AddColumn{} -> True + RenameColumn{} -> True + AlterColumnType{} -> True + AlterColumnDefault{} -> True + AlterColumnNull{} -> True + CreateDatabase{} -> True + DropDatabase{} -> True + _ -> False + +-- | 事务里只有数据语句走快照 +transactionData :: Statement -> Bool +transactionData statement = case statement of + Select{} -> True + SelectExpr{} -> True + Insert{} -> True + Update{} -> True + Delete{} -> True + _ -> False + +-- | 开事务:把当前库整库做一份快照当起点 +beginTransaction :: Session -> IO (Either SessionError QueryResult) +beginTransaction session = do + current <- readIORef (ssCurrent session) + existing <- readIORef (ssTransaction session) + case existing of + Just _ -> pure (Left (SessionError "query_error" "already in a transaction")) + Nothing + | T.null current -> pure (Left (SessionError "no_database" "no database selected")) + | otherwise -> do + backend <- currentBackend session + snapshotResult <- beSnapshot backend + case snapshotResult of + Left err -> pure (Left (engineErrorCode err)) + Right db -> do + writeIORef (ssTransaction session) (Just (Transaction db db)) + pure (Right (emptyResult (Just current))) + +-- | 提交:把快照差分交给存储层应用,成功才结束事务 +commitTransaction :: Session -> IO (Either SessionError QueryResult) +commitTransaction session = do + current <- readIORef (ssCurrent session) + existing <- readIORef (ssTransaction session) + case existing of + Nothing -> pure (Left (SessionError "query_error" "no transaction in progress")) + Just transaction -> do + backend <- currentBackend session + liveResult <- beSnapshot backend + case liveResult of + Left err -> pure (Left (engineErrorCode err)) + Right live -> do + applied <- beApplyTransaction backend (transactionOps (txBase transaction) (txStaged transaction) live) + case applied of + Left err -> pure (Left (engineErrorCode err)) + Right () -> do + writeIORef (ssTransaction session) Nothing + pure (Right (emptyResult (Just current))) + +-- | 回滚:快照丢掉,存储层没被改过 +rollbackTransaction :: Session -> IO (Either SessionError QueryResult) +rollbackTransaction session = do + current <- readIORef (ssCurrent session) + existing <- readIORef (ssTransaction session) + case existing of + Nothing -> pure (Left (SessionError "query_error" "no transaction in progress")) + Just _ -> do + writeIORef (ssTransaction session) Nothing + pure (Right (emptyResult (Just current))) + +-- | 事务里跑一条数据语句:只动内存快照,不碰存储层 +runInTransaction :: Session -> Statement -> IO (Either SessionError QueryResult) +runInTransaction session statement = do + existing <- readIORef (ssTransaction session) + case existing of + Nothing -> pure (Left (SessionError "query_error" "no transaction in progress")) + Just transaction -> case Engine.runStatement (txStaged transaction) statement of + Left err -> pure (Left (engineErrorCode err)) + Right (staged, rows) -> do + writeIORef (ssTransaction session) (Just transaction {txStaged = staged}) + pure (Right (resultOfStaged (txStaged transaction) statement rows)) + +-- | 事务内的结果集:空结果回落到解析出来的列名 +resultOfStaged :: Database -> Statement -> [Row] -> QueryResult +resultOfStaged db statement rows = resultOf (StatementResult cols rows) + where + cols = case rows of + [] -> columnsFromStatement db statement + (r : _) -> map fst r + +-- | 提交差分:相对快照列出要删、要写与要替换的行 +transactionOps :: Database -> Database -> Database -> [TxnOp] +transactionOps base staged live = + concatMap opsFor names + where + -- | 事务里碰得着的表:数据语句不增删表 + names = nubKeys (map fst base ++ map fst staged) + + -- | 一张表的差分 + opsFor name + | all (isJust . rowKey) (baseRows ++ stagedRows) = keyedOps + | stagedRows == baseRows = [] + | otherwise = [TxnReplace name stagedRows] + where + baseRows = rowsOf name base + stagedRows = rowsOf name staged + liveRows = rowsOf name live + + liveByKey = [(k, r) | r <- liveRows, Just k <- [rowKey r]] + stagedByKey = [(k, r) | r <- stagedRows, Just k <- [rowKey r]] + baseKeys = [k | r <- baseRows, Just k <- [rowKey r]] + + -- 开事务时就存在、现在没了:删;内容变了的行也先删再写 + gone = [k | (k, _) <- liveByKey, k `elem` baseKeys, not (any ((== k) . fst) stagedByKey)] + rewritten = [k | (k, r) <- stagedByKey, Just liveRow <- [lookup k liveByKey], liveRow /= r] + deleted = gone ++ rewritten + + -- 新增的行,以及跟库里不一样的行;并发插进来的行不动 + written = + [ r + | (k, r) <- stagedByKey + , case lookup k liveByKey of + Nothing -> True + Just liveRow -> liveRow /= r + ] + + keyedOps = + [TxnDelete name deleted | not (null deleted)] + ++ [TxnUpsert name written | not (null written)] + + -- | 表名去重但保持顺序 + nubKeys = foldr (\name acc -> if name `elem` acc then acc else name : acc) [] + + -- | 库里某张表的行 + rowsOf name db = maybe [] tableRows (lookup name db) + +-- | 行 id:只有整数 id 才认,跟存储层删行的口径一致 +rowKey :: Row -> Maybe Int +rowKey row = case lookup "id" row of + Just (VInt k) -> Just k + _ -> Nothing + -- | 普通身份跑语句前的权限判定;管理员与未登录放行 authorized :: Session -> Statement -> IO (Either SessionError ()) authorized session statement = do diff --git a/chusql-server/test/Spec.hs b/chusql-server/test/Spec.hs index ff20890..85a05f0 100644 --- a/chusql-server/test/Spec.hs +++ b/chusql-server/test/Spec.hs @@ -3,14 +3,20 @@ module Main (main) where -import ChuSQL.Core.Engine.Storage.IPC (Account (..), Request (ReqAccountReset)) +import ChuSQL.Core.Engine.Storage.IPC ( + Account (..), + Request (ReqAccountReset), + closeConnection, + localStorageLink, + setStorageLink, + ) import ChuSQL.Core.Engine.Syntax.Parser (parseStatement) -import ChuSQL.Core.Model (Database, Table (..), Value (..), pattern TInt, pattern TStr) +import ChuSQL.Core.Model (Database, Row, Table (..), Value (..), pattern TInt, pattern TStr) import ChuSQL.Interface.Auth (hashPasswordWith) import ChuSQL.Interface.Protocol (ClientRequest (..), Grant (..), RoleView (..), ServerResponse (..), decodeRequest, encodeResponse) import ChuSQL.Interface.RateLimit (newRateLimiter) import ChuSQL.Server.Accounts (Principal (..), ensureRootAccount) -import ChuSQL.Server.Backend (Backend (..), memoryBackend) +import ChuSQL.Server.Backend (Backend (..), StatementResult (..), ipcBackend, memoryBackend) import ChuSQL.Server.Privileges ( PrivilegeCommand (..), PrivilegeError (..), @@ -20,7 +26,7 @@ import ChuSQL.Server.Privileges ( newPrivileges, runPrivilegeCommand, ) -import ChuSQL.Server.Session (QueryResult (..)) +import ChuSQL.Server.Session (QueryResult (..), Session, SessionError (..), newSession, runStatementCoded) import ChuSQL.Server.TCP ( ServerConfig (..), ServerEnv, @@ -30,8 +36,8 @@ import ChuSQL.Server.TCP ( newServerEnv, startServer, ) -import Control.Concurrent (threadDelay) -import Control.Concurrent.MVar (newMVar) +import Control.Concurrent (forkIO, threadDelay) +import Control.Concurrent.MVar (MVar, newEmptyMVar, newMVar, putMVar, takeMVar) import Control.Exception (IOException, bracket, finally, try) import Control.Monad (void) import Data.Aeson (decode) @@ -59,7 +65,7 @@ import Network.Socket ( socketToHandle, withSocketsDo, ) -import System.Directory (doesFileExist, getTemporaryDirectory, removeFile) +import System.Directory (createDirectoryIfMissing, doesFileExist, getTemporaryDirectory, removeFile, removePathForcibly) import System.FilePath (()) import System.IO ( BufferMode (LineBuffering), @@ -87,6 +93,8 @@ spec = do tcpSpec tcpServerSpec privilegeSpec + transactionSpec + ipcConcurrencySpec -- 夹具:内存后端装在 MVar 里,跟 Web 测试同一套表 @@ -545,3 +553,238 @@ privilegeSpec = describe "server privileges" $ do Left err -> expectationFailure err Right stmt -> authorize service root testDatabase stmt `shouldReturn` Right () +-- 真存储夹具:一个用例一份数据目录,跑完关链路删干净 + +-- | 本次用例的数据目录(用例串行跑,固定名够用) +ipcDataDir :: IO FilePath +ipcDataDir = do + tmp <- getTemporaryDirectory + pure (tmp "chusql-server-test-ipc") + +-- | 写一份只属于这次用例的存储配置:数据目录与日志级别都从文件走 +writeIpcConfig :: FilePath -> IO FilePath +writeIpcConfig dir = do + let cfgPath = dir ++ ".toml" + -- TOML 的普通字符串会吃反斜杠,路径统一用正斜杠 + slashed = map (\c -> if c == '\\' then '/' else c) dir + writeFile + cfgPath + ( unlines + [ "[storage]" + , "data_dir = \"" ++ slashed ++ "\"" + , "[log]" + , "level = \"error\"" + ] + ) + pure cfgPath + +-- | 删掉这次用例的数据目录与配置,删不掉也不算错 +cleanIpc :: FilePath -> IO () +cleanIpc dir = do + _ <- try (removePathForcibly dir) :: IO (Either IOException ()) + _ <- try (removePathForcibly (dir ++ ".toml")) :: IO (Either IOException ()) + pure () + +-- 事务用例:内存后端跑得快,真存储的提交再在 IPC 用例里验一遍 + +-- | 起一个内存后端会话,当前库已切到 test +withMemorySession :: (Session -> IO a) -> IO a +withMemorySession body = do + db <- newMVar testDb + settings <- tempSettingsPath "transaction" + removeIfExists settings + session <- newSession (memoryBackend testDatabaseName db) (T.pack testDatabaseName) settings + _ <- runStatementCoded session "use test" + body session + +-- | 两个会话共用一份内存库,用来验会话隔离 +withTwoSessions :: (Session -> Session -> IO a) -> IO a +withTwoSessions body = do + db <- newMVar testDb + settings <- tempSettingsPath "transaction-pair" + removeIfExists settings + let backend = memoryBackend testDatabaseName db + first <- newSession backend (T.pack testDatabaseName) settings + second <- newSession backend (T.pack testDatabaseName) settings + _ <- runStatementCoded first "use test" + _ <- runStatementCoded second "use test" + body first second + +-- | 跑一条语句;出错就算用例失败 +mustSql :: Session -> Text -> IO QueryResult +mustSql session sql = runStatementCoded session sql >>= either failed pure + where + failed err = expectationFailure (T.unpack (sessMessage err)) >> pure (QueryResult [] [] 0 False Nothing) + +-- | 取 users 的 id 列 +sessionIds :: Session -> IO [Int] +sessionIds session = do + result <- mustSql session "select * from users" + pure [n | row <- qrRows result, Just (VInt n) <- [nth 0 row]] + +-- | 取某个 id 的 age +sessionAge :: Session -> Int -> IO (Maybe Int) +sessionAge session wanted = do + result <- mustSql session "select * from users" + let ages = [(n, a) | row <- qrRows result, Just (VInt n) <- [nth 0 row], Just (VInt a) <- [nth 2 row]] + pure (lookup wanted ages) + +-- | 取失败里的文案 +failureMessage :: Either SessionError QueryResult -> Text +failureMessage outcome = either sessMessage (const "") outcome + +-- | 取结果第一列的整数 +intValues :: [[Value]] -> [Int] +intValues rows = [n | row <- rows, Just (VInt n) <- [nth 0 row]] + +-- | 显式事务用例 +transactionSpec :: Spec +transactionSpec = describe "server transactions" $ do + it "keeps uncommitted rows invisible to other sessions until COMMIT" $ withTwoSessions $ \writer reader -> do + _ <- mustSql writer "begin" + _ <- mustSql writer "insert into users (id, name, age) values (6, 'six', 26)" + sessionIds writer `shouldReturn` [1, 2, 3, 4, 5, 6] + sessionIds reader `shouldReturn` [1, 2, 3, 4, 5] + _ <- mustSql writer "commit" + ids <- sessionIds writer + ids `shouldMatchList` [1, 2, 3, 4, 5, 6] + seen <- sessionIds reader + seen `shouldMatchList` [1, 2, 3, 4, 5, 6] + + it "drops every change on ROLLBACK" $ withMemorySession $ \session -> do + _ <- mustSql session "begin" + _ <- mustSql session "insert into users (id, name, age) values (6, 'six', 26)" + _ <- mustSql session "update users set age = 99 where id = 1" + _ <- mustSql session "delete from users where id = 2" + rollback <- mustSql session "rollback" + qrRowCount rollback `shouldBe` 0 + kept <- sessionIds session + kept `shouldMatchList` [1, 2, 3, 4, 5] + + it "commits the whole write set at once" $ withMemorySession $ \session -> do + _ <- mustSql session "begin" + _ <- mustSql session "insert into users (id, name, age) values (6, 'six', 26)" + _ <- mustSql session "update users set age = 99 where id = 1" + _ <- mustSql session "delete from users where id = 2" + _ <- mustSql session "commit" + ids <- sessionIds session + ids `shouldMatchList` [1, 3, 4, 5, 6] + sessionAge session 1 `shouldReturn` Just 99 + + it "rejects DDL inside a transaction and takes it after ROLLBACK" $ withMemorySession $ \session -> do + _ <- mustSql session "begin" + denied <- runStatementCoded session "create table blocked (id int)" + failureMessage denied `shouldBe` "DDL is not allowed in a transaction" + _ <- mustSql session "rollback" + _ <- mustSql session "create table fresh (id int)" + _ <- mustSql session "insert into fresh (id) values (1)" + rows <- mustSql session "select * from fresh" + qrRowCount rows `shouldBe` 1 + + it "refuses nested BEGIN and COMMIT outside a transaction" $ withMemorySession $ \session -> do + _ <- mustSql session "begin" + nested <- runStatementCoded session "begin" + failureMessage nested `shouldBe` "already in a transaction" + _ <- mustSql session "rollback" + late <- runStatementCoded session "commit" + failureMessage late `shouldBe` "no transaction in progress" + + it "does not switch database inside a transaction" $ withMemorySession $ \session -> do + _ <- mustSql session "begin" + denied <- runStatementCoded session "use other" + failureMessage denied `shouldBe` "cannot switch database inside a transaction" + + it "needs a current database to start" $ do + db <- newMVar testDb + settings <- tempSettingsPath "transaction-bare" + removeIfExists settings + session <- newSession (memoryBackend testDatabaseName db) (T.pack testDatabaseName) settings + result <- runStatementCoded session "begin" + failureMessage result `shouldBe` "no database selected" + +-- | 起一份真存储并装成当前链路;动态库打不开就当用例待定 +withIpcStorage :: IO () -> IO () +withIpcStorage body = do + dir <- ipcDataDir + cleanIpc dir + createDirectoryIfMissing True dir + cfgPath <- writeIpcConfig dir + opened <- localStorageLink (Just cfgPath) + case opened of + Left err -> cleanIpc dir >> pendingWith ("cannot open the storage library: " ++ err) + Right link -> do + setStorageLink link + body `finally` (closeConnection >> cleanIpc dir) + +-- | 跑一条语句,出错就算用例失败 +mustRun :: IO (Either String StatementResult) -> IO () +mustRun action = action >>= either expectationFailure (const (pure ())) + +-- | 取结果行,出错就算用例失败 +rowsOf :: IO (Either String StatementResult) -> IO [Row] +rowsOf action = action >>= either (\err -> expectationFailure err >> pure []) (pure . srRows) + +-- | 取结果里某一列的整数 +intColumn :: String -> [Row] -> [Int] +intColumn name rows = [n | row <- rows, Just (VInt n) <- [lookup name row]] + +-- | 往 p 表连插一批行,结果放 MVar 交回主线程 +writeRows :: Backend -> [Int] -> MVar [Either String StatementResult] -> IO () +writeRows session ids done = do + outs <- mapM (\i -> beStatement session ("INSERT INTO p (id) VALUES (" ++ show i ++ ")")) ids + putMVar done outs + +-- | 同一进程里开两个库:会话之间不串数据,也不互相挡路 +ipcConcurrencySpec :: Spec +ipcConcurrencySpec = describe "server backend on the real storage" $ do + it "keeps two databases apart" $ withIpcStorage $ do + base <- ipcBackend + mustRun (beStatement base "CREATE DATABASE alpha") + mustRun (beStatement base "CREATE DATABASE beta") + let alpha = beWithDatabase base "alpha" + beta = beWithDatabase base "beta" + mustRun (beStatement alpha "CREATE TABLE t (id int)") + mustRun (beStatement beta "CREATE TABLE t (id int)") + mustRun (beStatement alpha "INSERT INTO t (id) VALUES (1)") + mustRun (beStatement beta "INSERT INTO t (id) VALUES (2)") + rowsA <- rowsOf (beStatement alpha "SELECT id FROM t") + rowsB <- rowsOf (beStatement beta "SELECT id FROM t") + intColumn "id" rowsA `shouldBe` [1] + intColumn "id" rowsB `shouldBe` [2] + + it "runs two sessions at once without mixing their rows" $ withIpcStorage $ do + base <- ipcBackend + mustRun (beStatement base "CREATE DATABASE alpha") + mustRun (beStatement base "CREATE DATABASE beta") + let alpha = beWithDatabase base "alpha" + beta = beWithDatabase base "beta" + mustRun (beStatement alpha "CREATE TABLE p (id int)") + mustRun (beStatement beta "CREATE TABLE p (id int)") + doneA <- newEmptyMVar + doneB <- newEmptyMVar + _ <- forkIO (writeRows alpha [1 .. 20] doneA) + _ <- forkIO (writeRows beta [101 .. 120] doneB) + outs <- sequence [takeMVar doneA, takeMVar doneB] + [err | Left err <- concat outs] `shouldBe` [] + rowsA <- rowsOf (beStatement alpha "SELECT id FROM p") + rowsB <- rowsOf (beStatement beta "SELECT id FROM p") + intColumn "id" rowsA `shouldMatchList` [1 .. 20] + intColumn "id" rowsB `shouldMatchList` [101 .. 120] + + it "commits an explicit transaction to the real storage" $ withIpcStorage $ do + base <- ipcBackend + mustRun (beStatement base "CREATE DATABASE alpha") + let alpha = beWithDatabase base "alpha" + settings <- tempSettingsPath "transaction-ipc" + removeIfExists settings + session <- newSession alpha (T.pack testDatabaseName) settings + _ <- mustSql session "use alpha" + _ <- mustSql session "create table t (id int)" + _ <- mustSql session "begin" + _ <- mustSql session "insert into t (id) values (1)" + inside <- mustSql session "select * from t" + intValues (qrRows inside) `shouldBe` [1] + _ <- mustSql session "commit" + rows <- rowsOf (beStatement alpha "SELECT id FROM t") + intColumn "id" rows `shouldBe` [1] + diff --git a/docs/architecture.md b/docs/architecture.md index 1e5a640..522ff04 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -45,7 +45,13 @@ chusql-server `chusql-server` 默认监听 `127.0.0.1:7777`,客户端建立连接后完成登录、数据库选择和 SQL 执行。 -每条连接拥有独立会话状态,例如当前数据库选择不会影响其他连接。 +每条连接拥有独立会话状态,例如当前数据库选择不会影响其他连接。不同连接可以同时执行语句:读请求之间可以并行,写请求之间由存储层排队。 + +语句里的库前缀和表限定符由引擎在语义层解析:存储层只接受精确的表名键,因此 `FROM sales.orders` 与选定 `sales` 后的 `FROM orders` 指向同一张表。 + +显式事务在会话层实现:`BEGIN` 把当前数据库整库读成内存快照,事务内的语句都在快照上执行;`COMMIT` 计算快照与库的差分,用一条 `apply_transaction` 请求交给存储层应用;`ROLLBACK` 直接丢弃快照。因此隔离级别是快照读加原子提交:事务里看不到别的事务未提交或后来提交的改动,提交对外一次性可见。 + +WAL 负责崩溃恢复:每条记录带自增 LSN,一组写操作与一条提交标记一次落盘;进程重启时按标记分组重放已提交的记录、丢弃没有标记的整组,重放前逐条做幂等检查,恢复结束把最大 LSN 写进 `wal.checkpoint` 并清空 WAL。`SystemStatus` 应答里的 `last_lsn` 报告该检查点 LSN。 系统数据库及部分管理操作仅允许管理员访问。 diff --git a/docs/commands.md b/docs/commands.md index eb21cfb..b78a175 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -38,6 +38,22 @@ csql -d sales -e "SELECT * FROM orders LIMIT 5" csql -f json -e "SELECT count(*) FROM orders" ``` +表名与列名的写法: + +- 选定数据库后语句里直接写表名即可,例如 `SELECT name FROM orders WHERE orders.id = 1`。 +- 表名可以带库前缀(`sales.orders`),列名可以带表限定符(`orders.id`)或库限定符(`sales.orders.id`),这几种写法指向同一张表和同一个列。 +- 只有候选唯一时才自动推导前缀;同名的表或列出现在多个候选上会以 `ambiguous table` / `ambiguous column` 报错,需要写全。 +- `FROM orders AS o` 之后只能用 `o.id`,不能再写 `orders.id`。 + +事务: + +- `BEGIN`(或 `START TRANSACTION`)开始显式事务,`COMMIT` 提交,`ROLLBACK` 回滚。 +- 事务内只允许数据语句;DDL 与 `USE` 会被拒绝,`SHOW DATABASES` 这类不碰表数据的语句照常可用。 +- 事务里的读都看开事务那一刻的快照:别的连接在事务期间提交的改动不会出现在事务里,本会话未提交的改动别的连接也看不见。 +- 同一连接同时只能有一个事务,没有保存点与嵌套事务;开事务前要先选好数据库。 +- 提交时按行 `id` 写回本事务改动过的行,事务没碰过的行不受影响;没有整数 `id` 的表按整表替换提交。 +- 提交是持久的:整批改动与一条提交标记一次落盘,提交后进程被强杀也会在重启时重放;提交前崩溃则整批丢弃,不会只落一半。 + ### 交互模式 SQL 语句以分号结束,可跨多行输入。