v26.10.2 修复bug,简化项目
This commit is contained in:
1 parent
ff3dbddd13
commit
fc9bbc20e4
26 files changed
+557
-299
No files matched your search
@@ -78,7 +78,9 @@ resources/chusql.ui.settings.json
|
|||||||
.\#*
|
.\#*
|
||||||
AGENTS.md
|
AGENTS.md
|
||||||
agent-credientials
|
agent-credientials
|
||||||
|
skills-lock.json
|
||||||
.dsh
|
.dsh
|
||||||
|
.agents
|
||||||
# 注意:.zed/tasks.json 是本仓库共享的构建任务(stack build / run / test / repl),
|
# 注意:.zed/tasks.json 是本仓库共享的构建任务(stack build / run / test / repl),
|
||||||
# 需要入库,所以这里不忽略 .zed/
|
# 需要入库,所以这里不忽略 .zed/
|
||||||
|
|
||||||
|
|||||||
@@ -32,10 +32,10 @@ runStatementM (DropDatabase name) = fmap (fmap (const [])) (dropDatabase name)
|
|||||||
runStatementM (UseDatabase name) = fmap (fmap (const [])) (useDatabase name)
|
runStatementM (UseDatabase name) = fmap (fmap (const [])) (useDatabase name)
|
||||||
runStatementM ShowDatabases = fmap (fmap (map (\name -> [("database", VStr name)]))) listDatabases
|
runStatementM ShowDatabases = fmap (fmap (map (\name -> [("database", VStr name)]))) listDatabases
|
||||||
runStatementM q = do
|
runStatementM q = do
|
||||||
db <- schema
|
result <- schema
|
||||||
case prepare db q of
|
case result >>= \db -> (,) db <$> prepare db q of
|
||||||
Left err -> pure (Left err)
|
Left err -> pure (Left err)
|
||||||
Right resolved -> runStatementUncheckedM db resolved
|
Right (db, resolved) -> runStatementUncheckedM db resolved
|
||||||
|
|
||||||
-- | 按语句类型分发(已检查过)
|
-- | 按语句类型分发(已检查过)
|
||||||
runStatementUncheckedM :: (MonadStorage m) => Database -> Statement -> m (Either String [Row])
|
runStatementUncheckedM :: (MonadStorage m) => Database -> Statement -> m (Either String [Row])
|
||||||
|
|||||||
@@ -6,8 +6,10 @@ module ChuSQL.Core.Engine.Parallel (
|
|||||||
) where
|
) where
|
||||||
|
|
||||||
import Control.Concurrent (forkIO)
|
import Control.Concurrent (forkIO)
|
||||||
import Control.Concurrent.MVar (MVar, modifyMVar, modifyMVar_, newEmptyMVar, newMVar, putMVar, takeMVar)
|
import Control.Concurrent.Chan (Chan, newChan, readChan, writeChan)
|
||||||
|
import Control.Concurrent.MVar (MVar, modifyMVar, newEmptyMVar, newMVar, putMVar, takeMVar)
|
||||||
import Control.Exception (SomeException, throwIO, try)
|
import Control.Exception (SomeException, throwIO, try)
|
||||||
|
import Control.Monad (forever)
|
||||||
import System.IO.Unsafe (unsafePerformIO)
|
import System.IO.Unsafe (unsafePerformIO)
|
||||||
|
|
||||||
-- 并行执行:常驻工作线程池,结果按提交顺序收回。
|
-- 并行执行:常驻工作线程池,结果按提交顺序收回。
|
||||||
@@ -16,11 +18,8 @@ import System.IO.Unsafe (unsafePerformIO)
|
|||||||
parallelShardLimit :: Int
|
parallelShardLimit :: Int
|
||||||
parallelShardLimit = 8
|
parallelShardLimit = 8
|
||||||
|
|
||||||
-- | 池:待办队列与唤醒信号
|
-- | 池:工作线程共享的待办队列
|
||||||
data Pool = Pool
|
newtype Pool = Pool (Chan (IO ()))
|
||||||
{ poolJobs :: MVar [IO ()]
|
|
||||||
, poolReady :: MVar ()
|
|
||||||
}
|
|
||||||
|
|
||||||
{-# NOINLINE poolRef #-}
|
{-# NOINLINE poolRef #-}
|
||||||
-- | 进程全局的池,按容量复用
|
-- | 进程全局的池,按容量复用
|
||||||
@@ -31,11 +30,14 @@ poolRef = unsafePerformIO (newMVar Nothing)
|
|||||||
workerPool :: Int -> IO Pool
|
workerPool :: Int -> IO Pool
|
||||||
workerPool size = modifyMVar poolRef reuse
|
workerPool size = modifyMVar poolRef reuse
|
||||||
where
|
where
|
||||||
-- | 现有池够大就留着,否则换新的
|
-- | 复用队列并按需补充工作线程
|
||||||
reuse current = case current of
|
reuse current = case current of
|
||||||
Just (capacity, pool)
|
Just (capacity, pool@(Pool jobs))
|
||||||
| capacity >= wanted -> pure (current, pool)
|
| capacity >= wanted -> pure (current, pool)
|
||||||
_ -> do
|
| otherwise -> do
|
||||||
|
sequence_ [forkIO (worker jobs) | _ <- [capacity + 1 .. wanted]]
|
||||||
|
pure (Just (wanted, pool), pool)
|
||||||
|
Nothing -> do
|
||||||
pool <- startPool wanted
|
pool <- startPool wanted
|
||||||
pure (Just (wanted, pool), pool)
|
pure (Just (wanted, pool), pool)
|
||||||
|
|
||||||
@@ -44,29 +46,17 @@ workerPool size = modifyMVar poolRef reuse
|
|||||||
|
|
||||||
-- | 起 wanted 个常驻线程吃队列
|
-- | 起 wanted 个常驻线程吃队列
|
||||||
startPool n = do
|
startPool n = do
|
||||||
jobs <- newMVar []
|
jobs <- newChan
|
||||||
ready <- newMVar ()
|
sequence_ [forkIO (worker jobs) | _ <- [1 .. n]]
|
||||||
sequence_ [forkIO (worker jobs ready) | _ <- [1 .. n]]
|
pure (Pool jobs)
|
||||||
pure (Pool jobs ready)
|
|
||||||
|
|
||||||
-- | 一个工作线程:被唤醒一次就取一个待办
|
-- | 一个工作线程:等待并执行队首待办
|
||||||
worker :: MVar [IO ()] -> MVar () -> IO ()
|
worker :: Chan (IO ()) -> IO ()
|
||||||
worker jobs ready = do
|
worker jobs = forever (readChan jobs >>= id)
|
||||||
takeMVar ready
|
|
||||||
job <- modifyMVar jobs pick
|
|
||||||
case job of
|
|
||||||
Nothing -> worker jobs ready
|
|
||||||
Just act -> act >> worker jobs ready
|
|
||||||
where
|
|
||||||
-- | 取队首,空队列就回 Nothing
|
|
||||||
pick [] = pure ([], Nothing)
|
|
||||||
pick (j : js) = pure (js, Just j)
|
|
||||||
|
|
||||||
-- | 往池里丢一个待办
|
-- | 往池里丢一个待办
|
||||||
submit :: Pool -> IO () -> IO ()
|
submit :: Pool -> IO () -> IO ()
|
||||||
submit pool job = do
|
submit (Pool jobs) = writeChan jobs
|
||||||
modifyMVar_ (poolJobs pool) (pure . (++ [job]))
|
|
||||||
putMVar (poolReady pool) ()
|
|
||||||
|
|
||||||
-- | 跑一个动作,结果或异常都填进格子
|
-- | 跑一个动作,结果或异常都填进格子
|
||||||
fillCell :: MVar (Either SomeException a) -> IO a -> IO ()
|
fillCell :: MVar (Either SomeException a) -> IO a -> IO ()
|
||||||
|
|||||||
@@ -32,12 +32,7 @@ builtinFunctions types = do
|
|||||||
|
|
||||||
-- | 按名称和完整参数类型解析函数
|
-- | 按名称和完整参数类型解析函数
|
||||||
resolveFunction :: FunctionRegistry -> String -> [TypeId] -> Either String FunctionSignature
|
resolveFunction :: FunctionRegistry -> String -> [TypeId] -> Either String FunctionSignature
|
||||||
resolveFunction (FunctionRegistry functions) name arguments =
|
resolveFunction registry name arguments = resolveNullableFunction registry name (map Just arguments)
|
||||||
case [sig | (sig, _) <- functions, map toLower (signatureName sig) == map toLower name
|
|
||||||
, signatureArguments sig == arguments] of
|
|
||||||
[sig] -> Right sig
|
|
||||||
[] -> Left ("no matching function signature: " ++ name)
|
|
||||||
_ -> Left ("ambiguous function signature: " ++ name)
|
|
||||||
|
|
||||||
-- | 按可空 SQL 参数解析确定签名
|
-- | 按可空 SQL 参数解析确定签名
|
||||||
resolveNullableFunction :: FunctionRegistry -> String -> [Maybe TypeId] -> Either String FunctionSignature
|
resolveNullableFunction :: FunctionRegistry -> String -> [Maybe TypeId] -> Either String FunctionSignature
|
||||||
|
|||||||
@@ -108,11 +108,11 @@ class (Monad m) => MonadStorage m where
|
|||||||
replaceSchema :: String -> [(String, Column)] -> [Row] -> m (Either String ())
|
replaceSchema :: String -> [(String, Column)] -> [Row] -> m (Either String ())
|
||||||
|
|
||||||
-- \| 取全库快照(表名 + 列 + 全部行)
|
-- \| 取全库快照(表名 + 列 + 全部行)
|
||||||
snapshot :: m Database
|
snapshot :: m (Either String Database)
|
||||||
|
|
||||||
-- \| 取全库结构:只要表名和列,不要行
|
-- \| 取全库结构:只要表名和列,不要行
|
||||||
schema :: m Database
|
schema :: m (Either String Database)
|
||||||
schema = fmap (map withoutRows) snapshot
|
schema = fmap (fmap (map withoutRows)) snapshot
|
||||||
where
|
where
|
||||||
-- | 表定义去掉所有行
|
-- | 表定义去掉所有行
|
||||||
withoutRows (name, t) = (name, t {tableRows = []})
|
withoutRows (name, t) = (name, t {tableRows = []})
|
||||||
@@ -10,7 +10,8 @@ module ChuSQL.Core.Engine.Storage.FFI (
|
|||||||
maintainStorage,
|
maintainStorage,
|
||||||
) where
|
) where
|
||||||
|
|
||||||
import Control.Exception (SomeException, try)
|
import Control.Concurrent (rtsSupportsBoundThreads, runInBoundThread)
|
||||||
|
import Control.Exception (SomeException, finally, mask_, try)
|
||||||
import Data.ByteString qualified as BS
|
import Data.ByteString qualified as BS
|
||||||
import Data.Text qualified as T
|
import Data.Text qualified as T
|
||||||
import Data.Text.Encoding qualified as TE
|
import Data.Text.Encoding qualified as TE
|
||||||
@@ -19,6 +20,7 @@ import Foreign
|
|||||||
import Foreign.C
|
import Foreign.C
|
||||||
|
|
||||||
-- Rust 存储库 cdylib 的 C ABI 绑定(ffi.rs),收发单行 JSON。
|
-- Rust 存储库 cdylib 的 C ABI 绑定(ffi.rs),收发单行 JSON。
|
||||||
|
-- 错误读取绑定同一 OS 线程;响应复制期间屏蔽取消并归还缓冲。
|
||||||
|
|
||||||
-- | 存储实例句柄(Rust 侧的 *mut Storage)
|
-- | 存储实例句柄(Rust 侧的 *mut Storage)
|
||||||
newtype StorageHandle = StorageHandle (Ptr ())
|
newtype StorageHandle = StorageHandle (Ptr ())
|
||||||
@@ -46,15 +48,8 @@ foreign import ccall safe "chusql_storage_maintenance"
|
|||||||
|
|
||||||
-- | 离线修复系统目录并返回结果
|
-- | 离线修复系统目录并返回结果
|
||||||
maintainStorage :: FilePath -> BS.ByteString -> IO (Either String BS.ByteString)
|
maintainStorage :: FilePath -> BS.ByteString -> IO (Either String BS.ByteString)
|
||||||
maintainStorage path payload = withCString path $ \config ->
|
maintainStorage path payload = BS.useAsCString (TE.encodeUtf8 (T.pack path)) $ \config ->
|
||||||
BS.useAsCStringLen payload $ \(input, size) -> alloca $ \outRef -> alloca $ \lenRef -> do
|
callStorage (cStorageMaintenance config) payload
|
||||||
code <- cStorageMaintenance config input (fromIntegral size) outRef lenRef
|
|
||||||
if code /= 0 then Left <$> lastError else do
|
|
||||||
output <- peek outRef
|
|
||||||
len <- peek lenRef
|
|
||||||
bytes <- BS.packCStringLen (output, fromIntegral len)
|
|
||||||
cStorageFree output len
|
|
||||||
pure (Right bytes)
|
|
||||||
|
|
||||||
-- | 库版本(Rust 侧的 CARGO_PKG_VERSION)
|
-- | 库版本(Rust 侧的 CARGO_PKG_VERSION)
|
||||||
storageVersion :: IO String
|
storageVersion :: IO String
|
||||||
@@ -62,7 +57,7 @@ storageVersion = cStorageVersion >>= peekCString
|
|||||||
|
|
||||||
-- | 打开存储实例,失败给错误文本
|
-- | 打开存储实例,失败给错误文本
|
||||||
openStorage :: Maybe FilePath -> IO (Either String StorageHandle)
|
openStorage :: Maybe FilePath -> IO (Either String StorageHandle)
|
||||||
openStorage path = do
|
openStorage path = onStorageThread $ mask_ $ do
|
||||||
result <- try (withCStringOrNull path cStorageOpen) :: IO (Either SomeException (Ptr ()))
|
result <- try (withCStringOrNull path cStorageOpen) :: IO (Either SomeException (Ptr ()))
|
||||||
case result of
|
case result of
|
||||||
Left e -> pure (Left ("storage library call failed: " ++ show e))
|
Left e -> pure (Left ("storage library call failed: " ++ show e))
|
||||||
@@ -72,7 +67,7 @@ openStorage path = do
|
|||||||
where
|
where
|
||||||
-- | 没有路径就传空指针,否则转成 C 字符串
|
-- | 没有路径就传空指针,否则转成 C 字符串
|
||||||
withCStringOrNull Nothing f = f nullPtr
|
withCStringOrNull Nothing f = f nullPtr
|
||||||
withCStringOrNull (Just s) f = withCString s f
|
withCStringOrNull (Just s) f = BS.useAsCString (TE.encodeUtf8 (T.pack s)) f
|
||||||
|
|
||||||
-- | 关掉存储实例
|
-- | 关掉存储实例
|
||||||
closeStorage :: StorageHandle -> IO ()
|
closeStorage :: StorageHandle -> IO ()
|
||||||
@@ -80,19 +75,25 @@ closeStorage (StorageHandle handle) = cStorageClose handle
|
|||||||
|
|
||||||
-- | 一条 JSON 请求换一条 JSON 响应
|
-- | 一条 JSON 请求换一条 JSON 响应
|
||||||
storageRequest :: StorageHandle -> BS.ByteString -> IO (Either String BS.ByteString)
|
storageRequest :: StorageHandle -> BS.ByteString -> IO (Either String BS.ByteString)
|
||||||
storageRequest (StorageHandle handle) payload =
|
storageRequest (StorageHandle handle) = callStorage (cStorageRequest handle)
|
||||||
BS.useAsCStringLen payload $ \(inPtr, inLen) ->
|
|
||||||
alloca $ \outPtrRef ->
|
-- | 调用存储并复制和归还响应缓冲
|
||||||
alloca $ \outLenRef -> do
|
callStorage :: (Ptr CChar -> CSize -> Ptr (Ptr CChar) -> Ptr CSize -> IO CInt)
|
||||||
code <- cStorageRequest handle (castPtr inPtr) (fromIntegral inLen) outPtrRef outLenRef
|
-> BS.ByteString -> IO (Either String BS.ByteString)
|
||||||
if code /= 0
|
callStorage call payload = onStorageThread $ mask_ $
|
||||||
then Left <$> lastError
|
BS.useAsCStringLen payload $ \(input, size) ->
|
||||||
else do
|
alloca $ \outRef -> alloca $ \lenRef -> do
|
||||||
outPtr <- peek outPtrRef
|
code <- call input (fromIntegral size) outRef lenRef
|
||||||
outLen <- peek outLenRef
|
if code /= 0 then Left <$> lastError else do
|
||||||
bytes <- BS.packCStringLen (castPtr outPtr, fromIntegral outLen)
|
output <- peek outRef
|
||||||
cStorageFree outPtr outLen
|
len <- peek lenRef
|
||||||
pure (Right bytes)
|
bytes <- BS.packCStringLen (output, fromIntegral len)
|
||||||
|
`finally` cStorageFree output len
|
||||||
|
pure (Right bytes)
|
||||||
|
|
||||||
|
-- | 多线程运行时绑定存储调用线程
|
||||||
|
onStorageThread :: IO a -> IO a
|
||||||
|
onStorageThread = if rtsSupportsBoundThreads then runInBoundThread else id
|
||||||
|
|
||||||
-- | 当前线程最近一次失败的文本
|
-- | 当前线程最近一次失败的文本
|
||||||
lastError :: IO String
|
lastError :: IO String
|
||||||
|
|||||||
@@ -36,7 +36,8 @@ import ChuSQL.Core.Engine.Parallel (parallelShardLimit, poolRun, workerPool)
|
|||||||
import ChuSQL.Core.Engine.Storage
|
import ChuSQL.Core.Engine.Storage
|
||||||
import ChuSQL.Core.Engine.Storage.FFI (closeStorage, openStorage, storageRequest)
|
import ChuSQL.Core.Engine.Storage.FFI (closeStorage, openStorage, storageRequest)
|
||||||
import Control.Concurrent (getNumCapabilities)
|
import Control.Concurrent (getNumCapabilities)
|
||||||
import Control.Concurrent.MVar (MVar, modifyMVar_, newMVar, readMVar)
|
import Control.Concurrent.MVar (MVar, modifyMVar_, newMVar, putMVar, readMVar, takeMVar, withMVar)
|
||||||
|
import Control.Exception (bracket, mask_)
|
||||||
import Data.Aeson (eitherDecodeStrict, encode)
|
import Data.Aeson (eitherDecodeStrict, encode)
|
||||||
import Data.ByteString qualified as BS
|
import Data.ByteString qualified as BS
|
||||||
import Data.ByteString.Lazy qualified as BL
|
import Data.ByteString.Lazy qualified as BL
|
||||||
@@ -89,9 +90,33 @@ data StorageLink = StorageLink
|
|||||||
linkRef :: MVar (Maybe StorageLink)
|
linkRef :: MVar (Maybe StorageLink)
|
||||||
linkRef = unsafePerformIO (newMVar Nothing)
|
linkRef = unsafePerformIO (newMVar Nothing)
|
||||||
|
|
||||||
|
{-# NOINLINE linkUsers #-}
|
||||||
|
-- | 在途请求计数与空闲信号
|
||||||
|
linkUsers :: (MVar Int, MVar ())
|
||||||
|
linkUsers = unsafePerformIO $ (,) <$> newMVar 0 <*> newMVar ()
|
||||||
|
|
||||||
|
-- | 登记请求并在结束时释放链路占用
|
||||||
|
withStorageLink :: (Maybe StorageLink -> IO a) -> IO a
|
||||||
|
withStorageLink = bracket enter leave
|
||||||
|
where
|
||||||
|
(users, idle) = linkUsers
|
||||||
|
-- | 登记当前链路的在途请求
|
||||||
|
enter = withMVar linkRef $ \current -> do
|
||||||
|
case current of
|
||||||
|
Nothing -> pure ()
|
||||||
|
Just _ -> modifyMVar_ users $ \count -> do
|
||||||
|
if count == 0 then takeMVar idle else pure ()
|
||||||
|
pure (count + 1)
|
||||||
|
pure current
|
||||||
|
-- | 最后一个请求结束后发出空闲信号
|
||||||
|
leave Nothing = pure ()
|
||||||
|
leave (Just _) = mask_ $ modifyMVar_ users $ \count -> do
|
||||||
|
if count == 1 then putMVar idle () else pure ()
|
||||||
|
pure (count - 1)
|
||||||
|
|
||||||
-- | 装入一条链路;一个进程只装一次
|
-- | 装入一条链路;一个进程只装一次
|
||||||
setStorageLink :: StorageLink -> IO ()
|
setStorageLink :: StorageLink -> IO ()
|
||||||
setStorageLink link = modifyMVar_ linkRef (const (pure (Just link)))
|
setStorageLink link = modifyMVar_ linkRef $ \_ -> withMVar (snd linkUsers) (const (pure (Just link)))
|
||||||
|
|
||||||
-- | 打开本地的存储动态库,得到一条链路
|
-- | 打开本地的存储动态库,得到一条链路
|
||||||
localStorageLink :: Maybe FilePath -> IO (Either String StorageLink)
|
localStorageLink :: Maybe FilePath -> IO (Either String StorageLink)
|
||||||
@@ -106,7 +131,7 @@ localStorageLink path = fmap (fmap toLink) (openStorage path)
|
|||||||
|
|
||||||
-- | 关掉当前链路(进程退出前调用);没有链路就什么都不做
|
-- | 关掉当前链路(进程退出前调用);没有链路就什么都不做
|
||||||
closeConnection :: IO ()
|
closeConnection :: IO ()
|
||||||
closeConnection = modifyMVar_ linkRef $ \current -> do
|
closeConnection = modifyMVar_ linkRef $ \current -> withMVar (snd linkUsers) $ \_ -> do
|
||||||
case current of
|
case current of
|
||||||
Nothing -> pure ()
|
Nothing -> pure ()
|
||||||
Just link -> slClose link
|
Just link -> slClose link
|
||||||
@@ -114,22 +139,19 @@ closeConnection = modifyMVar_ linkRef $ \current -> do
|
|||||||
|
|
||||||
-- | 发一条请求并解码应答;链路没装就报错,不崩
|
-- | 发一条请求并解码应答;链路没装就报错,不崩
|
||||||
sendRequest :: Request -> IO Response
|
sendRequest :: Request -> IO Response
|
||||||
sendRequest req = do
|
sendRequest req = withStorageLink $ \current -> case current of
|
||||||
current <- readMVar linkRef
|
Nothing -> pure (RespError "storage not configured")
|
||||||
case current of
|
Just link -> do
|
||||||
Nothing -> pure (RespError "storage not configured")
|
raw <- slRequest link (BL.toStrict (encode req))
|
||||||
Just link -> do
|
case raw of
|
||||||
raw <- slRequest link (BL.toStrict (encode req))
|
Left e -> pure (RespError ("storage error: " ++ e))
|
||||||
case raw of
|
Right bytes -> case eitherDecodeStrict (BS.dropWhileEnd (== 13) bytes) of
|
||||||
Left e -> pure (RespError ("storage error: " ++ e))
|
Left e -> pure (RespError ("decode error: " ++ e))
|
||||||
Right bytes -> case eitherDecodeStrict (BS.dropWhileEnd (== 13) bytes) of
|
Right resp -> pure resp
|
||||||
Left e -> pure (RespError ("decode error: " ++ e))
|
|
||||||
Right resp -> pure resp
|
|
||||||
|
|
||||||
-- | 原样转发一条载荷给存储库
|
-- | 原样转发一条载荷给存储库
|
||||||
sendRawRequest :: BS.ByteString -> IO (Either String BS.ByteString)
|
sendRawRequest :: BS.ByteString -> IO (Either String BS.ByteString)
|
||||||
sendRawRequest payload = do
|
sendRawRequest payload = withStorageLink $ \current -> do
|
||||||
current <- readMVar linkRef
|
|
||||||
case current of
|
case current of
|
||||||
Nothing -> pure (Left "storage not configured")
|
Nothing -> pure (Left "storage not configured")
|
||||||
Just link -> slRequest link payload
|
Just link -> slRequest link payload
|
||||||
@@ -212,10 +234,6 @@ doInsertMany env t rs = ask env "insert_batch" (ReqInsertBatch t rs) okOnly
|
|||||||
doDeleteKeys :: Env -> String -> [Int] -> IO (Either String ())
|
doDeleteKeys :: Env -> String -> [Int] -> IO (Either String ())
|
||||||
doDeleteKeys env t ks = ask env "delete_keys" (ReqDeleteKeys t ks) okOnly
|
doDeleteKeys env t ks = ask env "delete_keys" (ReqDeleteKeys t ks) okOnly
|
||||||
|
|
||||||
-- | 发 ReplaceAll
|
|
||||||
doReplaceAll :: Env -> String -> [Row] -> IO (Either String ())
|
|
||||||
doReplaceAll env t rs = ask env "replace_all" (ReqReplaceAll t rs) okOnly
|
|
||||||
|
|
||||||
-- | 发 ListTables
|
-- | 发 ListTables
|
||||||
doListTables :: Env -> IO (Either String [String])
|
doListTables :: Env -> IO (Either String [String])
|
||||||
doListTables env = ask env "list_tables" ReqListTables $ \resp -> case resp of
|
doListTables env = ask env "list_tables" ReqListTables $ \resp -> case resp of
|
||||||
@@ -254,26 +272,6 @@ doDescribeTable env t = ask env "describe_table" (ReqDescribeTable t) $ \resp ->
|
|||||||
RespSchema info -> Just info
|
RespSchema info -> Just info
|
||||||
_ -> Nothing
|
_ -> Nothing
|
||||||
|
|
||||||
-- | 给某一列建索引
|
|
||||||
doCreateIndex :: Env -> String -> String -> IO (Either String ())
|
|
||||||
doCreateIndex env t c = ask env "create_index" (ReqCreateIndex t c) okOnly
|
|
||||||
|
|
||||||
-- | 去掉某一列的索引
|
|
||||||
doDropIndex :: Env -> String -> String -> IO (Either String ())
|
|
||||||
doDropIndex env t c = ask env "drop_index" (ReqDropIndex t c) okOnly
|
|
||||||
|
|
||||||
-- | 删一列(列定义、这一列上的索引、每行里的那一格一起没)
|
|
||||||
doDropColumn :: Env -> String -> String -> IO (Either String ())
|
|
||||||
doDropColumn env t c = ask env "drop_column" (ReqDropColumn t c) okOnly
|
|
||||||
|
|
||||||
-- | 发 ReplaceSchema(ALTER 用)
|
|
||||||
doReplaceSchema :: Env -> String -> [SchemaColumn] -> [Row] -> IO (Either String ())
|
|
||||||
doReplaceSchema env t cols rs = ask env "replace_schema" (ReqReplaceSchema t cols rs) okOnly
|
|
||||||
|
|
||||||
-- | 发 CreateTable
|
|
||||||
doCreateTable :: Env -> String -> [SchemaColumn] -> IO (Either String ())
|
|
||||||
doCreateTable env t cols = ask env "create_table" (ReqCreateTable t cols) okOnly
|
|
||||||
|
|
||||||
-- | 发 DropTable
|
-- | 发 DropTable
|
||||||
doDropTable :: Env -> String -> IO (Either String ())
|
doDropTable :: Env -> String -> IO (Either String ())
|
||||||
doDropTable env t = ask env "drop_table" (ReqDropTable t) okOnly
|
doDropTable env t = ask env "drop_table" (ReqDropTable t) okOnly
|
||||||
@@ -350,7 +348,7 @@ instance MonadStorage IPCStorage where
|
|||||||
deleteKeys t ks = IPCStorage (\env -> doDeleteKeys env t ks)
|
deleteKeys t ks = IPCStorage (\env -> doDeleteKeys env t ks)
|
||||||
|
|
||||||
-- \| 发 ReplaceAll
|
-- \| 发 ReplaceAll
|
||||||
replaceAll t rs = IPCStorage (\env -> doReplaceAll env t rs)
|
replaceAll t rs = IPCStorage $ \env -> ask env "replace_all" (ReqReplaceAll t rs) okOnly
|
||||||
|
|
||||||
-- \| 按某一列的索引取一行(没有索引就回 NoIndex)
|
-- \| 按某一列的索引取一行(没有索引就回 NoIndex)
|
||||||
lookupByColumn t c k = IPCStorage (\env -> doLookupByColumn env t c k)
|
lookupByColumn t c k = IPCStorage (\env -> doLookupByColumn env t c k)
|
||||||
@@ -359,45 +357,44 @@ instance MonadStorage IPCStorage where
|
|||||||
scanRange t c lo hi = IPCStorage (\env -> doScanRange env t c lo hi)
|
scanRange t c lo hi = IPCStorage (\env -> doScanRange env t c lo hi)
|
||||||
|
|
||||||
-- \| 发 CreateTable
|
-- \| 发 CreateTable
|
||||||
createTable name cols = IPCStorage (\env -> doCreateTable env name (map toWire cols))
|
createTable name cols = IPCStorage $ \env -> ask env "create_table" (ReqCreateTable name (map toWire cols)) okOnly
|
||||||
|
|
||||||
-- \| ALTER:整表换列定义与全部行
|
-- \| ALTER:整表换列定义与全部行
|
||||||
replaceSchema name cols rows = IPCStorage (\env -> doReplaceSchema env name (map toWire cols) rows)
|
replaceSchema name cols rows = IPCStorage $ \env -> ask env "replace_schema" (ReqReplaceSchema name (map toWire cols) rows) okOnly
|
||||||
|
|
||||||
-- \| 发 DropTable
|
-- \| 发 DropTable
|
||||||
dropTable name = IPCStorage (\env -> doDropTable env name)
|
dropTable name = IPCStorage (\env -> doDropTable env name)
|
||||||
|
|
||||||
-- \| 建索引
|
-- \| 建索引
|
||||||
createIndex t c = IPCStorage (\env -> doCreateIndex env t c)
|
createIndex t c = IPCStorage $ \env -> ask env "create_index" (ReqCreateIndex t c) okOnly
|
||||||
|
|
||||||
-- \| 删索引
|
-- \| 删索引
|
||||||
dropIndex t c = IPCStorage (\env -> doDropIndex env t c)
|
dropIndex t c = IPCStorage $ \env -> ask env "drop_index" (ReqDropIndex t c) okOnly
|
||||||
|
|
||||||
-- \| 删一列
|
-- \| 删一列
|
||||||
dropColumn t c = IPCStorage (\env -> doDropColumn env t c)
|
dropColumn t c = IPCStorage $ \env -> ask env "drop_column" (ReqDropColumn t c) okOnly
|
||||||
|
|
||||||
-- \| 列表 + 逐表扫描拼库
|
-- \| 列表 + 逐表扫描拼库
|
||||||
snapshot = IPCStorage $ \env -> do
|
snapshot = IPCStorage $ \env -> do
|
||||||
result <- doListCatalog env
|
result <- doListCatalog env
|
||||||
case result of
|
case result of
|
||||||
Left _ -> pure []
|
Left err -> pure (Left err)
|
||||||
Right xs -> mapM (loadEntry env) xs
|
Right xs -> fmap sequence (mapM (loadEntry env) xs)
|
||||||
where
|
where
|
||||||
|
-- | 读取表数据并保留扫描错误
|
||||||
loadEntry env info = do
|
loadEntry env info = do
|
||||||
rows <- doScan env (tiTable info)
|
rows <- doScan env (tiTable info)
|
||||||
pure
|
pure $ fmap (\rs ->
|
||||||
( tiTable info
|
( tiTable info
|
||||||
, Table (tiTable info) (schemaToColumns (tiColumns info)) (either (const []) id rows) (Just (metaOf info))
|
, Table (tiTable info) (schemaToColumns (tiColumns info)) rs (Just (metaOf info))
|
||||||
)
|
)) rows
|
||||||
|
|
||||||
-- \| 只问数据字典要结构,设了当前库只留它的表
|
-- \| 只问数据字典要结构,设了当前库只留它的表
|
||||||
schema = IPCStorage $ \env -> do
|
schema = IPCStorage $ \env -> do
|
||||||
result <- ask env "all_catalogs" ReqAllCatalog $ \resp -> case resp of
|
result <- ask env "all_catalogs" ReqAllCatalog $ \resp -> case resp of
|
||||||
RespCatalog infos -> Just infos
|
RespCatalog infos -> Just infos
|
||||||
_ -> Nothing
|
_ -> Nothing
|
||||||
pure $ case result of
|
pure (fmap (concatMap (visibleTables (envDatabase env))) result)
|
||||||
Left _ -> []
|
|
||||||
Right xs -> concatMap (visibleTables (envDatabase env)) xs
|
|
||||||
|
|
||||||
-- | 线上表信息转引擎侧统计
|
-- | 线上表信息转引擎侧统计
|
||||||
metaOf :: TableInfo -> TableMeta
|
metaOf :: TableInfo -> TableMeta
|
||||||
|
|||||||
@@ -73,10 +73,10 @@ instance MonadStorage MemoryStorage where
|
|||||||
)
|
)
|
||||||
|
|
||||||
-- \| 原样返回当前库
|
-- \| 原样返回当前库
|
||||||
snapshot = MemoryStorage $ \db -> Right (db, db)
|
snapshot = MemoryStorage $ \db -> Right (Right db, db)
|
||||||
|
|
||||||
-- \| 返回实时行数与去重统计的无数据结构
|
-- \| 返回实时行数与去重统计的无数据结构
|
||||||
schema = MemoryStorage $ \db -> Right (map describe db, db)
|
schema = MemoryStorage $ \db -> Right (Right (map describe db), db)
|
||||||
where
|
where
|
||||||
-- | 将存储数据汇总为结构统计
|
-- | 将存储数据汇总为结构统计
|
||||||
describe (name, table) =
|
describe (name, table) =
|
||||||
|
|||||||
@@ -26,22 +26,20 @@ spec = do
|
|||||||
fmap (map verdict . findings) (analyzeNormalization schema
|
fmap (map verdict . findings) (analyzeNormalization schema
|
||||||
(Declaration [["a", "b"]] [Dependency ["a"] ["c"]] True (Just True)))
|
(Declaration [["a", "b"]] [Dependency ["a"] ["c"]] True (Just True)))
|
||||||
`shouldBe` Right [Satisfied, Violated, Violated, Violated]
|
`shouldBe` Right [Satisfied, Violated, Violated, Violated]
|
||||||
it "detects transitive dependencies" $ do
|
it "detects transitive dependencies and a preserving split" $ do
|
||||||
fmap (map verdict . findings) (analyzeNormalization schema
|
let analysis = analyzeNormalization schema
|
||||||
(Declaration [["a"]] [Dependency ["b"] ["c"]] True (Just True)))
|
(Declaration [["a"]] [Dependency ["b"] ["c"]] True (Just True))
|
||||||
|
fmap (map verdict . findings) analysis
|
||||||
`shouldBe` Right [Satisfied, Satisfied, Violated, Violated]
|
`shouldBe` Right [Satisfied, Satisfied, Violated, Violated]
|
||||||
it "distinguishes 3NF from BCNF using all candidate keys" $ do
|
fmap suggestions analysis
|
||||||
fmap (map verdict . findings) (analyzeNormalization schema
|
|
||||||
(Declaration [["a", "b"]] [Dependency ["c"] ["b"]] True (Just True)))
|
|
||||||
`shouldBe` Right [Satisfied, Satisfied, Satisfied, Violated]
|
|
||||||
it "reports lossless decomposition that loses dependencies" $ do
|
|
||||||
fmap suggestions (analyzeNormalization schema
|
|
||||||
(Declaration [["a", "b"]] [Dependency ["c"] ["b"]] True (Just True)))
|
|
||||||
`shouldBe` Right [Decomposition [["b", "c"], ["a", "c"]] Satisfied Violated]
|
|
||||||
it "reports dependency preserving transitive split" $ do
|
|
||||||
fmap suggestions (analyzeNormalization schema
|
|
||||||
(Declaration [["a"]] [Dependency ["b"] ["c"]] True (Just True)))
|
|
||||||
`shouldBe` Right [Decomposition [["b", "c"], ["a", "b"]] Satisfied Satisfied]
|
`shouldBe` Right [Decomposition [["b", "c"], ["a", "b"]] Satisfied Satisfied]
|
||||||
|
it "distinguishes 3NF from BCNF and detects dependency loss" $ do
|
||||||
|
let analysis = analyzeNormalization schema
|
||||||
|
(Declaration [["a", "b"]] [Dependency ["c"] ["b"]] True (Just True))
|
||||||
|
fmap (map verdict . findings) analysis
|
||||||
|
`shouldBe` Right [Satisfied, Satisfied, Satisfied, Violated]
|
||||||
|
fmap suggestions analysis
|
||||||
|
`shouldBe` Right [Decomposition [["b", "c"], ["a", "c"]] Satisfied Violated]
|
||||||
it "does not certify incomplete dependencies" $ do
|
it "does not certify incomplete dependencies" $ do
|
||||||
fmap (map verdict . findings) (analyzeNormalization schema
|
fmap (map verdict . findings) (analyzeNormalization schema
|
||||||
(Declaration [["a"]] [] False (Just True)))
|
(Declaration [["a"]] [] False (Just True)))
|
||||||
|
|||||||
@@ -29,7 +29,6 @@ spec = do
|
|||||||
parseTypeExpr "MaybeInt" `shouldSatisfy` isFailure
|
parseTypeExpr "MaybeInt" `shouldSatisfy` isFailure
|
||||||
it "restores stable nested identities from bootstrap metadata" $ do
|
it "restores stable nested identities from bootstrap metadata" $ do
|
||||||
let ty = tupleType [listType intType, maybeType stringType]
|
let ty = tupleType [listType intType, maybeType stringType]
|
||||||
resolveType builtinTypes ty `shouldBe` resolveType builtinTypes ty
|
|
||||||
fmap descriptorPhysical (resolveType builtinTypes ty)
|
fmap descriptorPhysical (resolveType builtinTypes ty)
|
||||||
`shouldBe` Right (Product [Sequence SignedInteger, Optional Utf8])
|
`shouldBe` Right (Product [Sequence SignedInteger, Optional Utf8])
|
||||||
it "distinguishes logical String from List Int" $
|
it "distinguishes logical String from List Int" $
|
||||||
|
|||||||
+167
-42
@@ -17,17 +17,23 @@ import ChuSQL.Core.Model
|
|||||||
import ChuSQL.Core.Engine.Semantic (prepare)
|
import ChuSQL.Core.Engine.Semantic (prepare)
|
||||||
import ChuSQL.Core.Engine.Storage (IndexResult (..), MonadStorage (..))
|
import ChuSQL.Core.Engine.Storage (IndexResult (..), MonadStorage (..))
|
||||||
import ChuSQL.Core.Engine.Storage.Memory (MemoryStorage (runMemoryStorage))
|
import ChuSQL.Core.Engine.Storage.Memory (MemoryStorage (runMemoryStorage))
|
||||||
import ChuSQL.Core.Engine.Storage.IPC (IPCStorage (..), closeConnection, defaultEnv, doListTables, envForDatabase, getDatabaseName, localStorageLink, runIPCStorage, setDatabaseName, setStorageLink)
|
import ChuSQL.Core.Engine.Storage.FFI (closeStorage, maintainStorage, openStorage)
|
||||||
|
import ChuSQL.Core.Engine.Storage.IPC (IPCStorage (..), StorageLink (..), closeConnection, defaultEnv, doListTables, envForDatabase, getDatabaseName, localStorageLink, runIPCStorage, sendRawRequest, setDatabaseName, setStorageLink)
|
||||||
import ChuSQL.Core.Engine.Syntax.AST
|
import ChuSQL.Core.Engine.Syntax.AST
|
||||||
import ChuSQL.Core.Engine.Syntax.Parser
|
import ChuSQL.Core.Engine.Syntax.Parser
|
||||||
import Control.Concurrent (threadDelay)
|
import Control.Concurrent (forkFinally, forkIO, killThread, threadDelay)
|
||||||
import Control.Concurrent.MVar (MVar, modifyMVar_, newMVar, readMVar)
|
import Control.Concurrent.Chan (newChan, readChan, writeChan)
|
||||||
import Control.Exception (ErrorCall, IOException, finally, try)
|
import Control.Concurrent.MVar (MVar, modifyMVar_, newEmptyMVar, newMVar, putMVar, readMVar, takeMVar, tryPutMVar)
|
||||||
|
import Control.Monad (replicateM_, void)
|
||||||
|
import Control.Exception (ErrorCall, IOException, bracket_, finally, try)
|
||||||
import Data.Hashable (hash)
|
import Data.Hashable (hash)
|
||||||
|
import qualified Data.ByteString as BS
|
||||||
|
import qualified Data.ByteString.Char8 as BSC
|
||||||
import Data.List (isInfixOf, sortOn)
|
import Data.List (isInfixOf, sortOn)
|
||||||
import Data.Unique (hashUnique, newUnique)
|
import Data.Unique (hashUnique, newUnique)
|
||||||
|
import GHC.IO.Encoding (getForeignEncoding, latin1, setForeignEncoding)
|
||||||
import System.CPUTime (getCPUTime)
|
import System.CPUTime (getCPUTime)
|
||||||
import System.Directory (createDirectoryIfMissing, getTemporaryDirectory, removePathForcibly)
|
import System.Directory (createDirectoryIfMissing, getTemporaryDirectory, removePathForcibly, renameFile)
|
||||||
import System.FilePath ((</>))
|
import System.FilePath ((</>))
|
||||||
import System.Timeout (timeout)
|
import System.Timeout (timeout)
|
||||||
import Test.Hspec
|
import Test.Hspec
|
||||||
@@ -118,6 +124,81 @@ domainSql db (sql : rest) = do
|
|||||||
-- | 跑引擎层全部 hspec 用例
|
-- | 跑引擎层全部 hspec 用例
|
||||||
main :: IO ()
|
main :: IO ()
|
||||||
main = hspec $ do
|
main = hspec $ do
|
||||||
|
describe "Storage read failures" $ do
|
||||||
|
it "propagates catalog failures through snapshot, schema and SQL" $ do
|
||||||
|
setStorageLink StorageLink
|
||||||
|
{ slRequest = \_ -> pure (Right (BSC.pack "{\"status\":\"error\",\"message\":\"catalog unavailable\"}"))
|
||||||
|
, slClose = pure ()
|
||||||
|
}
|
||||||
|
flip finally closeConnection $ do
|
||||||
|
fmap (fmap (map fst)) (runIPCStorage snapshot) `shouldReturn` Left "catalog unavailable"
|
||||||
|
fmap (fmap (map fst)) (runIPCStorage schema) `shouldReturn` Left "catalog unavailable"
|
||||||
|
runIPCStorage (runStatementM (CreateTable "blocked" [("id", TInt)]))
|
||||||
|
`shouldReturn` Left "catalog unavailable"
|
||||||
|
|
||||||
|
it "rejects the entire snapshot when a table scan fails" $ do
|
||||||
|
setStorageLink StorageLink
|
||||||
|
{ slRequest = \payload -> pure $ Right $ BSC.pack $
|
||||||
|
if BSC.pack "list_catalog" `BS.isInfixOf` payload
|
||||||
|
then "{\"status\":\"catalog\",\"schemas\":[{\"table\":\"broken\",\"columns\":[],\"row_count\":1}]}"
|
||||||
|
else "{\"status\":\"error\",\"message\":\"data page unreadable\"}"
|
||||||
|
, slClose = pure ()
|
||||||
|
}
|
||||||
|
(fmap (fmap (map fst)) (runIPCStorage snapshot) `shouldReturn` Left "data page unreadable") `finally` closeConnection
|
||||||
|
|
||||||
|
describe "Storage lifecycle and cancellation" $ do
|
||||||
|
it "waits for concurrent requests before closing the storage link" $ do
|
||||||
|
started <- newChan
|
||||||
|
release <- newEmptyMVar
|
||||||
|
finished <- newChan
|
||||||
|
closing <- newEmptyMVar
|
||||||
|
closed <- newEmptyMVar
|
||||||
|
setStorageLink StorageLink
|
||||||
|
{ slRequest = \_ -> writeChan started () >> readMVar release >> pure (Right BS.empty)
|
||||||
|
, slClose = putMVar closed ()
|
||||||
|
}
|
||||||
|
flip finally (void (tryPutMVar release ()) >> closeConnection) $ do
|
||||||
|
replicateM_ 2 (void (forkIO (sendRawRequest BS.empty >>= writeChan finished)))
|
||||||
|
timeout 1000000 (replicateM_ 2 (readChan started)) `shouldReturn` Just ()
|
||||||
|
void (forkIO (putMVar closing () >> closeConnection))
|
||||||
|
takeMVar closing
|
||||||
|
timeout 100000 (readMVar closed) `shouldReturn` Nothing
|
||||||
|
putMVar release ()
|
||||||
|
replicateM_ 2 (timeout 1000000 (readChan finished) `shouldReturn` Just (Right BS.empty))
|
||||||
|
timeout 1000000 (readMVar closed) `shouldReturn` Just ()
|
||||||
|
sendRawRequest BS.empty `shouldReturn` Left "storage not configured"
|
||||||
|
closeConnection
|
||||||
|
|
||||||
|
it "keeps the pool usable after cancellation while all workers are busy" $ do
|
||||||
|
pool <- workerPool 4
|
||||||
|
started <- newChan
|
||||||
|
release <- newEmptyMVar
|
||||||
|
finished <- newEmptyMVar
|
||||||
|
void (forkIO (poolRun pool (replicate 4 (writeChan started () >> readMVar release)) >> putMVar finished ()))
|
||||||
|
flip finally (void (tryPutMVar release ())) $ do
|
||||||
|
timeout 1000000 (replicateM_ 4 (readChan started)) `shouldReturn` Just ()
|
||||||
|
replicateM_ 2 (timeout 100000 (poolRun pool (replicate 3 (pure ()))) `shouldReturn` Nothing)
|
||||||
|
putMVar release ()
|
||||||
|
timeout 1000000 (takeMVar finished) `shouldReturn` Just ()
|
||||||
|
timeout 1000000 (poolRun pool [pure ()]) `shouldReturn` Just [()]
|
||||||
|
|
||||||
|
it "releases the storage link when an in-flight request is cancelled" $ do
|
||||||
|
started <- newEmptyMVar
|
||||||
|
release <- newEmptyMVar
|
||||||
|
finished <- newEmptyMVar
|
||||||
|
closed <- newEmptyMVar
|
||||||
|
setStorageLink StorageLink
|
||||||
|
{ slRequest = \_ -> putMVar started () >> takeMVar release >> pure (Right BS.empty)
|
||||||
|
, slClose = putMVar closed ()
|
||||||
|
}
|
||||||
|
thread <- forkFinally (sendRawRequest BS.empty) (const (putMVar finished ()))
|
||||||
|
flip finally (killThread thread >> closeConnection) $ do
|
||||||
|
timeout 1000000 (takeMVar started) `shouldReturn` Just ()
|
||||||
|
killThread thread
|
||||||
|
timeout 1000000 (takeMVar finished) `shouldReturn` Just ()
|
||||||
|
timeout 1000000 closeConnection `shouldReturn` Just ()
|
||||||
|
readMVar closed `shouldReturn` ()
|
||||||
|
|
||||||
NormalizationSpec.spec
|
NormalizationSpec.spec
|
||||||
RuntimeSpec.spec
|
RuntimeSpec.spec
|
||||||
describe "identity syntax" $ do
|
describe "identity syntax" $ do
|
||||||
@@ -1772,9 +1853,10 @@ main = hspec $ do
|
|||||||
sameResultAsUnoptimized "SELECT d.name FROM (SELECT name, age FROM users) d WHERE d.age > 20 ORDER BY d.name"
|
sameResultAsUnoptimized "SELECT d.name FROM (SELECT name, age FROM users) d WHERE d.age > 20 ORDER BY d.name"
|
||||||
|
|
||||||
it "supplies memory schema statistics without exposing rows" $ do
|
it "supplies memory schema statistics without exposing rows" $ do
|
||||||
case runMemoryStorage (schema :: MemoryStorage Database) testDB of
|
case runMemoryStorage schema testDB of
|
||||||
Left err -> expectationFailure err
|
Left err -> expectationFailure err
|
||||||
Right (structures, _) -> do
|
Right (Left err, _) -> expectationFailure err
|
||||||
|
Right (Right structures, _) -> do
|
||||||
map (tableRows . snd) structures `shouldBe` [[], []]
|
map (tableRows . snd) structures `shouldBe` [[], []]
|
||||||
fmap (fmap metaRowCount . tableMeta) (lookup "users" structures) `shouldBe` Just (Just 3)
|
fmap (fmap metaRowCount . tableMeta) (lookup "users" structures) `shouldBe` Just (Just 3)
|
||||||
|
|
||||||
@@ -2094,6 +2176,51 @@ main = hspec $ do
|
|||||||
]
|
]
|
||||||
|
|
||||||
describe "ChuSQL.Core.Engine.Storage.IPC" $ do
|
describe "ChuSQL.Core.Engine.Storage.IPC" $ do
|
||||||
|
it "uses UTF-8 config paths regardless of the foreign string encoding" $ do
|
||||||
|
tmp <- getTemporaryDirectory
|
||||||
|
stamp <- uniqueStamp
|
||||||
|
let dir = tmp </> ("chusql-utf8-" ++ stamp)
|
||||||
|
config = dir </> "settings-é.toml"
|
||||||
|
createDirectoryIfMissing True dir
|
||||||
|
flip finally (cleanSessionDir dir) $ do
|
||||||
|
original <- writeSessionConfig dir
|
||||||
|
renameFile original config
|
||||||
|
encoding <- getForeignEncoding
|
||||||
|
bracket_ (setForeignEncoding latin1) (setForeignEncoding encoding) $ do
|
||||||
|
opened <- openStorage (Just config)
|
||||||
|
case opened of
|
||||||
|
Left err -> expectationFailure err
|
||||||
|
Right handle -> closeStorage handle
|
||||||
|
maintainStorage config (BSC.pack "{\"mode\":\"unknown\"}") `shouldReturn` Left "invalid maintenance mode"
|
||||||
|
|
||||||
|
it "preserves each worker's storage open error" $ do
|
||||||
|
tmp <- getTemporaryDirectory
|
||||||
|
stamp <- uniqueStamp
|
||||||
|
pool <- workerPool 4
|
||||||
|
let paths = [tmp </> ("missing-ffi-" ++ stamp ++ "-" ++ show n) | n <- [1 .. 16 :: Int]]
|
||||||
|
-- | 检查打开失败并归还意外成功的句柄
|
||||||
|
checkOpen path = do
|
||||||
|
opened <- openStorage (Just path)
|
||||||
|
case opened of
|
||||||
|
Left err -> do
|
||||||
|
err `shouldContain` "config file not found"
|
||||||
|
err `shouldContain` path
|
||||||
|
Right handle -> closeStorage handle >> expectationFailure "expected missing config error"
|
||||||
|
poolRun pool (map checkOpen paths) >> pure ()
|
||||||
|
|
||||||
|
it "returns FFI input errors from workers and remains usable" $ do
|
||||||
|
withTestSession $ \srv -> do
|
||||||
|
pool <- workerPool 4
|
||||||
|
-- | 检查无效 UTF-8 的请求与维护错误
|
||||||
|
let checkErrors = do
|
||||||
|
requested <- sendRawRequest (BS.pack [255])
|
||||||
|
requested `shouldSatisfy` either (isInfixOf "request is not valid UTF-8") (const False)
|
||||||
|
maintained <- maintainStorage (dataDir srv ++ ".toml") (BS.pack [255])
|
||||||
|
maintained `shouldSatisfy` either (isInfixOf "invalid utf-8") (const False)
|
||||||
|
poolRun pool (replicate 16 checkErrors) >> pure ()
|
||||||
|
sendRawRequest (BSC.pack "{\"method\":\"ping\"}")
|
||||||
|
`shouldReturn` Right (BSC.pack "{\"status\":\"pong\"}")
|
||||||
|
|
||||||
it "persists runtime type identities and nested values across restart" $ do
|
it "persists runtime type identities and nested values across restart" $ do
|
||||||
withTestSession $ \srv -> do
|
withTestSession $ \srv -> do
|
||||||
-- | 解析并通过真实存储执行 SQL
|
-- | 解析并通过真实存储执行 SQL
|
||||||
@@ -2143,42 +2270,21 @@ main = hspec $ do
|
|||||||
query "SELECT secret FROM reused" `shouldReturn` Right [[("secret", VStr "new")]]
|
query "SELECT secret FROM reused" `shouldReturn` Right [[("secret", VStr "new")]]
|
||||||
query "SELECT secret FROM reused WHERE id = 1" `shouldReturn` Right [[("secret", VStr "new")]]
|
query "SELECT secret FROM reused WHERE id = 1" `shouldReturn` Right [[("secret", VStr "new")]]
|
||||||
|
|
||||||
it "scan returns rows with all three value types after insert" $ do
|
it "scans stored value types and reports missing tables" $ do
|
||||||
withTestSession $ \srv -> withServerEnv srv $ do
|
withTestSession $ \srv -> withServerEnv srv $ do
|
||||||
let row = [("id", VInt 7), ("name", VStr "Alice"), ("flag", VBool True)]
|
let row = [("id", VInt 7), ("name", VStr "Alice"), ("flag", VBool True)]
|
||||||
runIPCStorage (insert "users" row) `shouldReturn` Right ()
|
runIPCStorage (insert "users" row) `shouldReturn` Right ()
|
||||||
rows <- runIPCStorage (scan "users")
|
rows <- runIPCStorage (scan "users")
|
||||||
(map sortRow <$> rows) `shouldBe` Right [sortRow row]
|
(map sortRow <$> rows) `shouldBe` Right [sortRow row]
|
||||||
|
|
||||||
it "scan on a missing table returns Left" $ do
|
|
||||||
withTestSession $ \srv -> withServerEnv srv $ do
|
|
||||||
result <- runIPCStorage (scan "no_such_table")
|
result <- runIPCStorage (scan "no_such_table")
|
||||||
result `shouldSatisfy` isLeft
|
result `shouldSatisfy` isLeft
|
||||||
|
|
||||||
it "scan returns all 20 inserted rows in order" $ do
|
it "scan returns all 20 inserted rows in order" $ do
|
||||||
withTestSession $ \srv -> withServerEnv srv $ do
|
withTestSession $ \srv -> withServerEnv srv $ do
|
||||||
mapM_ (\i -> runIPCStorage (insert "many" [("id", VInt i)])) [1 .. 20 :: Int]
|
runIPCStorage (insertMany "many" [[("id", VInt i)] | i <- [1 .. 20 :: Int]]) `shouldReturn` Right ()
|
||||||
rows <- runIPCStorage (scan "many")
|
rows <- runIPCStorage (scan "many")
|
||||||
fmap length rows `shouldBe` Right 20
|
fmap length rows `shouldBe` Right 20
|
||||||
fmap (sortOn show . map (lookup "id")) rows
|
fmap (map (lookup "id")) rows `shouldBe` Right (map (Just . VInt) [1 .. 20])
|
||||||
`shouldBe` Right (sortOn show (map (Just . VInt) [1 .. 20]))
|
|
||||||
|
|
||||||
it "replaceAll leaves only the new rows" $ do
|
|
||||||
withTestSession $ \srv -> withServerEnv srv $ do
|
|
||||||
runIPCStorage (insert "t" [("id", VInt 1)]) `shouldReturn` Right ()
|
|
||||||
runIPCStorage (replaceAll "t" [[("id", VInt 9)], [("id", VInt 8)]]) `shouldReturn` Right ()
|
|
||||||
rows <- runIPCStorage (scan "t")
|
|
||||||
fmap (sortOn show . map (lookup "id")) rows
|
|
||||||
`shouldBe` Right (sortOn show [Just (VInt 9), Just (VInt 8)])
|
|
||||||
|
|
||||||
it "lookupByColumn finds a row inserted with an id" $ do
|
|
||||||
withTestSession $ \srv -> withServerEnv srv $ do
|
|
||||||
let row = [("id", VInt 7), ("name", VStr "Zoe")]
|
|
||||||
runIPCStorage (insert "keyed" row) `shouldReturn` Right ()
|
|
||||||
found <- runIPCStorage (lookupByColumn "keyed" "id" (VInt 7))
|
|
||||||
fmap sortRow (indexRow found) `shouldBe` Just (sortRow row)
|
|
||||||
missing <- runIPCStorage (lookupByColumn "keyed" "id" (VInt 8))
|
|
||||||
missing `shouldBe` Right (IndexRows [])
|
|
||||||
|
|
||||||
it "looks up a string index" $ do
|
it "looks up a string index" $ do
|
||||||
withTestSession $ \srv -> withServerEnv srv $ do
|
withTestSession $ \srv -> withServerEnv srv $ do
|
||||||
@@ -2202,21 +2308,27 @@ main = hspec $ do
|
|||||||
fmap (fmap (map (lookup "id"))) open `shouldBe` Right (Just [Just (VInt 3)])
|
fmap (fmap (map (lookup "id"))) open `shouldBe` Right (Just [Just (VInt 3)])
|
||||||
runIPCStorage (scanRange "rng" "nope" Nothing Nothing) `shouldReturn` Right Nothing
|
runIPCStorage (scanRange "rng" "nope" Nothing Nothing) `shouldReturn` Right Nothing
|
||||||
|
|
||||||
it "replaceAll rebuilds the index" $ do
|
it "looks up inserted ids and replaces rows with rebuilt indexes" $ do
|
||||||
withTestSession $ \srv -> withServerEnv srv $ do
|
withTestSession $ \srv -> withServerEnv srv $ do
|
||||||
runIPCStorage (insert "rb" [("id", VInt 1)]) `shouldReturn` Right ()
|
let original = [("id", VInt 1), ("name", VStr "Alice")]
|
||||||
runIPCStorage (replaceAll "rb" [[("id", VInt 9), ("name", VStr "Zoe")]]) `shouldReturn` Right ()
|
replacement = [("id", VInt 9), ("name", VStr "Zoe")]
|
||||||
|
runIPCStorage (insert "rb" original) `shouldReturn` Right ()
|
||||||
|
found <- runIPCStorage (lookupByColumn "rb" "id" (VInt 1))
|
||||||
|
fmap sortRow (indexRow found) `shouldBe` Just (sortRow original)
|
||||||
|
runIPCStorage (lookupByColumn "rb" "id" (VInt 8)) `shouldReturn` Right (IndexRows [])
|
||||||
|
runIPCStorage (replaceAll "rb" [replacement, [("id", VInt 8)]]) `shouldReturn` Right ()
|
||||||
|
rows <- runIPCStorage (scan "rb")
|
||||||
|
fmap (map sortRow) rows `shouldBe` Right [sortRow replacement, [("id", VInt 8)]]
|
||||||
old <- runIPCStorage (lookupByColumn "rb" "id" (VInt 1))
|
old <- runIPCStorage (lookupByColumn "rb" "id" (VInt 1))
|
||||||
old `shouldBe` Right (IndexRows [])
|
old `shouldBe` Right (IndexRows [])
|
||||||
new <- runIPCStorage (lookupByColumn "rb" "id" (VInt 9))
|
new <- runIPCStorage (lookupByColumn "rb" "id" (VInt 9))
|
||||||
fmap sortRow (indexRow new)
|
fmap sortRow (indexRow new) `shouldBe` Just (sortRow replacement)
|
||||||
`shouldBe` Just (sortRow [("id", VInt 9), ("name", VStr "Zoe")])
|
|
||||||
|
|
||||||
it "snapshot lists tables and scans each one to build the database" $ do
|
it "snapshot lists tables and scans each one to build the database" $ do
|
||||||
withTestSession $ \srv -> withServerEnv srv $ do
|
withTestSession $ \srv -> withServerEnv srv $ do
|
||||||
runIPCStorage (insert "a" [("id", VInt 1)]) `shouldReturn` Right ()
|
runIPCStorage (insert "a" [("id", VInt 1)]) `shouldReturn` Right ()
|
||||||
runIPCStorage (insert "b" [("id", VInt 2)]) `shouldReturn` Right ()
|
runIPCStorage (insert "b" [("id", VInt 2)]) `shouldReturn` Right ()
|
||||||
db <- runIPCStorage snapshot
|
Right db <- runIPCStorage snapshot
|
||||||
map fst db `shouldBe` ["a", "b"]
|
map fst db `shouldBe` ["a", "b"]
|
||||||
map (length . tableRows . snd) db `shouldBe` [1, 1]
|
map (length . tableRows . snd) db `shouldBe` [1, 1]
|
||||||
map (map (lookup "id") . tableRows . snd) db `shouldBe` [[Just (VInt 1)], [Just (VInt 2)]]
|
map (map (lookup "id") . tableRows . snd) db `shouldBe` [[Just (VInt 1)], [Just (VInt 2)]]
|
||||||
@@ -2346,7 +2458,7 @@ main = hspec $ do
|
|||||||
(Insert "meta_t" ["id", "code"] [[LitInt 1, LitInt 500], [LitInt 2, LitInt 501]])
|
(Insert "meta_t" ["id", "code"] [[LitInt 1, LitInt 500], [LitInt 2, LitInt 501]])
|
||||||
)
|
)
|
||||||
_ <- runIPCStorage (runStatementM (CreateIndex "meta_t" "code"))
|
_ <- runIPCStorage (runStatementM (CreateIndex "meta_t" "code"))
|
||||||
db <- runIPCStorage schema
|
Right db <- runIPCStorage schema
|
||||||
case lookup "meta_t" db of
|
case lookup "meta_t" db of
|
||||||
Nothing -> expectationFailure "meta_t missing from schema"
|
Nothing -> expectationFailure "meta_t missing from schema"
|
||||||
Just t -> do
|
Just t -> do
|
||||||
@@ -2857,6 +2969,19 @@ main = hspec $ do
|
|||||||
errorCategory "unsupported protocol version" `shouldBe` ProtocolError
|
errorCategory "unsupported protocol version" `shouldBe` ProtocolError
|
||||||
errorCategory "something else" `shouldBe` UnknownError
|
errorCategory "something else" `shouldBe` UnknownError
|
||||||
|
|
||||||
|
describe "Worker pool capacity" $ do
|
||||||
|
it "grows the existing queue so old references use the additional workers" $ do
|
||||||
|
original <- workerPool 4
|
||||||
|
_ <- workerPool 8
|
||||||
|
started <- newChan
|
||||||
|
release <- newEmptyMVar
|
||||||
|
finished <- newEmptyMVar
|
||||||
|
void (forkIO (poolRun original (replicate 8 (writeChan started () >> readMVar release)) >> putMVar finished ()))
|
||||||
|
flip finally (void (tryPutMVar release ())) $ do
|
||||||
|
timeout 1000000 (replicateM_ 8 (readChan started)) `shouldReturn` Just ()
|
||||||
|
putMVar release ()
|
||||||
|
timeout 1000000 (takeMVar finished) `shouldReturn` Just ()
|
||||||
|
|
||||||
-- | 每个测试表都清空行后的库
|
-- | 每个测试表都清空行后的库
|
||||||
emptyDB :: Database
|
emptyDB :: Database
|
||||||
emptyDB = [(n, t{tableRows = []}) | (n, t) <- testDB]
|
emptyDB = [(n, t{tableRows = []}) | (n, t) <- testDB]
|
||||||
@@ -2912,7 +3037,7 @@ cleanSessionDir dir = do
|
|||||||
_ <- try (removePathForcibly (dir ++ ".toml")) :: IO (Either IOException ())
|
_ <- try (removePathForcibly (dir ++ ".toml")) :: IO (Either IOException ())
|
||||||
pure ()
|
pure ()
|
||||||
|
|
||||||
-- | 一个用例一份干净数据目录;动态库打不开则 pending
|
-- | 每例独立目录,存储打开失败则测试失败
|
||||||
withTestSession :: (StorageSession -> IO ()) -> IO ()
|
withTestSession :: (StorageSession -> IO ()) -> IO ()
|
||||||
withTestSession act = do
|
withTestSession act = do
|
||||||
tmp <- getTemporaryDirectory
|
tmp <- getTemporaryDirectory
|
||||||
@@ -2921,7 +3046,7 @@ withTestSession act = do
|
|||||||
createDirectoryIfMissing True dir
|
createDirectoryIfMissing True dir
|
||||||
opened <- openSessionAt dir
|
opened <- openSessionAt dir
|
||||||
case opened of
|
case opened of
|
||||||
Left err -> cleanSessionDir dir >> pendingWith err
|
Left err -> cleanSessionDir dir >> expectationFailure err
|
||||||
Right session -> act session `finally` (closeConnection >> cleanSessionDir dir)
|
Right session -> act session `finally` (closeConnection >> cleanSessionDir dir)
|
||||||
|
|
||||||
-- | 语句不带库名,先建好测试库再切进去
|
-- | 语句不带库名,先建好测试库再切进去
|
||||||
@@ -3050,7 +3175,7 @@ instance MonadStorage ProbeStorage where
|
|||||||
dropTable _ = ProbeStorage (\_ -> pure (Left "probe: dropTable is not supported"))
|
dropTable _ = ProbeStorage (\_ -> pure (Left "probe: dropTable is not supported"))
|
||||||
dropColumn _ _ = ProbeStorage (\_ -> pure (Left "probe: dropColumn is not supported"))
|
dropColumn _ _ = ProbeStorage (\_ -> pure (Left "probe: dropColumn is not supported"))
|
||||||
replaceSchema _ _ _ = ProbeStorage (\_ -> pure (Left "probe: replaceSchema is not supported"))
|
replaceSchema _ _ _ = ProbeStorage (\_ -> pure (Left "probe: replaceSchema is not supported"))
|
||||||
snapshot = ProbeStorage (\_ -> pure [])
|
snapshot = ProbeStorage (\_ -> pure (Right []))
|
||||||
|
|
||||||
-- | 判断 Either 是 Right
|
-- | 判断 Either 是 Right
|
||||||
isRight :: Either a b -> Bool
|
isRight :: Either a b -> Bool
|
||||||
@@ -3271,7 +3396,7 @@ instance MonadStorage Counting where
|
|||||||
-- \| 换结构:走内存实现
|
-- \| 换结构:走内存实现
|
||||||
replaceSchema name cols rows = memory (memReplaceSchema name cols rows)
|
replaceSchema name cols rows = memory (memReplaceSchema name cols rows)
|
||||||
-- \| 快照:把当前库原样交出去
|
-- \| 快照:把当前库原样交出去
|
||||||
snapshot = Counting $ \db -> Right (db, db, [])
|
snapshot = Counting $ \db -> Right (Right db, db, [])
|
||||||
|
|
||||||
-- | 解析一条 SQL 并在计数存储上跑,返回结果与扫描过的表
|
-- | 解析一条 SQL 并在计数存储上跑,返回结果与扫描过的表
|
||||||
countedRun :: String -> (Either String [Row], [String])
|
countedRun :: String -> (Either String [Row], [String])
|
||||||
|
|||||||
@@ -167,23 +167,47 @@ fn put_node(p: &mut Page, node: &Node) -> io::Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// 从一页读出节点
|
/// 从一页读出节点
|
||||||
fn get_node(p: &Page) -> Node {
|
fn get_node(p: &Page) -> io::Result<Node> {
|
||||||
|
if p.data.len() < HEADER {
|
||||||
|
return Err(io::Error::new(io::ErrorKind::InvalidData, "truncated index node header"));
|
||||||
|
}
|
||||||
let kind = read_u8(p, OFF_KIND);
|
let kind = read_u8(p, OFF_KIND);
|
||||||
let count = read_u16(p, OFF_COUNT) as usize;
|
let count = read_u16(p, OFF_COUNT) as usize;
|
||||||
|
let slots_end = HEADER + count * SLOT;
|
||||||
|
if !matches!(kind, NODE_LEAF | NODE_INTERNAL) || slots_end > p.data.len() {
|
||||||
|
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index node kind or slot count"));
|
||||||
|
}
|
||||||
let trailer = read_u64(p, OFF_NEXT);
|
let trailer = read_u64(p, OFF_NEXT);
|
||||||
let mut keys = Vec::with_capacity(count);
|
let mut keys = Vec::with_capacity(count);
|
||||||
let mut values = Vec::with_capacity(count + 1);
|
let mut values = Vec::with_capacity(count + 1);
|
||||||
|
let mut cell_end = p.data.len();
|
||||||
for i in 0..count {
|
for i in 0..count {
|
||||||
let off = read_u16(p, HEADER + i * SLOT) as usize;
|
let off = read_u16(p, HEADER + i * SLOT) as usize;
|
||||||
|
if off < slots_end || off + CELL_FIXED > p.data.len() {
|
||||||
|
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index cell offset"));
|
||||||
|
}
|
||||||
let klen = read_u16(p, off) as usize;
|
let klen = read_u16(p, off) as usize;
|
||||||
|
if klen > MAX_KEY_BYTES || off + CELL_FIXED + klen > p.data.len() {
|
||||||
|
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index key length"));
|
||||||
|
}
|
||||||
|
if off + CELL_FIXED + klen > cell_end {
|
||||||
|
return Err(io::Error::new(io::ErrorKind::InvalidData, "overlapping index cells"));
|
||||||
|
}
|
||||||
|
cell_end = off;
|
||||||
keys.push(p.data[off + 2..off + 2 + klen].to_vec());
|
keys.push(p.data[off + 2..off + 2 + klen].to_vec());
|
||||||
values.push(read_u64(p, off + 2 + klen));
|
values.push(read_u64(p, off + 2 + klen));
|
||||||
}
|
}
|
||||||
|
if keys.windows(2).any(|pair| pair[0] > pair[1]) {
|
||||||
|
return Err(io::Error::new(io::ErrorKind::InvalidData, "index keys are not sorted"));
|
||||||
|
}
|
||||||
if kind == NODE_INTERNAL {
|
if kind == NODE_INTERNAL {
|
||||||
values.push(trailer);
|
values.push(trailer);
|
||||||
Node { kind, next: 0, keys, values }
|
if values.contains(&META_PAGE) {
|
||||||
|
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index child page"));
|
||||||
|
}
|
||||||
|
Ok(Node { kind, next: 0, keys, values })
|
||||||
} else {
|
} else {
|
||||||
Node { kind, next: trailer, keys, values }
|
Ok(Node { kind, next: trailer, keys, values })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -299,7 +323,7 @@ impl DiskBTree {
|
|||||||
|
|
||||||
/// 按页号读节点
|
/// 按页号读节点
|
||||||
fn load(&mut self, id: PageId) -> io::Result<Node> {
|
fn load(&mut self, id: PageId) -> io::Result<Node> {
|
||||||
self.file.with_page(id, get_node)
|
self.file.with_page(id, get_node)?
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 按页号写节点
|
/// 按页号写节点
|
||||||
@@ -338,24 +362,9 @@ impl DiskBTree {
|
|||||||
|
|
||||||
/// 按 key 取全部位置(重复键按值升序)
|
/// 按 key 取全部位置(重复键按值升序)
|
||||||
pub fn get_all(&mut self, key: &[u8]) -> io::Result<Vec<u64>> {
|
pub fn get_all(&mut self, key: &[u8]) -> io::Result<Vec<u64>> {
|
||||||
let Some(mut id) = self.root()? else {
|
let mut values: Vec<_> = self.scan(Some(key), Some(key))?.into_iter().map(|(_, value)| value).collect();
|
||||||
return Ok(Vec::new());
|
values.sort_unstable();
|
||||||
};
|
Ok(values)
|
||||||
loop {
|
|
||||||
let node = self.load(id)?;
|
|
||||||
if node.kind == NODE_LEAF {
|
|
||||||
let start = node.keys.partition_point(|k| k.as_slice() < key);
|
|
||||||
let mut out = Vec::new();
|
|
||||||
for i in start..node.keys.len() {
|
|
||||||
if node.keys[i].as_slice() != key {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
out.push(node.values[i]);
|
|
||||||
}
|
|
||||||
return Ok(out);
|
|
||||||
}
|
|
||||||
id = node.values[child_slot(&node.keys, key)];
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 插入一个键值对(键可重复)
|
/// 插入一个键值对(键可重复)
|
||||||
@@ -412,10 +421,9 @@ impl DiskBTree {
|
|||||||
self.store(id, &node)?;
|
self.store(id, &node)?;
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
let (sep, mut right) = split_node(&mut node);
|
let (sep, right) = split_node(&mut node);
|
||||||
let right_id = self.alloc(&right)?;
|
let right_id = self.alloc(&right)?;
|
||||||
if node.kind == NODE_LEAF {
|
if node.kind == NODE_LEAF {
|
||||||
right.next = node.next;
|
|
||||||
node.next = right_id;
|
node.next = right_id;
|
||||||
}
|
}
|
||||||
self.store(id, &node)?;
|
self.store(id, &node)?;
|
||||||
@@ -443,8 +451,14 @@ impl DiskBTree {
|
|||||||
}
|
}
|
||||||
return Ok(false);
|
return Ok(false);
|
||||||
}
|
}
|
||||||
let child = node.values[child_slot(&node.keys, key)];
|
let first = node.keys.partition_point(|k| k.as_slice() < key);
|
||||||
self.delete_rec(child, key, value)
|
let last = child_slot(&node.keys, key);
|
||||||
|
for &child in &node.values[first..=last] {
|
||||||
|
if self.delete_rec(child, key, value)? {
|
||||||
|
return Ok(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 叶子链上按顺序取全部 (键, 值)
|
/// 叶子链上按顺序取全部 (键, 值)
|
||||||
@@ -475,7 +489,7 @@ impl DiskBTree {
|
|||||||
}
|
}
|
||||||
id = match lo {
|
id = match lo {
|
||||||
None => node.values[0],
|
None => node.values[0],
|
||||||
Some(start) => node.values[child_slot(&node.keys, start)],
|
Some(start) => node.values[node.keys.partition_point(|k| k.as_slice() < start)],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
|
|||||||
@@ -229,17 +229,17 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
|
|||||||
DEFAULT_POOL_SIZE,
|
DEFAULT_POOL_SIZE,
|
||||||
&file_origin,
|
&file_origin,
|
||||||
);
|
);
|
||||||
|
let storage = file.storage.unwrap_or_default();
|
||||||
let (data_dir, dir_origin) = pick(
|
let (data_dir, dir_origin) = pick(
|
||||||
file.storage.as_ref().and_then(|s| s.data_dir.clone()),
|
storage.data_dir,
|
||||||
default_data_dir().to_string_lossy().into_owned(),
|
default_data_dir().to_string_lossy().into_owned(),
|
||||||
&file_origin,
|
&file_origin,
|
||||||
);
|
);
|
||||||
let (log_files, log_files_origin) = pick(
|
let (log_files, log_files_origin) = pick(
|
||||||
file.storage.as_ref().and_then(|s| s.log_files.clone()),
|
storage.log_files,
|
||||||
DEFAULT_LOG_FILES.to_string(),
|
DEFAULT_LOG_FILES.to_string(),
|
||||||
&file_origin,
|
&file_origin,
|
||||||
);
|
);
|
||||||
// 管名已经没有了:存储不再是独立进程,管道/套接字那一层随 FFI 一起删掉
|
|
||||||
let (log_level, log_origin) = pick(
|
let (log_level, log_origin) = pick(
|
||||||
file.log.and_then(|l| l.level),
|
file.log.and_then(|l| l.level),
|
||||||
DEFAULT_LOG_LEVEL.to_string(),
|
DEFAULT_LOG_LEVEL.to_string(),
|
||||||
@@ -248,9 +248,9 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
|
|||||||
|
|
||||||
let data_dir = data_dir.trim().to_string();
|
let data_dir = data_dir.trim().to_string();
|
||||||
let log_files = log_files.trim().to_string();
|
let log_files = log_files.trim().to_string();
|
||||||
let log_level = log_level.trim().to_string();
|
let log_level = log_level.trim();
|
||||||
|
|
||||||
let log_level = Level::parse(&log_level)
|
let log_level = Level::parse(log_level)
|
||||||
.ok_or_else(|| format!("log.level is not one of off/error/warn/info/debug: {}", log_level))?;
|
.ok_or_else(|| format!("log.level is not one of off/error/warn/info/debug: {}", log_level))?;
|
||||||
|
|
||||||
if data_dir.is_empty() {
|
if data_dir.is_empty() {
|
||||||
@@ -318,38 +318,3 @@ fn validate_layout(page_size: usize, btree_order: usize) -> Result<(), String> {
|
|||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
/// 别的层的分区不能被当成错误
|
|
||||||
#[test]
|
|
||||||
fn ignores_sections_owned_by_other_layers() {
|
|
||||||
let text = r#"
|
|
||||||
[page]
|
|
||||||
size = 8192
|
|
||||||
|
|
||||||
[storage]
|
|
||||||
data_dir = "../data"
|
|
||||||
|
|
||||||
[server]
|
|
||||||
listen_host = "127.0.0.1"
|
|
||||||
port = 7777
|
|
||||||
max_message = 1048576
|
|
||||||
|
|
||||||
[web]
|
|
||||||
port = 7778
|
|
||||||
"#;
|
|
||||||
let loaded = resolve(Some(text), Some(PathBuf::from("settings.toml"))).unwrap();
|
|
||||||
assert_eq!(loaded.config.page_size, 8192);
|
|
||||||
assert_eq!(loaded.config.data_dir, PathBuf::from("../data"));
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 自己分区里的拼写错误仍然要报错
|
|
||||||
#[test]
|
|
||||||
fn still_rejects_typos_inside_its_own_sections() {
|
|
||||||
let text = "[page]\npage_size = 8192\n";
|
|
||||||
assert!(resolve(Some(text), None).is_err());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1021,17 +1021,12 @@ impl Server {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 系统目录里账号表已登记
|
/// 读取系统初始化状态与检查点 LSN
|
||||||
fn system_initialized(&self) -> bool {
|
fn system_status(&self) -> std::io::Result<Response> {
|
||||||
self.catalog
|
let initialized = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?
|
||||||
.lock()
|
.describe(USERS_TABLE).is_some_and(|schema| schema.system);
|
||||||
.map(|c| c.describe(USERS_TABLE).is_some_and(|schema| schema.system))
|
let last_lsn = self.wal.read_checkpoint()?.unwrap_or(0);
|
||||||
.unwrap_or(false)
|
Ok(Response::System { initialized, last_lsn })
|
||||||
}
|
|
||||||
|
|
||||||
/// 最近一次落盘的检查点 LSN
|
|
||||||
fn last_checkpoint(&self) -> u64 {
|
|
||||||
self.wal.read_checkpoint().ok().flatten().unwrap_or(0)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 读账号表;表没登记或数据文件缺失都算错
|
/// 读账号表;表没登记或数据文件缺失都算错
|
||||||
@@ -1346,12 +1341,12 @@ impl Server {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
Request::BootstrapSystem { user, password_hash } => {
|
Request::BootstrapSystem { user, password_hash } => {
|
||||||
match self.bootstrap_system(user.as_deref(), password_hash.as_deref()) {
|
match self.bootstrap_system(user.as_deref(), password_hash.as_deref()).and_then(|_| self.system_status()) {
|
||||||
Ok(_) => Response::System { initialized: true, last_lsn: self.last_checkpoint() },
|
Ok(response) => response,
|
||||||
Err(e) => Response::Error { message: e.to_string() },
|
Err(e) => Response::Error { message: e.to_string() },
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Request::SystemStatus => Response::System { initialized: self.system_initialized(), last_lsn: self.last_checkpoint() },
|
Request::SystemStatus => self.system_status().unwrap_or_else(|e| Response::Error { message: e.to_string() }),
|
||||||
Request::Ping => {
|
Request::Ping => {
|
||||||
log_debug!(request, "ping");
|
log_debug!(request, "ping");
|
||||||
Response::Pong
|
Response::Pong
|
||||||
|
|||||||
@@ -1,9 +1,45 @@
|
|||||||
use chusql_core_storage::btree::{fits_in_page, DiskBTree, MAX_KEY_BYTES};
|
use chusql_core_storage::btree::{fits_in_page, DiskBTree, MAX_KEY_BYTES};
|
||||||
use chusql_core_storage::config::{DEFAULT_BTREE_ORDER, DEFAULT_PAGE_SIZE};
|
use chusql_core_storage::config::{DEFAULT_BTREE_ORDER, DEFAULT_PAGE_SIZE};
|
||||||
use chusql_core_storage::page::DEFAULT_POOL_SIZE;
|
use chusql_core_storage::page::DEFAULT_POOL_SIZE;
|
||||||
|
use std::io::{ErrorKind, Seek, SeekFrom, Write};
|
||||||
|
|
||||||
// B+ 树测试:字节键、重复键、范围扫描、重开与配置校验。
|
// B+ 树测试:字节键、重复键、范围扫描、重开与配置校验。
|
||||||
|
|
||||||
|
/// 损坏节点返回数据错误且重复读取不 panic
|
||||||
|
#[test]
|
||||||
|
fn malformed_node_fields_return_invalid_data() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("corrupt.idx");
|
||||||
|
{
|
||||||
|
let mut tree = open(dir.path(), "corrupt.idx");
|
||||||
|
tree.insert(b"x", 1).unwrap();
|
||||||
|
tree.insert(b"y", 2).unwrap();
|
||||||
|
}
|
||||||
|
let original = std::fs::read(&path).unwrap();
|
||||||
|
let cell = u16::from_le_bytes([original[DEFAULT_PAGE_SIZE + 11], original[DEFAULT_PAGE_SIZE + 12]]) as usize;
|
||||||
|
for (name, offset, bytes) in [
|
||||||
|
("slot count", 1, vec![255, 255]),
|
||||||
|
("node kind", 0, vec![2]),
|
||||||
|
("cell overlaps slots", 11, vec![0, 0]),
|
||||||
|
("cell beyond page", 11, vec![255, 15]),
|
||||||
|
("overlapping cells", 13, (cell as u16).to_le_bytes().to_vec()),
|
||||||
|
("key length", cell, vec![255, 255]),
|
||||||
|
("key order", cell + 2, vec![b'z']),
|
||||||
|
("zero child", 0, vec![1]),
|
||||||
|
] {
|
||||||
|
std::fs::write(&path, &original).unwrap();
|
||||||
|
let mut file = std::fs::OpenOptions::new().write(true).open(&path).unwrap();
|
||||||
|
file.seek(SeekFrom::Start((DEFAULT_PAGE_SIZE + offset) as u64)).unwrap();
|
||||||
|
file.write_all(&bytes).unwrap();
|
||||||
|
drop(file);
|
||||||
|
let mut tree = open(dir.path(), "corrupt.idx");
|
||||||
|
for _ in 0..2 {
|
||||||
|
let error = tree.get(b"x").unwrap_err();
|
||||||
|
assert_eq!(error.kind(), ErrorKind::InvalidData, "{name}: {error}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// 按默认参数打开指定索引文件
|
/// 按默认参数打开指定索引文件
|
||||||
fn open(dir: &std::path::Path, name: &str) -> DiskBTree {
|
fn open(dir: &std::path::Path, name: &str) -> DiskBTree {
|
||||||
DiskBTree::open(dir.join(name), DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).unwrap()
|
DiskBTree::open(dir.join(name), DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).unwrap()
|
||||||
@@ -14,6 +50,29 @@ fn key(n: i64) -> [u8; 8] {
|
|||||||
(n as u64 ^ (1u64 << 63)).to_be_bytes()
|
(n as u64 ^ (1u64 << 63)).to_be_bytes()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 重复键跨页后仍能完整查找和删除
|
||||||
|
#[test]
|
||||||
|
fn duplicate_keys_across_splits() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("split-duplicates.idx");
|
||||||
|
let mut tree = DiskBTree::open(&path, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
|
||||||
|
for value in (0..100).rev() {
|
||||||
|
tree.insert(b"same", value).unwrap();
|
||||||
|
}
|
||||||
|
drop(tree);
|
||||||
|
let mut tree = DiskBTree::open(&path, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
|
||||||
|
assert_eq!(tree.get(b"same").unwrap(), Some(0));
|
||||||
|
assert_eq!(tree.get_all(b"same").unwrap(), (0..100).collect::<Vec<_>>());
|
||||||
|
assert_eq!(tree.scan(Some(b"same"), Some(b"same")).unwrap().len(), 100);
|
||||||
|
for value in 0..100 {
|
||||||
|
assert!(tree.delete(b"same", value).unwrap());
|
||||||
|
}
|
||||||
|
assert!(!tree.delete(b"same", 0).unwrap());
|
||||||
|
drop(tree);
|
||||||
|
let mut tree = DiskBTree::open(&path, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
|
||||||
|
assert!(tree.get_all(b"same").unwrap().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
/// 空树查不到,遍历为空
|
/// 空树查不到,遍历为空
|
||||||
#[test]
|
#[test]
|
||||||
fn empty_tree_get() {
|
fn empty_tree_get() {
|
||||||
|
|||||||
@@ -187,13 +187,14 @@ fn load_without_explicit_path_uses_builtin_defaults() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// [server] 分区归别的层管;本层拼写错误要报错
|
/// 忽略其他层分区并拒绝存储配置拼写错误
|
||||||
#[test]
|
#[test]
|
||||||
fn server_section_belongs_to_another_layer() {
|
fn server_section_belongs_to_another_layer() {
|
||||||
let _guard = env_guard();
|
let _guard = env_guard();
|
||||||
let db_server = "[server]\nlisten_host = \"0.0.0.0\"\nport = 7778\nmax_rows = 10\n";
|
let db_server = "[page]\nsize = 8192\n[storage]\ndata_dir = \"../data\"\n[server]\nlisten_host = \"0.0.0.0\"\nport = 7778\nmax_rows = 10\n[web]\nport = 7778\n";
|
||||||
let loaded = config::resolve(Some(db_server), None).unwrap();
|
let loaded = config::resolve(Some(db_server), None).unwrap();
|
||||||
assert_eq!(loaded.config.data_dir, config::default_data_dir());
|
assert_eq!(loaded.config.page_size, 8192);
|
||||||
|
assert_eq!(loaded.config.data_dir, PathBuf::from("../data"));
|
||||||
|
|
||||||
let typo = "[storage]\ndata_dirs = \"oops\"\n";
|
let typo = "[storage]\ndata_dirs = \"oops\"\n";
|
||||||
assert!(config::resolve(Some(typo), None).is_err());
|
assert!(config::resolve(Some(typo), None).is_err());
|
||||||
|
|||||||
@@ -5,6 +5,38 @@ use chusql_core_storage::runtime::Storage;
|
|||||||
|
|
||||||
// 进程内存储集成测试:协议请求、索引、账号与配置。
|
// 进程内存储集成测试:协议请求、索引、账号与配置。
|
||||||
|
|
||||||
|
/// 检查点损坏不冒充零 LSN且可恢复读取
|
||||||
|
#[test]
|
||||||
|
fn system_status_reports_checkpoint_read_errors() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let (_server, data) = start_server();
|
||||||
|
let mut connection = connect()?;
|
||||||
|
let checkpoint = data.path().join("system/wal.checkpoint");
|
||||||
|
let initial: serde_json::Value = serde_json::from_str(&send_raw(&mut connection, r#"{"method":"system_status"}"#))?;
|
||||||
|
assert_eq!(initial["last_lsn"], 0);
|
||||||
|
request_ok(&mut connection, serde_json::json!({"method":"object_acl_replace","expected":null,"payload":"{}"}))?;
|
||||||
|
let original = std::fs::read(&checkpoint)?;
|
||||||
|
let before: serde_json::Value = serde_json::from_str(&send_raw(&mut connection, r#"{"method":"system_status"}"#))?;
|
||||||
|
assert_eq!(before["status"], "system");
|
||||||
|
assert!(before["last_lsn"].as_u64().unwrap() > 0);
|
||||||
|
std::fs::write(&checkpoint, b"bad")?;
|
||||||
|
let broken: serde_json::Value = serde_json::from_str(&send_raw(&mut connection, r#"{"method":"system_status"}"#))?;
|
||||||
|
assert_eq!(broken["status"], "error");
|
||||||
|
assert!(broken["message"].as_str().unwrap().contains("invalid WAL checkpoint length"));
|
||||||
|
assert!(broken.get("last_lsn").is_none());
|
||||||
|
assert!(send_raw(&mut connection, r#"{"method":"ping"}"#).contains("pong"));
|
||||||
|
std::fs::write(&checkpoint, &original)?;
|
||||||
|
let restored: serde_json::Value = serde_json::from_str(&send_raw(&mut connection, r#"{"method":"system_status"}"#))?;
|
||||||
|
assert_eq!(restored, before);
|
||||||
|
std::fs::remove_file(&checkpoint)?;
|
||||||
|
std::fs::create_dir(&checkpoint)?;
|
||||||
|
let unreadable: serde_json::Value = serde_json::from_str(&send_raw(&mut connection, r#"{"method":"system_status"}"#))?;
|
||||||
|
assert_eq!(unreadable["status"], "error");
|
||||||
|
assert!(unreadable.get("last_lsn").is_none());
|
||||||
|
std::fs::remove_dir(&checkpoint)?;
|
||||||
|
std::fs::write(&checkpoint, original)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// 权限快照支持跨页文本并拒绝旧版本覆盖
|
/// 权限快照支持跨页文本并拒绝旧版本覆盖
|
||||||
#[test]
|
#[test]
|
||||||
fn object_acl_snapshots_are_private_and_compare_versions() -> Result<(), Box<dyn std::error::Error>> {
|
fn object_acl_snapshots_are_private_and_compare_versions() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
|||||||
@@ -108,7 +108,7 @@ ipcBackend = do
|
|||||||
RespError err -> Left err
|
RespError err -> Left err
|
||||||
_ -> Left "unexpected database response"
|
_ -> Left "unexpected database response"
|
||||||
, beStorage = \payload -> runRaw payload
|
, beStorage = \payload -> runRaw payload
|
||||||
, beSnapshot = Right <$> runIPCStorageIn (snapshot :: IPCStorage Database) (sessionEnv current)
|
, beSnapshot = runIPCStorageIn snapshot (sessionEnv current)
|
||||||
, beApplyTransaction = \ops -> do
|
, beApplyTransaction = \ops -> do
|
||||||
response <- sendRequest (ReqInDatabase current (ReqApplyTransaction ops))
|
response <- sendRequest (ReqInDatabase current (ReqApplyTransaction ops))
|
||||||
pure $ case response of
|
pure $ case response of
|
||||||
@@ -148,10 +148,10 @@ ipcBackend = do
|
|||||||
Right rows -> do
|
Right rows -> do
|
||||||
cols <- case rows of
|
cols <- case rows of
|
||||||
[] -> do
|
[] -> do
|
||||||
db <- runIPCStorageIn (schema :: IPCStorage Database) env
|
db <- runIPCStorageIn schema env
|
||||||
pure (columnsFromStatement db stmt)
|
pure (fmap (\structures -> columnsFromStatement structures stmt) db)
|
||||||
(r : _) -> pure (map fst r)
|
(r : _) -> pure (Right (map fst r))
|
||||||
pure (Right (StatementResult cols rows))
|
pure (fmap (\names -> StatementResult names rows) cols)
|
||||||
|
|
||||||
-- | 测试用内存后端,状态是一个 Database
|
-- | 测试用内存后端,状态是一个 Database
|
||||||
memoryBackend :: String -> MVar Database -> Backend
|
memoryBackend :: String -> MVar Database -> Backend
|
||||||
|
|||||||
@@ -59,8 +59,8 @@ import Network.Socket (
|
|||||||
socket,
|
socket,
|
||||||
socketToHandle,
|
socketToHandle,
|
||||||
)
|
)
|
||||||
import System.IO (BufferMode (LineBuffering), IOMode (ReadWriteMode), hClose, hFlush, hPutStrLn, hSetBuffering, hSetNewlineMode, noNewlineTranslation, stderr)
|
import System.IO (BufferMode (LineBuffering), Handle, IOMode (ReadWriteMode), hClose, hFlush, hPutStrLn, hSetBuffering, hSetNewlineMode, noNewlineTranslation, stderr)
|
||||||
import System.IO.Error (isEOFError)
|
import System.IO.Error (eofErrorType, isEOFError, mkIOError)
|
||||||
|
|
||||||
-- TCP 服务器:监听端口,一连接一会话,按一行一 JSON 收发协议消息。
|
-- TCP 服务器:监听端口,一连接一会话,按一行一 JSON 收发协议消息。
|
||||||
|
|
||||||
@@ -255,20 +255,19 @@ serveConnection env conn = do
|
|||||||
entry <- newLiveEntry (isLocalPeer peer)
|
entry <- newLiveEntry (isLocalPeer peer)
|
||||||
unregister <- registerConnection (srvLive env) entry
|
unregister <- registerConnection (srvLive env) entry
|
||||||
session <- newSessionWith (srvBackend env) (srvRootName env) (srvSettingsFile env) (\names -> void (dropConnectionsOf (srvLive env) names))
|
session <- newSessionWith (srvBackend env) (srvRootName env) (srvSettingsFile env) (\names -> void (dropConnectionsOf (srvLive env) names))
|
||||||
outcome <- try (loop h session entry `finally` unregister) :: IO (Either IOException ())
|
outcome <- try (loop h session entry BSC.empty `finally` unregister) :: IO (Either IOException ())
|
||||||
case outcome of
|
case outcome of
|
||||||
Left err | not (isEOFError err) -> hPutStrLn stderr ("connection error: " ++ show err)
|
Left err | not (isEOFError err) -> hPutStrLn stderr ("connection error: " ++ show err)
|
||||||
_ -> pure ()
|
_ -> pure ()
|
||||||
void (try (hClose h) :: IO (Either IOException ()))
|
void (try (hClose h) :: IO (Either IOException ()))
|
||||||
where
|
where
|
||||||
-- | 逐行读请求并应答
|
-- | 逐行读请求并应答
|
||||||
loop h session entry = do
|
loop h session entry pending = do
|
||||||
line <- BSC.hGetLine h
|
frame <- readRequestLine h (scMaxMessage (srvConfig env)) pending
|
||||||
let cleaned = BSC.dropWhileEnd (== '\r') line
|
case frame of
|
||||||
if BSC.length cleaned > scMaxMessage (srvConfig env)
|
Nothing -> send h (RespError "too_large" "request line too large")
|
||||||
then send h (RespError "too_large" "request line too large") >> loop h session entry
|
Just (cleaned, remaining) -> case decodeRequest cleaned of
|
||||||
else case decodeRequest cleaned of
|
Left message -> send h (RespError "bad_request" message) >> loop h session entry remaining
|
||||||
Left message -> send h (RespError "bad_request" message) >> loop h session entry
|
|
||||||
Right request -> do
|
Right request -> do
|
||||||
expired <- readIORef (leDropped entry)
|
expired <- readIORef (leDropped entry)
|
||||||
if expired
|
if expired
|
||||||
@@ -276,13 +275,36 @@ serveConnection env conn = do
|
|||||||
else do
|
else do
|
||||||
(response, keepGoing) <- dispatch env session entry request
|
(response, keepGoing) <- dispatch env session entry request
|
||||||
send h response
|
send h response
|
||||||
if keepGoing then loop h session entry else pure ()
|
if keepGoing then loop h session entry remaining else pure ()
|
||||||
-- | 发一条响应并冲缓冲
|
-- | 发一条响应并冲缓冲
|
||||||
send h response = do
|
send h response = do
|
||||||
BLC.hPutStr h (encodeResponse response)
|
BLC.hPutStr h (encodeResponse response)
|
||||||
BLC.hPutStr h "\n"
|
BLC.hPutStr h "\n"
|
||||||
hFlush h
|
hFlush h
|
||||||
|
|
||||||
|
-- | 限长读取一行并保留后续帧
|
||||||
|
readRequestLine :: Handle -> Int -> BSC.ByteString -> IO (Maybe (BSC.ByteString, BSC.ByteString))
|
||||||
|
readRequestLine handle limit = collect [] 0
|
||||||
|
where
|
||||||
|
-- | 按块读取至换行、超限或 EOF
|
||||||
|
collect parts total bytes = case BSC.elemIndex '\n' bytes of
|
||||||
|
Just offset -> finish (BSC.take offset bytes : parts) (BSC.drop (offset + 1) bytes)
|
||||||
|
Nothing
|
||||||
|
| used > limit && (used - limit > 1 || BSC.null bytes || BSC.last bytes /= '\r') -> pure Nothing
|
||||||
|
| otherwise -> do
|
||||||
|
next <- BSC.hGetSome handle (max 1 (min 4094 (limit - used) + 2))
|
||||||
|
if BSC.null next
|
||||||
|
then if used == 0
|
||||||
|
then ioError (mkIOError eofErrorType "read request" (Just handle) Nothing)
|
||||||
|
else finish (bytes : parts) BSC.empty
|
||||||
|
else collect (bytes : parts) used next
|
||||||
|
where
|
||||||
|
used = total + BSC.length bytes
|
||||||
|
-- | 合并并校验一行的有效长度
|
||||||
|
finish parts remaining = do
|
||||||
|
let line = BSC.dropWhileEnd (== '\r') (BSC.concat (reverse parts))
|
||||||
|
pure (if BSC.length line > limit then Nothing else Just (line, remaining))
|
||||||
|
|
||||||
-- | 一条请求一条响应,quit 后停连接
|
-- | 一条请求一条响应,quit 后停连接
|
||||||
dispatch :: ServerEnv -> Session -> LiveEntry -> ClientRequest -> IO (ServerResponse, Bool)
|
dispatch :: ServerEnv -> Session -> LiveEntry -> ClientRequest -> IO (ServerResponse, Bool)
|
||||||
dispatch env session entry request = case request of
|
dispatch env session entry request = case request of
|
||||||
|
|||||||
@@ -51,7 +51,7 @@ import Control.Concurrent (forkIO, threadDelay)
|
|||||||
import Control.Concurrent.MVar (MVar, modifyMVar_, newEmptyMVar, newMVar, putMVar, takeMVar)
|
import Control.Concurrent.MVar (MVar, modifyMVar_, newEmptyMVar, newMVar, putMVar, takeMVar)
|
||||||
import System.Timeout (timeout)
|
import System.Timeout (timeout)
|
||||||
import Control.Exception (IOException, bracket, finally, try)
|
import Control.Exception (IOException, bracket, finally, try)
|
||||||
import Control.Monad (void)
|
import Control.Monad (replicateM_, void)
|
||||||
import Data.List (sort)
|
import Data.List (sort)
|
||||||
import Data.Aeson (decode)
|
import Data.Aeson (decode)
|
||||||
import qualified Data.Aeson as A
|
import qualified Data.Aeson as A
|
||||||
@@ -729,6 +729,24 @@ tcpServerSpec = do
|
|||||||
withConnection handle $ \h -> do
|
withConnection handle $ \h -> do
|
||||||
reply <- tcpTalk h (BSC.pack ("{\"method\":\"query\",\"sql\":\"" ++ replicate 40 'a' ++ "\"}"))
|
reply <- tcpTalk h (BSC.pack ("{\"method\":\"query\",\"sql\":\"" ++ replicate 40 'a' ++ "\"}"))
|
||||||
asText (at "code" reply) `shouldBe` "too_large"
|
asText (at "code" reply) `shouldBe` "too_large"
|
||||||
|
it "refuses oversized input before a newline and closes the connection" $
|
||||||
|
withTcpServer defaultServerConfig{scMaxMessage = 32} $ \_ handle ->
|
||||||
|
withConnection handle $ \h -> do
|
||||||
|
BSC.hPutStr h (BSC.replicate 33 'a')
|
||||||
|
hFlush h
|
||||||
|
line <- timeout 1000000 (BSC.hGetLine h)
|
||||||
|
case line of
|
||||||
|
Nothing -> expectationFailure "oversized input waited for a newline"
|
||||||
|
Just raw -> asText (at "code" (fromMaybe A.Null (decode (BL.fromStrict raw)))) `shouldBe` "too_large"
|
||||||
|
tcpClosed h
|
||||||
|
it "accepts exact-limit CRLF frames and preserves pipelined requests" $
|
||||||
|
withTcpServer defaultServerConfig{scMaxMessage = 17} $ \_ handle ->
|
||||||
|
withConnection handle $ \h -> do
|
||||||
|
BSC.hPutStr h "{\"method\":\"ping\"}\r\n{\"method\":\"ping\"}\n"
|
||||||
|
hFlush h
|
||||||
|
replicateM_ 2 $ do
|
||||||
|
raw <- timeout 1000000 (BSC.hGetLine h)
|
||||||
|
fmap (asText . at "status" . fromMaybe A.Null . decode . BL.fromStrict) raw `shouldBe` Just "pong"
|
||||||
it "ping answers pong and quit closes the connection" $
|
it "ping answers pong and quit closes the connection" $
|
||||||
withTcpServer defaultServerConfig $ \_ handle ->
|
withTcpServer defaultServerConfig $ \_ handle ->
|
||||||
withConnection handle $ \h -> do
|
withConnection handle $ \h -> do
|
||||||
@@ -1531,6 +1549,22 @@ transactionSpec = describe "server transactions" $ do
|
|||||||
rows <- mustSql session "select * from fresh"
|
rows <- mustSql session "select * from fresh"
|
||||||
qrRowCount rows `shouldBe` 1
|
qrRowCount rows `shouldBe` 1
|
||||||
|
|
||||||
|
it "does not start a transaction when its snapshot cannot be read" $ do
|
||||||
|
ref <- newMVar testDb
|
||||||
|
settings <- tempSettingsPath "snapshot-failure"
|
||||||
|
let base = memoryBackend testDatabaseName ref
|
||||||
|
-- | 保留测试后端并让快照读取失败
|
||||||
|
failing name = (beWithDatabase base name)
|
||||||
|
{ beSnapshot = pure (Left "data page unreadable")
|
||||||
|
, beWithDatabase = failing
|
||||||
|
}
|
||||||
|
session <- newSession (failing testDatabaseName) (T.pack testDatabaseName) settings
|
||||||
|
void (mustSql session "USE test")
|
||||||
|
result <- runStatementCoded session "BEGIN"
|
||||||
|
failureMessage result `shouldBe` "data page unreadable"
|
||||||
|
late <- runStatementCoded session "COMMIT"
|
||||||
|
failureMessage late `shouldBe` "no transaction in progress"
|
||||||
|
|
||||||
it "refuses nested BEGIN and COMMIT outside a transaction" $ withMemorySession $ \session -> do
|
it "refuses nested BEGIN and COMMIT outside a transaction" $ withMemorySession $ \session -> do
|
||||||
_ <- mustSql session "begin"
|
_ <- mustSql session "begin"
|
||||||
nested <- runStatementCoded session "begin"
|
nested <- runStatementCoded session "begin"
|
||||||
|
|||||||
@@ -11,6 +11,12 @@ function isObject(value) {
|
|||||||
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 校验结果列名及每行的列数。
|
||||||
|
function hasValidRows(body) {
|
||||||
|
return isObject(body) && Array.isArray(body.columns) && body.columns.every((column) => typeof column === 'string')
|
||||||
|
&& Array.isArray(body.rows) && body.rows.every((row) => Array.isArray(row) && row.length === body.columns.length);
|
||||||
|
}
|
||||||
|
|
||||||
// 发一次请求并处理错误。
|
// 发一次请求并处理错误。
|
||||||
async function request(url, init, database) {
|
async function request(url, init, database) {
|
||||||
const headers = new Headers(init && init.headers ? init.headers : undefined);
|
const headers = new Headers(init && init.headers ? init.headers : undefined);
|
||||||
@@ -114,21 +120,13 @@ export async function useDatabase(name, database) {
|
|||||||
|
|
||||||
// ---------------------------------------------------------------- 表结构
|
// ---------------------------------------------------------------- 表结构
|
||||||
|
|
||||||
// 取类型基名(去掉参数)。
|
|
||||||
function columnKind(type) {
|
|
||||||
return String(type).split('(')[0].trim().toLowerCase();
|
|
||||||
}
|
|
||||||
|
|
||||||
const knownTypes = ['int', 'bigint', 'smallint', 'str', 'text', 'varchar', 'char', 'float', 'double', 'decimal', 'numeric', 'bool', 'date', 'timestamp', 'blob'];
|
|
||||||
|
|
||||||
// 校验并规整列描述。
|
// 校验并规整列描述。
|
||||||
function asColumns(value) {
|
function asColumns(value) {
|
||||||
if (!Array.isArray(value)) throw new Error('服务器返回的列清单无法识别。');
|
if (!Array.isArray(value)) throw new Error('服务器返回的列清单无法识别。');
|
||||||
return value.map((entry) => {
|
return value.map((entry) => {
|
||||||
if (!isObject(entry) || typeof entry.name !== 'string' || typeof entry.type !== 'string') {
|
if (!isObject(entry) || typeof entry.name !== 'string' || typeof entry.type !== 'string' || !entry.type.trim()) {
|
||||||
throw new Error('服务器返回的列描述无法识别。');
|
throw new Error('服务器返回的列描述无法识别。');
|
||||||
}
|
}
|
||||||
if (!knownTypes.includes(columnKind(entry.type))) throw new Error(`不支持的列类型:${entry.type}`);
|
|
||||||
return { ...entry, name: entry.name, type: entry.type };
|
return { ...entry, name: entry.name, type: entry.type };
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -197,13 +195,11 @@ export async function fetchRows({ database, table, limit, offset, sort, dir, fil
|
|||||||
params.set('filter', JSON.stringify(filters.map((f) => ({ column: f.column, value: f.expression }))));
|
params.set('filter', JSON.stringify(filters.map((f) => ({ column: f.column, value: f.expression }))));
|
||||||
}
|
}
|
||||||
const body = await request(`/api/tables/${encodeURIComponent(table)}/rows?${params}`, undefined, database);
|
const body = await request(`/api/tables/${encodeURIComponent(table)}/rows?${params}`, undefined, database);
|
||||||
if (!isObject(body) || !Array.isArray(body.columns) || !body.columns.every((c) => typeof c === 'string')
|
if (!hasValidRows(body) || !Number.isSafeInteger(body.total) || body.total < 0) {
|
||||||
|| !Array.isArray(body.rows) || typeof body.total !== 'number') {
|
|
||||||
throw new Error('服务器返回的分页数据无法识别。');
|
throw new Error('服务器返回的分页数据无法识别。');
|
||||||
}
|
}
|
||||||
const columns = body.columns;
|
const columns = body.columns;
|
||||||
const rows = body.rows.map((raw) => {
|
const rows = body.rows.map((raw) => {
|
||||||
if (!Array.isArray(raw)) throw new Error('服务器返回的行数据无法识别。');
|
|
||||||
return Object.fromEntries(columns.map((column, index) => [column, raw[index]]));
|
return Object.fromEntries(columns.map((column, index) => [column, raw[index]]));
|
||||||
});
|
});
|
||||||
return { columns, rows, total: body.total };
|
return { columns, rows, total: body.total };
|
||||||
@@ -281,7 +277,7 @@ export async function query(sql, database, tabId) {
|
|||||||
const init = jsonInit('POST', { sql });
|
const init = jsonInit('POST', { sql });
|
||||||
init.headers['X-ChuSQL-Console'] = console;
|
init.headers['X-ChuSQL-Console'] = console;
|
||||||
const body = await request('/api/query', init, database);
|
const body = await request('/api/query', init, database);
|
||||||
if (!isObject(body) || !Array.isArray(body.columns) || !Array.isArray(body.rows)) {
|
if (!hasValidRows(body)) {
|
||||||
throw new Error('服务器返回的查询结果无法识别。');
|
throw new Error('服务器返回的查询结果无法识别。');
|
||||||
}
|
}
|
||||||
const durationMs = performance.now() - started;
|
const durationMs = performance.now() - started;
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import assert from 'node:assert/strict';
|
import assert from 'node:assert/strict';
|
||||||
import { readFile } from 'node:fs/promises';
|
import { readFile } from 'node:fs/promises';
|
||||||
|
|
||||||
// 检查前端复合值显示、SQL 生成和编码错误。
|
// 检查复合值、SQL 生成、类型元数据和编码错误。
|
||||||
|
|
||||||
const source = await readFile(new URL('../static/core.js', import.meta.url), 'utf8');
|
const source = await readFile(new URL('../static/core.js', import.meta.url), 'utf8');
|
||||||
const { cellText, sqlLiteral, typeLabel, editText } = await import(`data:text/javascript;base64,${Buffer.from(source).toString('base64')}`);
|
const { cellText, sqlLiteral, typeLabel, editText } = await import(`data:text/javascript;base64,${Buffer.from(source).toString('base64')}`);
|
||||||
@@ -17,4 +17,38 @@ assert.throws(() => sqlLiteral({ __chusql_runtime_v1: [1, [7, [typeInt]], [1, 2]
|
|||||||
assert.equal(typeLabel('runtime([1,[5,[[1,[]]]]])'), '[Int]');
|
assert.equal(typeLabel('runtime([1,[5,[[1,[]]]]])'), '[Int]');
|
||||||
assert.equal(editText({ __chusql_runtime_v1: [1, typeString, 'abc'] }), 'abc');
|
assert.equal(editText({ __chusql_runtime_v1: [1, typeString, 'abc'] }), 'abc');
|
||||||
assert.throws(() => sqlLiteral({ __chusql_runtime_v1: [1, typeInt, 1.5] }), /integer/);
|
assert.throws(() => sqlLiteral({ __chusql_runtime_v1: [1, typeInt, 1.5] }), /integer/);
|
||||||
console.log('runtime values: 9 assertions passed');
|
const apiSource = await readFile(new URL('../static/api.js', import.meta.url), 'utf8');
|
||||||
|
const api = await import(`data:text/javascript;base64,${Buffer.from(apiSource).toString('base64')}`);
|
||||||
|
const columns = ['int', 'runtime([1,[5,[[1,[]]]]])', 'runtime([1,[6,[[4,[]]]]])', 'runtime([1,[7,[[1,[]],[4,[]]]]])', 'domain(age,int)']
|
||||||
|
.map((type, index) => ({ name: `column_${index}`, type }));
|
||||||
|
const table = { table: 'typed', rowCount: 0, columns };
|
||||||
|
const originalFetch = globalThis.fetch;
|
||||||
|
try {
|
||||||
|
globalThis.fetch = async (url) => new Response(JSON.stringify(url === '/api/tables' ? [table] : table));
|
||||||
|
assert.deepEqual((await api.listTables('test'))[0].columns, columns);
|
||||||
|
assert.deepEqual((await api.getTable('test', 'typed')).columns, columns);
|
||||||
|
for (const type of [undefined, 42, ' ']) {
|
||||||
|
globalThis.fetch = async () => new Response(JSON.stringify([{ ...table, columns: [{ name: 'broken', type }] }]));
|
||||||
|
await assert.rejects(api.listTables('test'), /列描述/);
|
||||||
|
}
|
||||||
|
const page = { columns: ['id', 'value'], rows: [[1, null]], total: 1 };
|
||||||
|
globalThis.fetch = async () => new Response(JSON.stringify(page));
|
||||||
|
assert.deepEqual((await api.fetchRows({ database: 'test', table: 'typed', limit: 10, offset: 0 })).rows,
|
||||||
|
[{ id: 1, value: null }]);
|
||||||
|
assert.deepEqual((await api.query('SELECT id, value FROM typed', 'test')).rows, page.rows);
|
||||||
|
for (const invalid of [
|
||||||
|
{ ...page, rows: [[1]] }, { ...page, rows: [[1, null, 3]] },
|
||||||
|
{ ...page, rows: [null] }, { ...page, columns: ['id', 42] },
|
||||||
|
]) {
|
||||||
|
globalThis.fetch = async () => new Response(JSON.stringify(invalid));
|
||||||
|
await assert.rejects(api.fetchRows({ database: 'test', table: 'typed', limit: 10, offset: 0 }), /分页数据/);
|
||||||
|
assert.equal((await api.query('SELECT id, value FROM typed', 'test')).type, 'error');
|
||||||
|
}
|
||||||
|
for (const total of [-1, 1.5, '1']) {
|
||||||
|
globalThis.fetch = async () => new Response(JSON.stringify({ ...page, total }));
|
||||||
|
await assert.rejects(api.fetchRows({ database: 'test', table: 'typed', limit: 10, offset: 0 }), /分页数据/);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
globalThis.fetch = originalFetch;
|
||||||
|
}
|
||||||
|
console.log('runtime values and schemas: 27 assertions passed');
|
||||||
@@ -117,7 +117,7 @@ TCP 连接在下一次请求时返回 unauthorized 并关闭;空闲连接不
|
|||||||
|
|
||||||
## 查询引擎扩展设计(尚未实现)
|
## 查询引擎扩展设计(尚未实现)
|
||||||
|
|
||||||
按 bound IR → 子查询物化 / 递归 CTE 推进;DOMAIN 约束见[类型文档](types.md#domain-约束设计尚未实现)。以下设计尚未实施,当前能力见前文;实施状态以 project-plan.txt 为准。
|
按 bound IR → 子查询物化 / 递归 CTE 推进;DOMAIN 约束见[类型文档](types.md#domain-约束设计尚未实现)。以下设计尚未实施,当前能力见前文;实施状态以 agent-credientials/progress.txt 为准。
|
||||||
|
|
||||||
### bound IR
|
### bound IR
|
||||||
|
|
||||||
|
|||||||
@@ -92,7 +92,7 @@ schema 类型不证明业务意义上的原子性。不满足 1NF 时更高范
|
|||||||
|
|
||||||
## 后续入口设计(尚未实现)
|
## 后续入口设计(尚未实现)
|
||||||
|
|
||||||
以下 TCP、CLI 和 Web 接口尚未实现,拟议命令与端点目前不可用。实施状态以 project-plan.txt 为准。
|
以下 TCP、CLI 和 Web 接口尚未实现,拟议命令与端点目前不可用。实施状态以 agent-credientials/progress.txt 为准。
|
||||||
|
|
||||||
### 协议与鉴权
|
### 协议与鉴权
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# 发布签名与验证设计
|
# 发布签名与验证设计
|
||||||
|
|
||||||
本文定义尚未实现的发行安全方案。当前操作见[安装文档](install.md),实施状态以 project-plan.txt 为准。先实现离线验证,再接入在线安装;该工作可独立于查询和存储扩展推进。
|
本文定义尚未实现的发行安全方案。当前操作见[安装文档](install.md),实施状态以 agent-credientials/progress.txt 为准。先实现离线验证,再接入在线安装;该工作可独立于查询和存储扩展推进。
|
||||||
|
|
||||||
## 选择与信任入口
|
## 选择与信任入口
|
||||||
|
|
||||||
|
|||||||
@@ -42,5 +42,4 @@ Web 步骤会先显式刷新该工程里的 model、engine、interface 与 serve
|
|||||||
## 约定
|
## 约定
|
||||||
|
|
||||||
- 门禁由项目所有者手动运行;agent 只跑模块级测试,不跑全量门禁。
|
- 门禁由项目所有者手动运行;agent 只跑模块级测试,不跑全量门禁。
|
||||||
- 期望基线(随里程碑更新):Engine 373 / Server 102 / Web 56 / CLI 13 / Bootstrap 9 / Rust 154。
|
- 计数、验收状态与主线进度以 `agent-credientials/progress.txt` 为准。
|
||||||
- 计数与主线进度以 `agent-credientials/project-plan.txt` 为准。
|
|
||||||
Reference in new issue
Block a user