优化器直方图

This commit is contained in:
LIm1tZz7 committed 2026-10-03 21:03:17 +08:00
1 parent 7d96933e32
commit d59e65b639
17 files changed
+353 -49

No files matched your search

@@ -1,8 +1,8 @@
module ChuSQL.Core.Engine.Algebra.Cost (tableMetaOf, probeCost, pointRows, rangeRows, indexCheaper) where
module ChuSQL.Core.Engine.Algebra.Cost (tableMetaOf, probeCost, pointRows, rangeRows, selectivityOf, indexCheaper) where
import ChuSQL.Core.Model
-- 统计与成本估算:行数、列级不同值数与索引探路页数。
-- 统计与成本估算:行数、不同值数、列直方图与索引探路页数。
-- | 表统计,拿不到就是 Nothing
tableMetaOf :: Database -> String -> Maybe TableMeta
@@ -20,9 +20,42 @@ pointRows m col = case [d | (c, d, _) <- metaDistinct m, c == col, d > 1] of
where
rows = max 0 (metaRowCount m)
-- | 范围扫描的估算命中行数:没有直方图,按四分之一算
rangeRows :: TableMeta -> Int
rangeRows m = max 0 (metaRowCount m) `div` 4
-- | 范围扫描的估算命中行数:有直方图按桶算,没有按四分之一
rangeRows :: TableMeta -> String -> Maybe (Value, Bool) -> Maybe (Value, Bool) -> Int
rangeRows m col lo hi = case histogramFor m col of
Just h -> estimateRows h lo hi
Nothing -> max 0 (metaRowCount m) `div` 4
-- | 直方图估算区间命中行数:按桶算交集占比
estimateRows :: Histogram -> Maybe (Value, Bool) -> Maybe (Value, Bool) -> Int
estimateRows h lo hi = max 0 (min total (round (sum (zipWith overlap [0 ..] (histBuckets h)))))
where
total = sum (histBuckets h)
width = if histHigh h > histLow h then (histHigh h - histLow h) / fromIntegral (length (histBuckets h)) else 0
lower = maybe (-1 / 0) id (boundOf lo)
upper = maybe (1 / 0) id (boundOf hi)
overlap i count
| width <= 0 = if lower <= histLow h && histLow h <= upper then fromIntegral count else 0
| otherwise = fromIntegral count * hit / width
where
start = histLow h + width * fromIntegral i
hit = max 0 (min (start + width) upper - max start lower)
-- | 区间选择率:直方图命中的比例;没有直方图给 Nothing
selectivityOf :: TableMeta -> String -> Maybe (Value, Bool) -> Maybe (Value, Bool) -> Maybe Double
selectivityOf m col lo hi = case histogramFor m col of
Nothing -> Nothing
Just h
| total <= 0 -> Nothing
| otherwise -> Just (fromIntegral (estimateRows h lo hi) / fromIntegral total)
where
total = sum (histBuckets h)
-- | 边界值转成参与估值的小数;非数值给 Nothing
boundOf :: Maybe (Value, Bool) -> Maybe Double
boundOf (Just (VInt n, _)) = Just (fromIntegral n)
boundOf (Just (VFloat d, _)) = Just d
boundOf _ = Nothing
-- | 有索引且探路加命中比全表扫便宜
indexCheaper :: TableMeta -> String -> Int -> Bool
@@ -1,6 +1,6 @@
module ChuSQL.Core.Engine.Algebra.JoinOrder (reorderJoins) where
import ChuSQL.Core.Engine.Algebra.Cost (pointRows, rangeRows, tableMetaOf)
import ChuSQL.Core.Engine.Algebra.Cost (pointRows, rangeRows, selectivityOf, tableMetaOf)
import ChuSQL.Core.Engine.Algebra.Expr (colsInExpr, hasDivision)
import ChuSQL.Core.Engine.Algebra.Op (RelOp (..), relOpCols)
import ChuSQL.Core.Engine.Syntax.AST (Expr (..), JoinKind (..))
@@ -45,7 +45,7 @@ leafRows :: Database -> RelOp -> Maybe Int
leafRows db op = case op of
Scan _ t _ -> rowCountOf db t
Lookup _ t c _ -> pointRows <$> tableMetaOf db t <*> pure c
Range _ t _ _ _ -> rangeRows <$> tableMetaOf db t
Range _ t c lo hi -> (\m -> rangeRows m c lo hi) <$> tableMetaOf db t
Filter p x -> scaleRows db p x <$> leafRows db x
Project _ x -> leafRows db x
_ -> Nothing
@@ -54,12 +54,20 @@ leafRows db op = case op of
rowCountOf :: Database -> String -> Maybe Int
rowCountOf db t = metaRowCount <$> tableMetaOf db t
-- | 过滤之后的大致行数:等值看不同值数,其他按四分之一
-- | 行数估算:等值看不同值数,区间看直方图,其余四分之一
scaleRows :: Database -> Expr -> RelOp -> Int -> Int
scaleRows db p x rows = case p of
Eq (Col k) _ -> case distinctOf db x k of
Just d | d > 1 -> max 1 (rows `div` d)
_ -> quarter
Gt (Col k) (LitInt v) -> pick k (Just (VInt v, False)) Nothing
Gt (Col k) (LitStr v) -> pick k (Just (VStr v, False)) Nothing
GtE (Col k) (LitInt v) -> pick k (Just (VInt v, True)) Nothing
GtE (Col k) (LitStr v) -> pick k (Just (VStr v, True)) Nothing
Lt (Col k) (LitInt v) -> pick k Nothing (Just (VInt v, False))
Lt (Col k) (LitStr v) -> pick k Nothing (Just (VStr v, False))
LtE (Col k) (LitInt v) -> pick k Nothing (Just (VInt v, True))
LtE (Col k) (LitStr v) -> pick k Nothing (Just (VStr v, True))
Gt _ _ -> quarter
Lt _ _ -> quarter
GtE _ _ -> quarter
@@ -70,6 +78,11 @@ scaleRows db p x rows = case p of
_ -> quarter
where
quarter = max 1 (rows `div` 4)
pick k lo hi = case leafSource x of
Just (alias, t)
| Just m <- tableMetaOf db t
, Just s <- selectivityOf m (unqualify alias k) lo hi -> max 1 (round (fromIntegral rows * s))
_ -> quarter
-- | 叶子对应的别名与表名
leafSource :: RelOp -> Maybe (Maybe String, String)
@@ -159,10 +159,10 @@ worthLookup db t col = case tableMetaOf db t of
Just m -> indexCheaper m col (pointRows m col)
-- | 范围扫描值不值得改写;没有统计就按老规矩改
worthRange :: Database -> String -> String -> Bool
worthRange db t col = case tableMetaOf db t of
worthRange :: Database -> String -> String -> Maybe (Value, Bool) -> Maybe (Value, Bool) -> Bool
worthRange db t col lo hi = case tableMetaOf db t of
Nothing -> True
Just m -> indexCheaper m col (rangeRows m)
Just m -> indexCheaper m col (rangeRows m col lo hi)
-- | 单节点重写
rewriteNode :: Database -> RelOp -> RelOp
@@ -171,45 +171,53 @@ rewriteNode db (Filter (Eq (Col k) (LitInt v)) (Scan mAlias t Nothing))
rewriteNode db (Filter (Eq (Col k) (LitStr v)) (Scan mAlias t Nothing))
| worthLookup db t (unqualify mAlias k) = Lookup mAlias t (unqualify mAlias k) (VStr v)
rewriteNode db (Filter (And (Gt (Col k1) (LitInt lo)) (Lt (Col k2) (LitInt hi))) (Scan mAlias t Nothing))
| k1 == k2 && worthRange db t (unqualify mAlias k1) =
| k1 == k2 && worthRange db t (unqualify mAlias k1) (Just (VInt lo, False)) (Just (VInt hi, False)) =
Range mAlias t (unqualify mAlias k1) (Just (VInt lo, False)) (Just (VInt hi, False))
rewriteNode db (Filter (And (Gt (Col k1) (LitStr lo)) (Lt (Col k2) (LitStr hi))) (Scan mAlias t Nothing))
| k1 == k2 && worthRange db t (unqualify mAlias k1) =
| k1 == k2 && worthRange db t (unqualify mAlias k1) (Just (VStr lo, False)) (Just (VStr hi, False)) =
Range mAlias t (unqualify mAlias k1) (Just (VStr lo, False)) (Just (VStr hi, False))
rewriteNode db (Filter (Gt (Col k) (LitInt v)) (Scan mAlias t Nothing))
| worthRange db t (unqualify mAlias k) = Range mAlias t (unqualify mAlias k) (Just (VInt v, False)) Nothing
| worthRange db t (unqualify mAlias k) (Just (VInt v, False)) Nothing =
Range mAlias t (unqualify mAlias k) (Just (VInt v, False)) Nothing
rewriteNode db (Filter (Gt (Col k) (LitStr v)) (Scan mAlias t Nothing))
| worthRange db t (unqualify mAlias k) = Range mAlias t (unqualify mAlias k) (Just (VStr v, False)) Nothing
| worthRange db t (unqualify mAlias k) (Just (VStr v, False)) Nothing =
Range mAlias t (unqualify mAlias k) (Just (VStr v, False)) Nothing
rewriteNode db (Filter (Lt (Col k) (LitInt v)) (Scan mAlias t Nothing))
| worthRange db t (unqualify mAlias k) = Range mAlias t (unqualify mAlias k) Nothing (Just (VInt v, False))
| worthRange db t (unqualify mAlias k) Nothing (Just (VInt v, False)) =
Range mAlias t (unqualify mAlias k) Nothing (Just (VInt v, False))
rewriteNode db (Filter (Lt (Col k) (LitStr v)) (Scan mAlias t Nothing))
| worthRange db t (unqualify mAlias k) = Range mAlias t (unqualify mAlias k) Nothing (Just (VStr v, False))
| worthRange db t (unqualify mAlias k) Nothing (Just (VStr v, False)) =
Range mAlias t (unqualify mAlias k) Nothing (Just (VStr v, False))
rewriteNode db (Filter (And (GtE (Col k1) (LitInt lo)) (LtE (Col k2) (LitInt hi))) (Scan mAlias t Nothing))
| k1 == k2 && worthRange db t (unqualify mAlias k1) =
| k1 == k2 && worthRange db t (unqualify mAlias k1) (Just (VInt lo, True)) (Just (VInt hi, True)) =
Range mAlias t (unqualify mAlias k1) (Just (VInt lo, True)) (Just (VInt hi, True))
rewriteNode db (Filter (And (GtE (Col k1) (LitInt lo)) (Lt (Col k2) (LitInt hi))) (Scan mAlias t Nothing))
| k1 == k2 && worthRange db t (unqualify mAlias k1) =
| k1 == k2 && worthRange db t (unqualify mAlias k1) (Just (VInt lo, True)) (Just (VInt hi, False)) =
Range mAlias t (unqualify mAlias k1) (Just (VInt lo, True)) (Just (VInt hi, False))
rewriteNode db (Filter (And (Gt (Col k1) (LitInt lo)) (LtE (Col k2) (LitInt hi))) (Scan mAlias t Nothing))
| k1 == k2 && worthRange db t (unqualify mAlias k1) =
| k1 == k2 && worthRange db t (unqualify mAlias k1) (Just (VInt lo, False)) (Just (VInt hi, True)) =
Range mAlias t (unqualify mAlias k1) (Just (VInt lo, False)) (Just (VInt hi, True))
rewriteNode db (Filter (And (GtE (Col k1) (LitStr lo)) (LtE (Col k2) (LitStr hi))) (Scan mAlias t Nothing))
| k1 == k2 && worthRange db t (unqualify mAlias k1) =
| k1 == k2 && worthRange db t (unqualify mAlias k1) (Just (VStr lo, True)) (Just (VStr hi, True)) =
Range mAlias t (unqualify mAlias k1) (Just (VStr lo, True)) (Just (VStr hi, True))
rewriteNode db (Filter (And (GtE (Col k1) (LitStr lo)) (Lt (Col k2) (LitStr hi))) (Scan mAlias t Nothing))
| k1 == k2 && worthRange db t (unqualify mAlias k1) =
| k1 == k2 && worthRange db t (unqualify mAlias k1) (Just (VStr lo, True)) (Just (VStr hi, False)) =
Range mAlias t (unqualify mAlias k1) (Just (VStr lo, True)) (Just (VStr hi, False))
rewriteNode db (Filter (And (Gt (Col k1) (LitStr lo)) (LtE (Col k2) (LitStr hi))) (Scan mAlias t Nothing))
| k1 == k2 && worthRange db t (unqualify mAlias k1) =
| k1 == k2 && worthRange db t (unqualify mAlias k1) (Just (VStr lo, False)) (Just (VStr hi, True)) =
Range mAlias t (unqualify mAlias k1) (Just (VStr lo, False)) (Just (VStr hi, True))
rewriteNode db (Filter (GtE (Col k) (LitInt v)) (Scan mAlias t Nothing))
| worthRange db t (unqualify mAlias k) = Range mAlias t (unqualify mAlias k) (Just (VInt v, True)) Nothing
| worthRange db t (unqualify mAlias k) (Just (VInt v, True)) Nothing =
Range mAlias t (unqualify mAlias k) (Just (VInt v, True)) Nothing
rewriteNode db (Filter (GtE (Col k) (LitStr v)) (Scan mAlias t Nothing))
| worthRange db t (unqualify mAlias k) = Range mAlias t (unqualify mAlias k) (Just (VStr v, True)) Nothing
| worthRange db t (unqualify mAlias k) (Just (VStr v, True)) Nothing =
Range mAlias t (unqualify mAlias k) (Just (VStr v, True)) Nothing
rewriteNode db (Filter (LtE (Col k) (LitInt v)) (Scan mAlias t Nothing))
| worthRange db t (unqualify mAlias k) = Range mAlias t (unqualify mAlias k) Nothing (Just (VInt v, True))
| worthRange db t (unqualify mAlias k) Nothing (Just (VInt v, True)) =
Range mAlias t (unqualify mAlias k) Nothing (Just (VInt v, True))
rewriteNode db (Filter (LtE (Col k) (LitStr v)) (Scan mAlias t Nothing))
| worthRange db t (unqualify mAlias k) = Range mAlias t (unqualify mAlias k) Nothing (Just (VStr v, True))
| worthRange db t (unqualify mAlias k) Nothing (Just (VStr v, True)) =
Range mAlias t (unqualify mAlias k) Nothing (Just (VStr v, True))
rewriteNode db (Filter p (Join kind l r c))
| not (hasDivision p || planHasDivision (Join kind l r c)) =
case kind of
@@ -401,7 +401,7 @@ instance MonadStorage IPCStorage where
-- | 线上表信息转引擎侧统计
metaOf :: TableInfo -> TableMeta
metaOf info = TableMeta (tiRows info) (tiStats info) (tiIndexes info)
metaOf info = TableMeta (tiRows info) (tiStats info) (tiHistograms info) (tiIndexes info)
-- | 没设库名就照单全收;设了库名只收这个库的表,键名去掉库名前缀
visibleTables :: Maybe String -> TableInfo -> [(String, Table)]
+45 -6
View File
@@ -2,6 +2,7 @@
module Main where
import ChuSQL.Core.Engine.Algebra.Cost (rangeRows, selectivityOf)
import ChuSQL.Core.Engine.Algebra.Eval (evalRelOp, evalRelOpM)
import ChuSQL.Core.Engine.Algebra.Expr (colsInExpr, evalExpr)
import ChuSQL.Core.Engine.Algebra.Op (RelOp (..), renderPlan)
@@ -71,7 +72,7 @@ chainA =
{ tableName = "a"
, tableCols = [("id", TInt), ("name", TStr)]
, tableRows = [[("id", VInt n), ("name", VStr ("a" ++ show n))] | n <- [1 .. 6]]
, tableMeta = Just (TableMeta 6 [("id", 6, False)] ["id"])
, tableMeta = Just (TableMeta 6 [("id", 6, False)] [] ["id"])
}
-- | 连接链测试表 b:4 行,a_id 各不相同
@@ -81,7 +82,7 @@ chainB =
{ tableName = "b"
, tableCols = [("id", TInt), ("a_id", TInt)]
, tableRows = [[("id", VInt n), ("a_id", VInt n)] | n <- [1 .. 4]]
, tableMeta = Just (TableMeta 4 [("id", 4, False), ("a_id", 4, False)] ["id"])
, tableMeta = Just (TableMeta 4 [("id", 4, False), ("a_id", 4, False)] [] ["id"])
}
-- | 连接链测试表 c:2 行,b_id 各不相同
@@ -91,7 +92,7 @@ chainC =
{ tableName = "c"
, tableCols = [("id", TInt), ("b_id", TInt)]
, tableRows = [[("id", VInt n), ("b_id", VInt n)] | n <- [1, 2]]
, tableMeta = Just (TableMeta 2 [("id", 2, False), ("b_id", 2, False)] ["id"])
, tableMeta = Just (TableMeta 2 [("id", 2, False), ("b_id", 2, False)] [] ["id"])
}
-- | 三张表组成的连接链测试库
@@ -551,7 +552,7 @@ main = hspec $ do
executeOperator OpNe [VInt 1, VFloat 1.0] `shouldBe` Right (VBool False)
it "rewrites >= and <= into an inclusive index range" $ do
let stat rows distinct indexes = TableMeta rows [(c, d, False) | (c, d) <- distinct] indexes
let stat rows distinct indexes = TableMeta rows [(c, d, False) | (c, d) <- distinct] [] indexes
dbWith m = [("users", Table "users" [("id", TInt), ("name", TStr)] [] (Just m))]
indexed = dbWith (stat 10000 [("id", 10000)] ["id"])
low = Filter (GtE (Col "id") (LitInt 2)) (Scan Nothing "users" Nothing)
@@ -1509,6 +1510,44 @@ main = hspec $ do
[ [("name", VStr "Carol")]
]
describe "ChuSQL.Core.Engine.Algebra.Cost (直方图)" $ do
it "builds a 16-bucket histogram from the numeric values" $ do
let hist = histogramOf [VInt n | n <- [0 .. 99]]
fmap histLow hist `shouldBe` Just 0
fmap histHigh hist `shouldBe` Just 99
fmap (length . histBuckets) hist `shouldBe` Just 16
fmap (sum . histBuckets) hist `shouldBe` Just 100
-- 非数值与 NULL 不参与;只有一个值都落第 0 桶
histogramOf [VStr "a", VNull] `shouldBe` Nothing
histogramOf [VInt 5, VNull, VStr "a"] `shouldBe` Just (Histogram 5 5 (1 : replicate 15 0))
it "estimates a range from the histogram buckets" $ do
let hist = Histogram 0 1600 (replicate 16 100)
meta = TableMeta 1600 [] [("id", hist)] ["id"]
plain = TableMeta 1600 [] [] []
rangeRows meta "id" (Just (VInt 0, True)) (Just (VInt 100, True)) `shouldBe` 100
rangeRows meta "id" Nothing Nothing `shouldBe` 1600
selectivityOf meta "id" (Just (VInt 0, True)) (Just (VInt 100, True)) `shouldBe` Just 0.0625
-- 没有直方图退回四分之一
rangeRows plain "id" Nothing Nothing `shouldBe` 400
selectivityOf plain "id" Nothing Nothing `shouldBe` Nothing
it "uses the histogram to reject a range that would scan most rows" $ do
let buckets = replicate 8 63 ++ replicate 8 62
withHist = TableMeta 1000 [("id", 1000, False)] [("id", Histogram 0 1600 buckets)] ["id"]
plain = TableMeta 1000 [("id", 1000, False)] [] ["id"]
dbWith m = [("users", Table "users" [("id", TInt), ("name", TStr)] [] (Just m))]
narrow = Filter (And (GtE (Col "id") (LitInt 10)) (LtE (Col "id") (LitInt 20))) (Scan Nothing "users" Nothing)
wide = Filter (Gt (Col "id") (LitInt 0)) (Scan Nothing "users" Nothing)
-- 直方图说这个窄范围只命中十几行:改写
optimize (dbWith withHist) narrow
`shouldBe` Range Nothing "users" "id" (Just (VInt 10, True)) (Just (VInt 20, True))
-- 直方图说这条范围覆盖全表:索引不比全表扫便宜,不改写
optimize (dbWith withHist) wide `shouldBe` wide
-- 没有直方图按四分之一算,仍然改写
optimize (dbWith plain) wide
`shouldBe` Range Nothing "users" "id" (Just (VInt 0, False)) Nothing
describe "ChuSQL.Core.Engine.Algebra.Optimize" $ do
it "keeps single-table WHERE results identical" $ do
sameResultAsUnoptimized "SELECT name FROM users WHERE age > 18"
@@ -1559,7 +1598,7 @@ main = hspec $ do
`shouldBe` Right (Just (LitBool False))
it "picks the index only when the statistics make it cheaper" $ do
let stat rows distinct indexes = TableMeta rows [(c, d, False) | (c, d) <- distinct] indexes
let stat rows distinct indexes = TableMeta rows [(c, d, False) | (c, d) <- distinct] [] indexes
dbWith m = [("users", Table "users" [("id", TInt), ("name", TStr)] [] (Just m))]
unknown = [("users", Table "users" [("id", TInt), ("name", TStr)] [] Nothing)]
point = Filter (Eq (Col "id") (LitInt 1)) (Scan Nothing "users" Nothing)
@@ -2689,7 +2728,7 @@ rowShards width rows
-- | 声明 1000 行的大表
bigTable :: Table
bigTable = users {tableName = "big", tableMeta = Just (TableMeta 1000 [("id", 1000, False)] ["id"])}
bigTable = users {tableName = "big", tableMeta = Just (TableMeta 1000 [("id", 1000, False)] [] ["id"])}
-- | 只装大表的库
bigDB :: Database