优化器直方图
This commit is contained in:
LIm1tZz7 committed
1 parent
7d96933e32
commit
d59e65b639
17 files changed
+353
-49
No files matched your search
@@ -1,8 +1,8 @@
|
||||
module ChuSQL.Core.Engine.Algebra.Cost (tableMetaOf, probeCost, pointRows, rangeRows, indexCheaper) where
|
||||
module ChuSQL.Core.Engine.Algebra.Cost (tableMetaOf, probeCost, pointRows, rangeRows, selectivityOf, indexCheaper) where
|
||||
|
||||
import ChuSQL.Core.Model
|
||||
|
||||
-- 统计与成本估算：行数、列级不同值数与索引探路页数。
|
||||
-- 统计与成本估算：行数、不同值数、列直方图与索引探路页数。
|
||||
|
||||
-- | 表统计，拿不到就是 Nothing
|
||||
tableMetaOf :: Database -> String -> Maybe TableMeta
|
||||
@@ -20,9 +20,42 @@ pointRows m col = case [d | (c, d, _) <- metaDistinct m, c == col, d > 1] of
|
||||
where
|
||||
rows = max 0 (metaRowCount m)
|
||||
|
||||
-- | 范围扫描的估算命中行数：没有直方图，按四分之一算
|
||||
rangeRows :: TableMeta -> Int
|
||||
rangeRows m = max 0 (metaRowCount m) `div` 4
|
||||
-- | 范围扫描的估算命中行数：有直方图按桶算，没有按四分之一
|
||||
rangeRows :: TableMeta -> String -> Maybe (Value, Bool) -> Maybe (Value, Bool) -> Int
|
||||
rangeRows m col lo hi = case histogramFor m col of
|
||||
Just h -> estimateRows h lo hi
|
||||
Nothing -> max 0 (metaRowCount m) `div` 4
|
||||
|
||||
-- | 直方图估算区间命中行数：按桶算交集占比
|
||||
estimateRows :: Histogram -> Maybe (Value, Bool) -> Maybe (Value, Bool) -> Int
|
||||
estimateRows h lo hi = max 0 (min total (round (sum (zipWith overlap [0 ..] (histBuckets h)))))
|
||||
where
|
||||
total = sum (histBuckets h)
|
||||
width = if histHigh h > histLow h then (histHigh h - histLow h) / fromIntegral (length (histBuckets h)) else 0
|
||||
lower = maybe (-1 / 0) id (boundOf lo)
|
||||
upper = maybe (1 / 0) id (boundOf hi)
|
||||
overlap i count
|
||||
| width <= 0 = if lower <= histLow h && histLow h <= upper then fromIntegral count else 0
|
||||
| otherwise = fromIntegral count * hit / width
|
||||
where
|
||||
start = histLow h + width * fromIntegral i
|
||||
hit = max 0 (min (start + width) upper - max start lower)
|
||||
|
||||
-- | 区间选择率：直方图命中的比例；没有直方图给 Nothing
|
||||
selectivityOf :: TableMeta -> String -> Maybe (Value, Bool) -> Maybe (Value, Bool) -> Maybe Double
|
||||
selectivityOf m col lo hi = case histogramFor m col of
|
||||
Nothing -> Nothing
|
||||
Just h
|
||||
| total <= 0 -> Nothing
|
||||
| otherwise -> Just (fromIntegral (estimateRows h lo hi) / fromIntegral total)
|
||||
where
|
||||
total = sum (histBuckets h)
|
||||
|
||||
-- | 边界值转成参与估值的小数；非数值给 Nothing
|
||||
boundOf :: Maybe (Value, Bool) -> Maybe Double
|
||||
boundOf (Just (VInt n, _)) = Just (fromIntegral n)
|
||||
boundOf (Just (VFloat d, _)) = Just d
|
||||
boundOf _ = Nothing
|
||||
|
||||
-- | 有索引且探路加命中比全表扫便宜
|
||||
indexCheaper :: TableMeta -> String -> Int -> Bool
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module ChuSQL.Core.Engine.Algebra.JoinOrder (reorderJoins) where
|
||||
|
||||
import ChuSQL.Core.Engine.Algebra.Cost (pointRows, rangeRows, tableMetaOf)
|
||||
import ChuSQL.Core.Engine.Algebra.Cost (pointRows, rangeRows, selectivityOf, tableMetaOf)
|
||||
import ChuSQL.Core.Engine.Algebra.Expr (colsInExpr, hasDivision)
|
||||
import ChuSQL.Core.Engine.Algebra.Op (RelOp (..), relOpCols)
|
||||
import ChuSQL.Core.Engine.Syntax.AST (Expr (..), JoinKind (..))
|
||||
@@ -45,7 +45,7 @@ leafRows :: Database -> RelOp -> Maybe Int
|
||||
leafRows db op = case op of
|
||||
Scan _ t _ -> rowCountOf db t
|
||||
Lookup _ t c _ -> pointRows <$> tableMetaOf db t <*> pure c
|
||||
Range _ t _ _ _ -> rangeRows <$> tableMetaOf db t
|
||||
Range _ t c lo hi -> (\m -> rangeRows m c lo hi) <$> tableMetaOf db t
|
||||
Filter p x -> scaleRows db p x <$> leafRows db x
|
||||
Project _ x -> leafRows db x
|
||||
_ -> Nothing
|
||||
@@ -54,12 +54,20 @@ leafRows db op = case op of
|
||||
rowCountOf :: Database -> String -> Maybe Int
|
||||
rowCountOf db t = metaRowCount <$> tableMetaOf db t
|
||||
|
||||
-- | 过滤之后的大致行数：等值看不同值数，其他按四分之一
|
||||
-- | 行数估算：等值看不同值数，区间看直方图，其余四分之一
|
||||
scaleRows :: Database -> Expr -> RelOp -> Int -> Int
|
||||
scaleRows db p x rows = case p of
|
||||
Eq (Col k) _ -> case distinctOf db x k of
|
||||
Just d | d > 1 -> max 1 (rows `div` d)
|
||||
_ -> quarter
|
||||
Gt (Col k) (LitInt v) -> pick k (Just (VInt v, False)) Nothing
|
||||
Gt (Col k) (LitStr v) -> pick k (Just (VStr v, False)) Nothing
|
||||
GtE (Col k) (LitInt v) -> pick k (Just (VInt v, True)) Nothing
|
||||
GtE (Col k) (LitStr v) -> pick k (Just (VStr v, True)) Nothing
|
||||
Lt (Col k) (LitInt v) -> pick k Nothing (Just (VInt v, False))
|
||||
Lt (Col k) (LitStr v) -> pick k Nothing (Just (VStr v, False))
|
||||
LtE (Col k) (LitInt v) -> pick k Nothing (Just (VInt v, True))
|
||||
LtE (Col k) (LitStr v) -> pick k Nothing (Just (VStr v, True))
|
||||
Gt _ _ -> quarter
|
||||
Lt _ _ -> quarter
|
||||
GtE _ _ -> quarter
|
||||
@@ -70,6 +78,11 @@ scaleRows db p x rows = case p of
|
||||
_ -> quarter
|
||||
where
|
||||
quarter = max 1 (rows `div` 4)
|
||||
pick k lo hi = case leafSource x of
|
||||
Just (alias, t)
|
||||
| Just m <- tableMetaOf db t
|
||||
, Just s <- selectivityOf m (unqualify alias k) lo hi -> max 1 (round (fromIntegral rows * s))
|
||||
_ -> quarter
|
||||
|
||||
-- | 叶子对应的别名与表名
|
||||
leafSource :: RelOp -> Maybe (Maybe String, String)
|
||||
|
||||
@@ -159,10 +159,10 @@ worthLookup db t col = case tableMetaOf db t of
|
||||
Just m -> indexCheaper m col (pointRows m col)
|
||||
|
||||
-- | 范围扫描值不值得改写；没有统计就按老规矩改
|
||||
worthRange :: Database -> String -> String -> Bool
|
||||
worthRange db t col = case tableMetaOf db t of
|
||||
worthRange :: Database -> String -> String -> Maybe (Value, Bool) -> Maybe (Value, Bool) -> Bool
|
||||
worthRange db t col lo hi = case tableMetaOf db t of
|
||||
Nothing -> True
|
||||
Just m -> indexCheaper m col (rangeRows m)
|
||||
Just m -> indexCheaper m col (rangeRows m col lo hi)
|
||||
|
||||
-- | 单节点重写
|
||||
rewriteNode :: Database -> RelOp -> RelOp
|
||||
@@ -171,45 +171,53 @@ rewriteNode db (Filter (Eq (Col k) (LitInt v)) (Scan mAlias t Nothing))
|
||||
rewriteNode db (Filter (Eq (Col k) (LitStr v)) (Scan mAlias t Nothing))
|
||||
| worthLookup db t (unqualify mAlias k) = Lookup mAlias t (unqualify mAlias k) (VStr v)
|
||||
rewriteNode db (Filter (And (Gt (Col k1) (LitInt lo)) (Lt (Col k2) (LitInt hi))) (Scan mAlias t Nothing))
|
||||
| k1 == k2 && worthRange db t (unqualify mAlias k1) =
|
||||
| k1 == k2 && worthRange db t (unqualify mAlias k1) (Just (VInt lo, False)) (Just (VInt hi, False)) =
|
||||
Range mAlias t (unqualify mAlias k1) (Just (VInt lo, False)) (Just (VInt hi, False))
|
||||
rewriteNode db (Filter (And (Gt (Col k1) (LitStr lo)) (Lt (Col k2) (LitStr hi))) (Scan mAlias t Nothing))
|
||||
| k1 == k2 && worthRange db t (unqualify mAlias k1) =
|
||||
| k1 == k2 && worthRange db t (unqualify mAlias k1) (Just (VStr lo, False)) (Just (VStr hi, False)) =
|
||||
Range mAlias t (unqualify mAlias k1) (Just (VStr lo, False)) (Just (VStr hi, False))
|
||||
rewriteNode db (Filter (Gt (Col k) (LitInt v)) (Scan mAlias t Nothing))
|
||||
| worthRange db t (unqualify mAlias k) = Range mAlias t (unqualify mAlias k) (Just (VInt v, False)) Nothing
|
||||
| worthRange db t (unqualify mAlias k) (Just (VInt v, False)) Nothing =
|
||||
Range mAlias t (unqualify mAlias k) (Just (VInt v, False)) Nothing
|
||||
rewriteNode db (Filter (Gt (Col k) (LitStr v)) (Scan mAlias t Nothing))
|
||||
| worthRange db t (unqualify mAlias k) = Range mAlias t (unqualify mAlias k) (Just (VStr v, False)) Nothing
|
||||
| worthRange db t (unqualify mAlias k) (Just (VStr v, False)) Nothing =
|
||||
Range mAlias t (unqualify mAlias k) (Just (VStr v, False)) Nothing
|
||||
rewriteNode db (Filter (Lt (Col k) (LitInt v)) (Scan mAlias t Nothing))
|
||||
| worthRange db t (unqualify mAlias k) = Range mAlias t (unqualify mAlias k) Nothing (Just (VInt v, False))
|
||||
| worthRange db t (unqualify mAlias k) Nothing (Just (VInt v, False)) =
|
||||
Range mAlias t (unqualify mAlias k) Nothing (Just (VInt v, False))
|
||||
rewriteNode db (Filter (Lt (Col k) (LitStr v)) (Scan mAlias t Nothing))
|
||||
| worthRange db t (unqualify mAlias k) = Range mAlias t (unqualify mAlias k) Nothing (Just (VStr v, False))
|
||||
| worthRange db t (unqualify mAlias k) Nothing (Just (VStr v, False)) =
|
||||
Range mAlias t (unqualify mAlias k) Nothing (Just (VStr v, False))
|
||||
rewriteNode db (Filter (And (GtE (Col k1) (LitInt lo)) (LtE (Col k2) (LitInt hi))) (Scan mAlias t Nothing))
|
||||
| k1 == k2 && worthRange db t (unqualify mAlias k1) =
|
||||
| k1 == k2 && worthRange db t (unqualify mAlias k1) (Just (VInt lo, True)) (Just (VInt hi, True)) =
|
||||
Range mAlias t (unqualify mAlias k1) (Just (VInt lo, True)) (Just (VInt hi, True))
|
||||
rewriteNode db (Filter (And (GtE (Col k1) (LitInt lo)) (Lt (Col k2) (LitInt hi))) (Scan mAlias t Nothing))
|
||||
| k1 == k2 && worthRange db t (unqualify mAlias k1) =
|
||||
| k1 == k2 && worthRange db t (unqualify mAlias k1) (Just (VInt lo, True)) (Just (VInt hi, False)) =
|
||||
Range mAlias t (unqualify mAlias k1) (Just (VInt lo, True)) (Just (VInt hi, False))
|
||||
rewriteNode db (Filter (And (Gt (Col k1) (LitInt lo)) (LtE (Col k2) (LitInt hi))) (Scan mAlias t Nothing))
|
||||
| k1 == k2 && worthRange db t (unqualify mAlias k1) =
|
||||
| k1 == k2 && worthRange db t (unqualify mAlias k1) (Just (VInt lo, False)) (Just (VInt hi, True)) =
|
||||
Range mAlias t (unqualify mAlias k1) (Just (VInt lo, False)) (Just (VInt hi, True))
|
||||
rewriteNode db (Filter (And (GtE (Col k1) (LitStr lo)) (LtE (Col k2) (LitStr hi))) (Scan mAlias t Nothing))
|
||||
| k1 == k2 && worthRange db t (unqualify mAlias k1) =
|
||||
| k1 == k2 && worthRange db t (unqualify mAlias k1) (Just (VStr lo, True)) (Just (VStr hi, True)) =
|
||||
Range mAlias t (unqualify mAlias k1) (Just (VStr lo, True)) (Just (VStr hi, True))
|
||||
rewriteNode db (Filter (And (GtE (Col k1) (LitStr lo)) (Lt (Col k2) (LitStr hi))) (Scan mAlias t Nothing))
|
||||
| k1 == k2 && worthRange db t (unqualify mAlias k1) =
|
||||
| k1 == k2 && worthRange db t (unqualify mAlias k1) (Just (VStr lo, True)) (Just (VStr hi, False)) =
|
||||
Range mAlias t (unqualify mAlias k1) (Just (VStr lo, True)) (Just (VStr hi, False))
|
||||
rewriteNode db (Filter (And (Gt (Col k1) (LitStr lo)) (LtE (Col k2) (LitStr hi))) (Scan mAlias t Nothing))
|
||||
| k1 == k2 && worthRange db t (unqualify mAlias k1) =
|
||||
| k1 == k2 && worthRange db t (unqualify mAlias k1) (Just (VStr lo, False)) (Just (VStr hi, True)) =
|
||||
Range mAlias t (unqualify mAlias k1) (Just (VStr lo, False)) (Just (VStr hi, True))
|
||||
rewriteNode db (Filter (GtE (Col k) (LitInt v)) (Scan mAlias t Nothing))
|
||||
| worthRange db t (unqualify mAlias k) = Range mAlias t (unqualify mAlias k) (Just (VInt v, True)) Nothing
|
||||
| worthRange db t (unqualify mAlias k) (Just (VInt v, True)) Nothing =
|
||||
Range mAlias t (unqualify mAlias k) (Just (VInt v, True)) Nothing
|
||||
rewriteNode db (Filter (GtE (Col k) (LitStr v)) (Scan mAlias t Nothing))
|
||||
| worthRange db t (unqualify mAlias k) = Range mAlias t (unqualify mAlias k) (Just (VStr v, True)) Nothing
|
||||
| worthRange db t (unqualify mAlias k) (Just (VStr v, True)) Nothing =
|
||||
Range mAlias t (unqualify mAlias k) (Just (VStr v, True)) Nothing
|
||||
rewriteNode db (Filter (LtE (Col k) (LitInt v)) (Scan mAlias t Nothing))
|
||||
| worthRange db t (unqualify mAlias k) = Range mAlias t (unqualify mAlias k) Nothing (Just (VInt v, True))
|
||||
| worthRange db t (unqualify mAlias k) Nothing (Just (VInt v, True)) =
|
||||
Range mAlias t (unqualify mAlias k) Nothing (Just (VInt v, True))
|
||||
rewriteNode db (Filter (LtE (Col k) (LitStr v)) (Scan mAlias t Nothing))
|
||||
| worthRange db t (unqualify mAlias k) = Range mAlias t (unqualify mAlias k) Nothing (Just (VStr v, True))
|
||||
| worthRange db t (unqualify mAlias k) Nothing (Just (VStr v, True)) =
|
||||
Range mAlias t (unqualify mAlias k) Nothing (Just (VStr v, True))
|
||||
rewriteNode db (Filter p (Join kind l r c))
|
||||
| not (hasDivision p || planHasDivision (Join kind l r c)) =
|
||||
case kind of
|
||||
|
||||
@@ -401,7 +401,7 @@ instance MonadStorage IPCStorage where
|
||||
|
||||
-- | 线上表信息转引擎侧统计
|
||||
metaOf :: TableInfo -> TableMeta
|
||||
metaOf info = TableMeta (tiRows info) (tiStats info) (tiIndexes info)
|
||||
metaOf info = TableMeta (tiRows info) (tiStats info) (tiHistograms info) (tiIndexes info)
|
||||
|
||||
-- | 没设库名就照单全收；设了库名只收这个库的表，键名去掉库名前缀
|
||||
visibleTables :: Maybe String -> TableInfo -> [(String, Table)]
|
||||
|
||||
Reference in new issue
Block a user