优化器直方图

chusql-core/storage/src/catalog.rs

@@ -12,6 +12,7 @@ use crate::protocol::{Row, SchemaColumn};
 // 数据字典：记每张表有哪些列、什么类型、多少行、有哪些索引，以及列级统计。
 
 const STATS_DISTINCT_CAP: u64 = 4096;
+const HIST_BUCKETS: usize = 16;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct IndexSchema {
@@ -23,6 +24,12 @@ pub struct ColumnStat {
     pub distinct: u64,
     #[serde(default)]
     pub capped: bool,
+    #[serde(default)]
+    pub lo: Option<f64>,
+    #[serde(default)]
+    pub hi: Option<f64>,
+    #[serde(default)]
+    pub hist: Vec<u64>,
 }
 
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
@@ -152,6 +159,14 @@ impl Catalog {
                 if seen.remove(&fold(name, value_hash(v))) {
                     stat.distinct = stat.distinct.saturating_sub(1);
                 }
+                hist_remove(stat, v);
+            }
+        }
+        if entry.row_count == 0 {
+            for stat in entry.stats.values_mut() {
+                stat.lo = None;
+                stat.hi = None;
+                stat.hist.clear();
             }
         }
     }
@@ -187,6 +202,7 @@ impl Catalog {
                     seen.clear();
                 }
             }
+            hist_add(stat, v);
         }
     }
 
@@ -397,3 +413,63 @@ fn fold(column: &str, value_hash: u64) -> u64 {
     value_hash.hash(&mut h);
     h.finish()
 }
+
+/// 值落在第几个桶；区间退化时都算第 0 桶
+fn hist_index(lo: f64, hi: f64, v: f64) -> usize {
+    if hi <= lo {
+        return 0;
+    }
+    let scaled = ((v - lo) / (hi - lo) * HIST_BUCKETS as f64).floor();
+    scaled.clamp(0.0, (HIST_BUCKETS - 1) as f64) as usize
+}
+
+/// 区间拓宽时把旧桶按中点并进来，只保总量
+fn hist_merge(old: &[u64], old_lo: f64, old_hi: f64, new_lo: f64, new_hi: f64) -> Vec<u64> {
+    let mut out = vec![0u64; HIST_BUCKETS];
+    for (i, count) in old.iter().enumerate() {
+        if *count == 0 {
+            continue;
+        }
+        let mid = if old_hi > old_lo {
+            old_lo + (old_hi - old_lo) * (i as f64 + 0.5) / old.len() as f64
+        } else {
+            old_lo
+        };
+        out[hist_index(new_lo, new_hi, mid)] += *count;
+    }
+    out
+}
+
+/// 把一个值并进直方图；值跑出区间就拓宽区间
+fn hist_add(stat: &mut ColumnStat, v: &serde_json::Value) {
+    let Some(x) = v.as_f64() else { return };
+    match (stat.lo, stat.hi) {
+        (Some(lo), Some(hi)) if stat.hist.len() == HIST_BUCKETS => {
+            if x < lo || x > hi {
+                let (nlo, nhi) = (lo.min(x), hi.max(x));
+                stat.hist = hist_merge(&stat.hist, lo, hi, nlo, nhi);
+                stat.lo = Some(nlo);
+                stat.hi = Some(nhi);
+            }
+            let index = hist_index(stat.lo.unwrap_or(x), stat.hi.unwrap_or(x), x);
+            stat.hist[index] += 1;
+        }
+        _ => {
+            stat.lo = Some(x);
+            stat.hi = Some(x);
+            stat.hist = vec![0u64; HIST_BUCKETS];
+            stat.hist[0] = 1;
+        }
+    }
+}
+
+/// 从直方图里减掉一个值；值在区间外就不动
+fn hist_remove(stat: &mut ColumnStat, v: &serde_json::Value) {
+    let Some(x) = v.as_f64() else { return };
+    let (Some(lo), Some(hi)) = (stat.lo, stat.hi) else { return };
+    if x < lo || x > hi || stat.hist.len() != HIST_BUCKETS {
+        return;
+    }
+    let index = hist_index(lo, hi, x);
+    stat.hist[index] = stat.hist[index].saturating_sub(1);
+}

chusql-core/storage/src/protocol.rs

@@ -58,12 +58,18 @@ fn default_true() -> bool {
     true
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct ColumnStatWire {
     pub name: String,
     pub distinct: u64,
     #[serde(default)]
     pub capped: bool,
+    #[serde(default)]
+    pub lo: Option<f64>,
+    #[serde(default)]
+    pub hi: Option<f64>,
+    #[serde(default)]
+    pub hist: Vec<u64>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]

chusql-core/storage/src/runtime.rs

@@ -1602,6 +1602,9 @@ fn stat_wires(schema: &crate::catalog::TableSchema) -> Vec<ColumnStatWire> {
             name: name.clone(),
             distinct: s.distinct,
             capped: s.capped,
+            lo: s.lo,
+            hi: s.hi,
+            hist: s.hist.clone(),
         })
         .collect()
 }

chusql-core/storage/tests/catalog_test.rs

@@ -83,6 +83,47 @@ fn remove_column_drops_definition_stats_and_index() {
     assert!(c.remove_column("nope", "code").is_err());
 }
 
+/// 直方图随插入累计、随删除递减，值跑出区间就拓宽
+#[test]
+fn stats_track_a_histogram_for_numeric_columns() {
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("catalog.json");
+
+    let mut c = Catalog::load(&path).unwrap();
+    let rows: Vec<Row> = (0..100)
+        .map(|n| row(&[("id", json!(n)), ("name", json!(format!("u{}", n)))]))
+        .collect();
+    c.rebuild_stats("t", &rows);
+
+    let s = c.describe("t").unwrap();
+    let id = s.stats.get("id").unwrap();
+    assert_eq!(id.distinct, 100);
+    assert_eq!(id.lo, Some(0.0));
+    assert_eq!(id.hi, Some(99.0));
+    assert_eq!(id.hist.len(), 16);
+    assert_eq!(id.hist.iter().sum::<u64>(), 100);
+    let name = s.stats.get("name").unwrap();
+    assert!(name.hist.is_empty(), "字符串列不进直方图");
+    assert_eq!(name.lo, None);
+
+    c.record_insert("t", &row(&[("id", json!(1000)), ("name", json!("x"))]));
+    let s = c.describe("t").unwrap();
+    let id = s.stats.get("id").unwrap();
+    assert_eq!(id.lo, Some(0.0));
+    assert_eq!(id.hi, Some(1000.0));
+    assert_eq!(id.hist.iter().sum::<u64>(), 101);
+
+    c.record_delete("t", &[row(&[("id", json!(1000)), ("name", json!("x"))])]);
+    let s = c.describe("t").unwrap();
+    assert_eq!(s.stats.get("id").unwrap().hist.iter().sum::<u64>(), 100);
+
+    c.record_delete("t", &rows);
+    let s = c.describe("t").unwrap();
+    let id = s.stats.get("id").unwrap();
+    assert_eq!(id.lo, None, "删空之后直方图清掉");
+    assert_eq!(id.hist.iter().sum::<u64>(), 0);
+}
+
 /// 老类型名并到新写法，参数保留，未知类型原样
 #[test]
 fn normalize_type_maps_legacy_names() {

chusql-core/storage/tests/runtime_test.rs

@@ -1554,6 +1554,11 @@ fn stats_report_distinct_values() {
     assert!(r.contains(r#""row_count":3"#), "row_count: {}", r);
     assert!(r.contains(r#""name":"age","distinct":2"#), "age has 2 distinct values: {}", r);
     assert!(r.contains(r#""name":"id","distinct":3"#), "id has 3 distinct values: {}", r);
+    assert!(
+        r.contains(r#""name":"age","distinct":2,"capped":false,"lo":30.0,"hi":41.0"#),
+        "age carries a histogram range: {}",
+        r
+    );
 
     send(&mut c, r#"{"method":"delete_keys","table":"stat_t","keys":[1]}"#);
     let r = send(&mut c, r#"{"method":"describe_table","table":"stat_t"}"#);