rust部分推进：进程间通信（Windows平台命名管线）、内存页表、B+树、WAL恢复系统、动态配置
This commit is contained in:
LIm1tZz7 committed
1 parent
91ce26d7f0
commit
e428265357
41 files changed
+4546
-159
No files matched your search
@@ -0,0 +1,412 @@
|
||||
//! 命名管道服务：按行读写 JSON。Windows 上管道名映射为 \\.\pipe\<name>。
|
||||
//! 启动时读配置（环境变量 → TOML → 默认值）并把每项来源打进日志。
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
|
||||
use chusql_storage::catalog::Catalog;
|
||||
use chusql_storage::config::{self, Config, Loaded};
|
||||
use chusql_storage::heap::HeapTable;
|
||||
use chusql_storage::log;
|
||||
use chusql_storage::protocol::{Request, Response};
|
||||
use chusql_storage::wal::{Wal, WalOp};
|
||||
use chusql_storage::{log_debug, log_error, log_info, log_warn};
|
||||
use interprocess::local_socket::{
|
||||
prelude::*,
|
||||
GenericNamespaced,
|
||||
ListenerOptions,
|
||||
};
|
||||
use interprocess::TryClone;
|
||||
|
||||
// * 路径
|
||||
|
||||
/// 表名 -> 数据文件路径。
|
||||
fn table_path(cfg: &Config, table: &str) -> PathBuf {
|
||||
cfg.data_dir.join(format!("{}.db", table))
|
||||
}
|
||||
|
||||
/// 表名 -> 索引文件路径。
|
||||
fn index_path(cfg: &Config, table: &str) -> PathBuf {
|
||||
cfg.data_dir.join(format!("{}.idx", table))
|
||||
}
|
||||
|
||||
/// 数据字典文件路径。
|
||||
fn catalog_path(cfg: &Config) -> PathBuf {
|
||||
cfg.data_dir.join("catalog.json")
|
||||
}
|
||||
|
||||
/// WAL 文件路径。
|
||||
fn wal_path(cfg: &Config) -> PathBuf {
|
||||
cfg.data_dir.join("wal.log")
|
||||
}
|
||||
|
||||
// * 表操作
|
||||
|
||||
/// 打开一张表，执行闭包；表不存在就创建。
|
||||
fn with_table<R>(
|
||||
cfg: &Config,
|
||||
table: &str,
|
||||
f: impl FnOnce(&mut HeapTable) -> std::io::Result<R>,
|
||||
) -> std::io::Result<R> {
|
||||
std::fs::create_dir_all(&cfg.data_dir)?;
|
||||
let mut t = HeapTable::open(table_path(cfg, table), cfg.page_size)?;
|
||||
f(&mut t)
|
||||
}
|
||||
|
||||
/// 打开带索引的表，执行闭包。
|
||||
fn with_indexed_table<R>(
|
||||
cfg: &Config,
|
||||
table: &str,
|
||||
f: impl FnOnce(&mut HeapTable) -> std::io::Result<R>,
|
||||
) -> std::io::Result<R> {
|
||||
std::fs::create_dir_all(&cfg.data_dir)?;
|
||||
let mut t = HeapTable::open_indexed(
|
||||
table_path(cfg, table),
|
||||
index_path(cfg, table),
|
||||
cfg.page_size,
|
||||
cfg.btree_order,
|
||||
)?;
|
||||
f(&mut t)
|
||||
}
|
||||
|
||||
/// 表文件是否存在且非空。
|
||||
fn table_exists(cfg: &Config, table: &str) -> bool {
|
||||
let p = table_path(cfg, table);
|
||||
std::fs::metadata(&p).map(|m| m.len() > 0).unwrap_or(false)
|
||||
}
|
||||
|
||||
/// 打开已存在的表；不存在返回 NotFound。
|
||||
fn with_existing_table<R>(
|
||||
cfg: &Config,
|
||||
table: &str,
|
||||
f: impl FnOnce(&mut HeapTable) -> std::io::Result<R>,
|
||||
) -> std::io::Result<R> {
|
||||
if !table_exists(cfg, table) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::NotFound,
|
||||
format!("unknown table: {}", table),
|
||||
));
|
||||
}
|
||||
let mut t = HeapTable::open(table_path(cfg, table), cfg.page_size)?;
|
||||
f(&mut t)
|
||||
}
|
||||
|
||||
// * 数据字典操作
|
||||
|
||||
/// 读 catalog，按一行数据补列，再写回。
|
||||
fn catalog_add_row(
|
||||
cfg: &Config,
|
||||
table: &str,
|
||||
row: &serde_json::Map<String, serde_json::Value>,
|
||||
) -> std::io::Result<()> {
|
||||
let mut c = Catalog::load(catalog_path(cfg))?;
|
||||
let row: chusql_storage::protocol::Row = row.clone().into_iter().collect();
|
||||
c.ensure_columns(table, &row);
|
||||
c.save(catalog_path(cfg))
|
||||
}
|
||||
|
||||
/// 读 catalog，按多行数据补列，再写回。
|
||||
fn catalog_add_rows(
|
||||
cfg: &Config,
|
||||
table: &str,
|
||||
rows: &[serde_json::Map<String, serde_json::Value>],
|
||||
) -> std::io::Result<()> {
|
||||
let mut c = Catalog::load(catalog_path(cfg))?;
|
||||
for r in rows {
|
||||
let row: chusql_storage::protocol::Row = r.clone().into_iter().collect();
|
||||
c.ensure_columns(table, &row);
|
||||
}
|
||||
c.save(catalog_path(cfg))
|
||||
}
|
||||
|
||||
// * WAL 写路径与恢复
|
||||
|
||||
/// 把一条 WAL 操作真正落到磁盘（堆表 + 索引 + catalog）。
|
||||
fn apply_op(cfg: &Config, op: &WalOp) -> std::io::Result<()> {
|
||||
match op {
|
||||
WalOp::Insert { table, row, key } => {
|
||||
match key {
|
||||
None => with_table(cfg, table, |t| t.insert(row))?,
|
||||
Some(k) => with_indexed_table(cfg, table, |t| t.insert_keyed(row, *k))?,
|
||||
}
|
||||
let map: serde_json::Map<_, _> = row.clone().into_iter().collect();
|
||||
catalog_add_row(cfg, table, &map)?;
|
||||
Ok(())
|
||||
}
|
||||
WalOp::ReplaceAll { table, rows } => {
|
||||
with_indexed_table(cfg, table, |t| t.replace_all(rows))?;
|
||||
let maps: Vec<_> = rows.iter().map(|r| r.clone().into_iter().collect()).collect();
|
||||
catalog_add_rows(cfg, table, &maps)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 写前日志 + 写数据 + 清空 WAL。
|
||||
///
|
||||
/// 顺序：clear（清掉上次已完成的） → append+fsync → apply → clear。
|
||||
/// 若中途进程被杀，重启时 WAL 里能读到那条操作并重放。
|
||||
fn write_via_wal(cfg: &Config, op: &WalOp) -> std::io::Result<()> {
|
||||
let wal = Wal::new(wal_path(cfg));
|
||||
|
||||
// 清掉上一次遗留（正常情况下已经是空的）
|
||||
wal.clear()?;
|
||||
|
||||
// 先写 WAL 并 fsync
|
||||
wal.append(op)?;
|
||||
|
||||
// 再写数据
|
||||
let result = apply_op(cfg, op);
|
||||
|
||||
// 无论 apply 成功还是返回 Err，都清空 WAL：
|
||||
// 失败的操作不应被重放；若中途被杀，这里跑不到，WAL 会留到重启时重放。
|
||||
wal.clear()?;
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// 启动时重放 WAL 里的残留操作。
|
||||
fn recover_wal(cfg: &Config) -> std::io::Result<()> {
|
||||
let wal = Wal::new(wal_path(cfg));
|
||||
match wal.read()? {
|
||||
None => Ok(()),
|
||||
Some(op) => {
|
||||
log_warn!(core, "replaying incomplete WAL entry");
|
||||
apply_op(cfg, &op)?;
|
||||
wal.clear()?;
|
||||
log_info!(core, "WAL replay complete");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// * 请求分发
|
||||
|
||||
/// 把一条请求翻译成一条响应。
|
||||
fn handle_request(cfg: &Config, req: Request) -> Response {
|
||||
match req {
|
||||
Request::Ping => {
|
||||
log_debug!(pipe, "ping");
|
||||
Response::Pong
|
||||
}
|
||||
|
||||
Request::Scan { table } => {
|
||||
log_debug!(pipe, "scan table={}", table);
|
||||
match with_existing_table(cfg, &table, |t| t.scan()) {
|
||||
Ok(rows) => Response::Rows { rows },
|
||||
Err(e) => {
|
||||
log_warn!(pipe, "scan {}: {}", table, e);
|
||||
Response::Error {
|
||||
message: format!("scan {}: {}", table, e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Request::Insert { table, row, key } => {
|
||||
log_debug!(pipe, "insert table={} key={:?}", table, key);
|
||||
let tname = table.clone();
|
||||
let op = WalOp::Insert { table, row, key };
|
||||
match write_via_wal(cfg, &op) {
|
||||
Ok(()) => Response::Ok,
|
||||
Err(e) => {
|
||||
log_warn!(pipe, "insert {}: {}", tname, e);
|
||||
Response::Error {
|
||||
message: format!("insert {}: {}", tname, e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Request::LookupByIndex { table, key } => {
|
||||
log_debug!(pipe, "lookup_by_index table={} key={}", table, key);
|
||||
match with_indexed_table(cfg, &table, |t| t.get_by_key(key)) {
|
||||
Ok(Some(row)) => Response::Rows { rows: vec![row] },
|
||||
Ok(None) => Response::Rows { rows: vec![] },
|
||||
Err(e) => {
|
||||
log_warn!(pipe, "lookup_by_index {}: {}", table, e);
|
||||
Response::Error {
|
||||
message: format!("lookup_by_index {}: {}", table, e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Request::ReplaceAll { table, rows } => {
|
||||
log_debug!(pipe, "replace_all table={} rows={}", table, rows.len());
|
||||
let tname = table.clone();
|
||||
let op = WalOp::ReplaceAll { table, rows };
|
||||
match write_via_wal(cfg, &op) {
|
||||
Ok(()) => Response::Ok,
|
||||
Err(e) => {
|
||||
log_warn!(pipe, "replace_all {}: {}", tname, e);
|
||||
Response::Error {
|
||||
message: format!("replace_all {}: {}", tname, e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Request::ListTables => {
|
||||
log_debug!(pipe, "list_tables");
|
||||
match list_tables(cfg) {
|
||||
Ok(tables) => Response::Tables { tables },
|
||||
Err(e) => {
|
||||
log_warn!(pipe, "list_tables: {}", e);
|
||||
Response::Error {
|
||||
message: format!("list_tables: {}", e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Request::DescribeTable { table } => {
|
||||
log_debug!(pipe, "describe_table table={}", table);
|
||||
match describe_table(cfg, &table) {
|
||||
Ok(cols) => Response::Schema { columns: cols },
|
||||
Err(e) => {
|
||||
log_warn!(pipe, "describe_table {}: {}", table, e);
|
||||
Response::Error {
|
||||
message: format!("describe_table {}: {}", table, e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 列出表名：catalog 记录 + data 目录下的 .db 文件，取并集。
|
||||
fn list_tables(cfg: &Config) -> std::io::Result<Vec<String>> {
|
||||
let mut names = BTreeSet::new();
|
||||
|
||||
let c = Catalog::load(catalog_path(cfg))?;
|
||||
for t in c.table_names() {
|
||||
names.insert(t);
|
||||
}
|
||||
|
||||
let dir = &cfg.data_dir;
|
||||
if dir.exists() {
|
||||
for entry in std::fs::read_dir(dir)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
if let Some(stem) = name.strip_suffix(".db") {
|
||||
names.insert(stem.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(names.into_iter().collect())
|
||||
}
|
||||
|
||||
/// 查一张表的 schema。表在 catalog 里没有，但文件存在，返回空列。
|
||||
fn describe_table(
|
||||
cfg: &Config,
|
||||
table: &str,
|
||||
) -> std::io::Result<Vec<chusql_storage::protocol::SchemaColumn>> {
|
||||
let c = Catalog::load(catalog_path(cfg))?;
|
||||
if let Some(schema) = c.describe(table) {
|
||||
return Ok(schema.columns.clone());
|
||||
}
|
||||
if table_exists(cfg, table) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
Err(std::io::Error::new(
|
||||
std::io::ErrorKind::NotFound,
|
||||
format!("unknown table: {}", table),
|
||||
))
|
||||
}
|
||||
|
||||
// * 连接处理
|
||||
|
||||
/// 处理一条连接：逐行读 JSON 请求，逐行回 JSON 响应。
|
||||
fn handle(cfg: &Config, conn: LocalSocketStream) -> std::io::Result<()> {
|
||||
let mut writer = conn.try_clone()?;
|
||||
let reader = BufReader::new(conn);
|
||||
|
||||
for line in reader.lines() {
|
||||
let line = line?;
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let resp = match serde_json::from_str::<Request>(&line) {
|
||||
Ok(req) => handle_request(cfg, req),
|
||||
Err(e) => {
|
||||
log_warn!(pipe, "bad request line: {}", e);
|
||||
Response::Error {
|
||||
message: format!("parse error: {}", e),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
writeln!(writer, "{}", serde_json::to_string(&resp).unwrap())?;
|
||||
writer.flush()?;
|
||||
}
|
||||
log_debug!(pipe, "connection closed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// * 主入口
|
||||
|
||||
/// 把"配置从哪来"打进日志。
|
||||
fn log_config(loaded: &Loaded) {
|
||||
match &loaded.config_path {
|
||||
Some(p) => log_info!(core, "config file: {}", p.display()),
|
||||
None => log_info!(core, "config file: none"),
|
||||
}
|
||||
for (name, value, origin) in &loaded.origins {
|
||||
log_info!(core, " {:<17} {} [{}]", name, value, origin.describe());
|
||||
}
|
||||
}
|
||||
|
||||
fn main() -> std::io::Result<()> {
|
||||
let loaded = match config::load() {
|
||||
Ok(l) => l,
|
||||
Err(e) => {
|
||||
log_error!(core, "config error: {}", e);
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, e));
|
||||
}
|
||||
};
|
||||
log::init();
|
||||
log::set_level(loaded.config.log_level);
|
||||
|
||||
log_info!(core, "chusql-storage {} starting", env!("CARGO_PKG_VERSION"));
|
||||
log_config(&loaded);
|
||||
|
||||
let cfg = Arc::new(loaded.config);
|
||||
|
||||
// 启动时先重放 WAL，再做其他事
|
||||
if let Err(e) = recover_wal(&cfg) {
|
||||
log_error!(core, "WAL recovery failed: {}", e);
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
let ns_name = cfg
|
||||
.pipe_name
|
||||
.as_str()
|
||||
.to_ns_name::<GenericNamespaced>()
|
||||
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
let listener = ListenerOptions::new().name(ns_name).create_sync()?;
|
||||
|
||||
log_info!(
|
||||
core,
|
||||
"listening on pipe {} (\\\\.\\pipe\\{})",
|
||||
cfg.pipe_name,
|
||||
cfg.pipe_name
|
||||
);
|
||||
|
||||
for conn in listener.incoming() {
|
||||
let conn = conn?;
|
||||
let cfg = Arc::clone(&cfg);
|
||||
thread::spawn(move || {
|
||||
log_debug!(pipe, "connection accepted");
|
||||
if let Err(e) = handle(&cfg, conn) {
|
||||
log_error!(pipe, "connection error: {}", e);
|
||||
}
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,470 @@
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::page::{Page, PageFile, PageId};
|
||||
|
||||
// 磁盘版 B+ 树：每个节点占一页，page 0 存文件头（魔数 + 页大小 + order + 根页号）。
|
||||
// 节点页布局（order = 一个节点最多几个孩子，来自配置）：
|
||||
// [0] 节点类型：0=叶子，1=内部
|
||||
// [1..3] 键数量 u16
|
||||
// [3..11] 叶子的 next 页号 u64（内部保留）
|
||||
// [12..12+K*8] 键：K = order-1 个 i64
|
||||
// [后面] 叶子：K 个 u64 值；内部：order 个 u64 孩子页号
|
||||
|
||||
/// 文件头放在这一页。
|
||||
const META_PAGE: PageId = 0;
|
||||
/// 文件头魔数。
|
||||
const META_MAGIC: &[u8; 4] = b"CBTR";
|
||||
/// 文件头里各字段的偏移。
|
||||
const OFF_META_PAGE_SIZE: usize = 4;
|
||||
const OFF_META_ORDER: usize = 8;
|
||||
const OFF_META_ROOT: usize = 10;
|
||||
|
||||
/// 节点类型标记。
|
||||
const NODE_LEAF: u8 = 0;
|
||||
const NODE_INTERNAL: u8 = 1;
|
||||
/// 布局偏移。
|
||||
const OFF_TYPE: usize = 0;
|
||||
const OFF_COUNT: usize = 1;
|
||||
const OFF_NEXT: usize = 3;
|
||||
const OFF_KEYS: usize = 12;
|
||||
|
||||
/// 一个 order 的节点占多少字节（内部节点最坏情况：order-1 个键 + order 个孩子）。
|
||||
fn node_bytes(order: usize) -> usize {
|
||||
OFF_KEYS + max_keys(order) * 8 + order * 8
|
||||
}
|
||||
|
||||
/// 一个节点最多几个键。
|
||||
fn max_keys(order: usize) -> usize {
|
||||
order - 1
|
||||
}
|
||||
|
||||
/// 键后面那片区域的起点。
|
||||
fn off_payload(order: usize) -> usize {
|
||||
OFF_KEYS + max_keys(order) * 8
|
||||
}
|
||||
|
||||
/// 这个 order 放得进 size 字节的页吗。
|
||||
pub fn fits_in_page(page_size: usize, order: usize) -> bool {
|
||||
order >= 2 && node_bytes(order) <= page_size
|
||||
}
|
||||
|
||||
/// 给定页大小，order 最大能取多少（解 12 + (order-1)*8 + order*8 <= page_size）。
|
||||
pub fn max_order(page_size: usize) -> usize {
|
||||
page_size.saturating_sub(4) / 16
|
||||
}
|
||||
|
||||
/// 读一个 u8。
|
||||
fn read_u8(p: &Page, off: usize) -> u8 {
|
||||
p.data[off]
|
||||
}
|
||||
|
||||
/// 写一个 u8。
|
||||
fn write_u8(p: &mut Page, off: usize, v: u8) {
|
||||
p.data[off] = v;
|
||||
}
|
||||
|
||||
/// 读一个小端 u16。
|
||||
fn read_u16(p: &Page, off: usize) -> u16 {
|
||||
u16::from_le_bytes([p.data[off], p.data[off + 1]])
|
||||
}
|
||||
|
||||
/// 写一个小端 u16。
|
||||
fn write_u16(p: &mut Page, off: usize, v: u16) {
|
||||
p.data[off..off + 2].copy_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
|
||||
/// 读一个小端 u32。
|
||||
fn read_u32(p: &Page, off: usize) -> u32 {
|
||||
u32::from_le_bytes([p.data[off], p.data[off + 1], p.data[off + 2], p.data[off + 3]])
|
||||
}
|
||||
|
||||
/// 写一个小端 u32。
|
||||
fn write_u32(p: &mut Page, off: usize, v: u32) {
|
||||
p.data[off..off + 4].copy_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
|
||||
/// 读一个小端 u64。
|
||||
fn read_u64(p: &Page, off: usize) -> u64 {
|
||||
let mut b = [0u8; 8];
|
||||
b.copy_from_slice(&p.data[off..off + 8]);
|
||||
u64::from_le_bytes(b)
|
||||
}
|
||||
|
||||
/// 写一个小端 u64。
|
||||
fn write_u64(p: &mut Page, off: usize, v: u64) {
|
||||
p.data[off..off + 8].copy_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
|
||||
/// 写文件头。
|
||||
fn write_meta(page: &mut Page, page_size: usize, order: usize, root: PageId) {
|
||||
page.zero();
|
||||
page.data[0..4].copy_from_slice(META_MAGIC);
|
||||
write_u32(page, OFF_META_PAGE_SIZE, page_size as u32);
|
||||
write_u16(page, OFF_META_ORDER, order as u16);
|
||||
write_u64(page, OFF_META_ROOT, root);
|
||||
}
|
||||
|
||||
/// 检查文件头和当前配置是否一致（页大小 / order 改了就会对不上）。
|
||||
fn check_meta(page: &Page, page_size: usize, order: usize) -> io::Result<()> {
|
||||
if page.data[0..4] != *META_MAGIC {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"not a chusql index file (bad magic)",
|
||||
));
|
||||
}
|
||||
let stored_page = read_u32(page, OFF_META_PAGE_SIZE) as usize;
|
||||
let stored_order = read_u16(page, OFF_META_ORDER) as usize;
|
||||
if stored_page != page_size || stored_order != order {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!(
|
||||
"index file was built with page.size={} btree.order={}, config says page.size={} btree.order={}",
|
||||
stored_page, stored_order, page_size, order
|
||||
),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 一个节点在内存里的样子。
|
||||
enum NodeData {
|
||||
Leaf {
|
||||
keys: Vec<i64>,
|
||||
values: Vec<u64>,
|
||||
next: PageId,
|
||||
},
|
||||
Internal {
|
||||
keys: Vec<i64>,
|
||||
children: Vec<PageId>,
|
||||
},
|
||||
}
|
||||
|
||||
/// 从一页里读出节点。
|
||||
fn read_node(p: &Page, order: usize) -> NodeData {
|
||||
let ty = read_u8(p, OFF_TYPE);
|
||||
let n = read_u16(p, OFF_COUNT) as usize;
|
||||
let payload = off_payload(order);
|
||||
if ty == NODE_LEAF {
|
||||
let next = read_u64(p, OFF_NEXT);
|
||||
let mut keys = Vec::with_capacity(n);
|
||||
let mut values = Vec::with_capacity(n);
|
||||
for i in 0..n {
|
||||
keys.push(read_u64(p, OFF_KEYS + i * 8) as i64);
|
||||
values.push(read_u64(p, payload + i * 8));
|
||||
}
|
||||
NodeData::Leaf { keys, values, next }
|
||||
} else {
|
||||
let mut keys = Vec::with_capacity(n);
|
||||
let mut children = Vec::with_capacity(n + 1);
|
||||
for i in 0..n {
|
||||
keys.push(read_u64(p, OFF_KEYS + i * 8) as i64);
|
||||
}
|
||||
for i in 0..(n + 1) {
|
||||
children.push(read_u64(p, payload + i * 8));
|
||||
}
|
||||
NodeData::Internal { keys, children }
|
||||
}
|
||||
}
|
||||
|
||||
/// 把节点写回一页（先清零，避免残留）。
|
||||
fn write_node(p: &mut Page, node: &NodeData, order: usize) {
|
||||
p.zero();
|
||||
let payload = off_payload(order);
|
||||
match node {
|
||||
NodeData::Leaf { keys, values, next } => {
|
||||
write_u8(p, OFF_TYPE, NODE_LEAF);
|
||||
write_u16(p, OFF_COUNT, keys.len() as u16);
|
||||
write_u64(p, OFF_NEXT, *next);
|
||||
for (i, k) in keys.iter().enumerate() {
|
||||
write_u64(p, OFF_KEYS + i * 8, *k as u64);
|
||||
}
|
||||
for (i, v) in values.iter().enumerate() {
|
||||
write_u64(p, payload + i * 8, *v);
|
||||
}
|
||||
}
|
||||
NodeData::Internal { keys, children } => {
|
||||
write_u8(p, OFF_TYPE, NODE_INTERNAL);
|
||||
write_u16(p, OFF_COUNT, keys.len() as u16);
|
||||
for (i, k) in keys.iter().enumerate() {
|
||||
write_u64(p, OFF_KEYS + i * 8, *k as u64);
|
||||
}
|
||||
for (i, c) in children.iter().enumerate() {
|
||||
write_u64(p, payload + i * 8, *c);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 二分找 key 应该下探到的孩子下标。
|
||||
fn child_index(keys: &[i64], key: i64) -> usize {
|
||||
let mut lo = 0;
|
||||
let mut hi = keys.len();
|
||||
while lo < hi {
|
||||
let mid = (lo + hi) / 2;
|
||||
if keys[mid] <= key {
|
||||
lo = mid + 1;
|
||||
} else {
|
||||
hi = mid;
|
||||
}
|
||||
}
|
||||
lo
|
||||
}
|
||||
|
||||
/// 磁盘版 B+ 树。
|
||||
pub struct DiskBTree {
|
||||
file: PageFile,
|
||||
order: usize,
|
||||
}
|
||||
|
||||
impl DiskBTree {
|
||||
/// 打开或新建索引文件；页大小 / order 和文件头里记的不一致就报错。
|
||||
pub fn open<P: AsRef<Path>>(path: P, page_size: usize, order: usize) -> io::Result<Self> {
|
||||
if !fits_in_page(page_size, order) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
format!(
|
||||
"btree order {} does not fit in a {}-byte page (max {})",
|
||||
order,
|
||||
page_size,
|
||||
max_order(page_size)
|
||||
),
|
||||
));
|
||||
}
|
||||
let mut file = PageFile::open(path, page_size)?;
|
||||
if file.num_pages()? == 0 {
|
||||
let mut meta = Page::new(META_PAGE, page_size);
|
||||
write_meta(&mut meta, page_size, order, 0);
|
||||
file.write_page(&meta)?;
|
||||
} else {
|
||||
let meta = file.read_page(META_PAGE)?;
|
||||
check_meta(&meta, page_size, order)?;
|
||||
}
|
||||
Ok(DiskBTree { file, order })
|
||||
}
|
||||
|
||||
/// 读根页号；0 表示空树。
|
||||
fn root(&mut self) -> io::Result<Option<PageId>> {
|
||||
let meta = self.file.read_page(META_PAGE)?;
|
||||
let r = read_u64(&meta, OFF_META_ROOT);
|
||||
Ok(if r == 0 { None } else { Some(r) })
|
||||
}
|
||||
|
||||
/// 写根页号。
|
||||
fn set_root(&mut self, r: Option<PageId>) -> io::Result<()> {
|
||||
let mut meta = self.file.read_page(META_PAGE)?;
|
||||
write_u64(&mut meta, OFF_META_ROOT, r.unwrap_or(0));
|
||||
self.file.write_page(&meta)
|
||||
}
|
||||
|
||||
/// 分配一个新页。
|
||||
fn alloc(&mut self) -> io::Result<PageId> {
|
||||
Ok(self.file.append_page()?.id)
|
||||
}
|
||||
|
||||
/// 清空整棵树：截断文件后重写文件头（根页号归零）。
|
||||
pub fn clear(&mut self) -> io::Result<()> {
|
||||
let page_size = self.file.page_size();
|
||||
self.file.truncate()?;
|
||||
let mut meta = Page::new(META_PAGE, page_size);
|
||||
write_meta(&mut meta, page_size, self.order, 0);
|
||||
self.file.write_page(&meta)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 空页，用来拼一个新节点。
|
||||
fn new_page(&self, id: PageId) -> Page {
|
||||
Page::new(id, self.file.page_size())
|
||||
}
|
||||
|
||||
/// 按 key 找 value。
|
||||
pub fn get(&mut self, key: i64) -> io::Result<Option<u64>> {
|
||||
match self.root()? {
|
||||
None => Ok(None),
|
||||
Some(r) => self.get_rec(r, key),
|
||||
}
|
||||
}
|
||||
|
||||
/// 递归查找。
|
||||
fn get_rec(&mut self, page_id: PageId, key: i64) -> io::Result<Option<u64>> {
|
||||
let p = self.file.read_page(page_id)?;
|
||||
match read_node(&p, self.order) {
|
||||
NodeData::Leaf { keys, values, .. } => {
|
||||
Ok(keys.binary_search(&key).ok().map(|i| values[i]))
|
||||
}
|
||||
NodeData::Internal { keys, children } => {
|
||||
let i = child_index(&keys, key);
|
||||
self.get_rec(children[i], key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 插入 / 覆盖一个键值对。
|
||||
pub fn insert(&mut self, key: i64, value: u64) -> io::Result<()> {
|
||||
match self.root()? {
|
||||
None => {
|
||||
let id = self.alloc()?;
|
||||
let node = NodeData::Leaf {
|
||||
keys: vec![key],
|
||||
values: vec![value],
|
||||
next: 0,
|
||||
};
|
||||
let mut p = self.new_page(id);
|
||||
write_node(&mut p, &node, self.order);
|
||||
self.file.write_page(&p)?;
|
||||
self.set_root(Some(id))
|
||||
}
|
||||
Some(r) => {
|
||||
let (_, split) = self.insert_rec(r, key, value)?;
|
||||
if let Some((sep, right_id)) = split {
|
||||
// 根分裂：长高一层。
|
||||
let root_id = self.alloc()?;
|
||||
let node = NodeData::Internal {
|
||||
keys: vec![sep],
|
||||
children: vec![r, right_id],
|
||||
};
|
||||
let mut p = self.new_page(root_id);
|
||||
write_node(&mut p, &node, self.order);
|
||||
self.file.write_page(&p)?;
|
||||
self.set_root(Some(root_id))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 递归插入；返回可能上提的 (分隔键, 新右页号)。
|
||||
fn insert_rec(
|
||||
&mut self,
|
||||
page_id: PageId,
|
||||
key: i64,
|
||||
value: u64,
|
||||
) -> io::Result<(PageId, Option<(i64, PageId)>)> {
|
||||
let p = self.file.read_page(page_id)?;
|
||||
let limit = max_keys(self.order);
|
||||
match read_node(&p, self.order) {
|
||||
NodeData::Leaf { mut keys, mut values, next } => {
|
||||
match keys.binary_search(&key) {
|
||||
Ok(i) => values[i] = value,
|
||||
Err(i) => {
|
||||
keys.insert(i, key);
|
||||
values.insert(i, value);
|
||||
}
|
||||
}
|
||||
if keys.len() <= limit {
|
||||
let mut p2 = self.new_page(page_id);
|
||||
write_node(&mut p2, &NodeData::Leaf { keys, values, next }, self.order);
|
||||
self.file.write_page(&p2)?;
|
||||
Ok((page_id, None))
|
||||
} else {
|
||||
// 叶子分裂：右叶最小键复制上提。
|
||||
let mid = keys.len() / 2;
|
||||
let rkeys = keys.split_off(mid);
|
||||
let rvalues = values.split_off(mid);
|
||||
let sep = rkeys[0];
|
||||
|
||||
let right_id = self.alloc()?;
|
||||
let mut rp = self.new_page(right_id);
|
||||
write_node(
|
||||
&mut rp,
|
||||
&NodeData::Leaf { keys: rkeys, values: rvalues, next },
|
||||
self.order,
|
||||
);
|
||||
self.file.write_page(&rp)?;
|
||||
|
||||
let mut lp = self.new_page(page_id);
|
||||
write_node(
|
||||
&mut lp,
|
||||
&NodeData::Leaf { keys, values, next: right_id },
|
||||
self.order,
|
||||
);
|
||||
self.file.write_page(&lp)?;
|
||||
|
||||
Ok((page_id, Some((sep, right_id))))
|
||||
}
|
||||
}
|
||||
|
||||
NodeData::Internal { mut keys, mut children } => {
|
||||
let i = child_index(&keys, key);
|
||||
let (new_child, split) = self.insert_rec(children[i], key, value)?;
|
||||
children[i] = new_child;
|
||||
match split {
|
||||
None => {
|
||||
let mut p2 = self.new_page(page_id);
|
||||
write_node(&mut p2, &NodeData::Internal { keys, children }, self.order);
|
||||
self.file.write_page(&p2)?;
|
||||
Ok((page_id, None))
|
||||
}
|
||||
Some((sep, right_id)) => {
|
||||
keys.insert(i, sep);
|
||||
children.insert(i + 1, right_id);
|
||||
if keys.len() <= limit {
|
||||
let mut p2 = self.new_page(page_id);
|
||||
write_node(
|
||||
&mut p2,
|
||||
&NodeData::Internal { keys, children },
|
||||
self.order,
|
||||
);
|
||||
self.file.write_page(&p2)?;
|
||||
Ok((page_id, None))
|
||||
} else {
|
||||
// 内部节点分裂：中间键上提，不再保留原件。
|
||||
let mid = keys.len() / 2;
|
||||
let up = keys[mid];
|
||||
let rkeys = keys.split_off(mid + 1);
|
||||
keys.truncate(mid);
|
||||
let rchildren = children.split_off(mid + 1);
|
||||
|
||||
let right_id = self.alloc()?;
|
||||
let mut rp = self.new_page(right_id);
|
||||
write_node(
|
||||
&mut rp,
|
||||
&NodeData::Internal { keys: rkeys, children: rchildren },
|
||||
self.order,
|
||||
);
|
||||
self.file.write_page(&rp)?;
|
||||
|
||||
let mut lp = self.new_page(page_id);
|
||||
write_node(
|
||||
&mut lp,
|
||||
&NodeData::Internal { keys, children },
|
||||
self.order,
|
||||
);
|
||||
self.file.write_page(&lp)?;
|
||||
|
||||
Ok((page_id, Some((up, right_id))))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 按 key 升序返回所有键值对。
|
||||
pub fn iter_all(&mut self) -> io::Result<Vec<(i64, u64)>> {
|
||||
let mut out = Vec::new();
|
||||
if let Some(r) = self.root()? {
|
||||
self.walk(r, &mut out)?;
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// 中序遍历一棵子树。
|
||||
fn walk(&mut self, page_id: PageId, out: &mut Vec<(i64, u64)>) -> io::Result<()> {
|
||||
let p = self.file.read_page(page_id)?;
|
||||
match read_node(&p, self.order) {
|
||||
NodeData::Leaf { keys, values, .. } => {
|
||||
for i in 0..keys.len() {
|
||||
out.push((keys[i], values[i]));
|
||||
}
|
||||
}
|
||||
NodeData::Internal { children, .. } => {
|
||||
for c in children {
|
||||
self.walk(c, out)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
//! 数据字典：记录每张表有哪些列、什么类型。
|
||||
//! 存在 data/catalog.json。insert 时自动补列，describe 时读回。
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::protocol::{ColumnType, Row, SchemaColumn};
|
||||
|
||||
/// 一张表的 schema。
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct TableSchema {
|
||||
pub columns: Vec<SchemaColumn>,
|
||||
}
|
||||
|
||||
/// 整个数据字典。
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
pub struct Catalog {
|
||||
tables: BTreeMap<String, TableSchema>,
|
||||
}
|
||||
|
||||
impl Catalog {
|
||||
/// 从文件读；文件不存在返回空字典。
|
||||
pub fn load<P: AsRef<Path>>(path: P) -> io::Result<Self> {
|
||||
if !path.as_ref().exists() {
|
||||
return Ok(Catalog::default());
|
||||
}
|
||||
let bytes = std::fs::read(path)?;
|
||||
if bytes.is_empty() {
|
||||
return Ok(Catalog::default());
|
||||
}
|
||||
serde_json::from_slice(&bytes).map_err(io::Error::other)
|
||||
}
|
||||
|
||||
/// 写回文件。
|
||||
pub fn save<P: AsRef<Path>>(&self, path: P) -> io::Result<()> {
|
||||
let bytes = serde_json::to_vec_pretty(self).map_err(io::Error::other)?;
|
||||
std::fs::write(path, bytes)
|
||||
}
|
||||
|
||||
/// 一张表的 schema；没有就返回 None。
|
||||
pub fn describe(&self, table: &str) -> Option<&TableSchema> {
|
||||
self.tables.get(table)
|
||||
}
|
||||
|
||||
/// catalog 里记录的所有表名。
|
||||
pub fn table_names(&self) -> Vec<String> {
|
||||
self.tables.keys().cloned().collect()
|
||||
}
|
||||
|
||||
/// 按一行数据补列；已有的列不动。
|
||||
pub fn ensure_columns(&mut self, table: &str, row: &Row) {
|
||||
let entry = self.tables.entry(table.to_string()).or_default();
|
||||
for (k, v) in row {
|
||||
if entry.columns.iter().any(|c| &c.name == k) {
|
||||
continue;
|
||||
}
|
||||
if let Some(ty) = infer_type(v) {
|
||||
entry.columns.push(SchemaColumn {
|
||||
name: k.clone(),
|
||||
ty,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 按多行数据补列；replace_all 用。
|
||||
pub fn ensure_columns_from_rows(&mut self, table: &str, rows: &[Row]) {
|
||||
for r in rows {
|
||||
self.ensure_columns(table, r);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 从 JSON 值推断列类型；不支持的类型返回 None。
|
||||
fn infer_type(v: &serde_json::Value) -> Option<ColumnType> {
|
||||
match v {
|
||||
serde_json::Value::Number(_) => Some(ColumnType::Int),
|
||||
serde_json::Value::String(_) => Some(ColumnType::Str),
|
||||
serde_json::Value::Bool(_) => Some(ColumnType::Bool),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::log::Level;
|
||||
|
||||
// 配置：优先环境变量，其次 TOML 文件，最后内置默认值（下面写的就是默认值）。
|
||||
// 每个选项的来源都记在 Loaded::origins 里，server 启动时打进日志——读了哪个文件、
|
||||
// 哪些选项走了默认回退，一眼能看出来。
|
||||
|
||||
/// 默认配置文件名（相对当前工作目录）；可用 CHUSQL_CONFIG 指到别处。
|
||||
pub const DEFAULT_CONFIG_PATH: &str = "chusql-storage.toml";
|
||||
|
||||
/// 默认页大小（字节）。
|
||||
pub const DEFAULT_PAGE_SIZE: usize = 4096;
|
||||
/// 默认 B+ 树节点最大孩子数。
|
||||
pub const DEFAULT_BTREE_ORDER: usize = 4;
|
||||
/// 默认数据目录。
|
||||
pub const DEFAULT_DATA_DIR: &str = "data";
|
||||
/// 默认管道名。
|
||||
pub const DEFAULT_PIPE_NAME: &str = "chusql-storage";
|
||||
/// 默认日志等级。
|
||||
pub const DEFAULT_LOG_LEVEL: &str = "info";
|
||||
|
||||
/// 页大小允许的范围。
|
||||
const MIN_PAGE_SIZE: usize = 512;
|
||||
const MAX_PAGE_SIZE: usize = 65536;
|
||||
/// B+ 树节点至少要放得下 2 个键，分裂规则才成立。
|
||||
const MIN_BTREE_ORDER: usize = 3;
|
||||
|
||||
/// 生效的配置。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Config {
|
||||
pub page_size: usize,
|
||||
pub btree_order: usize,
|
||||
pub data_dir: PathBuf,
|
||||
pub pipe_name: String,
|
||||
pub log_level: Level,
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
fn default() -> Self {
|
||||
Config {
|
||||
page_size: DEFAULT_PAGE_SIZE,
|
||||
btree_order: DEFAULT_BTREE_ORDER,
|
||||
data_dir: PathBuf::from(DEFAULT_DATA_DIR),
|
||||
pipe_name: DEFAULT_PIPE_NAME.to_string(),
|
||||
log_level: Level::Info,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 一个选项的来源。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Origin {
|
||||
Default,
|
||||
File(PathBuf),
|
||||
Env(&'static str),
|
||||
}
|
||||
|
||||
impl Origin {
|
||||
/// 日志里那一列怎么写。
|
||||
pub fn describe(&self) -> String {
|
||||
match self {
|
||||
Origin::Default => "default".to_string(),
|
||||
Origin::File(p) => format!("file {}", p.display()),
|
||||
Origin::Env(k) => format!("env {}", k),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 生效配置 + 每个选项的来源。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Loaded {
|
||||
pub config: Config,
|
||||
pub config_path: Option<PathBuf>,
|
||||
pub origins: Vec<(&'static str, String, Origin)>,
|
||||
}
|
||||
|
||||
/// TOML 文件里的形状：字段全部可选，没写的就当"文件没提这一项"。
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct FileConfig {
|
||||
page: Option<FilePage>,
|
||||
btree: Option<FileBtree>,
|
||||
storage: Option<FileStorage>,
|
||||
server: Option<FileServer>,
|
||||
log: Option<FileLog>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct FilePage {
|
||||
size: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct FileBtree {
|
||||
order: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct FileStorage {
|
||||
data_dir: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct FileServer {
|
||||
pipe_name: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct FileLog {
|
||||
level: Option<String>,
|
||||
}
|
||||
|
||||
/// 从真实环境加载：CHUSQL_CONFIG（或当前目录的默认文件）+ 环境变量覆盖。
|
||||
pub fn load() -> Result<Loaded, String> {
|
||||
let path = match std::env::var("CHUSQL_CONFIG") {
|
||||
Ok(p) if !p.trim().is_empty() => Some(PathBuf::from(p)),
|
||||
_ => {
|
||||
let p = PathBuf::from(DEFAULT_CONFIG_PATH);
|
||||
if p.is_file() { Some(p) } else { None }
|
||||
}
|
||||
};
|
||||
let text = match &path {
|
||||
Some(p) => Some(
|
||||
std::fs::read_to_string(p)
|
||||
.map_err(|e| format!("cannot read config file {}: {}", p.display(), e))?,
|
||||
),
|
||||
None => None,
|
||||
};
|
||||
resolve(text.as_deref(), path, &|k| std::env::var(k).ok())
|
||||
}
|
||||
|
||||
/// 解析 TOML 文本 + 套环境变量覆盖；测试直接调这个，不碰真实环境。
|
||||
pub fn resolve(
|
||||
text: Option<&str>,
|
||||
path: Option<PathBuf>,
|
||||
env: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<Loaded, String> {
|
||||
let file: FileConfig = match text {
|
||||
None => FileConfig::default(),
|
||||
Some(t) => toml::from_str(t).map_err(|e| match &path {
|
||||
Some(p) => format!("bad config file {}: {}", p.display(), e),
|
||||
None => format!("bad config: {}", e),
|
||||
})?,
|
||||
};
|
||||
|
||||
let file_origin = Origin::File(
|
||||
path.clone()
|
||||
.unwrap_or_else(|| PathBuf::from(DEFAULT_CONFIG_PATH)),
|
||||
);
|
||||
|
||||
let (page_size, page_origin) = pick(
|
||||
"CHUSQL_PAGE_SIZE",
|
||||
file.page.and_then(|p| p.size),
|
||||
DEFAULT_PAGE_SIZE,
|
||||
&file_origin,
|
||||
env,
|
||||
|s| {
|
||||
s.trim()
|
||||
.parse::<usize>()
|
||||
.map_err(|_| format!("not a number: {}", s))
|
||||
},
|
||||
)?;
|
||||
let (btree_order, order_origin) = pick(
|
||||
"CHUSQL_BTREE_ORDER",
|
||||
file.btree.and_then(|b| b.order),
|
||||
DEFAULT_BTREE_ORDER,
|
||||
&file_origin,
|
||||
env,
|
||||
|s| {
|
||||
s.trim()
|
||||
.parse::<usize>()
|
||||
.map_err(|_| format!("not a number: {}", s))
|
||||
},
|
||||
)?;
|
||||
let (data_dir, dir_origin) = pick(
|
||||
"CHUSQL_DATA_DIR",
|
||||
file.storage.and_then(|s| s.data_dir),
|
||||
DEFAULT_DATA_DIR.to_string(),
|
||||
&file_origin,
|
||||
env,
|
||||
|s| Ok(s.trim().to_string()),
|
||||
)?;
|
||||
let (pipe_name, pipe_origin) = pick(
|
||||
"CHUSQL_PIPE",
|
||||
file.server.and_then(|s| s.pipe_name),
|
||||
DEFAULT_PIPE_NAME.to_string(),
|
||||
&file_origin,
|
||||
env,
|
||||
|s| Ok(s.trim().to_string()),
|
||||
)?;
|
||||
let (log_level, log_origin) = pick(
|
||||
"CHUSQL_LOG",
|
||||
file.log.and_then(|l| l.level),
|
||||
DEFAULT_LOG_LEVEL.to_string(),
|
||||
&file_origin,
|
||||
env,
|
||||
|s| Ok(s.trim().to_string()),
|
||||
)?;
|
||||
let log_level = Level::parse(&log_level)
|
||||
.ok_or_else(|| format!("log.level is not one of off/error/warn/info/debug: {}", log_level))?;
|
||||
|
||||
if data_dir.is_empty() {
|
||||
return Err("storage.data_dir must not be empty".to_string());
|
||||
}
|
||||
if pipe_name.is_empty() {
|
||||
return Err("server.pipe_name must not be empty".to_string());
|
||||
}
|
||||
validate_layout(page_size, btree_order)?;
|
||||
|
||||
let origins = vec![
|
||||
("page.size", page_size.to_string(), page_origin),
|
||||
("btree.order", btree_order.to_string(), order_origin),
|
||||
("storage.data_dir", data_dir.clone(), dir_origin),
|
||||
("server.pipe_name", pipe_name.clone(), pipe_origin),
|
||||
("log.level", log_level.name().to_string(), log_origin),
|
||||
];
|
||||
|
||||
Ok(Loaded {
|
||||
config: Config {
|
||||
page_size,
|
||||
btree_order,
|
||||
data_dir: PathBuf::from(data_dir),
|
||||
pipe_name,
|
||||
log_level,
|
||||
},
|
||||
config_path: if text.is_some() { path } else { None },
|
||||
origins,
|
||||
})
|
||||
}
|
||||
|
||||
/// 取一个选项：环境变量 → 配置文件 → 默认值，并记下它从哪来。
|
||||
fn pick<T: Clone>(
|
||||
env_key: &'static str,
|
||||
file_value: Option<T>,
|
||||
default: T,
|
||||
file_origin: &Origin,
|
||||
env: &dyn Fn(&str) -> Option<String>,
|
||||
parse: impl Fn(&str) -> Result<T, String>,
|
||||
) -> Result<(T, Origin), String> {
|
||||
if let Some(raw) = env(env_key) {
|
||||
let value = parse(&raw).map_err(|e| format!("{}: {}", env_key, e))?;
|
||||
return Ok((value, Origin::Env(env_key)));
|
||||
}
|
||||
if let Some(value) = file_value {
|
||||
return Ok((value, file_origin.clone()));
|
||||
}
|
||||
Ok((default, Origin::Default))
|
||||
}
|
||||
|
||||
/// 检查选项本身和选项之间是否说得通。
|
||||
fn validate_layout(page_size: usize, btree_order: usize) -> Result<(), String> {
|
||||
if !(MIN_PAGE_SIZE..=MAX_PAGE_SIZE).contains(&page_size) || !page_size.is_power_of_two() {
|
||||
return Err(format!(
|
||||
"page.size must be a power of two between {} and {}, got {}",
|
||||
MIN_PAGE_SIZE, MAX_PAGE_SIZE, page_size
|
||||
));
|
||||
}
|
||||
if btree_order < MIN_BTREE_ORDER {
|
||||
return Err(format!(
|
||||
"btree.order must be at least {}, got {}",
|
||||
MIN_BTREE_ORDER, btree_order
|
||||
));
|
||||
}
|
||||
if !crate::btree::fits_in_page(page_size, btree_order) {
|
||||
return Err(format!(
|
||||
"btree.order {} does not fit in a {}-byte page (max {})",
|
||||
btree_order,
|
||||
page_size,
|
||||
crate::btree::max_order(page_size)
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::btree::DiskBTree;
|
||||
use crate::page::PageId;
|
||||
use crate::page::{Page, PageFile};
|
||||
use crate::protocol::Row;
|
||||
|
||||
// 堆表：把行按插入顺序塞进页里，支持全表扫描。页大小由配置给（默认 4096）。
|
||||
// 页内布局（slotted page）：
|
||||
// [0..2] slot_count: u16（小端）
|
||||
// [2..] 槽位目录，每槽 8 字节：offset u32 + len u32
|
||||
// [中间] 空闲空间
|
||||
// [页尾] 行数据，从后往前长
|
||||
// 删除：把槽位的 len 置 0，数据不动（vacuum 以后再说）。
|
||||
|
||||
/// 页头：slot_count 占 2 字节。
|
||||
const HEADER_SIZE: usize = 2;
|
||||
/// 一个槽位占 8 字节：offset(u32) + len(u32)。
|
||||
const SLOT_SIZE: usize = 8;
|
||||
/// 槽位目录起点。
|
||||
const SLOT_DIR_START: usize = HEADER_SIZE;
|
||||
|
||||
/// 从字节里读一个小端 u16。
|
||||
fn read_u16(buf: &[u8], off: usize) -> u16 {
|
||||
u16::from_le_bytes([buf[off], buf[off + 1]])
|
||||
}
|
||||
|
||||
/// 往字节里写一个小端 u16。
|
||||
fn write_u16(buf: &mut [u8], off: usize, v: u16) {
|
||||
buf[off..off + 2].copy_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
|
||||
/// 从字节里读一个小端 u32。
|
||||
fn read_u32(buf: &[u8], off: usize) -> u32 {
|
||||
u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]])
|
||||
}
|
||||
|
||||
/// 往字节里写一个小端 u32。
|
||||
fn write_u32(buf: &mut [u8], off: usize, v: u32) {
|
||||
buf[off..off + 4].copy_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
|
||||
/// 一个 (页号, 槽位号) 打包成一个 u64。页号高 48 位，槽位号低 16 位。
|
||||
fn pack_position(page_id: PageId, slot: u16) -> u64 {
|
||||
(page_id << 16) | (slot as u64)
|
||||
}
|
||||
|
||||
/// 从 u64 解回 (页号, 槽位号)。
|
||||
fn unpack_position(v: u64) -> (PageId, u16) {
|
||||
(v >> 16, (v & 0xFFFF) as u16)
|
||||
}
|
||||
|
||||
/// 页内操作：不管文件，只管一页里的字节。
|
||||
pub struct HeapPage;
|
||||
|
||||
impl HeapPage {
|
||||
/// 这一页里有多少个槽位。
|
||||
pub fn slot_count(page: &Page) -> u16 {
|
||||
read_u16(&page.data, 0)
|
||||
}
|
||||
|
||||
/// 改写槽位数量。
|
||||
fn set_slot_count(page: &mut Page, v: u16) {
|
||||
write_u16(&mut page.data, 0, v);
|
||||
}
|
||||
|
||||
/// 空闲空间下界：槽位目录的末尾。
|
||||
fn free_start(page: &Page) -> usize {
|
||||
SLOT_DIR_START + (Self::slot_count(page) as usize) * SLOT_SIZE
|
||||
}
|
||||
|
||||
/// 空闲空间上界：所有存活行里最靠前的那个字节偏移（没存活行时就是页尾）。
|
||||
fn free_end(page: &Page) -> usize {
|
||||
let n = Self::slot_count(page);
|
||||
let mut min_off = page.data.len() as u32;
|
||||
for i in 0..n as usize {
|
||||
let slot_off = SLOT_DIR_START + i * SLOT_SIZE;
|
||||
let off = read_u32(&page.data, slot_off);
|
||||
let len = read_u32(&page.data, slot_off + 4);
|
||||
if len > 0 && off < min_off {
|
||||
min_off = off;
|
||||
}
|
||||
}
|
||||
min_off as usize
|
||||
}
|
||||
|
||||
/// 试插一行：成功返回槽位号，放不下返回 None。
|
||||
pub fn insert_tuple(page: &mut Page, tuple: &[u8]) -> Option<u16> {
|
||||
let n = Self::slot_count(page);
|
||||
let free_start = Self::free_start(page);
|
||||
let free_end = Self::free_end(page);
|
||||
|
||||
if free_start + SLOT_SIZE + tuple.len() > free_end {
|
||||
return None;
|
||||
}
|
||||
|
||||
// 行数据紧贴已有数据前面
|
||||
let tuple_off = free_end - tuple.len();
|
||||
page.data[tuple_off..tuple_off + tuple.len()].copy_from_slice(tuple);
|
||||
|
||||
// 追加槽位
|
||||
let slot_off = SLOT_DIR_START + (n as usize) * SLOT_SIZE;
|
||||
write_u32(&mut page.data, slot_off, tuple_off as u32);
|
||||
write_u32(&mut page.data, slot_off + 4, tuple.len() as u32);
|
||||
Self::set_slot_count(page, n + 1);
|
||||
|
||||
Some(n)
|
||||
}
|
||||
|
||||
/// 读槽位里的行字节；槽位不存在或已删除返回 None。
|
||||
pub fn get_tuple(page: &Page, slot: u16) -> Option<Vec<u8>> {
|
||||
let n = Self::slot_count(page);
|
||||
if slot >= n {
|
||||
return None;
|
||||
}
|
||||
let slot_off = SLOT_DIR_START + (slot as usize) * SLOT_SIZE;
|
||||
let off = read_u32(&page.data, slot_off) as usize;
|
||||
let len = read_u32(&page.data, slot_off + 4) as usize;
|
||||
if len == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(page.data[off..off + len].to_vec())
|
||||
}
|
||||
|
||||
/// 所有槽位号（含已删除的）。
|
||||
pub fn iter_slots(page: &Page) -> Vec<u16> {
|
||||
(0..Self::slot_count(page)).collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// 堆表：一个文件 = 若干页，顺序追加行。
|
||||
pub struct HeapTable {
|
||||
file: PageFile,
|
||||
index: Option<DiskBTree>,
|
||||
}
|
||||
|
||||
impl HeapTable {
|
||||
/// 打开（或创建）一张堆表。
|
||||
pub fn open<P: AsRef<Path>>(path: P, page_size: usize) -> io::Result<Self> {
|
||||
Ok(HeapTable {
|
||||
file: PageFile::open(path, page_size)?,
|
||||
index: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// 打开带 B+ 树索引的表；索引文件不存在会自动建。
|
||||
pub fn open_indexed<P: AsRef<Path>, Q: AsRef<Path>>(
|
||||
path: P,
|
||||
index_path: Q,
|
||||
page_size: usize,
|
||||
btree_order: usize,
|
||||
) -> io::Result<Self> {
|
||||
Ok(HeapTable {
|
||||
file: PageFile::open(path, page_size)?,
|
||||
index: Some(DiskBTree::open(index_path, page_size, btree_order)?),
|
||||
})
|
||||
}
|
||||
|
||||
/// 插入一行；不更新索引。
|
||||
pub fn insert(&mut self, row: &Row) -> io::Result<()> {
|
||||
self.insert_returning_position(row).map(|_| ())
|
||||
}
|
||||
|
||||
/// 插入一行并返回它的 (页号, 槽位号)。
|
||||
pub fn insert_returning_position(&mut self, row: &Row) -> io::Result<(PageId, u16)> {
|
||||
let tuple = serde_json::to_vec(row).map_err(io::Error::other)?;
|
||||
|
||||
if HEADER_SIZE + SLOT_SIZE + tuple.len() > self.file.page_size() {
|
||||
return Err(io::Error::other(format!(
|
||||
"tuple too large: {} bytes",
|
||||
tuple.len()
|
||||
)));
|
||||
}
|
||||
|
||||
let n = self.file.num_pages()?;
|
||||
if n > 0 {
|
||||
let last_id = n - 1;
|
||||
let mut p = self.file.read_page(last_id)?;
|
||||
if let Some(slot) = HeapPage::insert_tuple(&mut p, &tuple) {
|
||||
self.file.write_page(&p)?;
|
||||
return Ok((last_id, slot));
|
||||
}
|
||||
}
|
||||
|
||||
let mut p = self.file.append_page()?;
|
||||
let page_id = p.id;
|
||||
let slot = HeapPage::insert_tuple(&mut p, &tuple).expect("pre-checked fits");
|
||||
self.file.write_page(&p)?;
|
||||
Ok((page_id, slot))
|
||||
}
|
||||
|
||||
/// 插入一行，并把 (key, 位置) 写进索引。
|
||||
pub fn insert_keyed(&mut self, row: &Row, key: i64) -> io::Result<()> {
|
||||
let (page_id, slot) = self.insert_returning_position(row)?;
|
||||
if let Some(idx) = &mut self.index {
|
||||
idx.insert(key, pack_position(page_id, slot))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 按索引键取一行；没索引或键不存在返回 None。
|
||||
pub fn get_by_key(&mut self, key: i64) -> io::Result<Option<Row>> {
|
||||
let pos = match &mut self.index {
|
||||
None => return Ok(None),
|
||||
Some(idx) => match idx.get(key)? {
|
||||
None => return Ok(None),
|
||||
Some(p) => p,
|
||||
},
|
||||
};
|
||||
let (page_id, slot) = unpack_position(pos);
|
||||
self.read_at(page_id, slot)
|
||||
}
|
||||
|
||||
/// 按 (页号, 槽位号) 读一行。
|
||||
pub fn read_at(&mut self, page_id: PageId, slot: u16) -> io::Result<Option<Row>> {
|
||||
let p = self.file.read_page(page_id)?;
|
||||
match HeapPage::get_tuple(&p, slot) {
|
||||
None => Ok(None),
|
||||
Some(bytes) => {
|
||||
let row: Row = serde_json::from_slice(&bytes).map_err(io::Error::other)?;
|
||||
Ok(Some(row))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 全表扫描：按页顺序、按槽位顺序返回所有行。
|
||||
pub fn scan(&mut self) -> io::Result<Vec<Row>> {
|
||||
let n = self.file.num_pages()?;
|
||||
let mut rows = Vec::new();
|
||||
for i in 0..n {
|
||||
let p = self.file.read_page(i)?;
|
||||
for slot in HeapPage::iter_slots(&p) {
|
||||
if let Some(bytes) = HeapPage::get_tuple(&p, slot) {
|
||||
let row: Row = serde_json::from_slice(&bytes)
|
||||
.map_err(io::Error::other)?;
|
||||
rows.push(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// 整表改写：清空堆表和索引，再把新行逐行插回去。
|
||||
/// 行里有整数 id 就顺手重建索引 —— 堆表位置变了，旧索引必须作废，否则会按老位置读出别的行。
|
||||
pub fn replace_all(&mut self, rows: &[Row]) -> io::Result<()> {
|
||||
self.file.truncate()?;
|
||||
if let Some(idx) = &mut self.index {
|
||||
idx.clear()?;
|
||||
}
|
||||
for r in rows {
|
||||
let (page_id, slot) = self.insert_returning_position(r)?;
|
||||
if let (Some(idx), Some(key)) = (&mut self.index, row_key(r)) {
|
||||
idx.insert(key, pack_position(page_id, slot))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// 一行里的索引键：id 列是整数就用它（和 Haskell 侧"主键列叫 id"的约定一致）。
|
||||
fn row_key(row: &Row) -> Option<i64> {
|
||||
match row.get("id") {
|
||||
Some(serde_json::Value::Number(n)) => n.as_i64(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
// 存储层库入口：对外暴露 page / protocol / heap / btree / config / log（server 与集成测试都从这里引）。
|
||||
|
||||
pub mod btree;
|
||||
pub mod config;
|
||||
pub mod heap;
|
||||
pub mod log;
|
||||
pub mod page;
|
||||
pub mod protocol;
|
||||
pub mod catalog;
|
||||
pub mod wal;
|
||||
@@ -0,0 +1,387 @@
|
||||
use std::fmt;
|
||||
use std::io::{IsTerminal, Write};
|
||||
use std::sync::atomic::{AtomicU8, Ordering};
|
||||
use std::sync::OnceLock;
|
||||
|
||||
// 极简日志：等级存在一个原子变量里，被关掉的等级只花一次原子读——不求值参数、不做格式化。
|
||||
// 行格式：[等级 本地日期时间] [来源] 消息
|
||||
// - ERROR 加粗红、WARN 加粗橙、DEBUG 加粗不变色（只在 stderr 是终端时着色，重定向到文件时是纯文本）
|
||||
// - 来源区分 core（服务自身：启动、配置）和 pipe（由管道请求触发）
|
||||
// 等级由配置文件的 log.level 或环境变量 CHUSQL_LOG 决定。
|
||||
|
||||
/// 转义序列收尾。
|
||||
const RESET: &str = "\u{1b}[0m";
|
||||
/// 加粗红（ERROR）。
|
||||
const BOLD_RED: &str = "\u{1b}[1m\u{1b}[31m";
|
||||
/// 加粗橙（WARN）：256 色里的 208 号。
|
||||
const BOLD_ORANGE: &str = "\u{1b}[1m\u{1b}[38;5;208m";
|
||||
/// 只加粗（DEBUG）。
|
||||
const BOLD: &str = "\u{1b}[1m";
|
||||
|
||||
/// 日志等级；Off 表示全关。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum Level {
|
||||
Off,
|
||||
Error,
|
||||
Warn,
|
||||
Info,
|
||||
Debug,
|
||||
}
|
||||
|
||||
/// 日志来源：core = 服务自身，pipe = 由管道请求触发。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Channel {
|
||||
Core,
|
||||
Pipe,
|
||||
}
|
||||
|
||||
impl Channel {
|
||||
/// 日志行里的来源标签。
|
||||
fn tag(self) -> &'static str {
|
||||
match self {
|
||||
Channel::Core => "core",
|
||||
Channel::Pipe => "pipe",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Level {
|
||||
/// 解析等级名（大小写不敏感）。
|
||||
pub fn parse(s: &str) -> Option<Level> {
|
||||
match s.trim().to_ascii_lowercase().as_str() {
|
||||
"off" => Some(Level::Off),
|
||||
"error" => Some(Level::Error),
|
||||
"warn" | "warning" => Some(Level::Warn),
|
||||
"info" => Some(Level::Info),
|
||||
"debug" => Some(Level::Debug),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 等级名（写日志和写配置说明用）。
|
||||
pub fn name(self) -> &'static str {
|
||||
match self {
|
||||
Level::Off => "off",
|
||||
Level::Error => "error",
|
||||
Level::Warn => "warn",
|
||||
Level::Info => "info",
|
||||
Level::Debug => "debug",
|
||||
}
|
||||
}
|
||||
|
||||
/// 日志行里的标签，固定 5 字符宽，方便对齐。
|
||||
fn tag(self) -> &'static str {
|
||||
match self {
|
||||
Level::Error => "ERROR",
|
||||
Level::Warn => "WARN ",
|
||||
Level::Info => "INFO ",
|
||||
_ => "DEBUG",
|
||||
}
|
||||
}
|
||||
|
||||
/// 这个等级要不要着色（INFO 不着色）。
|
||||
fn color(self) -> &'static str {
|
||||
match self {
|
||||
Level::Error => BOLD_RED,
|
||||
Level::Warn => BOLD_ORANGE,
|
||||
Level::Debug => BOLD,
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 当前等级，默认 info。
|
||||
static LEVEL: AtomicU8 = AtomicU8::new(Level::Info as u8);
|
||||
|
||||
/// stderr 是不是终端（决定要不要着色），只问一次。
|
||||
static COLOR: OnceLock<bool> = OnceLock::new();
|
||||
|
||||
/// 设置全局等级。
|
||||
pub fn set_level(level: Level) {
|
||||
LEVEL.store(level as u8, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// 这个等级要不要输出——热路径上只走这一步（一次原子读 + 比较）。
|
||||
#[inline]
|
||||
pub fn enabled(level: Level) -> bool {
|
||||
level as u8 <= LEVEL.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// 启动时调一次：Windows 上给 stderr 打开 VT 处理，让 ANSI 颜色能被认识。
|
||||
pub fn init() {
|
||||
enable_vt();
|
||||
}
|
||||
|
||||
/// 写出一行；只在 enabled 为真时调用。
|
||||
pub fn write(channel: Channel, level: Level, args: fmt::Arguments) {
|
||||
let msg = args.to_string();
|
||||
let line = render(level, channel, color_enabled(), &now(), &msg);
|
||||
let stderr = std::io::stderr();
|
||||
let mut out = stderr.lock();
|
||||
let _ = writeln!(out, "{}", line);
|
||||
}
|
||||
|
||||
/// stderr 是终端才着色；CHUSQL_LOG_COLOR=always / never 可以强制开关。
|
||||
fn color_enabled() -> bool {
|
||||
*COLOR.get_or_init(|| match std::env::var("CHUSQL_LOG_COLOR").as_deref().map(str::trim) {
|
||||
Ok("always") | Ok("yes") | Ok("1") => true,
|
||||
Ok("never") | Ok("no") | Ok("0") => false,
|
||||
_ => std::io::stderr().is_terminal(),
|
||||
})
|
||||
}
|
||||
|
||||
/// 拼一行：[等级 时间] [来源] 消息。纯函数，方便测试。
|
||||
fn render(level: Level, channel: Channel, color: bool, stamp: &Stamp, msg: &str) -> String {
|
||||
let tag = level.tag();
|
||||
let paint = level.color();
|
||||
let level_text = if color && !paint.is_empty() {
|
||||
format!("{}{}{}", paint, tag, RESET)
|
||||
} else {
|
||||
tag.to_string()
|
||||
};
|
||||
format!("[{} {}] [{}] {}", level_text, stamp, channel.tag(), msg)
|
||||
}
|
||||
|
||||
/// 一个本地日期时间。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Stamp {
|
||||
pub year: u16,
|
||||
pub month: u16,
|
||||
pub day: u16,
|
||||
pub hour: u16,
|
||||
pub minute: u16,
|
||||
pub second: u16,
|
||||
pub milli: u16,
|
||||
}
|
||||
|
||||
impl fmt::Display for Stamp {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"{:04}-{:02}-{:02} {:02}:{:02}:{:02}.{:03}",
|
||||
self.year, self.month, self.day, self.hour, self.minute, self.second, self.milli
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// 当前本地时间：Windows 上走 kernel32::GetLocalTime。
|
||||
#[cfg(windows)]
|
||||
fn now() -> Stamp {
|
||||
#[repr(C)]
|
||||
struct SystemTime {
|
||||
year: u16,
|
||||
month: u16,
|
||||
day_of_week: u16,
|
||||
day: u16,
|
||||
hour: u16,
|
||||
minute: u16,
|
||||
second: u16,
|
||||
milliseconds: u16,
|
||||
}
|
||||
|
||||
#[link(name = "kernel32")]
|
||||
unsafe extern "system" {
|
||||
fn GetLocalTime(out: *mut SystemTime);
|
||||
}
|
||||
|
||||
let mut t = SystemTime {
|
||||
year: 0,
|
||||
month: 0,
|
||||
day_of_week: 0,
|
||||
day: 0,
|
||||
hour: 0,
|
||||
minute: 0,
|
||||
second: 0,
|
||||
milliseconds: 0,
|
||||
};
|
||||
unsafe { GetLocalTime(&mut t) };
|
||||
Stamp {
|
||||
year: t.year,
|
||||
month: t.month,
|
||||
day: t.day,
|
||||
hour: t.hour,
|
||||
minute: t.minute,
|
||||
second: t.second,
|
||||
milli: t.milliseconds,
|
||||
}
|
||||
}
|
||||
|
||||
/// 当前时间：其他平台没有取本地时区的零依赖办法，退回 UTC。
|
||||
#[cfg(not(windows))]
|
||||
fn now() -> Stamp {
|
||||
let d = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default();
|
||||
let secs = d.as_secs();
|
||||
let (year, month, day) = civil_from_days((secs / 86_400) as i64);
|
||||
Stamp {
|
||||
year: year as u16,
|
||||
month: month as u16,
|
||||
day: day as u16,
|
||||
hour: ((secs % 86_400) / 3600) as u16,
|
||||
minute: ((secs % 3_600) / 60) as u16,
|
||||
second: (secs % 60) as u16,
|
||||
milli: d.subsec_millis() as u16,
|
||||
}
|
||||
}
|
||||
|
||||
/// 把"距 1970-01-01 的天数"换算成 (年, 月, 日)：非 Windows 平台算时间用，单元测试也走这里。
|
||||
pub fn civil_from_days(days: i64) -> (i64, u16, u16) {
|
||||
let z = days + 719_468;
|
||||
let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
|
||||
let doe = z - era * 146_097;
|
||||
let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
|
||||
let y = yoe + era * 400;
|
||||
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
||||
let mp = (5 * doy + 2) / 153;
|
||||
let day = (doy - (153 * mp + 2) / 5 + 1) as u16;
|
||||
let month = if mp < 10 { mp + 3 } else { mp - 9 } as u16;
|
||||
(y + i64::from(month <= 2), month, day)
|
||||
}
|
||||
|
||||
/// Windows：打开 stderr 的 VT 处理。
|
||||
#[cfg(windows)]
|
||||
fn enable_vt() {
|
||||
#[link(name = "kernel32")]
|
||||
unsafe extern "system" {
|
||||
fn GetStdHandle(which: u32) -> *mut core::ffi::c_void;
|
||||
fn GetConsoleMode(handle: *mut core::ffi::c_void, mode: *mut u32) -> i32;
|
||||
fn SetConsoleMode(handle: *mut core::ffi::c_void, mode: u32) -> i32;
|
||||
}
|
||||
|
||||
/// STD_ERROR_HANDLE：-12 的无符号写法。
|
||||
const STD_ERROR_HANDLE: u32 = 0xFFFF_FFF4;
|
||||
/// 让控制台认识 ANSI 转义序列。
|
||||
const ENABLE_VIRTUAL_TERMINAL_PROCESSING: u32 = 0x0004;
|
||||
|
||||
unsafe {
|
||||
let handle = GetStdHandle(STD_ERROR_HANDLE);
|
||||
let mut mode = 0u32;
|
||||
if GetConsoleMode(handle, &mut mode) != 0 {
|
||||
let _ = SetConsoleMode(handle, mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 非 Windows：不用做任何事。
|
||||
#[cfg(not(windows))]
|
||||
fn enable_vt() {}
|
||||
|
||||
/// 打一条 debug 日志（关掉时只花一次原子读）。
|
||||
#[macro_export]
|
||||
macro_rules! log_debug {
|
||||
(core, $($arg:tt)*) => {
|
||||
$crate::log_line!($crate::log::Channel::Core, $crate::log::Level::Debug, $($arg)*)
|
||||
};
|
||||
(pipe, $($arg:tt)*) => {
|
||||
$crate::log_line!($crate::log::Channel::Pipe, $crate::log::Level::Debug, $($arg)*)
|
||||
};
|
||||
}
|
||||
|
||||
/// 打一条 info 日志。
|
||||
#[macro_export]
|
||||
macro_rules! log_info {
|
||||
(core, $($arg:tt)*) => {
|
||||
$crate::log_line!($crate::log::Channel::Core, $crate::log::Level::Info, $($arg)*)
|
||||
};
|
||||
(pipe, $($arg:tt)*) => {
|
||||
$crate::log_line!($crate::log::Channel::Pipe, $crate::log::Level::Info, $($arg)*)
|
||||
};
|
||||
}
|
||||
|
||||
/// 打一条 warn 日志。
|
||||
#[macro_export]
|
||||
macro_rules! log_warn {
|
||||
(core, $($arg:tt)*) => {
|
||||
$crate::log_line!($crate::log::Channel::Core, $crate::log::Level::Warn, $($arg)*)
|
||||
};
|
||||
(pipe, $($arg:tt)*) => {
|
||||
$crate::log_line!($crate::log::Channel::Pipe, $crate::log::Level::Warn, $($arg)*)
|
||||
};
|
||||
}
|
||||
|
||||
/// 打一条 error 日志。
|
||||
#[macro_export]
|
||||
macro_rules! log_error {
|
||||
(core, $($arg:tt)*) => {
|
||||
$crate::log_line!($crate::log::Channel::Core, $crate::log::Level::Error, $($arg)*)
|
||||
};
|
||||
(pipe, $($arg:tt)*) => {
|
||||
$crate::log_line!($crate::log::Channel::Pipe, $crate::log::Level::Error, $($arg)*)
|
||||
};
|
||||
}
|
||||
|
||||
/// 上面四个宏的公共部分：先看等级，再看要不要真的写。内部用，别直接调。
|
||||
#[doc(hidden)]
|
||||
#[macro_export]
|
||||
macro_rules! log_line {
|
||||
($chan:expr, $lvl:expr, $($arg:tt)*) => {
|
||||
if $crate::log::enabled($lvl) {
|
||||
$crate::log::write($chan, $lvl, format_args!($($arg)*))
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// 天数换算日期：几个已知点 + 两次闰年的 2 月 29 日。
|
||||
#[test]
|
||||
fn civil_from_days_known_dates() {
|
||||
assert_eq!(civil_from_days(0), (1970, 1, 1));
|
||||
assert_eq!(civil_from_days(10_957), (2000, 1, 1));
|
||||
assert_eq!(civil_from_days(11_016), (2000, 2, 29));
|
||||
assert_eq!(civil_from_days(19_723), (2024, 1, 1));
|
||||
assert_eq!(civil_from_days(19_782), (2024, 2, 29));
|
||||
}
|
||||
|
||||
/// 行格式：等级 + 本地日期时间 + 来源 + 消息，等级该粗的粗、该色的色。
|
||||
#[test]
|
||||
fn render_has_time_channel_and_colors() {
|
||||
let stamp = Stamp {
|
||||
year: 2026,
|
||||
month: 9,
|
||||
day: 24,
|
||||
hour: 13,
|
||||
minute: 45,
|
||||
second: 12,
|
||||
milli: 130,
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
render(Level::Error, Channel::Pipe, true, &stamp, "boom"),
|
||||
"[\u{1b}[1m\u{1b}[31mERROR\u{1b}[0m 2026-09-24 13:45:12.130] [pipe] boom"
|
||||
);
|
||||
assert_eq!(
|
||||
render(Level::Warn, Channel::Core, true, &stamp, "careful"),
|
||||
"[\u{1b}[1m\u{1b}[38;5;208mWARN \u{1b}[0m 2026-09-24 13:45:12.130] [core] careful"
|
||||
);
|
||||
assert_eq!(
|
||||
render(Level::Debug, Channel::Pipe, true, &stamp, "detail"),
|
||||
"[\u{1b}[1mDEBUG\u{1b}[0m 2026-09-24 13:45:12.130] [pipe] detail"
|
||||
);
|
||||
assert_eq!(
|
||||
render(Level::Info, Channel::Core, true, &stamp, "hello"),
|
||||
"[INFO 2026-09-24 13:45:12.130] [core] hello"
|
||||
);
|
||||
}
|
||||
|
||||
/// 关掉颜色时整行必须是纯文本（重定向到文件时用）。
|
||||
#[test]
|
||||
fn render_without_color_is_plain() {
|
||||
let stamp = Stamp {
|
||||
year: 2026,
|
||||
month: 9,
|
||||
day: 24,
|
||||
hour: 1,
|
||||
minute: 2,
|
||||
second: 3,
|
||||
milli: 4,
|
||||
};
|
||||
assert_eq!(
|
||||
render(Level::Error, Channel::Core, false, &stamp, "boom"),
|
||||
"[ERROR 2026-09-24 01:02:03.004] [core] boom"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, Read, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
// 页管理：数据库文件按固定大小的页组织，页大小由配置给（默认 4096，见 config.rs）。
|
||||
// 只负责"按页号读 / 写"，不理解页里的内容 —— 内容怎么解释由上层（堆表 / B+ 树 / WAL）决定。
|
||||
// 页号从 0 开始，第 n 页在文件中的偏移是 n * page_size。
|
||||
|
||||
/// 页号：从 0 开始。u64 足够大，不用担心中途用完。
|
||||
pub type PageId = u64;
|
||||
|
||||
/// 一页：固定 page_size 字节，外加自己的页号。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Page {
|
||||
pub id: PageId,
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Page {
|
||||
/// 新建一页，内容全 0。
|
||||
pub fn new(id: PageId, size: usize) -> Self {
|
||||
Page {
|
||||
id,
|
||||
data: vec![0u8; size],
|
||||
}
|
||||
}
|
||||
|
||||
/// 清零。复用时先清空，避免残留旧数据。
|
||||
pub fn zero(&mut self) {
|
||||
self.data.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
/// 页文件：负责把 Page 落到磁盘上（不做缓存，每次读写都直接 syscall；缓冲池是后面的阶段）。
|
||||
#[derive(Debug)]
|
||||
pub struct PageFile {
|
||||
file: File,
|
||||
page_size: usize,
|
||||
}
|
||||
|
||||
impl PageFile {
|
||||
/// 打开（或创建）一个页文件；页大小由配置传进来。
|
||||
pub fn open<P: AsRef<Path>>(path: P, page_size: usize) -> io::Result<Self> {
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.open(path)?;
|
||||
Ok(PageFile { file, page_size })
|
||||
}
|
||||
|
||||
/// 这个文件的页大小。
|
||||
pub fn page_size(&self) -> usize {
|
||||
self.page_size
|
||||
}
|
||||
|
||||
/// 读取指定页号的页；页号超出文件末尾时返回 UnexpectedEof。
|
||||
pub fn read_page(&mut self, id: PageId) -> io::Result<Page> {
|
||||
let mut data = vec![0u8; self.page_size];
|
||||
self.file.seek(SeekFrom::Start(id * self.page_size as u64))?;
|
||||
self.file.read_exact(&mut data)?;
|
||||
Ok(Page { id, data })
|
||||
}
|
||||
|
||||
/// 写入一页。若页号超出当前文件末尾，文件会自动扩展。
|
||||
pub fn write_page(&mut self, page: &Page) -> io::Result<()> {
|
||||
if page.data.len() != self.page_size {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
format!(
|
||||
"page has {} bytes, this file uses {}",
|
||||
page.data.len(),
|
||||
self.page_size
|
||||
),
|
||||
));
|
||||
}
|
||||
self.file
|
||||
.seek(SeekFrom::Start(page.id * self.page_size as u64))?;
|
||||
self.file.write_all(&page.data)?;
|
||||
self.file.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 在文件末尾追加一页，返回新页的页号。
|
||||
pub fn append_page(&mut self) -> io::Result<Page> {
|
||||
let id = self.num_pages()?;
|
||||
let page = Page::new(id, self.page_size);
|
||||
self.write_page(&page)?;
|
||||
Ok(page)
|
||||
}
|
||||
|
||||
/// 当前文件里有多少页；长度不是页大小整数倍，说明页大小和文件对不上。
|
||||
pub fn num_pages(&mut self) -> io::Result<PageId> {
|
||||
let len = self.file.metadata()?.len();
|
||||
let size = self.page_size as u64;
|
||||
if len % size != 0 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!(
|
||||
"file length {} is not a multiple of page size {} (did page.size change?)",
|
||||
len, size
|
||||
),
|
||||
));
|
||||
}
|
||||
Ok(len / size)
|
||||
}
|
||||
|
||||
/// 强制把文件缓冲刷到磁盘。写入关键路径时用。
|
||||
pub fn sync(&mut self) -> io::Result<()> {
|
||||
self.file.sync_all()
|
||||
}
|
||||
|
||||
/// 清空文件，游标归零。
|
||||
pub fn truncate(&mut self) -> io::Result<()> {
|
||||
self.file.set_len(0)?;
|
||||
self.file.seek(SeekFrom::Start(0))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
|
||||
// 协议定义：Haskell ↔ Rust 之间那条"行分隔 JSON"消息的类型（请求 / 响应）。
|
||||
|
||||
/// 一行：列名到 JSON 值的映射。
|
||||
pub type Row = HashMap<String, serde_json::Value>;
|
||||
|
||||
/// 列类型；跟 Haskell 侧 TInt / TStr / TBool 对应。
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ColumnType {
|
||||
Int,
|
||||
Str,
|
||||
Bool,
|
||||
}
|
||||
|
||||
/// 一列的 schema。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct SchemaColumn {
|
||||
pub name: String,
|
||||
pub ty: ColumnType,
|
||||
}
|
||||
|
||||
/// 请求；method 字段做 tag。
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "method", rename_all = "snake_case")]
|
||||
pub enum Request {
|
||||
Ping,
|
||||
Scan { table: String },
|
||||
Insert {
|
||||
table: String,
|
||||
row: Row,
|
||||
#[serde(default)]
|
||||
key: Option<i64>,
|
||||
},
|
||||
LookupByIndex { table: String, key: i64 },
|
||||
ReplaceAll { table: String, rows: Vec<Row> },
|
||||
ListTables,
|
||||
DescribeTable { table: String },
|
||||
}
|
||||
|
||||
/// 响应；status 字段做 tag。
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "status", rename_all = "lowercase")]
|
||||
pub enum Response {
|
||||
Pong,
|
||||
Rows { rows: Vec<Row> },
|
||||
Tables { tables: Vec<String> },
|
||||
Schema { columns: Vec<SchemaColumn> },
|
||||
Ok,
|
||||
Error { message: String },
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
//! 简化版 WAL：写前日志 + 崩溃恢复。
|
||||
//!
|
||||
//! 条目格式：
|
||||
//! [u32 body_len][body]
|
||||
//! body = [u64 LSN][u8 操作类型][u16 表名长度][表名字节][JSON 载荷]
|
||||
//!
|
||||
//! 简化版约束：
|
||||
//! - 最多只有一条未完成的操作。写成功就清空 WAL。
|
||||
//! - 不做 checkpoint / redo / undo / 事务。
|
||||
//!
|
||||
//! 崩溃恢复：
|
||||
//! - 启动时若 WAL 非空，重放那一条，然后清空。
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::protocol::Row;
|
||||
|
||||
/// 操作类型标记。
|
||||
const OP_INSERT: u8 = 1;
|
||||
const OP_REPLACE_ALL: u8 = 2;
|
||||
|
||||
/// 一条 WAL 操作。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub enum WalOp {
|
||||
Insert {
|
||||
table: String,
|
||||
row: Row,
|
||||
key: Option<i64>,
|
||||
},
|
||||
ReplaceAll {
|
||||
table: String,
|
||||
rows: Vec<Row>,
|
||||
},
|
||||
}
|
||||
|
||||
/// WAL 文件句柄。
|
||||
pub struct Wal {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl Wal {
|
||||
/// 绑定到 WAL 文件路径；首次 append 时自动创建文件。
|
||||
pub fn new<P: AsRef<Path>>(path: P) -> Self {
|
||||
Wal {
|
||||
path: path.as_ref().to_path_buf(),
|
||||
}
|
||||
}
|
||||
|
||||
/// 日志文件路径。
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
/// 清空 WAL；文件不存在则什么都不做。
|
||||
pub fn clear(&self) -> io::Result<()> {
|
||||
if self.path.exists() {
|
||||
let f = OpenOptions::new().write(true).open(&self.path)?;
|
||||
f.set_len(0)?;
|
||||
f.sync_all()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 追加一条操作并 fsync。
|
||||
pub fn append(&self, op: &WalOp) -> io::Result<()> {
|
||||
let bytes = encode(op)?;
|
||||
let mut f = OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&self.path)?;
|
||||
f.write_all(&bytes)?;
|
||||
f.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 读取当前 WAL；空文件 / 不存在返回 None。
|
||||
pub fn read(&self) -> io::Result<Option<WalOp>> {
|
||||
if !self.path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut f = File::open(&self.path)?;
|
||||
let mut buf = Vec::new();
|
||||
f.read_to_end(&mut buf)?;
|
||||
if buf.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
match decode(&buf) {
|
||||
Ok(opt) => Ok(opt),
|
||||
Err(e) => Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("corrupt WAL: {}", e),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 把一条操作编成字节。
|
||||
fn encode(op: &WalOp) -> io::Result<Vec<u8>> {
|
||||
// 先编出 (类型, 表名, 载荷)
|
||||
let (op_type, table, payload) = match op {
|
||||
WalOp::Insert { table, row, key } => {
|
||||
let p = serde_json::to_vec(&(row, key)).map_err(io::Error::other)?;
|
||||
(OP_INSERT, table.clone(), p)
|
||||
}
|
||||
WalOp::ReplaceAll { table, rows } => {
|
||||
let p = serde_json::to_vec(rows).map_err(io::Error::other)?;
|
||||
(OP_REPLACE_ALL, table.clone(), p)
|
||||
}
|
||||
};
|
||||
|
||||
let table_bytes = table.as_bytes();
|
||||
|
||||
// body = [u64 LSN][u8 类型][u16 表名长度][表名][载荷]
|
||||
// 简化版 LSN 固定为 1；完整版会改成真正递增。
|
||||
let lsn: u64 = 1;
|
||||
let mut body = Vec::with_capacity(8 + 1 + 2 + table_bytes.len() + payload.len());
|
||||
body.extend_from_slice(&lsn.to_le_bytes());
|
||||
body.push(op_type);
|
||||
body.extend_from_slice(&(table_bytes.len() as u16).to_le_bytes());
|
||||
body.extend_from_slice(table_bytes);
|
||||
body.extend_from_slice(&payload);
|
||||
|
||||
// 整条 = [u32 body_len][body]
|
||||
let mut out = Vec::with_capacity(4 + body.len());
|
||||
out.extend_from_slice(&(body.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(&body);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// 解析一条 WAL 条目。
|
||||
///
|
||||
/// - `Ok(Some(op))`：成功解析出操作
|
||||
/// - `Ok(None)`：尾部被截断（写入一半就断电），按"没有这条"处理
|
||||
/// - `Err(_)`：字节流本身损坏，抛错
|
||||
fn decode(buf: &[u8]) -> Result<Option<WalOp>, String> {
|
||||
if buf.len() < 4 {
|
||||
return Ok(None);
|
||||
}
|
||||
let body_len = u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]) as usize;
|
||||
if buf.len() < 4 + body_len {
|
||||
// 尾部被截断——忽略
|
||||
return Ok(None);
|
||||
}
|
||||
let body = &buf[4..4 + body_len];
|
||||
if body.len() < 8 + 1 + 2 {
|
||||
return Err("body too short".into());
|
||||
}
|
||||
// LSN 简化版忽略
|
||||
let _lsn = u64::from_le_bytes([
|
||||
body[0], body[1], body[2], body[3], body[4], body[5], body[6], body[7],
|
||||
]);
|
||||
let op_type = body[8];
|
||||
let table_len = u16::from_le_bytes([body[9], body[10]]) as usize;
|
||||
if body.len() < 11 + table_len {
|
||||
return Err("table name truncated".into());
|
||||
}
|
||||
let table = String::from_utf8(body[11..11 + table_len].to_vec())
|
||||
.map_err(|e| format!("bad utf8 table name: {}", e))?;
|
||||
let payload = &body[11 + table_len..];
|
||||
|
||||
let op = match op_type {
|
||||
OP_INSERT => {
|
||||
let (row, key): (Row, Option<i64>) = serde_json::from_slice(payload)
|
||||
.map_err(|e| format!("bad insert payload: {}", e))?;
|
||||
WalOp::Insert { table, row, key }
|
||||
}
|
||||
OP_REPLACE_ALL => {
|
||||
let rows: Vec<Row> = serde_json::from_slice(payload)
|
||||
.map_err(|e| format!("bad replace_all payload: {}", e))?;
|
||||
WalOp::ReplaceAll { table, rows }
|
||||
}
|
||||
other => return Err(format!("unknown op type: {}", other)),
|
||||
};
|
||||
Ok(Some(op))
|
||||
}
|
||||
Reference in new issue
Block a user