修bug,完善功能
build / build (push) Blocked by required conditions
tests / build (push) Waiting to run
tests / tests (push) Waiting to run

This commit is contained in:
ChuNa7a committed 2026-10-16 23:52:12 +08:00
1 parent 30364b1c4b
commit 6996e19694
125 files changed
+15343 -3746

No files matched your search

+104 -20
View File
@@ -22,7 +22,8 @@ const SLOT: usize = 2;
const CELL_FIXED: usize = 2 + 8;
/// 键长上限:节点里的格子必须放得进一页
pub const MAX_KEY_BYTES: usize = 64;
pub const MAX_KEY_BYTES: usize = 128;
const LEGACY_KEY_BYTES: usize = 64;
/// 一个节点最多几个键
fn max_keys(order: usize) -> usize {
@@ -31,7 +32,7 @@ fn max_keys(order: usize) -> usize {
/// 一个格子最多占多少字节
fn max_cell_bytes() -> usize {
CELL_FIXED + MAX_KEY_BYTES
CELL_FIXED + LEGACY_KEY_BYTES
}
/// 这个 order 放得进一页吗
@@ -44,6 +45,16 @@ pub fn max_order(page_size: usize) -> usize {
page_size.saturating_sub(HEADER) / (SLOT + max_cell_bytes()) + 1
}
/// 按页容量和分支数计算有效键长
pub fn key_limit(page_size: usize, order: usize) -> usize {
if order < 2 {
return 0;
}
(page_size.saturating_sub(HEADER) / (order - 1))
.saturating_sub(SLOT + CELL_FIXED)
.min(MAX_KEY_BYTES)
}
/// 读一个 u8
fn read_u8(p: &Page, off: usize) -> u8 {
p.data[off]
@@ -66,7 +77,12 @@ fn write_u16(p: &mut Page, off: usize, v: u16) {
/// 读一个小端 u32
fn read_u32(p: &Page, off: usize) -> u32 {
u32::from_le_bytes([p.data[off], p.data[off + 1], p.data[off + 2], p.data[off + 3]])
u32::from_le_bytes([
p.data[off],
p.data[off + 1],
p.data[off + 2],
p.data[off + 3],
])
}
/// 写一个小端 u32
@@ -91,7 +107,7 @@ fn write_meta(page: &mut Page, page_size: usize, order: usize, root: PageId) {
page.zero();
page.data[0..4].copy_from_slice(META_MAGIC);
write_u32(page, OFF_PAGE_SIZE, page_size as u32);
write_u16(page, OFF_MAX_KEY, MAX_KEY_BYTES as u16);
write_u16(page, OFF_MAX_KEY, key_limit(page_size, order) as u16);
write_u64(page, OFF_ROOT, root);
write_u16(page, OFF_ORDER, order as u16);
}
@@ -115,6 +131,13 @@ fn check_meta(page: &Page, page_size: usize, order: usize) -> io::Result<()> {
),
));
}
let stored_key = read_u16(page, OFF_MAX_KEY) as usize;
if stored_key < LEGACY_KEY_BYTES || stored_key > key_limit(page_size, order) {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"unsupported index key limit",
));
}
Ok(())
}
@@ -169,13 +192,19 @@ fn put_node(p: &mut Page, node: &Node) -> io::Result<()> {
/// 从一页读出节点
fn get_node(p: &Page) -> io::Result<Node> {
if p.data.len() < HEADER {
return Err(io::Error::new(io::ErrorKind::InvalidData, "truncated index node header"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"truncated index node header",
));
}
let kind = read_u8(p, OFF_KIND);
let count = read_u16(p, OFF_COUNT) as usize;
let slots_end = HEADER + count * SLOT;
if !matches!(kind, NODE_LEAF | NODE_INTERNAL) || slots_end > p.data.len() {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index node kind or slot count"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid index node kind or slot count",
));
}
let trailer = read_u64(p, OFF_NEXT);
let mut keys = Vec::with_capacity(count);
@@ -184,30 +213,55 @@ fn get_node(p: &Page) -> io::Result<Node> {
for i in 0..count {
let off = read_u16(p, HEADER + i * SLOT) as usize;
if off < slots_end || off + CELL_FIXED > p.data.len() {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index cell offset"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid index cell offset",
));
}
let klen = read_u16(p, off) as usize;
if klen > MAX_KEY_BYTES || off + CELL_FIXED + klen > p.data.len() {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index key length"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid index key length",
));
}
if off + CELL_FIXED + klen > cell_end {
return Err(io::Error::new(io::ErrorKind::InvalidData, "overlapping index cells"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"overlapping index cells",
));
}
cell_end = off;
keys.push(p.data[off + 2..off + 2 + klen].to_vec());
values.push(read_u64(p, off + 2 + klen));
}
if keys.windows(2).any(|pair| pair[0] > pair[1]) {
return Err(io::Error::new(io::ErrorKind::InvalidData, "index keys are not sorted"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"index keys are not sorted",
));
}
if kind == NODE_INTERNAL {
values.push(trailer);
if values.contains(&META_PAGE) {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index child page"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid index child page",
));
}
Ok(Node { kind, next: 0, keys, values })
Ok(Node {
kind,
next: 0,
keys,
values,
})
} else {
Ok(Node { kind, next: trailer, keys, values })
Ok(Node {
kind,
next: trailer,
keys,
values,
})
}
}
@@ -244,7 +298,12 @@ fn split_node(node: &mut Node) -> (Vec<u8>, Node) {
node.keys.pop();
(
sep,
Node { kind: NODE_INTERNAL, next: 0, keys: right_keys, values: right_values },
Node {
kind: NODE_INTERNAL,
next: 0,
keys: right_keys,
values: right_values,
},
)
}
}
@@ -287,20 +346,26 @@ impl DiskBTree {
}
/// 键太长直接报错,避免写坏节点
fn check_key(&self, key: &[u8]) -> io::Result<()> {
if key.len() > MAX_KEY_BYTES {
pub(crate) fn check_key(key: &[u8], page_size: usize, order: usize) -> io::Result<()> {
let limit = key_limit(page_size, order);
if key.len() > limit {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!(
"index key is {} bytes, at most {} are supported",
key.len(),
MAX_KEY_BYTES
limit
),
));
}
Ok(())
}
/// 按当前索引配置预检键长
pub(crate) fn validate_key(&self, key: &[u8]) -> io::Result<()> {
Self::check_key(key, self.file.page_size(), self.order)
}
/// 读根页页号;空树是 None
fn root(&mut self) -> io::Result<Option<PageId>> {
let r = self.file.with_page(META_PAGE, |p| read_u64(p, OFF_ROOT))?;
@@ -331,6 +396,11 @@ impl DiskBTree {
self.file.update_page(id, |p| put_node(p, node))?
}
/// 设置后续索引页的提交点
pub(crate) fn set_lsn(&mut self, lsn: u64) {
self.file.set_lsn(lsn);
}
/// 清空整棵树
pub fn clear(&mut self) -> io::Result<()> {
let page_size = self.file.page_size();
@@ -362,14 +432,26 @@ impl DiskBTree {
/// 按 key 取全部位置(重复键按值升序)
pub fn get_all(&mut self, key: &[u8]) -> io::Result<Vec<u64>> {
let mut values: Vec<_> = self.scan(Some(key), Some(key))?.into_iter().map(|(_, value)| value).collect();
let mut values: Vec<_> = self
.scan(Some(key), Some(key))?
.into_iter()
.map(|(_, value)| value)
.collect();
values.sort_unstable();
Ok(values)
}
/// 插入一个键值对(键可重复)
pub fn insert(&mut self, key: &[u8], value: u64) -> io::Result<()> {
self.check_key(key)?;
self.validate_key(key)?;
if key.len() > LEGACY_KEY_BYTES {
let limit = key_limit(self.file.page_size(), self.order);
let mut meta = self.file.read_page(META_PAGE)?;
if read_u16(&meta, OFF_MAX_KEY) as usize != limit {
write_u16(&mut meta, OFF_MAX_KEY, limit as u16);
self.file.write_page(&meta)?;
}
}
match self.root()? {
None => {
let node = Node {
@@ -412,7 +494,9 @@ impl DiskBTree {
} else {
let child = node.values[child_slot(&node.keys, key)];
if let Some((sep, right_id)) = self.insert_rec(child, key, value)? {
let at = node.keys.partition_point(|k| k.as_slice() <= sep.as_slice());
let at = node
.keys
.partition_point(|k| k.as_slice() <= sep.as_slice());
node.keys.insert(at, sep);
node.values.insert(at + 1, right_id);
}
+134 -41
View File
@@ -37,6 +37,8 @@ pub struct TableSchema {
#[serde(default)]
pub object_id: u64,
#[serde(default)]
pub file_generation: String,
#[serde(default)]
// 0 表示旧目录尚无可验证的结构版本。
pub schema_version: u64,
#[serde(default)]
@@ -78,6 +80,13 @@ impl Catalog {
return Ok(Catalog::default());
}
let mut catalog: Self = serde_json::from_slice(&bytes).map_err(io::Error::other)?;
if catalog
.tables
.values()
.any(|schema| !valid_generation(&schema.file_generation))
{
return Err(io::Error::other("invalid physical file generation"));
}
catalog.upgrade_object_ids()?;
Ok(catalog)
}
@@ -94,7 +103,9 @@ impl Catalog {
}
for schema in self.tables.values_mut() {
if schema.object_id == 0 {
self.object_high_water = self.object_high_water.checked_add(1)
self.object_high_water = self
.object_high_water
.checked_add(1)
.ok_or_else(|| io::Error::other("catalog object id exhausted"))?;
schema.object_id = self.object_high_water;
}
@@ -102,9 +113,37 @@ impl Catalog {
Ok(())
}
/// 切换物理文件代次而不改变逻辑结构版本
pub(crate) fn set_generation(
&mut self,
table: &str,
object: u64,
source: &str,
target: &str,
) -> io::Result<()> {
if !valid_generation(target) {
return Err(io::Error::other("invalid physical file generation"));
}
let schema = self
.tables
.get_mut(table)
.ok_or_else(|| io::Error::other("migration table missing"))?;
if schema.object_id != object
|| (schema.file_generation != source && schema.file_generation != target)
{
return Err(io::Error::other(
"serialization_failure: migration object changed",
));
}
schema.file_generation = target.to_owned();
Ok(())
}
/// 分配不复用的对象编号
pub fn reserve_object_id(&mut self) -> io::Result<u64> {
self.object_high_water = self.object_high_water.checked_add(1)
self.object_high_water = self
.object_high_water
.checked_add(1)
.ok_or_else(|| io::Error::other("catalog object id exhausted"))?;
Ok(self.object_high_water)
}
@@ -123,7 +162,11 @@ impl Catalog {
file.sync_all()?;
drop(file);
std::fs::rename(pending, &path)?;
std::fs::OpenOptions::new().read(true).write(true).open(path)?.sync_all()
std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(path)?
.sync_all()
}
/// 一张表的 schema
@@ -149,7 +192,12 @@ impl Catalog {
}
/// 按日志中的编号建立表
pub fn create_table_with_id(&mut self, table: &str, columns: Vec<SchemaColumn>, object_id: Option<u64>) -> io::Result<()> {
pub fn create_table_with_id(
&mut self,
table: &str,
columns: Vec<SchemaColumn>,
object_id: Option<u64>,
) -> io::Result<()> {
if self.tables.contains_key(table) {
return Err(io::Error::new(
io::ErrorKind::AlreadyExists,
@@ -160,13 +208,17 @@ impl Catalog {
Some(id) if id == 0 || self.tables.values().any(|schema| schema.object_id == id) => {
return Err(io::Error::other("invalid catalog object id"));
}
Some(id) => { self.object_high_water = self.object_high_water.max(id); id }
Some(id) => {
self.object_high_water = self.object_high_water.max(id);
id
}
None => self.reserve_object_id()?,
};
self.tables.insert(
table.to_string(),
TableSchema {
object_id,
file_generation: String::new(),
schema_version: 1,
system: false,
columns,
@@ -182,7 +234,9 @@ impl Catalog {
/// 标记内部账号表
pub fn mark_system(&mut self, table: &str) -> io::Result<()> {
let entry = self.tables.get_mut(table)
let entry = self
.tables
.get_mut(table)
.ok_or_else(|| io::Error::other("missing system table"))?;
entry.system = true;
Ok(())
@@ -306,7 +360,10 @@ impl Catalog {
));
}
let version = next_schema_version(entry)?;
let mut entry = self.tables.remove(from).ok_or_else(|| io::Error::other("missing rename source"))?;
let mut entry = self
.tables
.remove(from)
.ok_or_else(|| io::Error::other("missing rename source"))?;
entry.schema_version = version;
self.seen.remove(from);
self.tables.insert(to.to_string(), entry);
@@ -319,13 +376,19 @@ impl Catalog {
}
/// 用日志中的目标版本替换列定义
pub fn set_columns_version(&mut self, table: &str, columns: Vec<SchemaColumn>, version: Option<u64>) -> io::Result<()> {
let entry = self
.tables
.get_mut(table)
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?;
pub fn set_columns_version(
&mut self,
table: &str,
columns: Vec<SchemaColumn>,
version: Option<u64>,
) -> io::Result<()> {
let entry = self.tables.get_mut(table).ok_or_else(|| {
io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table))
})?;
if let Some(version) = version {
if version == 0 { return Err(io::Error::other("invalid WAL schema version")); }
if version == 0 {
return Err(io::Error::other("invalid WAL schema version"));
}
entry.schema_version = version;
} else if entry.columns != columns {
entry.schema_version = next_schema_version(entry)?;
@@ -342,7 +405,11 @@ impl Catalog {
pub fn normalize_types(&mut self) -> io::Result<bool> {
let mut changed = false;
for entry in self.tables.values_mut() {
if entry.columns.iter().any(|column| normalize_type(&column.ty) != column.ty) {
if entry
.columns
.iter()
.any(|column| normalize_type(&column.ty) != column.ty)
{
entry.schema_version = next_schema_version(entry)?;
}
for column in entry.columns.iter_mut() {
@@ -362,13 +429,19 @@ impl Catalog {
}
/// 用日志目标版本删除列定义
pub fn remove_column_version(&mut self, table: &str, column: &str, version: Option<u64>) -> io::Result<bool> {
let entry = self
.tables
.get_mut(table)
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?;
pub fn remove_column_version(
&mut self,
table: &str,
column: &str,
version: Option<u64>,
) -> io::Result<bool> {
let entry = self.tables.get_mut(table).ok_or_else(|| {
io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table))
})?;
if let Some(version) = version {
if version == 0 { return Err(io::Error::other("invalid WAL schema version")); }
if version == 0 {
return Err(io::Error::other("invalid WAL schema version"));
}
entry.schema_version = version;
} else if entry.columns.iter().any(|c| c.name == column) {
entry.schema_version = next_schema_version(entry)?;
@@ -388,10 +461,9 @@ impl Catalog {
/// 记住这一版已整理过(隐藏列名要留住)
pub fn mark_compacted(&mut self, table: &str) -> io::Result<()> {
let entry = self
.tables
.get_mut(table)
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?;
let entry = self.tables.get_mut(table).ok_or_else(|| {
io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table))
})?;
entry.compacted = true;
self.seen.remove(table);
Ok(())
@@ -399,7 +471,9 @@ impl Catalog {
/// 这张表还有隐藏列字节吗
pub fn has_dropped_columns(&self, table: &str) -> bool {
self.tables.get(table).is_some_and(|e| !e.dropped_columns.is_empty())
self.tables
.get(table)
.is_some_and(|e| !e.dropped_columns.is_empty())
}
/// 还有隐藏列字节待整理的表
@@ -413,10 +487,9 @@ impl Catalog {
/// 记下一条索引定义
pub fn add_index(&mut self, table: &str, column: &str) -> io::Result<()> {
let entry = self
.tables
.get_mut(table)
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?;
let entry = self.tables.get_mut(table).ok_or_else(|| {
io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table))
})?;
if entry.indexes.iter().any(|i| i.column == column) {
return Err(io::Error::new(
io::ErrorKind::AlreadyExists,
@@ -431,10 +504,9 @@ impl Catalog {
/// 去掉一条索引定义
pub fn remove_index(&mut self, table: &str, column: &str) -> io::Result<()> {
let entry = self
.tables
.get_mut(table)
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?;
let entry = self.tables.get_mut(table).ok_or_else(|| {
io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table))
})?;
match entry.indexes.iter().position(|i| i.column == column) {
None => Err(io::Error::new(
io::ErrorKind::NotFound,
@@ -450,11 +522,20 @@ impl Catalog {
/// 把行里"字典还不知道的列"补进来;返回有没有变化
fn add_missing_columns(entry: &mut TableSchema, row: &Row) -> io::Result<bool> {
let additions: Vec<_> = row.iter().filter(|(name, _)|
!entry.dropped_columns.contains(*name) && !entry.columns.iter().any(|c| &c.name == *name))
.filter_map(|(name, value)| infer_type(value).map(|ty| SchemaColumn {
name: name.clone(), ty, ..Default::default()
})).collect();
let additions: Vec<_> = row
.iter()
.filter(|(name, _)| {
!entry.dropped_columns.contains(*name)
&& !entry.columns.iter().any(|c| &c.name == *name)
})
.filter_map(|(name, value)| {
infer_type(value).map(|ty| SchemaColumn {
name: name.clone(),
ty,
..Default::default()
})
})
.collect();
if additions.is_empty() {
return Ok(false);
}
@@ -465,7 +546,9 @@ fn add_missing_columns(entry: &mut TableSchema, row: &Row) -> io::Result<bool> {
/// 递增结构版本,耗尽时拒绝变更
fn next_schema_version(entry: &TableSchema) -> io::Result<u64> {
entry.schema_version.checked_add(1)
entry
.schema_version
.checked_add(1)
.ok_or_else(|| io::Error::other("catalog schema version exhausted"))
}
@@ -576,12 +659,22 @@ fn hist_add(stat: &mut ColumnStat, v: &serde_json::Value) {
/// 从直方图里减掉一个值;值在区间外就不动
fn hist_remove(stat: &mut ColumnStat, v: &serde_json::Value) {
let Some(x) = v.as_f64() else { return };
let (Some(lo), Some(hi)) = (stat.lo, stat.hi) else { return };
let (Some(lo), Some(hi)) = (stat.lo, stat.hi) else {
return;
};
if x < lo || x > hi || stat.hist.len() != HIST_BUCKETS {
return;
}
let index = hist_index(lo, hi, x);
if let Some(bucket) = (0..stat.hist.len()).filter(|&i| stat.hist[i] > 0).min_by_key(|&i| i.abs_diff(index)) {
if let Some(bucket) = (0..stat.hist.len())
.filter(|&i| stat.hist[i] > 0)
.min_by_key(|&i| i.abs_diff(index))
{
stat.hist[bucket] -= 1;
}
}
/// 文件代次仅允许空值或固定长度十六进制
pub(crate) fn valid_generation(value: &str) -> bool {
value.is_empty() || (value.len() == 32 && value.bytes().all(|byte| byte.is_ascii_hexdigit()))
}
+13 -7
View File
@@ -33,7 +33,9 @@ pub fn default_config_path() -> PathBuf {
{
let appdata = std::env::var_os("APPDATA").filter(|v| !v.is_empty());
if let Some(appdata) = appdata {
return PathBuf::from(appdata).join(APP_DIR_NAME).join(CONFIG_FILE_NAME);
return PathBuf::from(appdata)
.join(APP_DIR_NAME)
.join(CONFIG_FILE_NAME);
}
}
#[cfg(not(windows))]
@@ -62,7 +64,9 @@ pub fn default_data_dir() -> PathBuf {
}
let roaming = std::env::var_os("APPDATA").filter(|v| !v.is_empty());
if let Some(roaming) = roaming {
return PathBuf::from(roaming).join(APP_DIR_NAME).join(DATA_DIR_NAME);
return PathBuf::from(roaming)
.join(APP_DIR_NAME)
.join(DATA_DIR_NAME);
}
PathBuf::from(APP_DIR_NAME).join(DATA_DIR_NAME)
}
@@ -210,9 +214,7 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
})?,
};
let file_origin = Origin::File(
path.clone().unwrap_or_else(default_config_path),
);
let file_origin = Origin::File(path.clone().unwrap_or_else(default_config_path));
let (page_size, page_origin) = pick(
file.page.and_then(|p| p.size),
@@ -250,8 +252,12 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
let log_files = log_files.trim().to_string();
let log_level = log_level.trim();
let log_level = Level::parse(log_level)
.ok_or_else(|| format!("log.level is not one of off/error/warn/info/debug: {}", log_level))?;
let log_level = Level::parse(log_level).ok_or_else(|| {
format!(
"log.level is not one of off/error/warn/info/debug: {}",
log_level
)
})?;
if data_dir.is_empty() {
return Err("storage.data_dir must not be empty".to_string());
+58 -15
View File
@@ -1,6 +1,6 @@
use std::cell::RefCell;
use std::ffi::{c_char, CStr};
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::ffi::{CStr, c_char};
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::ptr;
use std::slice;
@@ -42,7 +42,11 @@ pub unsafe extern "C" fn chusql_storage_open(config_path: *const c_char) -> *mut
let text = unsafe { CStr::from_ptr(config_path) }
.to_str()
.map_err(|_| "config path is not valid UTF-8".to_string())?;
if text.is_empty() { None } else { Some(text.to_string()) }
if text.is_empty() {
None
} else {
Some(text.to_string())
}
};
Storage::open(path.as_deref()).map(Box::new)
}));
@@ -78,27 +82,57 @@ pub unsafe extern "C" fn chusql_storage_close(handle: *mut Storage) {
/// 输出指针可写且不与输入重叠;响应使用 free 释放。
#[unsafe(no_mangle)]
pub unsafe extern "C" fn chusql_storage_maintenance(
config_path: *const c_char, request: *const u8, request_len: usize,
out: *mut *mut u8, out_len: *mut usize,
config_path: *const c_char,
request: *const u8,
request_len: usize,
out: *mut *mut u8,
out_len: *mut usize,
) -> i32 {
if !out.is_null() { unsafe { *out = ptr::null_mut(); } }
if !out_len.is_null() { unsafe { *out_len = 0; } }
if request.is_null() || out.is_null() || out_len.is_null() || request_len > isize::MAX as usize {
if !out.is_null() {
unsafe {
*out = ptr::null_mut();
}
}
if !out_len.is_null() {
unsafe {
*out_len = 0;
}
}
if request.is_null() || out.is_null() || out_len.is_null() || request_len > isize::MAX as usize
{
set_error("invalid maintenance argument");
return ERR;
}
let result = catch_unwind(AssertUnwindSafe(|| {
let path = if config_path.is_null() { None } else { Some(unsafe { CStr::from_ptr(config_path) }.to_str().map_err(|e| e.to_string())?) };
let text = std::str::from_utf8(unsafe { slice::from_raw_parts(request, request_len) }).map_err(|e| e.to_string())?;
let path = if config_path.is_null() {
None
} else {
Some(
unsafe { CStr::from_ptr(config_path) }
.to_str()
.map_err(|e| e.to_string())?,
)
};
let text = std::str::from_utf8(unsafe { slice::from_raw_parts(request, request_len) })
.map_err(|e| e.to_string())?;
Storage::maintenance(path, text)
}));
let response = match result {
Ok(Ok(response)) => response,
Ok(Err(err)) => { set_error(err); return ERR; }
Err(_) => { set_error("panic in system maintenance"); return ERR; }
Ok(Err(err)) => {
set_error(err);
return ERR;
}
Err(_) => {
set_error("panic in system maintenance");
return ERR;
}
};
let buffer = response.into_bytes().into_boxed_slice();
unsafe { *out_len = buffer.len(); *out = Box::into_raw(buffer) as *mut u8; }
unsafe {
*out_len = buffer.len();
*out = Box::into_raw(buffer) as *mut u8;
}
OK
}
@@ -121,7 +155,12 @@ pub unsafe extern "C" fn chusql_storage_request(
if !out_len.is_null() {
unsafe { *out_len = 0 };
}
if handle.is_null() || request.is_null() || out.is_null() || out_len.is_null() || request_len > isize::MAX as usize {
if handle.is_null()
|| request.is_null()
|| out.is_null()
|| out_len.is_null()
|| request_len > isize::MAX as usize
{
set_error("invalid argument");
return ERR;
}
@@ -178,6 +217,10 @@ pub unsafe extern "C" fn chusql_storage_last_error(out_len: *mut usize) -> *cons
if !out_len.is_null() {
unsafe { *out_len = bytes.len() };
}
if bytes.is_empty() { ptr::null() } else { bytes.as_ptr() }
if bytes.is_empty() {
ptr::null()
} else {
bytes.as_ptr()
}
})
}
+83 -15
View File
@@ -1,8 +1,8 @@
use serde::Deserializer;
use serde::de::{IgnoredAny, MapAccess, Visitor};
use std::collections::HashSet;
use std::io;
use std::path::Path;
use serde::de::{IgnoredAny, MapAccess, Visitor};
use serde::Deserializer;
use crate::btree::DiskBTree;
use crate::page::PageId;
@@ -11,12 +11,18 @@ use crate::protocol::Row;
// 堆表:行按插入顺序进页,支持全表扫描、按索引取行、行级删除。
pub(crate) const DUPLICATE_ID: &str = "duplicate value on the id index";
const HEADER_SIZE: usize = 2;
const SLOT_SIZE: usize = 8;
const SLOT_DIR_START: usize = HEADER_SIZE;
/// 未选字段只校验并跳过,不分配值。
fn decode_row(bytes: &[u8], columns: Option<&[String]>, hidden: &HashSet<String>) -> io::Result<Row> {
fn decode_row(
bytes: &[u8],
columns: Option<&[String]>,
hidden: &HashSet<String>,
) -> io::Result<Row> {
struct RowVisitor<'a>(Option<&'a [String]>, &'a HashSet<String>);
impl<'de> Visitor<'de> for RowVisitor<'_> {
type Value = Row;
@@ -38,7 +44,9 @@ fn decode_row(bytes: &[u8], columns: Option<&[String]>, hidden: &HashSet<String>
}
}
let mut decoder = serde_json::Deserializer::from_slice(bytes);
let row = decoder.deserialize_map(RowVisitor(columns, hidden)).map_err(io::Error::other)?;
let row = decoder
.deserialize_map(RowVisitor(columns, hidden))
.map_err(io::Error::other)?;
decoder.end().map_err(io::Error::other)?;
Ok(row)
}
@@ -93,7 +101,7 @@ fn row_int(row: &Row, column: &str) -> Option<i64> {
}
/// i64 键 → 8 字节大端(保序,负数也在前)
fn key_of(value: i64) -> [u8; 8] {
pub(crate) fn key_of(value: i64) -> [u8; 8] {
(value as u64 ^ (1u64 << 63)).to_be_bytes()
}
@@ -108,10 +116,23 @@ fn value_key(v: &serde_json::Value) -> Option<Vec<u8>> {
}
/// 取行里某一列的索引键;NULL 与缺列不入索引
fn row_key(row: &Row, column: &str) -> Option<Vec<u8>> {
pub(crate) fn row_key(row: &Row, column: &str) -> Option<Vec<u8>> {
row.get(column).and_then(value_key)
}
/// 按实际键编码校验一列的索引容量
pub(crate) fn check_index_key(
row: &Row,
column: &str,
page_size: usize,
order: usize,
) -> io::Result<()> {
if let Some(key) = row_key(row, column) {
DiskBTree::check_key(&key, page_size, order)?;
}
Ok(())
}
pub struct HeapPage;
impl HeapPage {
@@ -312,7 +333,7 @@ impl HeapTable {
Ok(()) => {}
Err(err) if err.kind() == io::ErrorKind::InvalidData => {
// 格式不兼容(例如上一版落的索引文件):删掉重开
let _ = std::fs::remove_file(p);
crate::page_wal::remove(p)?;
t.attach_index(col, p, page_size, btree_order, pool_size)?;
}
Err(err) => return Err(err),
@@ -373,6 +394,7 @@ impl HeapTable {
let Some(k) = row_key(&r, column) else {
continue;
};
DiskBTree::check_key(&k, page_size, btree_order)?;
// 行身份索引仍然唯一;其余列允许重复值
if column == "id" && !seen.insert(k.clone()) {
return Err(io::Error::new(
@@ -450,10 +472,11 @@ impl HeapTable {
let Some(k) = row_key(row, &idx.column) else {
continue;
};
idx.tree.validate_key(&k)?;
if idx.column == "id" && idx.tree.get(&k)?.is_some() {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"duplicate value on the id index".to_string(),
DUPLICATE_ID.to_string(),
));
}
pending.push((i, k));
@@ -511,6 +534,16 @@ impl HeapTable {
Ok((page_id, slot))
}
/// 读取身份索引中同键行数供提交预检
pub(crate) fn id_key_count(&mut self, key: &[u8]) -> io::Result<usize> {
let index = self
.indexes
.iter_mut()
.find(|index| index.column == "id")
.ok_or_else(|| io::Error::other("missing id index"))?;
Ok(index.tree.get_all(key)?.len())
}
/// 按 `id` 取一行
pub fn get_by_key(&mut self, key: i64) -> io::Result<Option<Row>> {
self.get_by_column_key("id", key)
@@ -577,13 +610,21 @@ impl HeapTable {
let mut rows = Vec::new();
for (key, pos) in pairs {
if let Some((bound, inclusive)) = &lo_bound {
let outside = if *inclusive { &key < bound } else { &key <= bound };
let outside = if *inclusive {
&key < bound
} else {
&key <= bound
};
if outside {
continue;
}
}
if let Some((bound, inclusive)) = &hi_bound {
let outside = if *inclusive { &key > bound } else { &key >= bound };
let outside = if *inclusive {
&key > bound
} else {
&key >= bound
};
if outside {
continue;
}
@@ -699,7 +740,10 @@ impl HeapTable {
}
/// 全表扫描,带位置与列投影
fn scan_projected_with_positions(&mut self, columns: Option<&[String]>) -> io::Result<Vec<(PageId, u16, Row)>> {
fn scan_projected_with_positions(
&mut self,
columns: Option<&[String]>,
) -> io::Result<Vec<(PageId, u16, Row)>> {
let n = self.file.num_pages()?;
self.scan_pages_with_positions(columns, 0, n)
}
@@ -729,6 +773,14 @@ impl HeapTable {
/// 整表改写并重建所有索引
pub fn replace_all(&mut self, rows: &[Row]) -> io::Result<()> {
for row in rows {
Self::encode_row(row, self.file.page_size())?;
for index in &self.indexes {
if let Some(key) = row_key(row, &index.column) {
index.tree.validate_key(&key)?;
}
}
}
self.file.truncate()?;
for idx in &mut self.indexes {
idx.tree.clear()?;
@@ -745,6 +797,14 @@ impl HeapTable {
Ok(())
}
/// 同时标记堆页与索引页的提交点
pub(crate) fn set_lsn(&mut self, lsn: u64) {
self.file.set_lsn(lsn);
for index in &mut self.indexes {
index.tree.set_lsn(lsn);
}
}
/// 把脏页写回(索引也一起)
pub fn flush(&mut self) -> io::Result<()> {
for idx in &mut self.indexes {
@@ -767,11 +827,19 @@ impl HeapTable {
}
/// 当前缓存页数
pub fn cached_pages(&self) -> usize { self.file.cached_pages() }
pub fn cached_pages(&self) -> usize {
self.file.cached_pages()
}
/// 命中次数
pub fn hits(&self) -> u64 { self.file.hits() }
pub fn hits(&self) -> u64 {
self.file.hits()
}
/// 未命中次数
pub fn misses(&self) -> u64 { self.file.misses() }
pub fn misses(&self) -> u64 {
self.file.misses()
}
/// 命中率
pub fn hit_rate(&self) -> f64 { self.file.hit_rate() }
pub fn hit_rate(&self) -> f64 {
self.file.hit_rate()
}
}
+2
View File
@@ -6,7 +6,9 @@ pub mod config;
pub mod ffi;
pub mod heap;
pub mod log;
mod mvcc;
pub mod page;
mod page_wal;
pub mod protocol;
pub mod runtime;
pub mod wal;
+13 -3
View File
@@ -1,7 +1,7 @@
use std::fmt;
use std::io::{IsTerminal, Write};
use std::sync::atomic::{AtomicU8, Ordering};
use std::sync::OnceLock;
use std::sync::atomic::{AtomicU8, Ordering};
// 极简日志:等级用原子变量,关掉的等级只花一次原子读。
@@ -123,8 +123,18 @@ fn streams_to_stderr(level: Level) -> bool {
/// 按目标流是否终端决定是否着色
fn color_enabled(to_stderr: bool) -> bool {
let cached = if to_stderr { &COLOR_STDERR } else { &COLOR_STDOUT };
*cached.get_or_init(|| if to_stderr { std::io::stderr().is_terminal() } else { std::io::stdout().is_terminal() })
let cached = if to_stderr {
&COLOR_STDERR
} else {
&COLOR_STDOUT
};
*cached.get_or_init(|| {
if to_stderr {
std::io::stderr().is_terminal()
} else {
std::io::stdout().is_terminal()
}
})
}
/// 拼一行(纯函数,可测)
File diff suppressed because it is too large. Load diff
+50 -33
View File
@@ -3,6 +3,8 @@ use std::fs::{File, OpenOptions};
use std::io::{self, Read, Seek, SeekFrom, Write};
use std::path::Path;
use crate::page_wal::PageWal;
// 页管理:定长页的读写,外加一层页缓冲池。
pub type PageId = u64;
@@ -33,6 +35,7 @@ impl Page {
#[derive(Debug)]
pub struct PageFile {
file: File,
journal: PageWal,
page_size: usize,
capacity: usize,
cache: HashMap<PageId, Page>,
@@ -55,14 +58,16 @@ impl PageFile {
page_size: usize,
capacity: usize,
) -> io::Result<Self> {
let file = OpenOptions::new()
let mut file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(path)?;
.open(path.as_ref())?;
let journal = PageWal::open(path.as_ref(), page_size, &mut file)?;
Ok(PageFile {
file,
journal,
page_size,
capacity,
cache: HashMap::new(),
@@ -74,6 +79,16 @@ impl PageFile {
})
}
/// 标记后续页修改所属的提交点
pub fn set_lsn(&mut self, lsn: u64) {
self.journal.set_lsn(lsn);
}
/// 读取已记录的页提交 LSN
pub fn page_lsn(&self, id: PageId) -> Option<u64> {
self.journal.page_lsn(id)
}
/// 页大小
pub fn page_size(&self) -> usize {
self.page_size
@@ -109,7 +124,6 @@ impl PageFile {
}
}
/// 标记为最近使用
fn touch(&mut self, id: PageId) {
self.clock += 1;
@@ -127,27 +141,28 @@ impl PageFile {
return Ok(false);
};
if self.dirty.contains(&victim) {
let p = self.cache[&victim].clone();
self.write_raw(&p)?;
self.dirty.remove(&victim);
let pages: Vec<_> = self
.dirty
.iter()
.filter_map(|id| self.cache.get(id).cloned())
.collect();
let images: Vec<_> = pages
.iter()
.map(|page| (page.id, page.data.as_slice()))
.collect();
self.journal.prepare(&images)?;
for page in &pages {
self.write_image(page)?;
}
self.dirty.clear();
}
self.cache.remove(&victim);
self.last_used.remove(&victim);
Ok(true)
}
/// 直接写文件
fn write_raw(&mut self, page: &Page) -> io::Result<()> {
if page.data.len() != self.page_size {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!(
"page has {} bytes, this file uses {}",
page.data.len(),
self.page_size
),
));
}
/// 将已记录镜像的页写入数据文件
fn write_image(&mut self, page: &Page) -> io::Result<()> {
self.file
.seek(SeekFrom::Start(page.id * self.page_size as u64))?;
self.file.write_all(&page.data)
@@ -156,12 +171,12 @@ impl PageFile {
/// 直接从文件读
fn read_raw(&mut self, id: PageId) -> io::Result<Page> {
let mut data = vec![0u8; self.page_size];
self.file.seek(SeekFrom::Start(id * self.page_size as u64))?;
self.file
.seek(SeekFrom::Start(id * self.page_size as u64))?;
self.file.read_exact(&mut data)?;
Ok(Page { id, data })
}
/// 读一页(走缓存)
pub fn read_page(&mut self, id: PageId) -> io::Result<Page> {
self.note_access(id)?;
@@ -241,26 +256,27 @@ impl PageFile {
));
}
let on_disk = len / size;
let cached_max = self
.cache
.keys()
.copied()
.max()
.map(|m| m + 1)
.unwrap_or(0);
let cached_max = self.cache.keys().copied().max().map(|m| m + 1).unwrap_or(0);
Ok(on_disk.max(cached_max))
}
/// 把脏页写回磁盘
pub fn flush(&mut self) -> io::Result<()> {
let dirty: Vec<PageId> = self.dirty.iter().copied().collect();
for id in dirty {
if let Some(p) = self.cache.get(&id) {
let p = p.clone();
self.write_raw(&p)?;
}
let pages: Vec<Page> = self
.dirty
.iter()
.filter_map(|id| self.cache.get(id).cloned())
.collect();
let images: Vec<_> = pages
.iter()
.map(|page| (page.id, page.data.as_slice()))
.collect();
self.journal.prepare(&images)?;
for page in &pages {
self.write_image(page)?;
}
self.file.sync_all()?;
self.journal.checkpoint()?;
self.dirty.clear();
Ok(())
}
@@ -272,6 +288,7 @@ impl PageFile {
/// 清空文件
pub fn truncate(&mut self) -> io::Result<()> {
self.journal.reset()?;
self.cache.clear();
self.last_used.clear();
self.dirty.clear();
+294
View File
@@ -0,0 +1,294 @@
use std::collections::BTreeMap;
use std::fs::{File, OpenOptions};
use std::io::{self, Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
// 全页预写镜像修复撕裂写,页 LSN 在数据同步后原子发布。
const HEADER: usize = 40;
const RESET: u64 = u64::MAX;
#[derive(Debug, Serialize, Deserialize)]
struct State {
version: u32,
page_size: usize,
sequence: u64,
pages: BTreeMap<u64, u64>,
}
#[derive(Debug)]
pub(crate) struct PageWal {
path: PathBuf,
state: State,
lsn: u64,
pending: bool,
failed: bool,
file: Option<File>,
}
/// 为页文件生成独立的元数据路径
fn sidecar(path: &Path, suffix: &str) -> PathBuf {
let mut name = path.as_os_str().to_os_string();
name.push(suffix);
PathBuf::from(name)
}
/// 计算帧校验和以拒绝损坏的完整镜像
fn checksum(bytes: &[u8]) -> u64 {
bytes.iter().fold(0xcbf29ce484222325, |hash, byte| {
(hash ^ u64::from(*byte)).wrapping_mul(0x100000001b3)
})
}
/// 同步目录中的创建、替换与删除
fn sync_parent(path: &Path) -> io::Result<()> {
#[cfg(unix)]
File::open(
path.parent()
.filter(|p| !p.as_os_str().is_empty())
.unwrap_or(Path::new(".")),
)?
.sync_all()?;
Ok(())
}
impl PageWal {
/// 加载页版本并先恢复未完成的全页写入
pub(crate) fn open(path: &Path, page_size: usize, data: &mut File) -> io::Result<Self> {
let metadata = sidecar(path, ".pages");
let state = match std::fs::read(&metadata) {
Ok(bytes) => serde_json::from_slice::<State>(&bytes).map_err(io::Error::other)?,
Err(error) if error.kind() == io::ErrorKind::NotFound => State {
version: 1,
page_size,
sequence: 0,
pages: BTreeMap::new(),
},
Err(error) => return Err(error),
};
if state.version != 1 || state.page_size != page_size {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"unsupported page LSN metadata",
));
}
let mut journal = Self {
path: path.to_path_buf(),
state,
lsn: 0,
pending: false,
failed: false,
file: None,
};
journal.recover(data)?;
Ok(journal)
}
/// 设置后续脏页所属的逻辑 WAL 提交点
pub(crate) fn set_lsn(&mut self, lsn: u64) {
self.lsn = lsn;
}
/// 读取已写镜像记录的页 LSN
pub(crate) fn page_lsn(&self, page: u64) -> Option<u64> {
self.state.pages.get(&page).copied()
}
/// 延迟创建日志并同步目录项
fn handle(&mut self) -> io::Result<&mut File> {
if self.file.is_none() {
let path = sidecar(&self.path, ".page-wal");
let file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&path)?;
sync_parent(&path)?;
self.file = Some(file);
}
Ok(self.file.as_mut().expect("opened above"))
}
/// 持久化一批镜像后才允许修改数据页
pub(crate) fn prepare(&mut self, pages: &[(u64, &[u8])]) -> io::Result<()> {
if self.failed {
return Err(io::Error::other("page journal requires recovery"));
}
if pages.iter().any(|(page, bytes)| {
(*page == RESET && !bytes.is_empty())
|| (*page != RESET
&& (bytes.len() != self.state.page_size
|| page.checked_mul(self.state.page_size as u64).is_none()))
}) {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"invalid page image size or offset",
));
}
if pages.is_empty() {
return Ok(());
}
self.failed = true;
for (page, bytes) in pages {
let sequence = self
.state
.sequence
.checked_add(1)
.ok_or_else(|| io::Error::other("page write sequence exhausted"))?;
let lsn = self
.lsn
.max(self.state.pages.get(page).copied().unwrap_or(0));
let mut frame = Vec::with_capacity(HEADER + bytes.len());
frame.extend_from_slice(b"CPW1");
frame.extend_from_slice(&sequence.to_le_bytes());
frame.extend_from_slice(&lsn.to_le_bytes());
frame.extend_from_slice(&page.to_le_bytes());
frame.extend_from_slice(&(bytes.len() as u32).to_le_bytes());
let mut checked = frame.clone();
checked.extend_from_slice(bytes);
frame.extend_from_slice(&checksum(&checked).to_le_bytes());
frame.extend_from_slice(bytes);
let file = self.handle()?;
file.seek(SeekFrom::End(0))?;
file.write_all(&frame)?;
self.state.sequence = sequence;
if *page == RESET {
self.state.pages.clear();
} else {
self.state.pages.insert(*page, lsn);
}
self.pending = true;
}
self.handle()?.sync_all()?;
self.failed = false;
Ok(())
}
/// 先记录截断以免重放旧页复活已删除数据
pub(crate) fn reset(&mut self) -> io::Result<()> {
self.prepare(&[(RESET, &[])])
}
/// 数据同步后发布页版本并清理镜像日志
pub(crate) fn checkpoint(&mut self) -> io::Result<()> {
if self.failed {
return Err(io::Error::other("page journal requires recovery"));
}
if !self.pending {
return Ok(());
}
self.failed = true;
let temporary = sidecar(&self.path, ".pages.pending");
let metadata = sidecar(&self.path, ".pages");
let mut file = File::create(&temporary)?;
serde_json::to_writer(&mut file, &self.state).map_err(io::Error::other)?;
file.sync_all()?;
drop(file);
std::fs::rename(&temporary, &metadata)?;
sync_parent(&metadata)?;
self.handle()?.set_len(0)?;
self.handle()?.sync_all()?;
self.pending = false;
self.failed = false;
Ok(())
}
/// 验证日志帧后重放比页检查点更新的镜像
fn recover(&mut self, data: &mut File) -> io::Result<()> {
let path = sidecar(&self.path, ".page-wal");
let mut file = match File::open(path) {
Ok(file) => file,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error),
};
if file.metadata()?.len() == 0 {
return Ok(());
}
let checkpoint = self.state.sequence;
let mut previous = 0;
loop {
let mut header = [0; HEADER];
match file.read_exact(&mut header) {
Ok(()) => (),
Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => break,
Err(error) => return Err(error),
}
let sequence = u64::from_le_bytes(header[4..12].try_into().expect("fixed header"));
let lsn = u64::from_le_bytes(header[12..20].try_into().expect("fixed header"));
let page = u64::from_le_bytes(header[20..28].try_into().expect("fixed header"));
let len = u32::from_le_bytes(header[28..32].try_into().expect("fixed header")) as usize;
if &header[..4] != b"CPW1"
|| sequence <= previous
|| (page == RESET && len != 0)
|| (page != RESET && len != self.state.page_size)
{
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid page image header",
));
}
let mut bytes = vec![0; len];
match file.read_exact(&mut bytes) {
Ok(()) => (),
Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => break,
Err(error) => return Err(error),
}
let mut checked = header[..32].to_vec();
checked.extend_from_slice(&bytes);
if checksum(&checked)
!= u64::from_le_bytes(header[32..40].try_into().expect("fixed header"))
{
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"page image checksum mismatch",
));
}
previous = sequence;
if sequence <= checkpoint {
continue;
}
if page == RESET {
data.set_len(0)?;
self.state.pages.clear();
} else {
let offset = page
.checked_mul(self.state.page_size as u64)
.ok_or_else(|| io::Error::other("page offset overflow"))?;
data.seek(SeekFrom::Start(offset))?;
data.write_all(&bytes)?;
self.state.pages.insert(page, lsn);
}
self.state.sequence = sequence;
}
data.sync_all()?;
self.pending = true;
self.checkpoint()
}
}
/// 删除页文件及其版本与镜像文件
pub(crate) fn remove(path: &Path) -> io::Result<()> {
for suffix in [".page-wal", ".pages", ".pages.pending", ""] {
match std::fs::remove_file(sidecar(path, suffix)) {
Ok(()) => (),
Err(error) if error.kind() == io::ErrorKind::NotFound => (),
Err(error) => return Err(error),
}
}
sync_parent(path)
}
/// 改名时保留对应页版本与恢复材料
pub(crate) fn rename(from: &Path, to: &Path) -> io::Result<()> {
for suffix in [".page-wal", ".pages", ".pages.pending", ""] {
match std::fs::rename(sidecar(from, suffix), sidecar(to, suffix)) {
Ok(()) => (),
Err(error) if error.kind() == io::ErrorKind::NotFound => (),
Err(error) => return Err(error),
}
}
sync_parent(to)
}
+195 -42
View File
@@ -94,19 +94,40 @@ pub struct IndexWire {
#[derive(Debug, Serialize, Deserialize)]
#[serde(tag = "method", rename_all = "snake_case")]
pub enum Request {
CatalogManage { command: Box<Request> },
CatalogManage {
command: Box<Request>,
},
AccountsList,
IdentityInitialize { administrator: String },
RoleCreate { user: String },
IdentityAlter { user: String, can_login: Option<bool>, is_superuser: Option<bool>, enabled: Option<bool>, system_catalog_manager: Option<bool>, allow_sudo_auth: Option<bool> },
AccountCreate { user: String, password_hash: String },
AccountReset { user: String, password_hash: String },
IdentityInitialize {
administrator: String,
},
RoleCreate {
user: String,
},
IdentityAlter {
user: String,
can_login: Option<bool>,
is_superuser: Option<bool>,
enabled: Option<bool>,
system_catalog_manager: Option<bool>,
allow_sudo_auth: Option<bool>,
},
AccountCreate {
user: String,
password_hash: String,
},
AccountReset {
user: String,
password_hash: String,
},
AccountLogin {
user: String,
#[serde(default)]
at: Option<String>,
},
AccountDrop { user: String },
AccountDrop {
user: String,
},
/// 引导系统目录:建系统表,给了 user 就同时补首个管理员
BootstrapSystem {
#[serde(default)]
@@ -115,14 +136,18 @@ pub enum Request {
password_hash: Option<String>,
},
SystemStatus,
BootstrapTypes { types: Vec<Row> },
BootstrapTypes {
types: Vec<Row>,
},
Ping,
Scan {
table: String,
#[serde(default)]
columns: Option<Vec<String>>,
},
ValidateWrite { table: String },
ValidateWrite {
table: String,
},
/// 分片扫描:第 shard 片只读总页数按 shards 均分后的那一段
ScanShard {
table: String,
@@ -177,6 +202,37 @@ pub enum Request {
table: String,
},
ListCatalog,
DomainSnapshot,
MvccBegin {
#[serde(default)]
serializable: bool,
},
MvccLock {
transaction: String,
ops: Vec<StorageOp>,
#[serde(default)]
reads: Vec<String>,
},
MvccPreview {
transaction: String,
},
MvccCommit {
transaction: String,
checkpoint: String,
ops: Vec<StorageOp>,
},
MvccRollback {
transaction: String,
},
DatabaseSnapshot {
#[serde(default)]
checkpoint: Option<String>,
},
AlterDomain {
table: String,
columns: Vec<SchemaColumn>,
checkpoint: String,
},
CreateIndex {
table: String,
column: String,
@@ -204,7 +260,7 @@ pub enum Request {
}
/// 事务提交里的一条写操作
#[derive(Debug, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "op", rename_all = "snake_case")]
pub enum StorageOp {
/// 追加写入的行(提交端已先删掉要覆盖的行)
@@ -215,37 +271,89 @@ pub enum StorageOp {
Replace { table: String, rows: Vec<Row> },
}
/// 检查存储表名和列名的字符集
pub(crate) fn valid_storage_name(name: &str) -> bool {
!name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')
}
/// 检查一行顶层字段名
fn valid_row_columns(row: &Row) -> bool {
row.keys().all(|name| valid_storage_name(name))
}
impl Request {
/// 拒绝可能影响文件路径的列名
/// 校验列定义与写入载荷的顶层字段名
pub fn valid_columns(&self) -> bool {
let valid = |name: &str| !name.is_empty()
&& name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_');
match self {
Self::CreateIndex { column, .. } | Self::DropIndex { column, .. }
| Self::DropColumn { column, .. } | Self::LookupByIndex { column, .. }
| Self::RangeByIndex { column, .. } => valid(column),
Self::CreateTable { columns, .. } => columns.iter().all(|c| valid(&c.name)),
Self::ReplaceSchema { columns, .. } => columns.iter().all(|c| valid(&c.name)),
Self::CreateIndex { column, .. }
| Self::DropIndex { column, .. }
| Self::DropColumn { column, .. }
| Self::LookupByIndex { column, .. }
| Self::RangeByIndex { column, .. } => valid_storage_name(column),
Self::CreateTable { columns, .. } | Self::AlterDomain { columns, .. } => {
columns.iter().all(|c| valid_storage_name(&c.name))
}
Self::ReplaceSchema { columns, rows, .. } => {
columns.iter().all(|c| valid_storage_name(&c.name))
&& rows.iter().all(valid_row_columns)
}
Self::Insert { row, .. } => valid_row_columns(row),
Self::InsertBatch { rows, .. } | Self::ReplaceAll { rows, .. } => {
rows.iter().all(valid_row_columns)
}
Self::ApplyTransaction { ops }
| Self::MvccLock { ops, .. }
| Self::MvccCommit { ops, .. } => ops.iter().all(|op| match op {
StorageOp::Upsert { rows, .. } | StorageOp::Replace { rows, .. } => {
rows.iter().all(valid_row_columns)
}
StorageOp::Delete { .. } => true,
}),
_ => true,
}
}
/// 普通表操作的统一访问边界
pub fn table(&self) -> Option<&str> {
match self {
Self::Scan { table, .. } | Self::ScanShard { table, .. } | Self::ValidateWrite { table }
Self::Scan { table, .. }
| Self::ScanShard { table, .. }
| Self::ValidateWrite { table }
| Self::Insert { table, .. }
| Self::InsertBatch { table, .. } | Self::DeleteKeys { table, .. }
| Self::LookupByIndex { table, .. } | Self::ReplaceAll { table, .. }
| Self::InsertBatch { table, .. }
| Self::DeleteKeys { table, .. }
| Self::LookupByIndex { table, .. }
| Self::ReplaceAll { table, .. }
| Self::RangeByIndex { table, .. }
| Self::DescribeTable { table } | Self::CreateTable { table, .. }
| Self::DropTable { table } | Self::CreateIndex { table, .. }
| Self::DropIndex { table, .. } | Self::DropColumn { table, .. }
| Self::DescribeTable { table }
| Self::CreateTable { table, .. }
| Self::DropTable { table }
| Self::CreateIndex { table, .. }
| Self::DropIndex { table, .. }
| Self::DropColumn { table, .. }
| Self::Compact { table }
| Self::ReplaceSchema { table, .. } => Some(table),
Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList | Self::CatalogManage { .. }
| Self::IdentityInitialize { .. } | Self::RoleCreate { .. } | Self::IdentityAlter { .. }
| Self::AccountCreate { .. } | Self::AccountReset { .. } | Self::AccountLogin { .. }
| Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus
| Self::ReplaceSchema { table, .. }
| Self::AlterDomain { table, .. } => Some(table),
Self::Ping
| Self::ListTables
| Self::ListCatalog
| Self::DomainSnapshot
| Self::MvccPreview { .. }
| Self::MvccBegin { .. }
| Self::MvccLock { .. }
| Self::MvccCommit { .. }
| Self::MvccRollback { .. }
| Self::DatabaseSnapshot { .. }
| Self::AccountsList
| Self::CatalogManage { .. }
| Self::IdentityInitialize { .. }
| Self::RoleCreate { .. }
| Self::IdentityAlter { .. }
| Self::AccountCreate { .. }
| Self::AccountReset { .. }
| Self::AccountLogin { .. }
| Self::AccountDrop { .. }
| Self::BootstrapSystem { .. }
| Self::SystemStatus
| Self::BootstrapTypes { .. }
| Self::ApplyTransaction { .. } => None,
}
@@ -284,14 +392,26 @@ fn default_id_column() -> String {
#[derive(Debug, Serialize, Deserialize)]
#[serde(tag = "status", rename_all = "snake_case")]
pub enum Response {
Guarded { response: Box<Response> },
Accounts { accounts: Vec<Account> },
Guarded {
response: Box<Response>,
},
Accounts {
accounts: Vec<Account>,
},
Pong,
Rows { rows: Vec<Row> },
Tables { tables: Vec<String> },
Rows {
rows: Vec<Row>,
},
Tables {
tables: Vec<String>,
},
Schema {
table: String,
#[serde(default, serialize_with = "serialize_version", deserialize_with = "deserialize_version")]
#[serde(
default,
serialize_with = "serialize_version",
deserialize_with = "deserialize_version"
)]
schema_version: u64,
columns: Vec<SchemaColumn>,
#[serde(default)]
@@ -309,9 +429,30 @@ pub enum Response {
#[serde(default)]
last_lsn: u64,
},
Error { message: String },
Catalog { schemas: Vec<TableSchemaWire> },
Object { database_id: u64, object_id: Option<u64> },
Error {
message: String,
},
Catalog {
schemas: Vec<TableSchemaWire>,
},
DomainSnapshot {
schemas: Vec<TableSchemaWire>,
rows: Vec<(String, Vec<Row>)>,
checkpoint: String,
},
MvccSnapshot {
transaction: String,
schemas: Vec<TableSchemaWire>,
rows: Vec<(String, Vec<Row>)>,
checkpoint: String,
},
SnapshotUnchanged {
checkpoint: String,
},
Object {
database_id: u64,
object_id: Option<u64>,
},
}
#[derive(Debug, Deserialize)]
@@ -330,7 +471,11 @@ pub struct TableSchemaWire {
pub database_id: u64,
#[serde(default)]
pub object_id: u64,
#[serde(default, serialize_with = "serialize_version", deserialize_with = "deserialize_version")]
#[serde(
default,
serialize_with = "serialize_version",
deserialize_with = "deserialize_version"
)]
pub schema_version: u64,
pub columns: Vec<SchemaColumn>,
#[serde(default)]
@@ -342,7 +487,10 @@ pub struct TableSchemaWire {
}
/// 将结构版本编码成精确十进制字符串
fn serialize_version<S: serde::Serializer>(version: &u64, serializer: S) -> Result<S::Ok, S::Error> {
fn serialize_version<S: serde::Serializer>(
version: &u64,
serializer: S,
) -> Result<S::Ok, S::Error> {
serializer.serialize_str(&version.to_string())
}
@@ -350,9 +498,14 @@ fn serialize_version<S: serde::Serializer>(version: &u64, serializer: S) -> Resu
fn deserialize_version<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result<u64, D::Error> {
let value = serde_json::Value::deserialize(deserializer)?;
match value {
serde_json::Value::String(s) if !s.is_empty() && s.len() <= 20 && s.bytes().all(|c| c.is_ascii_digit()) =>
s.parse().map_err(serde::de::Error::custom),
serde_json::Value::Number(n) => n.as_u64().ok_or_else(|| serde::de::Error::custom("invalid schema version")),
serde_json::Value::String(s)
if !s.is_empty() && s.len() <= 20 && s.bytes().all(|c| c.is_ascii_digit()) =>
{
s.parse().map_err(serde::de::Error::custom)
}
serde_json::Value::Number(n) => n
.as_u64()
.ok_or_else(|| serde::de::Error::custom("invalid schema version")),
_ => Err(serde::de::Error::custom("invalid schema version")),
}
}
File diff suppressed because it is too large. Load diff
+129 -28
View File
@@ -26,12 +26,30 @@ const OP_CREATE_TABLE: u8 = 11;
const OP_DROP_TABLE: u8 = 12;
const OP_CREATE_INDEX: u8 = 13;
const OP_DROP_INDEX: u8 = 14;
const OP_MIGRATE: u8 = 15;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub enum WalOp {
HideColumn { table: String, column: String, schema_version: Option<u64> },
Compact { table: String, rows: Vec<Row> },
Accounts { accounts: Vec<Account>, removed: Vec<String>, migrate_roles: bool },
Migrate {
table: String,
object_id: u64,
source: String,
target: String,
},
HideColumn {
table: String,
column: String,
schema_version: Option<u64>,
},
Compact {
table: String,
rows: Vec<Row>,
},
Accounts {
accounts: Vec<Account>,
removed: Vec<String>,
migrate_roles: bool,
},
Insert {
table: String,
row: Row,
@@ -80,6 +98,28 @@ pub enum WalOp {
Commit,
}
impl WalOp {
/// 返回单表操作的目标,账号组另行展开
pub(crate) fn table(&self) -> Option<&str> {
match self {
Self::Migrate { table, .. }
| Self::HideColumn { table, .. }
| Self::Compact { table, .. }
| Self::Insert { table, .. }
| Self::InsertBatch { table, .. }
| Self::DeleteKeys { table, .. }
| Self::ReplaceAll { table, .. }
| Self::DropColumn { table, .. }
| Self::ReplaceSchema { table, .. }
| Self::CreateTable { table, .. }
| Self::DropTable { table }
| Self::CreateIndex { table, .. }
| Self::DropIndex { table, .. } => Some(table),
Self::Accounts { .. } | Self::Commit => None,
}
}
}
#[derive(Debug, Serialize, Deserialize)]
struct HideColumnPayload {
column: String,
@@ -255,13 +295,11 @@ impl Wal {
/// 读当前 WAL 的第一条操作;空则 None
pub fn read(&self) -> io::Result<Option<WalOp>> {
let scan = self.read_all()?;
Ok(scan.records.into_iter().find_map(|(_, op)| {
if op == WalOp::Commit {
None
} else {
Some(op)
}
}))
Ok(scan.records.into_iter().find_map(
|(_, op)| {
if op == WalOp::Commit { None } else { Some(op) }
},
))
}
/// 读全部记录,撕裂或损坏的尾巴按长度截断
@@ -287,9 +325,12 @@ impl Wal {
let mut frames = Vec::new();
let mut offset = 0usize;
while offset + 4 <= buf.len() {
let body_len =
u32::from_le_bytes([buf[offset], buf[offset + 1], buf[offset + 2], buf[offset + 3]])
as usize;
let body_len = u32::from_le_bytes([
buf[offset],
buf[offset + 1],
buf[offset + 2],
buf[offset + 3],
]) as usize;
let end = offset + 4 + body_len;
if end > buf.len() {
break;
@@ -324,7 +365,10 @@ impl Wal {
}
let bytes = std::fs::read(&path)?;
if bytes.len() != 8 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid WAL checkpoint length"));
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid WAL checkpoint length",
));
}
let mut raw = [0u8; 8];
raw.copy_from_slice(&bytes[..8]);
@@ -342,10 +386,14 @@ impl Wal {
if let Some((lsn, _, _, _)) = frames.last() {
base = base.max(*lsn);
}
*slot = base.checked_add(1).ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
*slot = base
.checked_add(1)
.ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
}
let lsn = *slot;
*slot = slot.checked_add(1).ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
*slot = slot
.checked_add(1)
.ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
Ok(lsn)
}
@@ -392,6 +440,7 @@ fn truncate(f: &mut File) -> io::Result<()> {
/// 把操作编成字节
fn encode(op: &WalOp, lsn: u64) -> io::Result<Vec<u8>> {
let (op_type, table, payload) = match op {
WalOp::Migrate { table, .. } => (OP_MIGRATE, table.clone(), serde_json::to_vec(op).map_err(io::Error::other)?),
WalOp::HideColumn { table, column, schema_version } => (OP_HIDE_COLUMN, table.clone(),
serde_json::to_vec(&serde_json::json!({"column":column,"schema_version":schema_version})).map_err(io::Error::other)?),
WalOp::Compact { table, rows } => {
@@ -487,6 +536,25 @@ fn decode_body(body: &[u8]) -> Result<(u64, WalOp), String> {
let payload = &body[11 + table_len..];
let op = match op_type {
OP_MIGRATE => {
let op: WalOp = serde_json::from_slice(payload)
.map_err(|e| format!("bad migration payload: {e}"))?;
match &op {
WalOp::Migrate {
table: name,
source,
target,
..
} if name == &table
&& crate::catalog::valid_generation(source)
&& !target.is_empty()
&& crate::catalog::valid_generation(target) =>
{
op
}
_ => return Err("invalid migration WAL".into()),
}
}
OP_COMMIT => WalOp::Commit,
OP_HIDE_COLUMN => {
let value: serde_json::Value = serde_json::from_slice(payload)
@@ -498,28 +566,61 @@ fn decode_body(body: &[u8]) -> Result<(u64, WalOp), String> {
.map_err(|e| format!("bad hide_column payload: {e}"))?;
(p.column, p.schema_version)
};
WalOp::HideColumn { table, column, schema_version }
WalOp::HideColumn {
table,
column,
schema_version,
}
}
OP_COMPACT => {
let rows: Vec<Row> = serde_json::from_slice(payload)
.map_err(|e| format!("bad compact payload: {e}"))?;
let rows: Vec<Row> =
serde_json::from_slice(payload).map_err(|e| format!("bad compact payload: {e}"))?;
WalOp::Compact { table, rows }
}
OP_ACCOUNTS => {
if table != USERS_TABLE { return Err("invalid account WAL table".into()); }
let value: serde_json::Value = serde_json::from_slice(payload).map_err(|e| format!("bad account payload: {e}"))?;
if table != USERS_TABLE {
return Err("invalid account WAL table".into());
}
let value: serde_json::Value =
serde_json::from_slice(payload).map_err(|e| format!("bad account payload: {e}"))?;
let (accounts, removed, migrate_roles) = if value.is_array() {
(serde_json::from_value(value).map_err(|e| format!("bad account payload: {e}"))?, Vec::new(), false)
(
serde_json::from_value(value)
.map_err(|e| format!("bad account payload: {e}"))?,
Vec::new(),
false,
)
} else {
(serde_json::from_value(value.get("accounts").cloned().ok_or("missing account payload")?).map_err(|e| format!("bad account payload: {e}"))?,
serde_json::from_value(value.get("removed").cloned().ok_or("missing identity cleanup")?).map_err(|e| format!("bad account cleanup: {e}"))?,
value.get("migrate_roles").and_then(serde_json::Value::as_bool).ok_or("missing role migration flag")?)
(
serde_json::from_value(
value
.get("accounts")
.cloned()
.ok_or("missing account payload")?,
)
.map_err(|e| format!("bad account payload: {e}"))?,
serde_json::from_value(
value
.get("removed")
.cloned()
.ok_or("missing identity cleanup")?,
)
.map_err(|e| format!("bad account cleanup: {e}"))?,
value
.get("migrate_roles")
.and_then(serde_json::Value::as_bool)
.ok_or("missing role migration flag")?,
)
};
WalOp::Accounts { accounts, removed, migrate_roles }
WalOp::Accounts {
accounts,
removed,
migrate_roles,
}
}
OP_INSERT => {
let row: Row =
serde_json::from_slice(payload).map_err(|e| format!("bad insert payload: {}", e))?;
let row: Row = serde_json::from_slice(payload)
.map_err(|e| format!("bad insert payload: {}", e))?;
WalOp::Insert { table, row }
}
OP_INSERT_BATCH => {
+125 -12
View File
@@ -1,4 +1,4 @@
use chusql_core_storage::btree::{fits_in_page, DiskBTree, MAX_KEY_BYTES};
use chusql_core_storage::btree::{DiskBTree, MAX_KEY_BYTES, fits_in_page};
use chusql_core_storage::config::{DEFAULT_BTREE_ORDER, DEFAULT_PAGE_SIZE};
use chusql_core_storage::page::DEFAULT_POOL_SIZE;
use std::io::{ErrorKind, Seek, SeekFrom, Write};
@@ -16,20 +16,28 @@ fn malformed_node_fields_return_invalid_data() {
tree.insert(b"y", 2).unwrap();
}
let original = std::fs::read(&path).unwrap();
let cell = u16::from_le_bytes([original[DEFAULT_PAGE_SIZE + 11], original[DEFAULT_PAGE_SIZE + 12]]) as usize;
let cell = u16::from_le_bytes([
original[DEFAULT_PAGE_SIZE + 11],
original[DEFAULT_PAGE_SIZE + 12],
]) as usize;
for (name, offset, bytes) in [
("slot count", 1, vec![255, 255]),
("node kind", 0, vec![2]),
("cell overlaps slots", 11, vec![0, 0]),
("cell beyond page", 11, vec![255, 15]),
("overlapping cells", 13, (cell as u16).to_le_bytes().to_vec()),
(
"overlapping cells",
13,
(cell as u16).to_le_bytes().to_vec(),
),
("key length", cell, vec![255, 255]),
("key order", cell + 2, vec![b'z']),
("zero child", 0, vec![1]),
] {
std::fs::write(&path, &original).unwrap();
let mut file = std::fs::OpenOptions::new().write(true).open(&path).unwrap();
file.seek(SeekFrom::Start((DEFAULT_PAGE_SIZE + offset) as u64)).unwrap();
file.seek(SeekFrom::Start((DEFAULT_PAGE_SIZE + offset) as u64))
.unwrap();
file.write_all(&bytes).unwrap();
drop(file);
let mut tree = open(dir.path(), "corrupt.idx");
@@ -42,7 +50,13 @@ fn malformed_node_fields_return_invalid_data() {
/// 按默认参数打开指定索引文件
fn open(dir: &std::path::Path, name: &str) -> DiskBTree {
DiskBTree::open(dir.join(name), DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).unwrap()
DiskBTree::open(
dir.join(name),
DEFAULT_PAGE_SIZE,
DEFAULT_BTREE_ORDER,
DEFAULT_POOL_SIZE,
)
.unwrap()
}
/// 整数键编码成保序的 8 字节(与堆表一致)
@@ -101,12 +115,24 @@ fn reopen_persists() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("t.idx");
{
let mut t = DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).unwrap();
let mut t = DiskBTree::open(
&path,
DEFAULT_PAGE_SIZE,
DEFAULT_BTREE_ORDER,
DEFAULT_POOL_SIZE,
)
.unwrap();
for i in 0..50 {
t.insert(&key(i), i as u64).unwrap();
}
}
let mut t = DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).unwrap();
let mut t = DiskBTree::open(
&path,
DEFAULT_PAGE_SIZE,
DEFAULT_BTREE_ORDER,
DEFAULT_POOL_SIZE,
)
.unwrap();
for i in 0..50 {
assert_eq!(t.get(&key(i)).unwrap(), Some(i as u64));
}
@@ -148,12 +174,26 @@ fn config_mismatch_errors() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("t.idx");
{
let mut t = DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).unwrap();
let mut t = DiskBTree::open(
&path,
DEFAULT_PAGE_SIZE,
DEFAULT_BTREE_ORDER,
DEFAULT_POOL_SIZE,
)
.unwrap();
t.insert(&key(1), 1).unwrap();
}
assert!(DiskBTree::open(&path, DEFAULT_PAGE_SIZE, 8, DEFAULT_POOL_SIZE).is_err());
assert!(DiskBTree::open(&path, 8192, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).is_err());
assert!(DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).is_ok());
assert!(
DiskBTree::open(
&path,
DEFAULT_PAGE_SIZE,
DEFAULT_BTREE_ORDER,
DEFAULT_POOL_SIZE
)
.is_ok()
);
}
/// 不是索引文件要报错
@@ -162,7 +202,15 @@ fn not_an_index_file_errors() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("junk.idx");
std::fs::write(&path, vec![7u8; DEFAULT_PAGE_SIZE]).unwrap();
assert!(DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).is_err());
assert!(
DiskBTree::open(
&path,
DEFAULT_PAGE_SIZE,
DEFAULT_BTREE_ORDER,
DEFAULT_POOL_SIZE
)
.is_err()
);
}
/// order 放不进一页要报错
@@ -182,7 +230,10 @@ fn delete_removes_only_that_key() {
t.insert(&key(i), i as u64 + 1).unwrap();
}
assert!(t.delete(&key(50), 51).unwrap(), "deleting an existing entry should return true");
assert!(
t.delete(&key(50), 51).unwrap(),
"deleting an existing entry should return true"
);
assert_eq!(t.get(&key(50)).unwrap(), None);
assert_eq!(t.get(&key(49)).unwrap(), Some(50));
assert_eq!(t.get(&key(51)).unwrap(), Some(52));
@@ -262,7 +313,11 @@ fn splits_keep_keys_sorted() {
let all: Vec<Vec<u8>> = t.iter_all().unwrap().into_iter().map(|(k, _)| k).collect();
assert_eq!(all, expected);
for k in &expected {
assert!(t.get(k).unwrap().is_some(), "missing {:?}", String::from_utf8_lossy(k));
assert!(
t.get(k).unwrap().is_some(),
"missing {:?}",
String::from_utf8_lossy(k)
);
}
}
@@ -314,3 +369,61 @@ fn clear_resets_the_tree() {
t.insert(b"c", 3).unwrap();
assert_eq!(t.get(b"c").unwrap(), Some(3));
}
/// 长键跨页后可重启查询并保留旧配置
#[test]
fn long_keys_split_reopen_and_preserve_dense_configuration() {
use chusql_core_storage::btree::key_limit;
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("long.idx");
let mut tree = DiskBTree::open(&path, 512, 4, DEFAULT_POOL_SIZE).unwrap();
for n in 0..80_u64 {
let key = format!("{n:03}{}", "a".repeat(125));
tree.insert(key.as_bytes(), n).unwrap();
}
drop(tree);
let mut tree = DiskBTree::open(&path, 512, 4, DEFAULT_POOL_SIZE).unwrap();
assert_eq!(tree.scan(None, None).unwrap().len(), 80);
for n in 0..80_u64 {
let key = format!("{n:03}{}", "a".repeat(125));
assert_eq!(tree.get(key.as_bytes()).unwrap(), Some(n));
}
let dense_path = dir.path().join("dense.idx");
let mut dense = DiskBTree::open(&dense_path, 512, 7, DEFAULT_POOL_SIZE).unwrap();
let limit = key_limit(512, 7);
assert!((64..MAX_KEY_BYTES).contains(&limit));
dense.insert(&vec![b'a'; limit], 1).unwrap();
assert_eq!(
dense.insert(&vec![b'b'; limit + 1], 2).unwrap_err().kind(),
ErrorKind::InvalidInput
);
assert_eq!(dense.get(&vec![b'a'; limit]).unwrap(), Some(1));
}
/// 旧文件头可无损打开并首次写入长键
#[test]
fn legacy_key_header_upgrades_only_when_long_keys_are_written() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("legacy.idx");
let mut tree = open(dir.path(), "legacy.idx");
tree.insert(b"old", 1).unwrap();
drop(tree);
let mut bytes = std::fs::read(&path).unwrap();
bytes[8..10].copy_from_slice(&64_u16.to_le_bytes());
std::fs::write(&path, &bytes).unwrap();
let mut tree = open(dir.path(), "legacy.idx");
assert_eq!(tree.get(b"old").unwrap(), Some(1));
drop(tree);
assert_eq!(std::fs::read(&path).unwrap(), bytes);
let mut tree = open(dir.path(), "legacy.idx");
let key = "author/".to_string() + &"p".repeat(80);
tree.insert(key.as_bytes(), 2).unwrap();
drop(tree);
let mut tree = open(dir.path(), "legacy.idx");
assert_eq!(tree.get(key.as_bytes()).unwrap(), Some(2));
assert_eq!(tree.get(b"old").unwrap(), Some(1));
assert_eq!(
&std::fs::read(&path).unwrap()[8..10],
&128_u16.to_le_bytes()
);
}
+83 -31
View File
@@ -33,9 +33,14 @@ fn schema_versions_track_structure_and_survive_reload() {
fn legacy_schema_version_remains_unknown_until_change() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("catalog.json");
std::fs::write(&path, json!({"tables": {
"t": {"object_id": 42, "columns": []}
}}).to_string()).unwrap();
std::fs::write(
&path,
json!({"tables": {
"t": {"object_id": 42, "columns": []}
}})
.to_string(),
)
.unwrap();
let mut c = Catalog::load(&path).unwrap();
assert_eq!(c.describe("t").unwrap().schema_version, 0);
c.save(&path).unwrap();
@@ -51,10 +56,15 @@ fn legacy_schema_version_remains_unknown_until_change() {
fn exhausted_schema_version_preserves_structure() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("catalog.json");
std::fs::write(&path, json!({"tables": {
"t": {"object_id": 42, "schema_version": u64::MAX,
"columns": [{"name": "id", "ty": "int"}]}
}}).to_string()).unwrap();
std::fs::write(
&path,
json!({"tables": {
"t": {"object_id": 42, "schema_version": u64::MAX,
"columns": [{"name": "id", "ty": "int"}]}
}})
.to_string(),
)
.unwrap();
let mut c = Catalog::load(&path).unwrap();
assert!(c.remove_column("t", "id").is_err());
assert!(c.set_columns("t", vec![]).is_err());
@@ -90,20 +100,42 @@ fn object_ids_survive_reload_and_are_not_reused() {
fn legacy_object_ids_upgrade_without_collisions() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("catalog.json");
std::fs::write(&path, json!({"tables": {
"a": {"columns": [], "object_id": 42},
"b": {"columns": []}
}}).to_string()).unwrap();
std::fs::write(
&path,
json!({"tables": {
"a": {"columns": [], "object_id": 42},
"b": {"columns": []}
}})
.to_string(),
)
.unwrap();
let catalog = Catalog::load(&path).unwrap();
assert_eq!(catalog.describe("a").unwrap().object_id, 42);
assert_eq!(catalog.describe("b").unwrap().object_id, 43);
catalog.save(&path).unwrap();
assert_eq!(Catalog::load(&path).unwrap().describe("b").unwrap().object_id, 43);
std::fs::write(&path, json!({"tables": {
"a": {"columns": [], "object_id": 42},
"b": {"columns": [], "object_id": 42}
}}).to_string()).unwrap();
assert!(Catalog::load(&path).unwrap_err().to_string().contains("duplicate"));
assert_eq!(
Catalog::load(&path)
.unwrap()
.describe("b")
.unwrap()
.object_id,
43
);
std::fs::write(
&path,
json!({"tables": {
"a": {"columns": [], "object_id": 42},
"b": {"columns": [], "object_id": 42}
}})
.to_string(),
)
.unwrap();
assert!(
Catalog::load(&path)
.unwrap_err()
.to_string()
.contains("duplicate")
);
}
/// 编号耗尽明确失败且不建立对象
@@ -115,8 +147,6 @@ fn exhausted_object_ids_fail_without_creating_table() {
assert!(catalog.describe("t").is_none());
}
/// 用键值对构造一行
fn row(pairs: &[(&str, serde_json::Value)]) -> Row {
let mut m = Row::new();
@@ -128,7 +158,8 @@ fn row(pairs: &[(&str, serde_json::Value)]) -> Row {
/// 空文件得到空字典,未知表返回 None
#[test]
fn empty_file_gives_empty_catalog_and_unknown_table_returns_none() { let dir = tempfile::tempdir().unwrap();
fn empty_file_gives_empty_catalog_and_unknown_table_returns_none() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("catalog.json");
let c = Catalog::load(&path).unwrap();
assert_eq!(c.table_names().len(), 0);
@@ -142,7 +173,8 @@ fn ensure_then_describe() {
let path = dir.path().join("catalog.json");
let mut c = Catalog::load(&path).unwrap();
c.ensure_columns("users", &row(&[("id", json!(1)), ("name", json!("Alice"))])).unwrap();
c.ensure_columns("users", &row(&[("id", json!(1)), ("name", json!("Alice"))]))
.unwrap();
c.save(&path).unwrap();
let c2 = Catalog::load(&path).unwrap();
@@ -164,8 +196,10 @@ fn second_insert_does_not_duplicate_columns() {
let path = dir.path().join("catalog.json");
let mut c = Catalog::load(&path).unwrap();
c.ensure_columns("users", &row(&[("id", json!(1))])).unwrap();
c.ensure_columns("users", &row(&[("id", json!(2))])).unwrap();
c.ensure_columns("users", &row(&[("id", json!(1))]))
.unwrap();
c.ensure_columns("users", &row(&[("id", json!(2))]))
.unwrap();
assert_eq!(c.describe("users").unwrap().columns.len(), 1);
}
@@ -176,9 +210,11 @@ fn remove_column_drops_definition_stats_and_index() {
let path = dir.path().join("catalog.json");
let mut c = Catalog::load(&path).unwrap();
c.ensure_columns("t", &row(&[("id", json!(1)), ("code", json!(5))])).unwrap();
c.ensure_columns("t", &row(&[("id", json!(1)), ("code", json!(5))]))
.unwrap();
c.add_index("t", "code").unwrap();
c.rebuild_stats("t", &[row(&[("id", json!(1)), ("code", json!(5))])]).unwrap();
c.rebuild_stats("t", &[row(&[("id", json!(1)), ("code", json!(5))])])
.unwrap();
let removed = c.remove_column("t", "code").unwrap();
assert!(removed, "first remove should hit the column");
@@ -186,7 +222,10 @@ fn remove_column_drops_definition_stats_and_index() {
assert_eq!(s.columns.len(), 1);
assert_eq!(s.columns[0].name, "id");
assert!(s.indexes.is_empty(), "index on code should be gone");
assert!(!s.stats.contains_key("code"), "stats for code should be gone");
assert!(
!s.stats.contains_key("code"),
"stats for code should be gone"
);
let again = c.remove_column("t", "code").unwrap();
assert!(!again);
@@ -217,7 +256,8 @@ fn stats_track_a_histogram_for_numeric_columns() {
assert!(name.hist.is_empty(), "字符串列不进直方图");
assert_eq!(name.lo, None);
c.record_insert("t", &row(&[("id", json!(1000)), ("name", json!("x"))])).unwrap();
c.record_insert("t", &row(&[("id", json!(1000)), ("name", json!("x"))]))
.unwrap();
let s = c.describe("t").unwrap();
let id = s.stats.get("id").unwrap();
assert_eq!(id.lo, Some(0.0));
@@ -240,7 +280,9 @@ fn stats_track_a_histogram_for_numeric_columns() {
fn duplicate_deletion_keeps_distinct_until_last_occurrence() {
let mut catalog = Catalog::default();
let duplicate = row(&[("id", json!(1)), ("v", json!(7))]);
catalog.rebuild_stats("t", &[duplicate.clone(), duplicate.clone()]).unwrap();
catalog
.rebuild_stats("t", &[duplicate.clone(), duplicate.clone()])
.unwrap();
catalog.record_delete("t", std::slice::from_ref(&duplicate));
assert_eq!(catalog.describe("t").unwrap().stats["v"].distinct, 1);
catalog.record_delete("t", &[duplicate]);
@@ -251,7 +293,9 @@ fn duplicate_deletion_keeps_distinct_until_last_occurrence() {
#[test]
fn capped_distinct_keeps_histogram_and_other_columns_accurate() {
let mut catalog = Catalog::default();
let rows: Vec<Row> = (0..5000).map(|n| row(&[("id", json!(n)), ("v", json!(n % 2))])).collect();
let rows: Vec<Row> = (0..5000)
.map(|n| row(&[("id", json!(n)), ("v", json!(n % 2))]))
.collect();
catalog.rebuild_stats("t", &rows).unwrap();
let schema = catalog.describe("t").unwrap();
assert!(schema.stats["id"].capped);
@@ -277,7 +321,9 @@ fn rebuilt_histogram_has_exact_buckets_in_any_order() {
let stat = &forward.describe("t").unwrap().stats["v"];
assert_eq!(stat.hist, vec![10; 16]);
assert_eq!(stat.hist, reverse.describe("t").unwrap().stats["v"].hist);
forward.record_insert("t", &row(&[("v", json!(10000))])).unwrap();
forward
.record_insert("t", &row(&[("v", json!(10000))]))
.unwrap();
forward.record_delete("t", &[row(&[("v", json!(10000))])]);
forward.rebuild_stats("t", &rows[80..]).unwrap();
let repaired = &forward.describe("t").unwrap().stats["v"];
@@ -316,7 +362,13 @@ fn legacy_catalog_loads_and_normalizes() {
assert_eq!(s.columns.len(), 2);
assert!(s.columns[0].nullable, "老列默认可空");
assert!(c.normalize_types().unwrap());
let types: Vec<&str> = c.describe("legacy").unwrap().columns.iter().map(|col| col.ty.as_str()).collect();
let types: Vec<&str> = c
.describe("legacy")
.unwrap()
.columns
.iter()
.map(|col| col.ty.as_str())
.collect();
assert_eq!(types, vec!["int", "str"]);
assert!(!c.normalize_types().unwrap(), "第二次没有可改的");
}
+21 -6
View File
@@ -36,8 +36,7 @@ fn defaults_without_file() {
fn file_overrides_only_written_keys() {
let _guard = env_guard();
let path = PathBuf::from("t.toml");
let loaded = config::resolve(Some("[page]\nsize = 8192\n"), Some(path.clone()))
.unwrap();
let loaded = config::resolve(Some("[page]\nsize = 8192\n"), Some(path.clone())).unwrap();
assert_eq!(loaded.config.page_size, 8192);
assert_eq!(loaded.config.btree_order, DEFAULT_BTREE_ORDER);
@@ -45,9 +44,17 @@ fn file_overrides_only_written_keys() {
assert_eq!(loaded.config.log_files, PathBuf::from(DEFAULT_LOG_FILES));
assert_eq!(loaded.config_path, Some(path.clone()));
let page = loaded.origins.iter().find(|(n, _, _)| *n == "page.size").unwrap();
let page = loaded
.origins
.iter()
.find(|(n, _, _)| *n == "page.size")
.unwrap();
assert_eq!(page.2, Origin::File(path));
let order = loaded.origins.iter().find(|(n, _, _)| *n == "btree.order").unwrap();
let order = loaded
.origins
.iter()
.find(|(n, _, _)| *n == "btree.order")
.unwrap();
assert_eq!(order.2, Origin::Default);
}
@@ -110,7 +117,11 @@ fn default_config_path_follows_system_convention() {
let path = config::default_config_path();
assert_eq!(path.file_name().unwrap(), "settings.toml");
assert_eq!(path.parent().unwrap().file_name().unwrap(), "ChuSQL");
assert!(path.ends_with(format!("ChuSQL{}settings.toml", std::path::MAIN_SEPARATOR)), "got {}", path.display());
assert!(
path.ends_with(format!("ChuSQL{}settings.toml", std::path::MAIN_SEPARATOR)),
"got {}",
path.display()
);
// 本机 APPDATA 存在时必须是 %APPDATA%\ChuSQL\settings.toml
#[cfg(windows)]
@@ -168,7 +179,11 @@ fn explicit_missing_path_is_an_error() {
"got {}",
err
);
assert!(err.contains("chusql-no-such-config-9f21.toml"), "got {}", err);
assert!(
err.contains("chusql-no-such-config-9f21.toml"),
"got {}",
err
);
}
/// 不给 --config 且无配置文件时用内置默认
+64 -10
View File
@@ -26,16 +26,40 @@ fn maintenance_rejects_invalid_arguments_and_resets_outputs() {
let mut len = 99;
// SAFETY: 输出均可写;无效输入应在任何读取之前被拒绝。
unsafe {
assert_eq!(chusql_storage_maintenance(ptr::null(), ptr::null(), 0, &mut out, &mut len), -1);
assert_eq!(
chusql_storage_maintenance(ptr::null(), ptr::null(), 0, &mut out, &mut len),
-1
);
assert!(out.is_null());
assert_eq!(len, 0);
len = 99;
assert_eq!(chusql_storage_maintenance(ptr::null(), input.as_ptr(), input.len(), ptr::null_mut(), &mut len), -1);
assert_eq!(
chusql_storage_maintenance(
ptr::null(),
input.as_ptr(),
input.len(),
ptr::null_mut(),
&mut len
),
-1
);
assert_eq!(len, 0);
out = input.as_ptr() as *mut u8;
assert_eq!(chusql_storage_maintenance(ptr::null(), input.as_ptr(), input.len(), &mut out, ptr::null_mut()), -1);
assert_eq!(
chusql_storage_maintenance(
ptr::null(),
input.as_ptr(),
input.len(),
&mut out,
ptr::null_mut()
),
-1
);
assert!(out.is_null());
assert_eq!(chusql_storage_maintenance(ptr::null(), input.as_ptr(), usize::MAX, &mut out, &mut len), -1);
assert_eq!(
chusql_storage_maintenance(ptr::null(), input.as_ptr(), usize::MAX, &mut out, &mut len),
-1
);
}
assert!(out.is_null());
assert_eq!(len, 0);
@@ -47,20 +71,36 @@ fn maintenance_rejects_invalid_arguments_and_resets_outputs() {
fn request_validates_input_and_returns_owned_response() -> Result<(), Box<dyn std::error::Error>> {
let temp = tempfile::tempdir()?;
let config = temp.path().join("settings.toml");
std::fs::write(&config, format!("[storage]\ndata_dir = '{}'\nlog_files = '{}'\n[log]\nlevel = 'off'\n", temp.path().join("data").display(), temp.path().join("logs").display()))?;
std::fs::write(
&config,
format!(
"[storage]\ndata_dir = '{}'\nlog_files = '{}'\n[log]\nlevel = 'off'\n",
temp.path().join("data").display(),
temp.path().join("logs").display()
),
)?;
let storage = Storage::open(config.to_str())?;
let input = br#"{"method":"ping"}"#;
let mut out = ptr::null_mut();
let mut len = 0;
// SAFETY: 句柄和实际请求有效;超长长度必须在构造切片前被拒绝。
unsafe {
assert_eq!(chusql_storage_request(&storage, input.as_ptr(), usize::MAX, &mut out, &mut len), -1);
assert_eq!(
chusql_storage_request(&storage, input.as_ptr(), usize::MAX, &mut out, &mut len),
-1
);
assert!(out.is_null());
assert_eq!(len, 0);
assert_eq!(last_error(), "invalid argument");
assert_eq!(chusql_storage_request(&storage, [0xff].as_ptr(), 1, &mut out, &mut len), -1);
assert_eq!(
chusql_storage_request(&storage, [0xff].as_ptr(), 1, &mut out, &mut len),
-1
);
assert!(last_error().starts_with("request is not valid UTF-8:"));
assert_eq!(chusql_storage_request(&storage, input.as_ptr(), input.len(), &mut out, &mut len), 0);
assert_eq!(
chusql_storage_request(&storage, input.as_ptr(), input.len(), &mut out, &mut len),
0
);
assert!(!out.is_null());
let response = slice::from_raw_parts(out, len).to_vec();
chusql_storage_free(out, len);
@@ -71,7 +111,10 @@ fn request_validates_input_and_returns_owned_response() -> Result<(), Box<dyn st
let config = CString::new(config.to_str().unwrap())?;
// SAFETY: 路径与输出有效;失败不产生需要释放的缓冲。
unsafe {
assert_eq!(chusql_storage_maintenance(config.as_ptr(), [0xff].as_ptr(), 1, &mut out, &mut len), -1);
assert_eq!(
chusql_storage_maintenance(config.as_ptr(), [0xff].as_ptr(), 1, &mut out, &mut len),
-1
);
}
assert!(out.is_null());
assert_eq!(len, 0);
@@ -79,7 +122,18 @@ fn request_validates_input_and_returns_owned_response() -> Result<(), Box<dyn st
let maintenance = br#"{"mode":"reset","user":"root","password_hash":"hash","types":[{"name":"int","base_type":"int","parameterized":false}]}"#;
// SAFETY: 维护仅使用本测试临时目录;成功响应复制后由配套函数释放。
unsafe {
assert_eq!(chusql_storage_maintenance(config.as_ptr(), maintenance.as_ptr(), maintenance.len(), &mut out, &mut len), 0, "{}", last_error());
assert_eq!(
chusql_storage_maintenance(
config.as_ptr(),
maintenance.as_ptr(),
maintenance.len(),
&mut out,
&mut len
),
0,
"{}",
last_error()
);
assert!(!out.is_null());
let response = slice::from_raw_parts(out, len).to_vec();
chusql_storage_free(out, len);
+135 -37
View File
@@ -25,20 +25,61 @@ fn delete_propagates_index_read_errors() {
let original = row(&[("id", json!(1))]);
t.insert_row(&original).unwrap();
t.flush().unwrap();
let file = std::fs::OpenOptions::new().write(true).open(&index).unwrap();
let file = std::fs::OpenOptions::new()
.write(true)
.open(&index)
.unwrap();
file.set_len(DEFAULT_PAGE_SIZE as u64).unwrap();
let err = t.delete_by_keys(&[1]).unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
assert_eq!(t.scan().unwrap(), vec![original]);
}
/// 索引键失败不得留下堆行或清空旧表
#[test]
fn invalid_index_keys_leave_heap_and_indexes_unchanged() {
let dir = tempfile::tempdir().unwrap();
let mut table = HeapTable::open(dir.path().join("t.db"), DEFAULT_PAGE_SIZE, 1).unwrap();
let original = row(&[("id", json!(1)), ("name", json!("old"))]);
table.insert_row(&original).unwrap();
table
.build_index(
"name",
&dir.path().join("t.name.idx"),
DEFAULT_PAGE_SIZE,
4,
1,
)
.unwrap();
let invalid = row(&[("id", json!(2)), ("name", json!("x".repeat(129)))]);
assert!(table.insert_row(&invalid).is_err());
assert_eq!(table.scan().unwrap(), vec![original.clone()]);
assert!(table.replace_all(&[invalid]).is_err());
assert_eq!(table.scan().unwrap(), vec![original.clone()]);
assert_eq!(
table.get_by_string_key("name", "old").unwrap(),
Some(original)
);
assert!(
table
.replace_all(&[row(&[("name", json!("x".repeat(10_000)))])])
.is_err()
);
assert_eq!(table.scan().unwrap().len(), 1);
}
/// 跨页投影与分片保持行序及隐藏列
#[test]
fn projected_shards_preserve_order_and_hidden_columns() {
let dir = tempfile::tempdir().unwrap();
let mut t = HeapTable::open(dir.path().join("t.db"), 256, 1).unwrap();
for i in 0..40 {
t.insert_row(&row(&[("id", json!(i)), ("name", json!("alice")), ("secret", json!(i))])).unwrap();
t.insert_row(&row(&[
("id", json!(i)),
("name", json!("alice")),
("secret", json!(i)),
]))
.unwrap();
}
t.set_hidden_columns(["secret".to_string()].into_iter().collect());
let columns = vec!["id".to_string(), "secret".to_string()];
@@ -47,9 +88,15 @@ fn projected_shards_preserve_order_and_hidden_columns() {
let pages = t.num_pages().unwrap();
assert!(pages > 1);
let mut shards = t.scan_columns_pages(Some(&columns), 0, pages / 2).unwrap();
shards.extend(t.scan_columns_pages(Some(&columns), pages / 2, pages + 1).unwrap());
shards.extend(
t.scan_columns_pages(Some(&columns), pages / 2, pages + 1)
.unwrap(),
);
assert_eq!(shards, expected);
assert_eq!(t.read_at(0, 0).unwrap().unwrap(), row(&[("id", json!(0)), ("name", json!("alice"))]));
assert_eq!(
t.read_at(0, 0).unwrap().unwrap(),
row(&[("id", json!(0)), ("name", json!("alice"))])
);
}
/// 扫描与点读均返回行解码错误
@@ -78,15 +125,20 @@ fn empty_index_file_is_rebuilt_from_rows() {
let data = dir.path().join("t.db");
let index = dir.path().join("t.idx");
{
let mut t = HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
let mut t = HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE)
.unwrap();
for i in 1..=5 {
t.insert_row(&row(&[("id", json!(i))])).unwrap();
}
t.flush().unwrap();
}
std::fs::write(&index, b"").unwrap();
let mut t = HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
assert!(t.get_by_column_key("id", 3).unwrap().is_some(), "空索引文件要从堆数据重建");
let mut t =
HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
assert!(
t.get_by_column_key("id", 3).unwrap().is_some(),
"空索引文件要从堆数据重建"
);
assert!(t.get_by_column_key("id", 9).unwrap().is_none());
}
@@ -97,14 +149,24 @@ fn string_index_looks_up_by_value() {
let data = dir.path().join("s.db");
let index = dir.path().join("s.idx");
let mut t = HeapTable::open(&data, DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap();
t.insert_row(&row(&[("id", json!(1)), ("name", json!("alice"))])).unwrap();
t.insert_row(&row(&[("id", json!(2)), ("name", json!("bob"))])).unwrap();
t.insert_row(&row(&[("id", json!(3)), ("name", json!(null))])).unwrap();
t.build_index("name", &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
t.insert_row(&row(&[("id", json!(1)), ("name", json!("alice"))]))
.unwrap();
t.insert_row(&row(&[("id", json!(2)), ("name", json!("bob"))]))
.unwrap();
t.insert_row(&row(&[("id", json!(3)), ("name", json!(null))]))
.unwrap();
t.build_index("name", &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE)
.unwrap();
assert_eq!(t.get_by_string_key("name", "bob").unwrap().unwrap()["id"], json!(2));
assert_eq!(
t.get_by_string_key("name", "bob").unwrap().unwrap()["id"],
json!(2)
);
assert!(t.get_by_string_key("name", "dave").unwrap().is_none());
assert!(t.get_by_string_key("name", "").unwrap().is_none(), "NULL 不入索引");
assert!(
t.get_by_string_key("name", "").unwrap().is_none(),
"NULL 不入索引"
);
}
/// 允许重复值建索引:点查把重复的行都取回来
@@ -114,9 +176,12 @@ fn duplicate_values_are_indexed_and_all_returned() {
let data = dir.path().join("d.db");
let index = dir.path().join("d.idx");
let mut t = HeapTable::open(&data, DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap();
t.insert_row(&row(&[("id", json!(1)), ("name", json!("alice"))])).unwrap();
t.insert_row(&row(&[("id", json!(2)), ("name", json!("alice"))])).unwrap();
t.build_index("name", &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
t.insert_row(&row(&[("id", json!(1)), ("name", json!("alice"))]))
.unwrap();
t.insert_row(&row(&[("id", json!(2)), ("name", json!("alice"))]))
.unwrap();
t.build_index("name", &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE)
.unwrap();
let mut ids: Vec<i64> = t
.get_all_by_string_key("name", "alice")
@@ -218,9 +283,15 @@ fn replace_all_rebuilds_index() {
])
.unwrap();
assert!(t.get_by_key(1).unwrap().is_none(), "old key must be invalidated");
assert!(
t.get_by_key(1).unwrap().is_none(),
"old key must be invalidated"
);
assert!(t.get_by_key(2).unwrap().is_none());
let got = t.get_by_key(7).unwrap().expect("new row should be indexed by id");
let got = t
.get_by_key(7)
.unwrap()
.expect("new row should be indexed by id");
assert_eq!(got["name"], json!("Zoe"));
assert_eq!(t.scan().unwrap().len(), 2);
}
@@ -239,8 +310,11 @@ fn delete_by_keys_removes_rows() {
.unwrap();
for i in 1..=5 {
t.insert_row(&row(&[("id", json!(i)), ("name", json!(format!("u{}", i)))]))
.unwrap();
t.insert_row(&row(&[
("id", json!(i)),
("name", json!(format!("u{}", i))),
]))
.unwrap();
}
let deleted = t.delete_by_keys(&[2, 4, 99]).unwrap();
@@ -254,7 +328,10 @@ fn delete_by_keys_removes_rows() {
.collect();
assert_eq!(ids, vec![1, 3, 5], "remaining rows must keep their order");
assert!(t.get_by_key(2).unwrap().is_none(), "index entry should be deleted too");
assert!(
t.get_by_key(2).unwrap().is_none(),
"index entry should be deleted too"
);
assert!(t.get_by_key(4).unwrap().is_none());
assert!(t.get_by_key(3).unwrap().is_some());
@@ -284,10 +361,7 @@ fn insert_rows_batch() {
assert_eq!(t.scan().unwrap().len(), 20);
assert_eq!(t.get_by_key(20).unwrap().unwrap()["name"], json!("u20"));
let dup = vec![
row(&[("id", json!(100))]),
row(&[("id", json!(100))]),
];
let dup = vec![row(&[("id", json!(100))]), row(&[("id", json!(100))])];
assert!(t.insert_rows(&dup).is_err());
}
@@ -320,16 +394,23 @@ fn secondary_index_is_maintained() {
)
.unwrap();
assert!(t.has_index("code"));
assert_eq!(t.get_by_column_key("code", 7002).unwrap().unwrap()["id"], json!(2));
assert_eq!(
t.get_by_column_key("code", 7002).unwrap().unwrap()["id"],
json!(2)
);
t.insert_row(&row(&[("id", json!(3)), ("code", json!(7003))]))
.unwrap();
assert_eq!(t.get_by_column_key("code", 7003).unwrap().unwrap()["id"], json!(3));
assert_eq!(
t.get_by_column_key("code", 7003).unwrap().unwrap()["id"],
json!(3)
);
t.delete_by_keys(&[2]).unwrap();
assert!(t.get_by_column_key("code", 7002).unwrap().is_none());
t.insert_row(&row(&[("id", json!(4)), ("code", json!(7001))])).unwrap();
t.insert_row(&row(&[("id", json!(4)), ("code", json!(7001))]))
.unwrap();
let mut dup_ids: Vec<i64> = t
.get_all_by_column_key("code", 7001)
.unwrap()
@@ -339,7 +420,12 @@ fn secondary_index_is_maintained() {
dup_ids.sort_unstable();
assert_eq!(dup_ids, vec![1, 4], "duplicate index values keep both rows");
let mut t2 = HeapTable::open(dir.path().join("d.db"), DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap();
let mut t2 = HeapTable::open(
dir.path().join("d.db"),
DEFAULT_PAGE_SIZE,
DEFAULT_POOL_SIZE,
)
.unwrap();
t2.insert_rows(&[
row(&[("id", json!(1)), ("age", json!(30))]),
row(&[("id", json!(2)), ("age", json!(30))]),
@@ -360,7 +446,6 @@ fn secondary_index_is_maintained() {
);
}
/// 整表整理要收回删行留下的页内空洞
#[test]
fn compact_pages_reclaims_dead_space() {
@@ -370,7 +455,8 @@ fn compact_pages_reclaims_dead_space() {
let pad = "x".repeat(200);
for i in 0..10 {
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap();
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))]))
.unwrap();
}
let pages = t.num_pages().unwrap();
@@ -395,7 +481,8 @@ fn deletes_compact_pages_so_space_is_reused() {
let pad = "y".repeat(80);
for i in 0..30 {
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap();
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))]))
.unwrap();
}
assert_eq!(t.num_pages().unwrap(), 1, "30 行要先塞进一页");
@@ -404,7 +491,8 @@ fn deletes_compact_pages_so_space_is_reused() {
t.delete_by_keys(&keys).unwrap();
for i in 100..119 {
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap();
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))]))
.unwrap();
}
assert_eq!(t.num_pages().unwrap(), 1, "删掉的空间要能被复用");
assert_eq!(t.scan().unwrap().len(), 30);
@@ -416,11 +504,13 @@ fn compact_pages_keeps_index_positions_valid() {
let dir = tempfile::tempdir().unwrap();
let data = dir.path().join("t.db");
let index = dir.path().join("t.idx");
let mut t = HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
let mut t =
HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
let pad = "z".repeat(80);
for i in 0..30 {
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap();
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))]))
.unwrap();
}
let keys: Vec<i64> = (5..25).collect();
t.delete_by_keys(&keys).unwrap();
@@ -428,9 +518,17 @@ fn compact_pages_keeps_index_positions_valid() {
for i in [0, 1, 2, 3, 4, 25, 26, 27, 28, 29] {
let found = t.get_by_column_key("id", i).unwrap();
assert_eq!(found.as_ref().map(|r| r["pad"].clone()), Some(json!(pad)), "整理后 id={} 仍要能点查", i);
assert_eq!(
found.as_ref().map(|r| r["pad"].clone()),
Some(json!(pad)),
"整理后 id={} 仍要能点查",
i
);
}
for i in 5..25 {
assert!(t.get_by_column_key("id", i).unwrap().is_none(), "删掉的行不该再查出来");
assert!(
t.get_by_column_key("id", i).unwrap().is_none(),
"删掉的行不该再查出来"
);
}
}
+144
View File
@@ -142,3 +142,147 @@ fn dirty_page_flushed_on_drop() {
let p = pf2.read_page(0).unwrap();
assert_eq!(p.data[0], 42);
}
/// 已同步镜像恢复撕裂页且页 LSN 持久保留
#[test]
fn full_page_image_repairs_torn_write_before_read() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("torn.db");
let metadata = dir.path().join("torn.db.pages");
let journal = dir.path().join("torn.db.page-wal");
let mut file = PageFile::with_options(&path, 512, 1).unwrap();
file.set_lsn(7);
file.write_page(&Page::new(0, 512)).unwrap();
file.flush().unwrap();
let old_metadata = std::fs::read(&metadata).unwrap();
file.set_lsn(13);
let mut changed = Page::new(0, 512);
changed.data.fill(0x5a);
file.write_page(&changed).unwrap();
file.append_page().unwrap(); // 淘汰页零前镜像已同步
let image = std::fs::read(&journal).unwrap();
assert!(!image.is_empty());
drop(file);
std::fs::write(&metadata, old_metadata).unwrap();
std::fs::write(&journal, &image).unwrap();
std::fs::write(&path, [0x11; 213]).unwrap();
let mut reopened = PageFile::open(&path, 512).unwrap();
assert_eq!(reopened.read_page(0).unwrap().data, changed.data);
assert_eq!(reopened.page_lsn(0), Some(13));
drop(reopened);
assert_eq!(std::fs::metadata(&journal).unwrap().len(), 0);
let mut reopened = PageFile::open(&path, 512).unwrap();
assert_eq!(reopened.read_page(0).unwrap().data, changed.data);
assert_eq!(reopened.page_lsn(0), Some(13));
}
/// 撕裂日志尾不发布,完整损坏帧明确拒绝
#[test]
fn page_image_tail_and_corruption_are_distinguished() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("tail.db");
let metadata = dir.path().join("tail.db.pages");
let journal = dir.path().join("tail.db.page-wal");
let mut file = PageFile::with_options(&path, 512, 1).unwrap();
file.write_page(&Page::new(0, 512)).unwrap();
file.flush().unwrap();
let old_metadata = std::fs::read(&metadata).unwrap();
let old_data = std::fs::read(&path).unwrap();
file.set_lsn(20);
let mut changed = Page::new(0, 512);
changed.data.fill(9);
file.write_page(&changed).unwrap();
file.append_page().unwrap();
let image = std::fs::read(&journal).unwrap();
drop(file);
for end in [1, 39, image.len() - 1] {
std::fs::write(&metadata, &old_metadata).unwrap();
std::fs::write(&path, &old_data).unwrap();
std::fs::write(&journal, &image[..end]).unwrap();
let mut reopened = PageFile::open(&path, 512).unwrap();
assert_eq!(reopened.read_page(0).unwrap().data, vec![0; 512]);
}
let mut corrupt = image;
corrupt[40] ^= 1;
std::fs::write(&metadata, &old_metadata).unwrap();
std::fs::write(&journal, &corrupt).unwrap();
assert!(
PageFile::open(&path, 512)
.unwrap_err()
.to_string()
.contains("checksum")
);
assert_eq!(std::fs::read(&journal).unwrap(), corrupt);
}
/// 截断标记阻止旧页重放后复活
#[test]
fn truncate_clears_page_versions_and_survives_reopen() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("reset.db");
let mut file = PageFile::open(&path, 512).unwrap();
file.set_lsn(5);
for _ in 0..3 {
file.append_page().unwrap();
}
file.flush().unwrap();
file.set_lsn(8);
file.truncate().unwrap();
file.append_page().unwrap();
file.flush().unwrap();
assert_eq!(file.page_lsn(0), Some(8));
assert_eq!(file.page_lsn(2), None);
drop(file);
let mut file = PageFile::open(&path, 512).unwrap();
assert_eq!(file.num_pages().unwrap(), 1);
assert_eq!(file.page_lsn(0), Some(8));
assert_eq!(file.page_lsn(2), None);
}
/// 镜像写入失败后禁止析构或重试绕过恢复
#[test]
fn page_journal_error_fences_further_flushes() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("fenced.db");
let journal = dir.path().join("fenced.db.page-wal");
std::fs::write(&path, [0; 512]).unwrap();
let mut file = PageFile::open(&path, 512).unwrap();
let mut page = Page::new(0, 512);
page.data.fill(7);
file.write_page(&page).unwrap();
std::fs::create_dir(&journal).unwrap();
assert!(file.flush().is_err());
std::fs::remove_dir(&journal).unwrap();
assert!(
file.flush()
.unwrap_err()
.to_string()
.contains("requires recovery")
);
drop(file);
let mut file = PageFile::open(&path, 512).unwrap();
assert_eq!(file.read_page(0).unwrap().data, vec![0; 512]);
}
/// 小缓存淘汰不会反复记录同一批未改页
#[test]
fn eviction_batches_images_without_pool_sized_write_amplification() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("bounded.db");
let journal = dir.path().join("bounded.db.page-wal");
let mut file = PageFile::with_options(&path, 512, 2).unwrap();
file.set_lsn(9);
for id in 0..100 {
let mut page = Page::new(id, 512);
page.data.fill(id as u8);
file.write_page(&page).unwrap();
}
assert!(std::fs::metadata(&journal).unwrap().len() <= 100 * (512 + 40));
file.flush().unwrap();
drop(file);
let mut file = PageFile::open(&path, 512).unwrap();
for id in 0..100 {
assert_eq!(file.read_page(id).unwrap().data, vec![id as u8; 512]);
assert_eq!(file.page_lsn(id), Some(9));
}
}
File diff suppressed because it is too large. Load diff
+68 -17
View File
@@ -1,5 +1,5 @@
use chusql_core_storage::protocol::Row;
use chusql_core_storage::wal::{committed_ops, records_after, Wal, WalOp};
use chusql_core_storage::wal::{Wal, WalOp, committed_ops, records_after};
use serde_json::json;
// WAL 测试:追加、读取、清空、截断尾巴、LSN 与提交标记。
@@ -11,12 +11,32 @@ fn schema_versions_round_trip_and_legacy_hide_decodes() {
let path = dir.path().join("wal.log");
let wal = Wal::new(&path);
let ops = [
WalOp::HideColumn { table: "t".into(), column: "name".into(), schema_version: Some(u64::MAX) },
WalOp::DropColumn { table: "t".into(), column: "name".into(), rows: vec![], schema_version: Some(3) },
WalOp::ReplaceSchema { table: "t".into(), columns: vec![], rows: vec![], schema_version: Some(4) },
WalOp::HideColumn {
table: "t".into(),
column: "name".into(),
schema_version: Some(u64::MAX),
},
WalOp::DropColumn {
table: "t".into(),
column: "name".into(),
rows: vec![],
schema_version: Some(3),
},
WalOp::ReplaceSchema {
table: "t".into(),
columns: vec![],
rows: vec![],
schema_version: Some(4),
},
];
wal.append_group(&ops).unwrap();
assert_eq!(committed_ops(&wal.read_all().unwrap().records).into_iter().map(|(_, op)| op).collect::<Vec<_>>(), ops);
assert_eq!(
committed_ops(&wal.read_all().unwrap().records)
.into_iter()
.map(|(_, op)| op)
.collect::<Vec<_>>(),
ops
);
drop(wal);
let mut body = 1_u64.to_le_bytes().to_vec();
body.push(8);
@@ -26,9 +46,14 @@ fn schema_versions_round_trip_and_legacy_hide_decodes() {
let mut frame = (body.len() as u32).to_le_bytes().to_vec();
frame.extend_from_slice(&body);
std::fs::write(&path, frame).unwrap();
assert_eq!(Wal::new(&path).read().unwrap(), Some(WalOp::HideColumn {
table: "t".into(), column: "name".into(), schema_version: None,
}));
assert_eq!(
Wal::new(&path).read().unwrap(),
Some(WalOp::HideColumn {
table: "t".into(),
column: "name".into(),
schema_version: None,
})
);
}
/// 建表编号完整往返且兼容旧载荷
@@ -37,7 +62,11 @@ fn create_table_ids_round_trip_and_legacy_payloads_decode() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("wal.log");
let wal = Wal::new(&path);
let op = WalOp::CreateTable { table: "t".into(), columns: vec![], object_id: Some(42) };
let op = WalOp::CreateTable {
table: "t".into(),
columns: vec![],
object_id: Some(42),
};
wal.append(&op).unwrap();
assert_eq!(wal.read_all().unwrap().records[0].1, op);
drop(wal);
@@ -50,11 +79,16 @@ fn create_table_ids_round_trip_and_legacy_payloads_decode() {
frame.extend_from_slice(&body);
std::fs::write(&path, frame).unwrap();
let wal = Wal::new(&path);
assert_eq!(wal.read_all().unwrap().records[0].1,
WalOp::CreateTable { table: "t".into(), columns: vec![], object_id: None });
assert_eq!(
wal.read_all().unwrap().records[0].1,
WalOp::CreateTable {
table: "t".into(),
columns: vec![],
object_id: None
}
);
}
/// 用键值对构造一行
fn row(pairs: &[(&str, serde_json::Value)]) -> Row {
let mut m = Row::new();
@@ -167,7 +201,10 @@ fn every_record_gets_an_increasing_lsn() {
let scan = wal.read_all().unwrap();
let lsns: Vec<u64> = scan.records.iter().map(|(lsn, _)| *lsn).collect();
assert_eq!(lsns, vec![1, 2, 3]);
assert_eq!(scan.good_len as u64, std::fs::metadata(&path).unwrap().len());
assert_eq!(
scan.good_len as u64,
std::fs::metadata(&path).unwrap().len()
);
let reopened = Wal::new(&path);
reopened
@@ -254,7 +291,10 @@ fn trailing_torn_frame_can_be_truncated() {
assert!(scan.good_len < full.len() - 3);
wal.truncate_to(scan.good_len).unwrap();
assert_eq!(std::fs::metadata(&path).unwrap().len(), scan.good_len as u64);
assert_eq!(
std::fs::metadata(&path).unwrap().len(),
scan.good_len as u64
);
assert_eq!(wal.read_all().unwrap().records.len(), 1);
}
@@ -360,9 +400,21 @@ fn lsn_continues_from_the_checkpoint() {
#[test]
fn records_after_keeps_only_the_tail() {
let records = vec![
(1u64, WalOp::Insert { table: "t".into(), row: row(&[("id", json!(1))]) }),
(
1u64,
WalOp::Insert {
table: "t".into(),
row: row(&[("id", json!(1))]),
},
),
(2, WalOp::Commit),
(3, WalOp::Insert { table: "t".into(), row: row(&[("id", json!(2))]) }),
(
3,
WalOp::Insert {
table: "t".into(),
row: row(&[("id", json!(2))]),
},
),
(4, WalOp::Commit),
];
let kept = records_after(&records, 2);
@@ -371,4 +423,3 @@ fn records_after_keeps_only_the_tail() {
assert_eq!(records_after(&records, 4).len(), 0);
assert_eq!(records_after(&records, 0).len(), 4);
}