修bug,完善功能
This commit is contained in:
1 parent
30364b1c4b
commit
6996e19694
125 files changed
+15343
-3746
No files matched your search
@@ -22,7 +22,8 @@ const SLOT: usize = 2;
|
||||
const CELL_FIXED: usize = 2 + 8;
|
||||
|
||||
/// 键长上限:节点里的格子必须放得进一页
|
||||
pub const MAX_KEY_BYTES: usize = 64;
|
||||
pub const MAX_KEY_BYTES: usize = 128;
|
||||
const LEGACY_KEY_BYTES: usize = 64;
|
||||
|
||||
/// 一个节点最多几个键
|
||||
fn max_keys(order: usize) -> usize {
|
||||
@@ -31,7 +32,7 @@ fn max_keys(order: usize) -> usize {
|
||||
|
||||
/// 一个格子最多占多少字节
|
||||
fn max_cell_bytes() -> usize {
|
||||
CELL_FIXED + MAX_KEY_BYTES
|
||||
CELL_FIXED + LEGACY_KEY_BYTES
|
||||
}
|
||||
|
||||
/// 这个 order 放得进一页吗
|
||||
@@ -44,6 +45,16 @@ pub fn max_order(page_size: usize) -> usize {
|
||||
page_size.saturating_sub(HEADER) / (SLOT + max_cell_bytes()) + 1
|
||||
}
|
||||
|
||||
/// 按页容量和分支数计算有效键长
|
||||
pub fn key_limit(page_size: usize, order: usize) -> usize {
|
||||
if order < 2 {
|
||||
return 0;
|
||||
}
|
||||
(page_size.saturating_sub(HEADER) / (order - 1))
|
||||
.saturating_sub(SLOT + CELL_FIXED)
|
||||
.min(MAX_KEY_BYTES)
|
||||
}
|
||||
|
||||
/// 读一个 u8
|
||||
fn read_u8(p: &Page, off: usize) -> u8 {
|
||||
p.data[off]
|
||||
@@ -66,7 +77,12 @@ fn write_u16(p: &mut Page, off: usize, v: u16) {
|
||||
|
||||
/// 读一个小端 u32
|
||||
fn read_u32(p: &Page, off: usize) -> u32 {
|
||||
u32::from_le_bytes([p.data[off], p.data[off + 1], p.data[off + 2], p.data[off + 3]])
|
||||
u32::from_le_bytes([
|
||||
p.data[off],
|
||||
p.data[off + 1],
|
||||
p.data[off + 2],
|
||||
p.data[off + 3],
|
||||
])
|
||||
}
|
||||
|
||||
/// 写一个小端 u32
|
||||
@@ -91,7 +107,7 @@ fn write_meta(page: &mut Page, page_size: usize, order: usize, root: PageId) {
|
||||
page.zero();
|
||||
page.data[0..4].copy_from_slice(META_MAGIC);
|
||||
write_u32(page, OFF_PAGE_SIZE, page_size as u32);
|
||||
write_u16(page, OFF_MAX_KEY, MAX_KEY_BYTES as u16);
|
||||
write_u16(page, OFF_MAX_KEY, key_limit(page_size, order) as u16);
|
||||
write_u64(page, OFF_ROOT, root);
|
||||
write_u16(page, OFF_ORDER, order as u16);
|
||||
}
|
||||
@@ -115,6 +131,13 @@ fn check_meta(page: &Page, page_size: usize, order: usize) -> io::Result<()> {
|
||||
),
|
||||
));
|
||||
}
|
||||
let stored_key = read_u16(page, OFF_MAX_KEY) as usize;
|
||||
if stored_key < LEGACY_KEY_BYTES || stored_key > key_limit(page_size, order) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"unsupported index key limit",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -169,13 +192,19 @@ fn put_node(p: &mut Page, node: &Node) -> io::Result<()> {
|
||||
/// 从一页读出节点
|
||||
fn get_node(p: &Page) -> io::Result<Node> {
|
||||
if p.data.len() < HEADER {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "truncated index node header"));
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"truncated index node header",
|
||||
));
|
||||
}
|
||||
let kind = read_u8(p, OFF_KIND);
|
||||
let count = read_u16(p, OFF_COUNT) as usize;
|
||||
let slots_end = HEADER + count * SLOT;
|
||||
if !matches!(kind, NODE_LEAF | NODE_INTERNAL) || slots_end > p.data.len() {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index node kind or slot count"));
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"invalid index node kind or slot count",
|
||||
));
|
||||
}
|
||||
let trailer = read_u64(p, OFF_NEXT);
|
||||
let mut keys = Vec::with_capacity(count);
|
||||
@@ -184,30 +213,55 @@ fn get_node(p: &Page) -> io::Result<Node> {
|
||||
for i in 0..count {
|
||||
let off = read_u16(p, HEADER + i * SLOT) as usize;
|
||||
if off < slots_end || off + CELL_FIXED > p.data.len() {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index cell offset"));
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"invalid index cell offset",
|
||||
));
|
||||
}
|
||||
let klen = read_u16(p, off) as usize;
|
||||
if klen > MAX_KEY_BYTES || off + CELL_FIXED + klen > p.data.len() {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index key length"));
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"invalid index key length",
|
||||
));
|
||||
}
|
||||
if off + CELL_FIXED + klen > cell_end {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "overlapping index cells"));
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"overlapping index cells",
|
||||
));
|
||||
}
|
||||
cell_end = off;
|
||||
keys.push(p.data[off + 2..off + 2 + klen].to_vec());
|
||||
values.push(read_u64(p, off + 2 + klen));
|
||||
}
|
||||
if keys.windows(2).any(|pair| pair[0] > pair[1]) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "index keys are not sorted"));
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"index keys are not sorted",
|
||||
));
|
||||
}
|
||||
if kind == NODE_INTERNAL {
|
||||
values.push(trailer);
|
||||
if values.contains(&META_PAGE) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index child page"));
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"invalid index child page",
|
||||
));
|
||||
}
|
||||
Ok(Node { kind, next: 0, keys, values })
|
||||
Ok(Node {
|
||||
kind,
|
||||
next: 0,
|
||||
keys,
|
||||
values,
|
||||
})
|
||||
} else {
|
||||
Ok(Node { kind, next: trailer, keys, values })
|
||||
Ok(Node {
|
||||
kind,
|
||||
next: trailer,
|
||||
keys,
|
||||
values,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -244,7 +298,12 @@ fn split_node(node: &mut Node) -> (Vec<u8>, Node) {
|
||||
node.keys.pop();
|
||||
(
|
||||
sep,
|
||||
Node { kind: NODE_INTERNAL, next: 0, keys: right_keys, values: right_values },
|
||||
Node {
|
||||
kind: NODE_INTERNAL,
|
||||
next: 0,
|
||||
keys: right_keys,
|
||||
values: right_values,
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -287,20 +346,26 @@ impl DiskBTree {
|
||||
}
|
||||
|
||||
/// 键太长直接报错,避免写坏节点
|
||||
fn check_key(&self, key: &[u8]) -> io::Result<()> {
|
||||
if key.len() > MAX_KEY_BYTES {
|
||||
pub(crate) fn check_key(key: &[u8], page_size: usize, order: usize) -> io::Result<()> {
|
||||
let limit = key_limit(page_size, order);
|
||||
if key.len() > limit {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
format!(
|
||||
"index key is {} bytes, at most {} are supported",
|
||||
key.len(),
|
||||
MAX_KEY_BYTES
|
||||
limit
|
||||
),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 按当前索引配置预检键长
|
||||
pub(crate) fn validate_key(&self, key: &[u8]) -> io::Result<()> {
|
||||
Self::check_key(key, self.file.page_size(), self.order)
|
||||
}
|
||||
|
||||
/// 读根页页号;空树是 None
|
||||
fn root(&mut self) -> io::Result<Option<PageId>> {
|
||||
let r = self.file.with_page(META_PAGE, |p| read_u64(p, OFF_ROOT))?;
|
||||
@@ -331,6 +396,11 @@ impl DiskBTree {
|
||||
self.file.update_page(id, |p| put_node(p, node))?
|
||||
}
|
||||
|
||||
/// 设置后续索引页的提交点
|
||||
pub(crate) fn set_lsn(&mut self, lsn: u64) {
|
||||
self.file.set_lsn(lsn);
|
||||
}
|
||||
|
||||
/// 清空整棵树
|
||||
pub fn clear(&mut self) -> io::Result<()> {
|
||||
let page_size = self.file.page_size();
|
||||
@@ -362,14 +432,26 @@ impl DiskBTree {
|
||||
|
||||
/// 按 key 取全部位置(重复键按值升序)
|
||||
pub fn get_all(&mut self, key: &[u8]) -> io::Result<Vec<u64>> {
|
||||
let mut values: Vec<_> = self.scan(Some(key), Some(key))?.into_iter().map(|(_, value)| value).collect();
|
||||
let mut values: Vec<_> = self
|
||||
.scan(Some(key), Some(key))?
|
||||
.into_iter()
|
||||
.map(|(_, value)| value)
|
||||
.collect();
|
||||
values.sort_unstable();
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
/// 插入一个键值对(键可重复)
|
||||
pub fn insert(&mut self, key: &[u8], value: u64) -> io::Result<()> {
|
||||
self.check_key(key)?;
|
||||
self.validate_key(key)?;
|
||||
if key.len() > LEGACY_KEY_BYTES {
|
||||
let limit = key_limit(self.file.page_size(), self.order);
|
||||
let mut meta = self.file.read_page(META_PAGE)?;
|
||||
if read_u16(&meta, OFF_MAX_KEY) as usize != limit {
|
||||
write_u16(&mut meta, OFF_MAX_KEY, limit as u16);
|
||||
self.file.write_page(&meta)?;
|
||||
}
|
||||
}
|
||||
match self.root()? {
|
||||
None => {
|
||||
let node = Node {
|
||||
@@ -412,7 +494,9 @@ impl DiskBTree {
|
||||
} else {
|
||||
let child = node.values[child_slot(&node.keys, key)];
|
||||
if let Some((sep, right_id)) = self.insert_rec(child, key, value)? {
|
||||
let at = node.keys.partition_point(|k| k.as_slice() <= sep.as_slice());
|
||||
let at = node
|
||||
.keys
|
||||
.partition_point(|k| k.as_slice() <= sep.as_slice());
|
||||
node.keys.insert(at, sep);
|
||||
node.values.insert(at + 1, right_id);
|
||||
}
|
||||
|
||||
@@ -37,6 +37,8 @@ pub struct TableSchema {
|
||||
#[serde(default)]
|
||||
pub object_id: u64,
|
||||
#[serde(default)]
|
||||
pub file_generation: String,
|
||||
#[serde(default)]
|
||||
// 0 表示旧目录尚无可验证的结构版本。
|
||||
pub schema_version: u64,
|
||||
#[serde(default)]
|
||||
@@ -78,6 +80,13 @@ impl Catalog {
|
||||
return Ok(Catalog::default());
|
||||
}
|
||||
let mut catalog: Self = serde_json::from_slice(&bytes).map_err(io::Error::other)?;
|
||||
if catalog
|
||||
.tables
|
||||
.values()
|
||||
.any(|schema| !valid_generation(&schema.file_generation))
|
||||
{
|
||||
return Err(io::Error::other("invalid physical file generation"));
|
||||
}
|
||||
catalog.upgrade_object_ids()?;
|
||||
Ok(catalog)
|
||||
}
|
||||
@@ -94,7 +103,9 @@ impl Catalog {
|
||||
}
|
||||
for schema in self.tables.values_mut() {
|
||||
if schema.object_id == 0 {
|
||||
self.object_high_water = self.object_high_water.checked_add(1)
|
||||
self.object_high_water = self
|
||||
.object_high_water
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| io::Error::other("catalog object id exhausted"))?;
|
||||
schema.object_id = self.object_high_water;
|
||||
}
|
||||
@@ -102,9 +113,37 @@ impl Catalog {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 切换物理文件代次而不改变逻辑结构版本
|
||||
pub(crate) fn set_generation(
|
||||
&mut self,
|
||||
table: &str,
|
||||
object: u64,
|
||||
source: &str,
|
||||
target: &str,
|
||||
) -> io::Result<()> {
|
||||
if !valid_generation(target) {
|
||||
return Err(io::Error::other("invalid physical file generation"));
|
||||
}
|
||||
let schema = self
|
||||
.tables
|
||||
.get_mut(table)
|
||||
.ok_or_else(|| io::Error::other("migration table missing"))?;
|
||||
if schema.object_id != object
|
||||
|| (schema.file_generation != source && schema.file_generation != target)
|
||||
{
|
||||
return Err(io::Error::other(
|
||||
"serialization_failure: migration object changed",
|
||||
));
|
||||
}
|
||||
schema.file_generation = target.to_owned();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 分配不复用的对象编号
|
||||
pub fn reserve_object_id(&mut self) -> io::Result<u64> {
|
||||
self.object_high_water = self.object_high_water.checked_add(1)
|
||||
self.object_high_water = self
|
||||
.object_high_water
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| io::Error::other("catalog object id exhausted"))?;
|
||||
Ok(self.object_high_water)
|
||||
}
|
||||
@@ -123,7 +162,11 @@ impl Catalog {
|
||||
file.sync_all()?;
|
||||
drop(file);
|
||||
std::fs::rename(pending, &path)?;
|
||||
std::fs::OpenOptions::new().read(true).write(true).open(path)?.sync_all()
|
||||
std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(path)?
|
||||
.sync_all()
|
||||
}
|
||||
|
||||
/// 一张表的 schema
|
||||
@@ -149,7 +192,12 @@ impl Catalog {
|
||||
}
|
||||
|
||||
/// 按日志中的编号建立表
|
||||
pub fn create_table_with_id(&mut self, table: &str, columns: Vec<SchemaColumn>, object_id: Option<u64>) -> io::Result<()> {
|
||||
pub fn create_table_with_id(
|
||||
&mut self,
|
||||
table: &str,
|
||||
columns: Vec<SchemaColumn>,
|
||||
object_id: Option<u64>,
|
||||
) -> io::Result<()> {
|
||||
if self.tables.contains_key(table) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::AlreadyExists,
|
||||
@@ -160,13 +208,17 @@ impl Catalog {
|
||||
Some(id) if id == 0 || self.tables.values().any(|schema| schema.object_id == id) => {
|
||||
return Err(io::Error::other("invalid catalog object id"));
|
||||
}
|
||||
Some(id) => { self.object_high_water = self.object_high_water.max(id); id }
|
||||
Some(id) => {
|
||||
self.object_high_water = self.object_high_water.max(id);
|
||||
id
|
||||
}
|
||||
None => self.reserve_object_id()?,
|
||||
};
|
||||
self.tables.insert(
|
||||
table.to_string(),
|
||||
TableSchema {
|
||||
object_id,
|
||||
file_generation: String::new(),
|
||||
schema_version: 1,
|
||||
system: false,
|
||||
columns,
|
||||
@@ -182,7 +234,9 @@ impl Catalog {
|
||||
|
||||
/// 标记内部账号表
|
||||
pub fn mark_system(&mut self, table: &str) -> io::Result<()> {
|
||||
let entry = self.tables.get_mut(table)
|
||||
let entry = self
|
||||
.tables
|
||||
.get_mut(table)
|
||||
.ok_or_else(|| io::Error::other("missing system table"))?;
|
||||
entry.system = true;
|
||||
Ok(())
|
||||
@@ -306,7 +360,10 @@ impl Catalog {
|
||||
));
|
||||
}
|
||||
let version = next_schema_version(entry)?;
|
||||
let mut entry = self.tables.remove(from).ok_or_else(|| io::Error::other("missing rename source"))?;
|
||||
let mut entry = self
|
||||
.tables
|
||||
.remove(from)
|
||||
.ok_or_else(|| io::Error::other("missing rename source"))?;
|
||||
entry.schema_version = version;
|
||||
self.seen.remove(from);
|
||||
self.tables.insert(to.to_string(), entry);
|
||||
@@ -319,13 +376,19 @@ impl Catalog {
|
||||
}
|
||||
|
||||
/// 用日志中的目标版本替换列定义
|
||||
pub fn set_columns_version(&mut self, table: &str, columns: Vec<SchemaColumn>, version: Option<u64>) -> io::Result<()> {
|
||||
let entry = self
|
||||
.tables
|
||||
.get_mut(table)
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?;
|
||||
pub fn set_columns_version(
|
||||
&mut self,
|
||||
table: &str,
|
||||
columns: Vec<SchemaColumn>,
|
||||
version: Option<u64>,
|
||||
) -> io::Result<()> {
|
||||
let entry = self.tables.get_mut(table).ok_or_else(|| {
|
||||
io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table))
|
||||
})?;
|
||||
if let Some(version) = version {
|
||||
if version == 0 { return Err(io::Error::other("invalid WAL schema version")); }
|
||||
if version == 0 {
|
||||
return Err(io::Error::other("invalid WAL schema version"));
|
||||
}
|
||||
entry.schema_version = version;
|
||||
} else if entry.columns != columns {
|
||||
entry.schema_version = next_schema_version(entry)?;
|
||||
@@ -342,7 +405,11 @@ impl Catalog {
|
||||
pub fn normalize_types(&mut self) -> io::Result<bool> {
|
||||
let mut changed = false;
|
||||
for entry in self.tables.values_mut() {
|
||||
if entry.columns.iter().any(|column| normalize_type(&column.ty) != column.ty) {
|
||||
if entry
|
||||
.columns
|
||||
.iter()
|
||||
.any(|column| normalize_type(&column.ty) != column.ty)
|
||||
{
|
||||
entry.schema_version = next_schema_version(entry)?;
|
||||
}
|
||||
for column in entry.columns.iter_mut() {
|
||||
@@ -362,13 +429,19 @@ impl Catalog {
|
||||
}
|
||||
|
||||
/// 用日志目标版本删除列定义
|
||||
pub fn remove_column_version(&mut self, table: &str, column: &str, version: Option<u64>) -> io::Result<bool> {
|
||||
let entry = self
|
||||
.tables
|
||||
.get_mut(table)
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?;
|
||||
pub fn remove_column_version(
|
||||
&mut self,
|
||||
table: &str,
|
||||
column: &str,
|
||||
version: Option<u64>,
|
||||
) -> io::Result<bool> {
|
||||
let entry = self.tables.get_mut(table).ok_or_else(|| {
|
||||
io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table))
|
||||
})?;
|
||||
if let Some(version) = version {
|
||||
if version == 0 { return Err(io::Error::other("invalid WAL schema version")); }
|
||||
if version == 0 {
|
||||
return Err(io::Error::other("invalid WAL schema version"));
|
||||
}
|
||||
entry.schema_version = version;
|
||||
} else if entry.columns.iter().any(|c| c.name == column) {
|
||||
entry.schema_version = next_schema_version(entry)?;
|
||||
@@ -388,10 +461,9 @@ impl Catalog {
|
||||
|
||||
/// 记住这一版已整理过(隐藏列名要留住)
|
||||
pub fn mark_compacted(&mut self, table: &str) -> io::Result<()> {
|
||||
let entry = self
|
||||
.tables
|
||||
.get_mut(table)
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?;
|
||||
let entry = self.tables.get_mut(table).ok_or_else(|| {
|
||||
io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table))
|
||||
})?;
|
||||
entry.compacted = true;
|
||||
self.seen.remove(table);
|
||||
Ok(())
|
||||
@@ -399,7 +471,9 @@ impl Catalog {
|
||||
|
||||
/// 这张表还有隐藏列字节吗
|
||||
pub fn has_dropped_columns(&self, table: &str) -> bool {
|
||||
self.tables.get(table).is_some_and(|e| !e.dropped_columns.is_empty())
|
||||
self.tables
|
||||
.get(table)
|
||||
.is_some_and(|e| !e.dropped_columns.is_empty())
|
||||
}
|
||||
|
||||
/// 还有隐藏列字节待整理的表
|
||||
@@ -413,10 +487,9 @@ impl Catalog {
|
||||
|
||||
/// 记下一条索引定义
|
||||
pub fn add_index(&mut self, table: &str, column: &str) -> io::Result<()> {
|
||||
let entry = self
|
||||
.tables
|
||||
.get_mut(table)
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?;
|
||||
let entry = self.tables.get_mut(table).ok_or_else(|| {
|
||||
io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table))
|
||||
})?;
|
||||
if entry.indexes.iter().any(|i| i.column == column) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::AlreadyExists,
|
||||
@@ -431,10 +504,9 @@ impl Catalog {
|
||||
|
||||
/// 去掉一条索引定义
|
||||
pub fn remove_index(&mut self, table: &str, column: &str) -> io::Result<()> {
|
||||
let entry = self
|
||||
.tables
|
||||
.get_mut(table)
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?;
|
||||
let entry = self.tables.get_mut(table).ok_or_else(|| {
|
||||
io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table))
|
||||
})?;
|
||||
match entry.indexes.iter().position(|i| i.column == column) {
|
||||
None => Err(io::Error::new(
|
||||
io::ErrorKind::NotFound,
|
||||
@@ -450,11 +522,20 @@ impl Catalog {
|
||||
|
||||
/// 把行里"字典还不知道的列"补进来;返回有没有变化
|
||||
fn add_missing_columns(entry: &mut TableSchema, row: &Row) -> io::Result<bool> {
|
||||
let additions: Vec<_> = row.iter().filter(|(name, _)|
|
||||
!entry.dropped_columns.contains(*name) && !entry.columns.iter().any(|c| &c.name == *name))
|
||||
.filter_map(|(name, value)| infer_type(value).map(|ty| SchemaColumn {
|
||||
name: name.clone(), ty, ..Default::default()
|
||||
})).collect();
|
||||
let additions: Vec<_> = row
|
||||
.iter()
|
||||
.filter(|(name, _)| {
|
||||
!entry.dropped_columns.contains(*name)
|
||||
&& !entry.columns.iter().any(|c| &c.name == *name)
|
||||
})
|
||||
.filter_map(|(name, value)| {
|
||||
infer_type(value).map(|ty| SchemaColumn {
|
||||
name: name.clone(),
|
||||
ty,
|
||||
..Default::default()
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
if additions.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
@@ -465,7 +546,9 @@ fn add_missing_columns(entry: &mut TableSchema, row: &Row) -> io::Result<bool> {
|
||||
|
||||
/// 递增结构版本,耗尽时拒绝变更
|
||||
fn next_schema_version(entry: &TableSchema) -> io::Result<u64> {
|
||||
entry.schema_version.checked_add(1)
|
||||
entry
|
||||
.schema_version
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| io::Error::other("catalog schema version exhausted"))
|
||||
}
|
||||
|
||||
@@ -576,12 +659,22 @@ fn hist_add(stat: &mut ColumnStat, v: &serde_json::Value) {
|
||||
/// 从直方图里减掉一个值;值在区间外就不动
|
||||
fn hist_remove(stat: &mut ColumnStat, v: &serde_json::Value) {
|
||||
let Some(x) = v.as_f64() else { return };
|
||||
let (Some(lo), Some(hi)) = (stat.lo, stat.hi) else { return };
|
||||
let (Some(lo), Some(hi)) = (stat.lo, stat.hi) else {
|
||||
return;
|
||||
};
|
||||
if x < lo || x > hi || stat.hist.len() != HIST_BUCKETS {
|
||||
return;
|
||||
}
|
||||
let index = hist_index(lo, hi, x);
|
||||
if let Some(bucket) = (0..stat.hist.len()).filter(|&i| stat.hist[i] > 0).min_by_key(|&i| i.abs_diff(index)) {
|
||||
if let Some(bucket) = (0..stat.hist.len())
|
||||
.filter(|&i| stat.hist[i] > 0)
|
||||
.min_by_key(|&i| i.abs_diff(index))
|
||||
{
|
||||
stat.hist[bucket] -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// 文件代次仅允许空值或固定长度十六进制
|
||||
pub(crate) fn valid_generation(value: &str) -> bool {
|
||||
value.is_empty() || (value.len() == 32 && value.bytes().all(|byte| byte.is_ascii_hexdigit()))
|
||||
}
|
||||
@@ -33,7 +33,9 @@ pub fn default_config_path() -> PathBuf {
|
||||
{
|
||||
let appdata = std::env::var_os("APPDATA").filter(|v| !v.is_empty());
|
||||
if let Some(appdata) = appdata {
|
||||
return PathBuf::from(appdata).join(APP_DIR_NAME).join(CONFIG_FILE_NAME);
|
||||
return PathBuf::from(appdata)
|
||||
.join(APP_DIR_NAME)
|
||||
.join(CONFIG_FILE_NAME);
|
||||
}
|
||||
}
|
||||
#[cfg(not(windows))]
|
||||
@@ -62,7 +64,9 @@ pub fn default_data_dir() -> PathBuf {
|
||||
}
|
||||
let roaming = std::env::var_os("APPDATA").filter(|v| !v.is_empty());
|
||||
if let Some(roaming) = roaming {
|
||||
return PathBuf::from(roaming).join(APP_DIR_NAME).join(DATA_DIR_NAME);
|
||||
return PathBuf::from(roaming)
|
||||
.join(APP_DIR_NAME)
|
||||
.join(DATA_DIR_NAME);
|
||||
}
|
||||
PathBuf::from(APP_DIR_NAME).join(DATA_DIR_NAME)
|
||||
}
|
||||
@@ -210,9 +214,7 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
|
||||
})?,
|
||||
};
|
||||
|
||||
let file_origin = Origin::File(
|
||||
path.clone().unwrap_or_else(default_config_path),
|
||||
);
|
||||
let file_origin = Origin::File(path.clone().unwrap_or_else(default_config_path));
|
||||
|
||||
let (page_size, page_origin) = pick(
|
||||
file.page.and_then(|p| p.size),
|
||||
@@ -250,8 +252,12 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
|
||||
let log_files = log_files.trim().to_string();
|
||||
let log_level = log_level.trim();
|
||||
|
||||
let log_level = Level::parse(log_level)
|
||||
.ok_or_else(|| format!("log.level is not one of off/error/warn/info/debug: {}", log_level))?;
|
||||
let log_level = Level::parse(log_level).ok_or_else(|| {
|
||||
format!(
|
||||
"log.level is not one of off/error/warn/info/debug: {}",
|
||||
log_level
|
||||
)
|
||||
})?;
|
||||
|
||||
if data_dir.is_empty() {
|
||||
return Err("storage.data_dir must not be empty".to_string());
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::cell::RefCell;
|
||||
use std::ffi::{c_char, CStr};
|
||||
use std::panic::{catch_unwind, AssertUnwindSafe};
|
||||
use std::ffi::{CStr, c_char};
|
||||
use std::panic::{AssertUnwindSafe, catch_unwind};
|
||||
use std::ptr;
|
||||
use std::slice;
|
||||
|
||||
@@ -42,7 +42,11 @@ pub unsafe extern "C" fn chusql_storage_open(config_path: *const c_char) -> *mut
|
||||
let text = unsafe { CStr::from_ptr(config_path) }
|
||||
.to_str()
|
||||
.map_err(|_| "config path is not valid UTF-8".to_string())?;
|
||||
if text.is_empty() { None } else { Some(text.to_string()) }
|
||||
if text.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(text.to_string())
|
||||
}
|
||||
};
|
||||
Storage::open(path.as_deref()).map(Box::new)
|
||||
}));
|
||||
@@ -78,27 +82,57 @@ pub unsafe extern "C" fn chusql_storage_close(handle: *mut Storage) {
|
||||
/// 输出指针可写且不与输入重叠;响应使用 free 释放。
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn chusql_storage_maintenance(
|
||||
config_path: *const c_char, request: *const u8, request_len: usize,
|
||||
out: *mut *mut u8, out_len: *mut usize,
|
||||
config_path: *const c_char,
|
||||
request: *const u8,
|
||||
request_len: usize,
|
||||
out: *mut *mut u8,
|
||||
out_len: *mut usize,
|
||||
) -> i32 {
|
||||
if !out.is_null() { unsafe { *out = ptr::null_mut(); } }
|
||||
if !out_len.is_null() { unsafe { *out_len = 0; } }
|
||||
if request.is_null() || out.is_null() || out_len.is_null() || request_len > isize::MAX as usize {
|
||||
if !out.is_null() {
|
||||
unsafe {
|
||||
*out = ptr::null_mut();
|
||||
}
|
||||
}
|
||||
if !out_len.is_null() {
|
||||
unsafe {
|
||||
*out_len = 0;
|
||||
}
|
||||
}
|
||||
if request.is_null() || out.is_null() || out_len.is_null() || request_len > isize::MAX as usize
|
||||
{
|
||||
set_error("invalid maintenance argument");
|
||||
return ERR;
|
||||
}
|
||||
let result = catch_unwind(AssertUnwindSafe(|| {
|
||||
let path = if config_path.is_null() { None } else { Some(unsafe { CStr::from_ptr(config_path) }.to_str().map_err(|e| e.to_string())?) };
|
||||
let text = std::str::from_utf8(unsafe { slice::from_raw_parts(request, request_len) }).map_err(|e| e.to_string())?;
|
||||
let path = if config_path.is_null() {
|
||||
None
|
||||
} else {
|
||||
Some(
|
||||
unsafe { CStr::from_ptr(config_path) }
|
||||
.to_str()
|
||||
.map_err(|e| e.to_string())?,
|
||||
)
|
||||
};
|
||||
let text = std::str::from_utf8(unsafe { slice::from_raw_parts(request, request_len) })
|
||||
.map_err(|e| e.to_string())?;
|
||||
Storage::maintenance(path, text)
|
||||
}));
|
||||
let response = match result {
|
||||
Ok(Ok(response)) => response,
|
||||
Ok(Err(err)) => { set_error(err); return ERR; }
|
||||
Err(_) => { set_error("panic in system maintenance"); return ERR; }
|
||||
Ok(Err(err)) => {
|
||||
set_error(err);
|
||||
return ERR;
|
||||
}
|
||||
Err(_) => {
|
||||
set_error("panic in system maintenance");
|
||||
return ERR;
|
||||
}
|
||||
};
|
||||
let buffer = response.into_bytes().into_boxed_slice();
|
||||
unsafe { *out_len = buffer.len(); *out = Box::into_raw(buffer) as *mut u8; }
|
||||
unsafe {
|
||||
*out_len = buffer.len();
|
||||
*out = Box::into_raw(buffer) as *mut u8;
|
||||
}
|
||||
OK
|
||||
}
|
||||
|
||||
@@ -121,7 +155,12 @@ pub unsafe extern "C" fn chusql_storage_request(
|
||||
if !out_len.is_null() {
|
||||
unsafe { *out_len = 0 };
|
||||
}
|
||||
if handle.is_null() || request.is_null() || out.is_null() || out_len.is_null() || request_len > isize::MAX as usize {
|
||||
if handle.is_null()
|
||||
|| request.is_null()
|
||||
|| out.is_null()
|
||||
|| out_len.is_null()
|
||||
|| request_len > isize::MAX as usize
|
||||
{
|
||||
set_error("invalid argument");
|
||||
return ERR;
|
||||
}
|
||||
@@ -178,6 +217,10 @@ pub unsafe extern "C" fn chusql_storage_last_error(out_len: *mut usize) -> *cons
|
||||
if !out_len.is_null() {
|
||||
unsafe { *out_len = bytes.len() };
|
||||
}
|
||||
if bytes.is_empty() { ptr::null() } else { bytes.as_ptr() }
|
||||
if bytes.is_empty() {
|
||||
ptr::null()
|
||||
} else {
|
||||
bytes.as_ptr()
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
use serde::Deserializer;
|
||||
use serde::de::{IgnoredAny, MapAccess, Visitor};
|
||||
use std::collections::HashSet;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
use serde::de::{IgnoredAny, MapAccess, Visitor};
|
||||
use serde::Deserializer;
|
||||
|
||||
use crate::btree::DiskBTree;
|
||||
use crate::page::PageId;
|
||||
@@ -11,12 +11,18 @@ use crate::protocol::Row;
|
||||
|
||||
// 堆表:行按插入顺序进页,支持全表扫描、按索引取行、行级删除。
|
||||
|
||||
pub(crate) const DUPLICATE_ID: &str = "duplicate value on the id index";
|
||||
|
||||
const HEADER_SIZE: usize = 2;
|
||||
const SLOT_SIZE: usize = 8;
|
||||
const SLOT_DIR_START: usize = HEADER_SIZE;
|
||||
|
||||
/// 未选字段只校验并跳过,不分配值。
|
||||
fn decode_row(bytes: &[u8], columns: Option<&[String]>, hidden: &HashSet<String>) -> io::Result<Row> {
|
||||
fn decode_row(
|
||||
bytes: &[u8],
|
||||
columns: Option<&[String]>,
|
||||
hidden: &HashSet<String>,
|
||||
) -> io::Result<Row> {
|
||||
struct RowVisitor<'a>(Option<&'a [String]>, &'a HashSet<String>);
|
||||
impl<'de> Visitor<'de> for RowVisitor<'_> {
|
||||
type Value = Row;
|
||||
@@ -38,7 +44,9 @@ fn decode_row(bytes: &[u8], columns: Option<&[String]>, hidden: &HashSet<String>
|
||||
}
|
||||
}
|
||||
let mut decoder = serde_json::Deserializer::from_slice(bytes);
|
||||
let row = decoder.deserialize_map(RowVisitor(columns, hidden)).map_err(io::Error::other)?;
|
||||
let row = decoder
|
||||
.deserialize_map(RowVisitor(columns, hidden))
|
||||
.map_err(io::Error::other)?;
|
||||
decoder.end().map_err(io::Error::other)?;
|
||||
Ok(row)
|
||||
}
|
||||
@@ -93,7 +101,7 @@ fn row_int(row: &Row, column: &str) -> Option<i64> {
|
||||
}
|
||||
|
||||
/// i64 键 → 8 字节大端(保序,负数也在前)
|
||||
fn key_of(value: i64) -> [u8; 8] {
|
||||
pub(crate) fn key_of(value: i64) -> [u8; 8] {
|
||||
(value as u64 ^ (1u64 << 63)).to_be_bytes()
|
||||
}
|
||||
|
||||
@@ -108,10 +116,23 @@ fn value_key(v: &serde_json::Value) -> Option<Vec<u8>> {
|
||||
}
|
||||
|
||||
/// 取行里某一列的索引键;NULL 与缺列不入索引
|
||||
fn row_key(row: &Row, column: &str) -> Option<Vec<u8>> {
|
||||
pub(crate) fn row_key(row: &Row, column: &str) -> Option<Vec<u8>> {
|
||||
row.get(column).and_then(value_key)
|
||||
}
|
||||
|
||||
/// 按实际键编码校验一列的索引容量
|
||||
pub(crate) fn check_index_key(
|
||||
row: &Row,
|
||||
column: &str,
|
||||
page_size: usize,
|
||||
order: usize,
|
||||
) -> io::Result<()> {
|
||||
if let Some(key) = row_key(row, column) {
|
||||
DiskBTree::check_key(&key, page_size, order)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub struct HeapPage;
|
||||
|
||||
impl HeapPage {
|
||||
@@ -312,7 +333,7 @@ impl HeapTable {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == io::ErrorKind::InvalidData => {
|
||||
// 格式不兼容(例如上一版落的索引文件):删掉重开
|
||||
let _ = std::fs::remove_file(p);
|
||||
crate::page_wal::remove(p)?;
|
||||
t.attach_index(col, p, page_size, btree_order, pool_size)?;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
@@ -373,6 +394,7 @@ impl HeapTable {
|
||||
let Some(k) = row_key(&r, column) else {
|
||||
continue;
|
||||
};
|
||||
DiskBTree::check_key(&k, page_size, btree_order)?;
|
||||
// 行身份索引仍然唯一;其余列允许重复值
|
||||
if column == "id" && !seen.insert(k.clone()) {
|
||||
return Err(io::Error::new(
|
||||
@@ -450,10 +472,11 @@ impl HeapTable {
|
||||
let Some(k) = row_key(row, &idx.column) else {
|
||||
continue;
|
||||
};
|
||||
idx.tree.validate_key(&k)?;
|
||||
if idx.column == "id" && idx.tree.get(&k)?.is_some() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"duplicate value on the id index".to_string(),
|
||||
DUPLICATE_ID.to_string(),
|
||||
));
|
||||
}
|
||||
pending.push((i, k));
|
||||
@@ -511,6 +534,16 @@ impl HeapTable {
|
||||
Ok((page_id, slot))
|
||||
}
|
||||
|
||||
/// 读取身份索引中同键行数供提交预检
|
||||
pub(crate) fn id_key_count(&mut self, key: &[u8]) -> io::Result<usize> {
|
||||
let index = self
|
||||
.indexes
|
||||
.iter_mut()
|
||||
.find(|index| index.column == "id")
|
||||
.ok_or_else(|| io::Error::other("missing id index"))?;
|
||||
Ok(index.tree.get_all(key)?.len())
|
||||
}
|
||||
|
||||
/// 按 `id` 取一行
|
||||
pub fn get_by_key(&mut self, key: i64) -> io::Result<Option<Row>> {
|
||||
self.get_by_column_key("id", key)
|
||||
@@ -577,13 +610,21 @@ impl HeapTable {
|
||||
let mut rows = Vec::new();
|
||||
for (key, pos) in pairs {
|
||||
if let Some((bound, inclusive)) = &lo_bound {
|
||||
let outside = if *inclusive { &key < bound } else { &key <= bound };
|
||||
let outside = if *inclusive {
|
||||
&key < bound
|
||||
} else {
|
||||
&key <= bound
|
||||
};
|
||||
if outside {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if let Some((bound, inclusive)) = &hi_bound {
|
||||
let outside = if *inclusive { &key > bound } else { &key >= bound };
|
||||
let outside = if *inclusive {
|
||||
&key > bound
|
||||
} else {
|
||||
&key >= bound
|
||||
};
|
||||
if outside {
|
||||
continue;
|
||||
}
|
||||
@@ -699,7 +740,10 @@ impl HeapTable {
|
||||
}
|
||||
|
||||
/// 全表扫描,带位置与列投影
|
||||
fn scan_projected_with_positions(&mut self, columns: Option<&[String]>) -> io::Result<Vec<(PageId, u16, Row)>> {
|
||||
fn scan_projected_with_positions(
|
||||
&mut self,
|
||||
columns: Option<&[String]>,
|
||||
) -> io::Result<Vec<(PageId, u16, Row)>> {
|
||||
let n = self.file.num_pages()?;
|
||||
self.scan_pages_with_positions(columns, 0, n)
|
||||
}
|
||||
@@ -729,6 +773,14 @@ impl HeapTable {
|
||||
|
||||
/// 整表改写并重建所有索引
|
||||
pub fn replace_all(&mut self, rows: &[Row]) -> io::Result<()> {
|
||||
for row in rows {
|
||||
Self::encode_row(row, self.file.page_size())?;
|
||||
for index in &self.indexes {
|
||||
if let Some(key) = row_key(row, &index.column) {
|
||||
index.tree.validate_key(&key)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
self.file.truncate()?;
|
||||
for idx in &mut self.indexes {
|
||||
idx.tree.clear()?;
|
||||
@@ -745,6 +797,14 @@ impl HeapTable {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 同时标记堆页与索引页的提交点
|
||||
pub(crate) fn set_lsn(&mut self, lsn: u64) {
|
||||
self.file.set_lsn(lsn);
|
||||
for index in &mut self.indexes {
|
||||
index.tree.set_lsn(lsn);
|
||||
}
|
||||
}
|
||||
|
||||
/// 把脏页写回(索引也一起)
|
||||
pub fn flush(&mut self) -> io::Result<()> {
|
||||
for idx in &mut self.indexes {
|
||||
@@ -767,11 +827,19 @@ impl HeapTable {
|
||||
}
|
||||
|
||||
/// 当前缓存页数
|
||||
pub fn cached_pages(&self) -> usize { self.file.cached_pages() }
|
||||
pub fn cached_pages(&self) -> usize {
|
||||
self.file.cached_pages()
|
||||
}
|
||||
/// 命中次数
|
||||
pub fn hits(&self) -> u64 { self.file.hits() }
|
||||
pub fn hits(&self) -> u64 {
|
||||
self.file.hits()
|
||||
}
|
||||
/// 未命中次数
|
||||
pub fn misses(&self) -> u64 { self.file.misses() }
|
||||
pub fn misses(&self) -> u64 {
|
||||
self.file.misses()
|
||||
}
|
||||
/// 命中率
|
||||
pub fn hit_rate(&self) -> f64 { self.file.hit_rate() }
|
||||
pub fn hit_rate(&self) -> f64 {
|
||||
self.file.hit_rate()
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,9 @@ pub mod config;
|
||||
pub mod ffi;
|
||||
pub mod heap;
|
||||
pub mod log;
|
||||
mod mvcc;
|
||||
pub mod page;
|
||||
mod page_wal;
|
||||
pub mod protocol;
|
||||
pub mod runtime;
|
||||
pub mod wal;
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::fmt;
|
||||
use std::io::{IsTerminal, Write};
|
||||
use std::sync::atomic::{AtomicU8, Ordering};
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::atomic::{AtomicU8, Ordering};
|
||||
|
||||
// 极简日志:等级用原子变量,关掉的等级只花一次原子读。
|
||||
|
||||
@@ -123,8 +123,18 @@ fn streams_to_stderr(level: Level) -> bool {
|
||||
|
||||
/// 按目标流是否终端决定是否着色
|
||||
fn color_enabled(to_stderr: bool) -> bool {
|
||||
let cached = if to_stderr { &COLOR_STDERR } else { &COLOR_STDOUT };
|
||||
*cached.get_or_init(|| if to_stderr { std::io::stderr().is_terminal() } else { std::io::stdout().is_terminal() })
|
||||
let cached = if to_stderr {
|
||||
&COLOR_STDERR
|
||||
} else {
|
||||
&COLOR_STDOUT
|
||||
};
|
||||
*cached.get_or_init(|| {
|
||||
if to_stderr {
|
||||
std::io::stderr().is_terminal()
|
||||
} else {
|
||||
std::io::stdout().is_terminal()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// 拼一行(纯函数,可测)
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -3,6 +3,8 @@ use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, Read, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use crate::page_wal::PageWal;
|
||||
|
||||
// 页管理:定长页的读写,外加一层页缓冲池。
|
||||
|
||||
pub type PageId = u64;
|
||||
@@ -33,6 +35,7 @@ impl Page {
|
||||
#[derive(Debug)]
|
||||
pub struct PageFile {
|
||||
file: File,
|
||||
journal: PageWal,
|
||||
page_size: usize,
|
||||
capacity: usize,
|
||||
cache: HashMap<PageId, Page>,
|
||||
@@ -55,14 +58,16 @@ impl PageFile {
|
||||
page_size: usize,
|
||||
capacity: usize,
|
||||
) -> io::Result<Self> {
|
||||
let file = OpenOptions::new()
|
||||
let mut file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.open(path)?;
|
||||
.open(path.as_ref())?;
|
||||
let journal = PageWal::open(path.as_ref(), page_size, &mut file)?;
|
||||
Ok(PageFile {
|
||||
file,
|
||||
journal,
|
||||
page_size,
|
||||
capacity,
|
||||
cache: HashMap::new(),
|
||||
@@ -74,6 +79,16 @@ impl PageFile {
|
||||
})
|
||||
}
|
||||
|
||||
/// 标记后续页修改所属的提交点
|
||||
pub fn set_lsn(&mut self, lsn: u64) {
|
||||
self.journal.set_lsn(lsn);
|
||||
}
|
||||
|
||||
/// 读取已记录的页提交 LSN
|
||||
pub fn page_lsn(&self, id: PageId) -> Option<u64> {
|
||||
self.journal.page_lsn(id)
|
||||
}
|
||||
|
||||
/// 页大小
|
||||
pub fn page_size(&self) -> usize {
|
||||
self.page_size
|
||||
@@ -109,7 +124,6 @@ impl PageFile {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// 标记为最近使用
|
||||
fn touch(&mut self, id: PageId) {
|
||||
self.clock += 1;
|
||||
@@ -127,27 +141,28 @@ impl PageFile {
|
||||
return Ok(false);
|
||||
};
|
||||
if self.dirty.contains(&victim) {
|
||||
let p = self.cache[&victim].clone();
|
||||
self.write_raw(&p)?;
|
||||
self.dirty.remove(&victim);
|
||||
let pages: Vec<_> = self
|
||||
.dirty
|
||||
.iter()
|
||||
.filter_map(|id| self.cache.get(id).cloned())
|
||||
.collect();
|
||||
let images: Vec<_> = pages
|
||||
.iter()
|
||||
.map(|page| (page.id, page.data.as_slice()))
|
||||
.collect();
|
||||
self.journal.prepare(&images)?;
|
||||
for page in &pages {
|
||||
self.write_image(page)?;
|
||||
}
|
||||
self.dirty.clear();
|
||||
}
|
||||
self.cache.remove(&victim);
|
||||
self.last_used.remove(&victim);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// 直接写文件
|
||||
fn write_raw(&mut self, page: &Page) -> io::Result<()> {
|
||||
if page.data.len() != self.page_size {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
format!(
|
||||
"page has {} bytes, this file uses {}",
|
||||
page.data.len(),
|
||||
self.page_size
|
||||
),
|
||||
));
|
||||
}
|
||||
/// 将已记录镜像的页写入数据文件
|
||||
fn write_image(&mut self, page: &Page) -> io::Result<()> {
|
||||
self.file
|
||||
.seek(SeekFrom::Start(page.id * self.page_size as u64))?;
|
||||
self.file.write_all(&page.data)
|
||||
@@ -156,12 +171,12 @@ impl PageFile {
|
||||
/// 直接从文件读
|
||||
fn read_raw(&mut self, id: PageId) -> io::Result<Page> {
|
||||
let mut data = vec![0u8; self.page_size];
|
||||
self.file.seek(SeekFrom::Start(id * self.page_size as u64))?;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(id * self.page_size as u64))?;
|
||||
self.file.read_exact(&mut data)?;
|
||||
Ok(Page { id, data })
|
||||
}
|
||||
|
||||
|
||||
/// 读一页(走缓存)
|
||||
pub fn read_page(&mut self, id: PageId) -> io::Result<Page> {
|
||||
self.note_access(id)?;
|
||||
@@ -241,26 +256,27 @@ impl PageFile {
|
||||
));
|
||||
}
|
||||
let on_disk = len / size;
|
||||
let cached_max = self
|
||||
.cache
|
||||
.keys()
|
||||
.copied()
|
||||
.max()
|
||||
.map(|m| m + 1)
|
||||
.unwrap_or(0);
|
||||
let cached_max = self.cache.keys().copied().max().map(|m| m + 1).unwrap_or(0);
|
||||
Ok(on_disk.max(cached_max))
|
||||
}
|
||||
|
||||
/// 把脏页写回磁盘
|
||||
pub fn flush(&mut self) -> io::Result<()> {
|
||||
let dirty: Vec<PageId> = self.dirty.iter().copied().collect();
|
||||
for id in dirty {
|
||||
if let Some(p) = self.cache.get(&id) {
|
||||
let p = p.clone();
|
||||
self.write_raw(&p)?;
|
||||
}
|
||||
let pages: Vec<Page> = self
|
||||
.dirty
|
||||
.iter()
|
||||
.filter_map(|id| self.cache.get(id).cloned())
|
||||
.collect();
|
||||
let images: Vec<_> = pages
|
||||
.iter()
|
||||
.map(|page| (page.id, page.data.as_slice()))
|
||||
.collect();
|
||||
self.journal.prepare(&images)?;
|
||||
for page in &pages {
|
||||
self.write_image(page)?;
|
||||
}
|
||||
self.file.sync_all()?;
|
||||
self.journal.checkpoint()?;
|
||||
self.dirty.clear();
|
||||
Ok(())
|
||||
}
|
||||
@@ -272,6 +288,7 @@ impl PageFile {
|
||||
|
||||
/// 清空文件
|
||||
pub fn truncate(&mut self) -> io::Result<()> {
|
||||
self.journal.reset()?;
|
||||
self.cache.clear();
|
||||
self.last_used.clear();
|
||||
self.dirty.clear();
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, Read, Seek, SeekFrom, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
// 全页预写镜像修复撕裂写,页 LSN 在数据同步后原子发布。
|
||||
|
||||
const HEADER: usize = 40;
|
||||
const RESET: u64 = u64::MAX;
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct State {
|
||||
version: u32,
|
||||
page_size: usize,
|
||||
sequence: u64,
|
||||
pages: BTreeMap<u64, u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct PageWal {
|
||||
path: PathBuf,
|
||||
state: State,
|
||||
lsn: u64,
|
||||
pending: bool,
|
||||
failed: bool,
|
||||
file: Option<File>,
|
||||
}
|
||||
|
||||
/// 为页文件生成独立的元数据路径
|
||||
fn sidecar(path: &Path, suffix: &str) -> PathBuf {
|
||||
let mut name = path.as_os_str().to_os_string();
|
||||
name.push(suffix);
|
||||
PathBuf::from(name)
|
||||
}
|
||||
|
||||
/// 计算帧校验和以拒绝损坏的完整镜像
|
||||
fn checksum(bytes: &[u8]) -> u64 {
|
||||
bytes.iter().fold(0xcbf29ce484222325, |hash, byte| {
|
||||
(hash ^ u64::from(*byte)).wrapping_mul(0x100000001b3)
|
||||
})
|
||||
}
|
||||
|
||||
/// 同步目录中的创建、替换与删除
|
||||
fn sync_parent(path: &Path) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
File::open(
|
||||
path.parent()
|
||||
.filter(|p| !p.as_os_str().is_empty())
|
||||
.unwrap_or(Path::new(".")),
|
||||
)?
|
||||
.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl PageWal {
|
||||
/// 加载页版本并先恢复未完成的全页写入
|
||||
pub(crate) fn open(path: &Path, page_size: usize, data: &mut File) -> io::Result<Self> {
|
||||
let metadata = sidecar(path, ".pages");
|
||||
let state = match std::fs::read(&metadata) {
|
||||
Ok(bytes) => serde_json::from_slice::<State>(&bytes).map_err(io::Error::other)?,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => State {
|
||||
version: 1,
|
||||
page_size,
|
||||
sequence: 0,
|
||||
pages: BTreeMap::new(),
|
||||
},
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if state.version != 1 || state.page_size != page_size {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"unsupported page LSN metadata",
|
||||
));
|
||||
}
|
||||
let mut journal = Self {
|
||||
path: path.to_path_buf(),
|
||||
state,
|
||||
lsn: 0,
|
||||
pending: false,
|
||||
failed: false,
|
||||
file: None,
|
||||
};
|
||||
journal.recover(data)?;
|
||||
Ok(journal)
|
||||
}
|
||||
|
||||
/// 设置后续脏页所属的逻辑 WAL 提交点
|
||||
pub(crate) fn set_lsn(&mut self, lsn: u64) {
|
||||
self.lsn = lsn;
|
||||
}
|
||||
|
||||
/// 读取已写镜像记录的页 LSN
|
||||
pub(crate) fn page_lsn(&self, page: u64) -> Option<u64> {
|
||||
self.state.pages.get(&page).copied()
|
||||
}
|
||||
|
||||
/// 延迟创建日志并同步目录项
|
||||
fn handle(&mut self) -> io::Result<&mut File> {
|
||||
if self.file.is_none() {
|
||||
let path = sidecar(&self.path, ".page-wal");
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.open(&path)?;
|
||||
sync_parent(&path)?;
|
||||
self.file = Some(file);
|
||||
}
|
||||
Ok(self.file.as_mut().expect("opened above"))
|
||||
}
|
||||
|
||||
/// 持久化一批镜像后才允许修改数据页
|
||||
pub(crate) fn prepare(&mut self, pages: &[(u64, &[u8])]) -> io::Result<()> {
|
||||
if self.failed {
|
||||
return Err(io::Error::other("page journal requires recovery"));
|
||||
}
|
||||
if pages.iter().any(|(page, bytes)| {
|
||||
(*page == RESET && !bytes.is_empty())
|
||||
|| (*page != RESET
|
||||
&& (bytes.len() != self.state.page_size
|
||||
|| page.checked_mul(self.state.page_size as u64).is_none()))
|
||||
}) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"invalid page image size or offset",
|
||||
));
|
||||
}
|
||||
if pages.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
self.failed = true;
|
||||
for (page, bytes) in pages {
|
||||
let sequence = self
|
||||
.state
|
||||
.sequence
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| io::Error::other("page write sequence exhausted"))?;
|
||||
let lsn = self
|
||||
.lsn
|
||||
.max(self.state.pages.get(page).copied().unwrap_or(0));
|
||||
let mut frame = Vec::with_capacity(HEADER + bytes.len());
|
||||
frame.extend_from_slice(b"CPW1");
|
||||
frame.extend_from_slice(&sequence.to_le_bytes());
|
||||
frame.extend_from_slice(&lsn.to_le_bytes());
|
||||
frame.extend_from_slice(&page.to_le_bytes());
|
||||
frame.extend_from_slice(&(bytes.len() as u32).to_le_bytes());
|
||||
let mut checked = frame.clone();
|
||||
checked.extend_from_slice(bytes);
|
||||
frame.extend_from_slice(&checksum(&checked).to_le_bytes());
|
||||
frame.extend_from_slice(bytes);
|
||||
let file = self.handle()?;
|
||||
file.seek(SeekFrom::End(0))?;
|
||||
file.write_all(&frame)?;
|
||||
self.state.sequence = sequence;
|
||||
if *page == RESET {
|
||||
self.state.pages.clear();
|
||||
} else {
|
||||
self.state.pages.insert(*page, lsn);
|
||||
}
|
||||
self.pending = true;
|
||||
}
|
||||
self.handle()?.sync_all()?;
|
||||
self.failed = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 先记录截断以免重放旧页复活已删除数据
|
||||
pub(crate) fn reset(&mut self) -> io::Result<()> {
|
||||
self.prepare(&[(RESET, &[])])
|
||||
}
|
||||
|
||||
/// 数据同步后发布页版本并清理镜像日志
|
||||
pub(crate) fn checkpoint(&mut self) -> io::Result<()> {
|
||||
if self.failed {
|
||||
return Err(io::Error::other("page journal requires recovery"));
|
||||
}
|
||||
if !self.pending {
|
||||
return Ok(());
|
||||
}
|
||||
self.failed = true;
|
||||
let temporary = sidecar(&self.path, ".pages.pending");
|
||||
let metadata = sidecar(&self.path, ".pages");
|
||||
let mut file = File::create(&temporary)?;
|
||||
serde_json::to_writer(&mut file, &self.state).map_err(io::Error::other)?;
|
||||
file.sync_all()?;
|
||||
drop(file);
|
||||
std::fs::rename(&temporary, &metadata)?;
|
||||
sync_parent(&metadata)?;
|
||||
self.handle()?.set_len(0)?;
|
||||
self.handle()?.sync_all()?;
|
||||
self.pending = false;
|
||||
self.failed = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 验证日志帧后重放比页检查点更新的镜像
|
||||
fn recover(&mut self, data: &mut File) -> io::Result<()> {
|
||||
let path = sidecar(&self.path, ".page-wal");
|
||||
let mut file = match File::open(path) {
|
||||
Ok(file) => file,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if file.metadata()?.len() == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let checkpoint = self.state.sequence;
|
||||
let mut previous = 0;
|
||||
loop {
|
||||
let mut header = [0; HEADER];
|
||||
match file.read_exact(&mut header) {
|
||||
Ok(()) => (),
|
||||
Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => break,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
let sequence = u64::from_le_bytes(header[4..12].try_into().expect("fixed header"));
|
||||
let lsn = u64::from_le_bytes(header[12..20].try_into().expect("fixed header"));
|
||||
let page = u64::from_le_bytes(header[20..28].try_into().expect("fixed header"));
|
||||
let len = u32::from_le_bytes(header[28..32].try_into().expect("fixed header")) as usize;
|
||||
if &header[..4] != b"CPW1"
|
||||
|| sequence <= previous
|
||||
|| (page == RESET && len != 0)
|
||||
|| (page != RESET && len != self.state.page_size)
|
||||
{
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"invalid page image header",
|
||||
));
|
||||
}
|
||||
let mut bytes = vec![0; len];
|
||||
match file.read_exact(&mut bytes) {
|
||||
Ok(()) => (),
|
||||
Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => break,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
let mut checked = header[..32].to_vec();
|
||||
checked.extend_from_slice(&bytes);
|
||||
if checksum(&checked)
|
||||
!= u64::from_le_bytes(header[32..40].try_into().expect("fixed header"))
|
||||
{
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"page image checksum mismatch",
|
||||
));
|
||||
}
|
||||
previous = sequence;
|
||||
if sequence <= checkpoint {
|
||||
continue;
|
||||
}
|
||||
if page == RESET {
|
||||
data.set_len(0)?;
|
||||
self.state.pages.clear();
|
||||
} else {
|
||||
let offset = page
|
||||
.checked_mul(self.state.page_size as u64)
|
||||
.ok_or_else(|| io::Error::other("page offset overflow"))?;
|
||||
data.seek(SeekFrom::Start(offset))?;
|
||||
data.write_all(&bytes)?;
|
||||
self.state.pages.insert(page, lsn);
|
||||
}
|
||||
self.state.sequence = sequence;
|
||||
}
|
||||
data.sync_all()?;
|
||||
self.pending = true;
|
||||
self.checkpoint()
|
||||
}
|
||||
}
|
||||
|
||||
/// 删除页文件及其版本与镜像文件
|
||||
pub(crate) fn remove(path: &Path) -> io::Result<()> {
|
||||
for suffix in [".page-wal", ".pages", ".pages.pending", ""] {
|
||||
match std::fs::remove_file(sidecar(path, suffix)) {
|
||||
Ok(()) => (),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => (),
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
sync_parent(path)
|
||||
}
|
||||
|
||||
/// 改名时保留对应页版本与恢复材料
|
||||
pub(crate) fn rename(from: &Path, to: &Path) -> io::Result<()> {
|
||||
for suffix in [".page-wal", ".pages", ".pages.pending", ""] {
|
||||
match std::fs::rename(sidecar(from, suffix), sidecar(to, suffix)) {
|
||||
Ok(()) => (),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => (),
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
sync_parent(to)
|
||||
}
|
||||
@@ -94,19 +94,40 @@ pub struct IndexWire {
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "method", rename_all = "snake_case")]
|
||||
pub enum Request {
|
||||
CatalogManage { command: Box<Request> },
|
||||
CatalogManage {
|
||||
command: Box<Request>,
|
||||
},
|
||||
AccountsList,
|
||||
IdentityInitialize { administrator: String },
|
||||
RoleCreate { user: String },
|
||||
IdentityAlter { user: String, can_login: Option<bool>, is_superuser: Option<bool>, enabled: Option<bool>, system_catalog_manager: Option<bool>, allow_sudo_auth: Option<bool> },
|
||||
AccountCreate { user: String, password_hash: String },
|
||||
AccountReset { user: String, password_hash: String },
|
||||
IdentityInitialize {
|
||||
administrator: String,
|
||||
},
|
||||
RoleCreate {
|
||||
user: String,
|
||||
},
|
||||
IdentityAlter {
|
||||
user: String,
|
||||
can_login: Option<bool>,
|
||||
is_superuser: Option<bool>,
|
||||
enabled: Option<bool>,
|
||||
system_catalog_manager: Option<bool>,
|
||||
allow_sudo_auth: Option<bool>,
|
||||
},
|
||||
AccountCreate {
|
||||
user: String,
|
||||
password_hash: String,
|
||||
},
|
||||
AccountReset {
|
||||
user: String,
|
||||
password_hash: String,
|
||||
},
|
||||
AccountLogin {
|
||||
user: String,
|
||||
#[serde(default)]
|
||||
at: Option<String>,
|
||||
},
|
||||
AccountDrop { user: String },
|
||||
AccountDrop {
|
||||
user: String,
|
||||
},
|
||||
/// 引导系统目录:建系统表,给了 user 就同时补首个管理员
|
||||
BootstrapSystem {
|
||||
#[serde(default)]
|
||||
@@ -115,14 +136,18 @@ pub enum Request {
|
||||
password_hash: Option<String>,
|
||||
},
|
||||
SystemStatus,
|
||||
BootstrapTypes { types: Vec<Row> },
|
||||
BootstrapTypes {
|
||||
types: Vec<Row>,
|
||||
},
|
||||
Ping,
|
||||
Scan {
|
||||
table: String,
|
||||
#[serde(default)]
|
||||
columns: Option<Vec<String>>,
|
||||
},
|
||||
ValidateWrite { table: String },
|
||||
ValidateWrite {
|
||||
table: String,
|
||||
},
|
||||
/// 分片扫描:第 shard 片只读总页数按 shards 均分后的那一段
|
||||
ScanShard {
|
||||
table: String,
|
||||
@@ -177,6 +202,37 @@ pub enum Request {
|
||||
table: String,
|
||||
},
|
||||
ListCatalog,
|
||||
DomainSnapshot,
|
||||
MvccBegin {
|
||||
#[serde(default)]
|
||||
serializable: bool,
|
||||
},
|
||||
MvccLock {
|
||||
transaction: String,
|
||||
ops: Vec<StorageOp>,
|
||||
#[serde(default)]
|
||||
reads: Vec<String>,
|
||||
},
|
||||
MvccPreview {
|
||||
transaction: String,
|
||||
},
|
||||
MvccCommit {
|
||||
transaction: String,
|
||||
checkpoint: String,
|
||||
ops: Vec<StorageOp>,
|
||||
},
|
||||
MvccRollback {
|
||||
transaction: String,
|
||||
},
|
||||
DatabaseSnapshot {
|
||||
#[serde(default)]
|
||||
checkpoint: Option<String>,
|
||||
},
|
||||
AlterDomain {
|
||||
table: String,
|
||||
columns: Vec<SchemaColumn>,
|
||||
checkpoint: String,
|
||||
},
|
||||
CreateIndex {
|
||||
table: String,
|
||||
column: String,
|
||||
@@ -204,7 +260,7 @@ pub enum Request {
|
||||
}
|
||||
|
||||
/// 事务提交里的一条写操作
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "op", rename_all = "snake_case")]
|
||||
pub enum StorageOp {
|
||||
/// 追加写入的行(提交端已先删掉要覆盖的行)
|
||||
@@ -215,37 +271,89 @@ pub enum StorageOp {
|
||||
Replace { table: String, rows: Vec<Row> },
|
||||
}
|
||||
|
||||
/// 检查存储表名和列名的字符集
|
||||
pub(crate) fn valid_storage_name(name: &str) -> bool {
|
||||
!name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')
|
||||
}
|
||||
|
||||
/// 检查一行顶层字段名
|
||||
fn valid_row_columns(row: &Row) -> bool {
|
||||
row.keys().all(|name| valid_storage_name(name))
|
||||
}
|
||||
|
||||
impl Request {
|
||||
/// 拒绝可能影响文件路径的列名
|
||||
/// 校验列定义与写入载荷的顶层字段名
|
||||
pub fn valid_columns(&self) -> bool {
|
||||
let valid = |name: &str| !name.is_empty()
|
||||
&& name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_');
|
||||
match self {
|
||||
Self::CreateIndex { column, .. } | Self::DropIndex { column, .. }
|
||||
| Self::DropColumn { column, .. } | Self::LookupByIndex { column, .. }
|
||||
| Self::RangeByIndex { column, .. } => valid(column),
|
||||
Self::CreateTable { columns, .. } => columns.iter().all(|c| valid(&c.name)),
|
||||
Self::ReplaceSchema { columns, .. } => columns.iter().all(|c| valid(&c.name)),
|
||||
Self::CreateIndex { column, .. }
|
||||
| Self::DropIndex { column, .. }
|
||||
| Self::DropColumn { column, .. }
|
||||
| Self::LookupByIndex { column, .. }
|
||||
| Self::RangeByIndex { column, .. } => valid_storage_name(column),
|
||||
Self::CreateTable { columns, .. } | Self::AlterDomain { columns, .. } => {
|
||||
columns.iter().all(|c| valid_storage_name(&c.name))
|
||||
}
|
||||
Self::ReplaceSchema { columns, rows, .. } => {
|
||||
columns.iter().all(|c| valid_storage_name(&c.name))
|
||||
&& rows.iter().all(valid_row_columns)
|
||||
}
|
||||
Self::Insert { row, .. } => valid_row_columns(row),
|
||||
Self::InsertBatch { rows, .. } | Self::ReplaceAll { rows, .. } => {
|
||||
rows.iter().all(valid_row_columns)
|
||||
}
|
||||
Self::ApplyTransaction { ops }
|
||||
| Self::MvccLock { ops, .. }
|
||||
| Self::MvccCommit { ops, .. } => ops.iter().all(|op| match op {
|
||||
StorageOp::Upsert { rows, .. } | StorageOp::Replace { rows, .. } => {
|
||||
rows.iter().all(valid_row_columns)
|
||||
}
|
||||
StorageOp::Delete { .. } => true,
|
||||
}),
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
/// 普通表操作的统一访问边界
|
||||
pub fn table(&self) -> Option<&str> {
|
||||
match self {
|
||||
Self::Scan { table, .. } | Self::ScanShard { table, .. } | Self::ValidateWrite { table }
|
||||
Self::Scan { table, .. }
|
||||
| Self::ScanShard { table, .. }
|
||||
| Self::ValidateWrite { table }
|
||||
| Self::Insert { table, .. }
|
||||
| Self::InsertBatch { table, .. } | Self::DeleteKeys { table, .. }
|
||||
| Self::LookupByIndex { table, .. } | Self::ReplaceAll { table, .. }
|
||||
| Self::InsertBatch { table, .. }
|
||||
| Self::DeleteKeys { table, .. }
|
||||
| Self::LookupByIndex { table, .. }
|
||||
| Self::ReplaceAll { table, .. }
|
||||
| Self::RangeByIndex { table, .. }
|
||||
| Self::DescribeTable { table } | Self::CreateTable { table, .. }
|
||||
| Self::DropTable { table } | Self::CreateIndex { table, .. }
|
||||
| Self::DropIndex { table, .. } | Self::DropColumn { table, .. }
|
||||
| Self::DescribeTable { table }
|
||||
| Self::CreateTable { table, .. }
|
||||
| Self::DropTable { table }
|
||||
| Self::CreateIndex { table, .. }
|
||||
| Self::DropIndex { table, .. }
|
||||
| Self::DropColumn { table, .. }
|
||||
| Self::Compact { table }
|
||||
| Self::ReplaceSchema { table, .. } => Some(table),
|
||||
Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList | Self::CatalogManage { .. }
|
||||
| Self::IdentityInitialize { .. } | Self::RoleCreate { .. } | Self::IdentityAlter { .. }
|
||||
| Self::AccountCreate { .. } | Self::AccountReset { .. } | Self::AccountLogin { .. }
|
||||
| Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus
|
||||
| Self::ReplaceSchema { table, .. }
|
||||
| Self::AlterDomain { table, .. } => Some(table),
|
||||
Self::Ping
|
||||
| Self::ListTables
|
||||
| Self::ListCatalog
|
||||
| Self::DomainSnapshot
|
||||
| Self::MvccPreview { .. }
|
||||
| Self::MvccBegin { .. }
|
||||
| Self::MvccLock { .. }
|
||||
| Self::MvccCommit { .. }
|
||||
| Self::MvccRollback { .. }
|
||||
| Self::DatabaseSnapshot { .. }
|
||||
| Self::AccountsList
|
||||
| Self::CatalogManage { .. }
|
||||
| Self::IdentityInitialize { .. }
|
||||
| Self::RoleCreate { .. }
|
||||
| Self::IdentityAlter { .. }
|
||||
| Self::AccountCreate { .. }
|
||||
| Self::AccountReset { .. }
|
||||
| Self::AccountLogin { .. }
|
||||
| Self::AccountDrop { .. }
|
||||
| Self::BootstrapSystem { .. }
|
||||
| Self::SystemStatus
|
||||
| Self::BootstrapTypes { .. }
|
||||
| Self::ApplyTransaction { .. } => None,
|
||||
}
|
||||
@@ -284,14 +392,26 @@ fn default_id_column() -> String {
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "status", rename_all = "snake_case")]
|
||||
pub enum Response {
|
||||
Guarded { response: Box<Response> },
|
||||
Accounts { accounts: Vec<Account> },
|
||||
Guarded {
|
||||
response: Box<Response>,
|
||||
},
|
||||
Accounts {
|
||||
accounts: Vec<Account>,
|
||||
},
|
||||
Pong,
|
||||
Rows { rows: Vec<Row> },
|
||||
Tables { tables: Vec<String> },
|
||||
Rows {
|
||||
rows: Vec<Row>,
|
||||
},
|
||||
Tables {
|
||||
tables: Vec<String>,
|
||||
},
|
||||
Schema {
|
||||
table: String,
|
||||
#[serde(default, serialize_with = "serialize_version", deserialize_with = "deserialize_version")]
|
||||
#[serde(
|
||||
default,
|
||||
serialize_with = "serialize_version",
|
||||
deserialize_with = "deserialize_version"
|
||||
)]
|
||||
schema_version: u64,
|
||||
columns: Vec<SchemaColumn>,
|
||||
#[serde(default)]
|
||||
@@ -309,9 +429,30 @@ pub enum Response {
|
||||
#[serde(default)]
|
||||
last_lsn: u64,
|
||||
},
|
||||
Error { message: String },
|
||||
Catalog { schemas: Vec<TableSchemaWire> },
|
||||
Object { database_id: u64, object_id: Option<u64> },
|
||||
Error {
|
||||
message: String,
|
||||
},
|
||||
Catalog {
|
||||
schemas: Vec<TableSchemaWire>,
|
||||
},
|
||||
DomainSnapshot {
|
||||
schemas: Vec<TableSchemaWire>,
|
||||
rows: Vec<(String, Vec<Row>)>,
|
||||
checkpoint: String,
|
||||
},
|
||||
MvccSnapshot {
|
||||
transaction: String,
|
||||
schemas: Vec<TableSchemaWire>,
|
||||
rows: Vec<(String, Vec<Row>)>,
|
||||
checkpoint: String,
|
||||
},
|
||||
SnapshotUnchanged {
|
||||
checkpoint: String,
|
||||
},
|
||||
Object {
|
||||
database_id: u64,
|
||||
object_id: Option<u64>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -330,7 +471,11 @@ pub struct TableSchemaWire {
|
||||
pub database_id: u64,
|
||||
#[serde(default)]
|
||||
pub object_id: u64,
|
||||
#[serde(default, serialize_with = "serialize_version", deserialize_with = "deserialize_version")]
|
||||
#[serde(
|
||||
default,
|
||||
serialize_with = "serialize_version",
|
||||
deserialize_with = "deserialize_version"
|
||||
)]
|
||||
pub schema_version: u64,
|
||||
pub columns: Vec<SchemaColumn>,
|
||||
#[serde(default)]
|
||||
@@ -342,7 +487,10 @@ pub struct TableSchemaWire {
|
||||
}
|
||||
|
||||
/// 将结构版本编码成精确十进制字符串
|
||||
fn serialize_version<S: serde::Serializer>(version: &u64, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
fn serialize_version<S: serde::Serializer>(
|
||||
version: &u64,
|
||||
serializer: S,
|
||||
) -> Result<S::Ok, S::Error> {
|
||||
serializer.serialize_str(&version.to_string())
|
||||
}
|
||||
|
||||
@@ -350,9 +498,14 @@ fn serialize_version<S: serde::Serializer>(version: &u64, serializer: S) -> Resu
|
||||
fn deserialize_version<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result<u64, D::Error> {
|
||||
let value = serde_json::Value::deserialize(deserializer)?;
|
||||
match value {
|
||||
serde_json::Value::String(s) if !s.is_empty() && s.len() <= 20 && s.bytes().all(|c| c.is_ascii_digit()) =>
|
||||
s.parse().map_err(serde::de::Error::custom),
|
||||
serde_json::Value::Number(n) => n.as_u64().ok_or_else(|| serde::de::Error::custom("invalid schema version")),
|
||||
serde_json::Value::String(s)
|
||||
if !s.is_empty() && s.len() <= 20 && s.bytes().all(|c| c.is_ascii_digit()) =>
|
||||
{
|
||||
s.parse().map_err(serde::de::Error::custom)
|
||||
}
|
||||
serde_json::Value::Number(n) => n
|
||||
.as_u64()
|
||||
.ok_or_else(|| serde::de::Error::custom("invalid schema version")),
|
||||
_ => Err(serde::de::Error::custom("invalid schema version")),
|
||||
}
|
||||
}
|
||||
+2727
-541
File diff suppressed because it is too large.
Load diff
+129
-28
@@ -26,12 +26,30 @@ const OP_CREATE_TABLE: u8 = 11;
|
||||
const OP_DROP_TABLE: u8 = 12;
|
||||
const OP_CREATE_INDEX: u8 = 13;
|
||||
const OP_DROP_INDEX: u8 = 14;
|
||||
const OP_MIGRATE: u8 = 15;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub enum WalOp {
|
||||
HideColumn { table: String, column: String, schema_version: Option<u64> },
|
||||
Compact { table: String, rows: Vec<Row> },
|
||||
Accounts { accounts: Vec<Account>, removed: Vec<String>, migrate_roles: bool },
|
||||
Migrate {
|
||||
table: String,
|
||||
object_id: u64,
|
||||
source: String,
|
||||
target: String,
|
||||
},
|
||||
HideColumn {
|
||||
table: String,
|
||||
column: String,
|
||||
schema_version: Option<u64>,
|
||||
},
|
||||
Compact {
|
||||
table: String,
|
||||
rows: Vec<Row>,
|
||||
},
|
||||
Accounts {
|
||||
accounts: Vec<Account>,
|
||||
removed: Vec<String>,
|
||||
migrate_roles: bool,
|
||||
},
|
||||
Insert {
|
||||
table: String,
|
||||
row: Row,
|
||||
@@ -80,6 +98,28 @@ pub enum WalOp {
|
||||
Commit,
|
||||
}
|
||||
|
||||
impl WalOp {
|
||||
/// 返回单表操作的目标,账号组另行展开
|
||||
pub(crate) fn table(&self) -> Option<&str> {
|
||||
match self {
|
||||
Self::Migrate { table, .. }
|
||||
| Self::HideColumn { table, .. }
|
||||
| Self::Compact { table, .. }
|
||||
| Self::Insert { table, .. }
|
||||
| Self::InsertBatch { table, .. }
|
||||
| Self::DeleteKeys { table, .. }
|
||||
| Self::ReplaceAll { table, .. }
|
||||
| Self::DropColumn { table, .. }
|
||||
| Self::ReplaceSchema { table, .. }
|
||||
| Self::CreateTable { table, .. }
|
||||
| Self::DropTable { table }
|
||||
| Self::CreateIndex { table, .. }
|
||||
| Self::DropIndex { table, .. } => Some(table),
|
||||
Self::Accounts { .. } | Self::Commit => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct HideColumnPayload {
|
||||
column: String,
|
||||
@@ -255,13 +295,11 @@ impl Wal {
|
||||
/// 读当前 WAL 的第一条操作;空则 None
|
||||
pub fn read(&self) -> io::Result<Option<WalOp>> {
|
||||
let scan = self.read_all()?;
|
||||
Ok(scan.records.into_iter().find_map(|(_, op)| {
|
||||
if op == WalOp::Commit {
|
||||
None
|
||||
} else {
|
||||
Some(op)
|
||||
}
|
||||
}))
|
||||
Ok(scan.records.into_iter().find_map(
|
||||
|(_, op)| {
|
||||
if op == WalOp::Commit { None } else { Some(op) }
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
/// 读全部记录,撕裂或损坏的尾巴按长度截断
|
||||
@@ -287,9 +325,12 @@ impl Wal {
|
||||
let mut frames = Vec::new();
|
||||
let mut offset = 0usize;
|
||||
while offset + 4 <= buf.len() {
|
||||
let body_len =
|
||||
u32::from_le_bytes([buf[offset], buf[offset + 1], buf[offset + 2], buf[offset + 3]])
|
||||
as usize;
|
||||
let body_len = u32::from_le_bytes([
|
||||
buf[offset],
|
||||
buf[offset + 1],
|
||||
buf[offset + 2],
|
||||
buf[offset + 3],
|
||||
]) as usize;
|
||||
let end = offset + 4 + body_len;
|
||||
if end > buf.len() {
|
||||
break;
|
||||
@@ -324,7 +365,10 @@ impl Wal {
|
||||
}
|
||||
let bytes = std::fs::read(&path)?;
|
||||
if bytes.len() != 8 {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid WAL checkpoint length"));
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"invalid WAL checkpoint length",
|
||||
));
|
||||
}
|
||||
let mut raw = [0u8; 8];
|
||||
raw.copy_from_slice(&bytes[..8]);
|
||||
@@ -342,10 +386,14 @@ impl Wal {
|
||||
if let Some((lsn, _, _, _)) = frames.last() {
|
||||
base = base.max(*lsn);
|
||||
}
|
||||
*slot = base.checked_add(1).ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
|
||||
*slot = base
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
|
||||
}
|
||||
let lsn = *slot;
|
||||
*slot = slot.checked_add(1).ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
|
||||
*slot = slot
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
|
||||
Ok(lsn)
|
||||
}
|
||||
|
||||
@@ -392,6 +440,7 @@ fn truncate(f: &mut File) -> io::Result<()> {
|
||||
/// 把操作编成字节
|
||||
fn encode(op: &WalOp, lsn: u64) -> io::Result<Vec<u8>> {
|
||||
let (op_type, table, payload) = match op {
|
||||
WalOp::Migrate { table, .. } => (OP_MIGRATE, table.clone(), serde_json::to_vec(op).map_err(io::Error::other)?),
|
||||
WalOp::HideColumn { table, column, schema_version } => (OP_HIDE_COLUMN, table.clone(),
|
||||
serde_json::to_vec(&serde_json::json!({"column":column,"schema_version":schema_version})).map_err(io::Error::other)?),
|
||||
WalOp::Compact { table, rows } => {
|
||||
@@ -487,6 +536,25 @@ fn decode_body(body: &[u8]) -> Result<(u64, WalOp), String> {
|
||||
let payload = &body[11 + table_len..];
|
||||
|
||||
let op = match op_type {
|
||||
OP_MIGRATE => {
|
||||
let op: WalOp = serde_json::from_slice(payload)
|
||||
.map_err(|e| format!("bad migration payload: {e}"))?;
|
||||
match &op {
|
||||
WalOp::Migrate {
|
||||
table: name,
|
||||
source,
|
||||
target,
|
||||
..
|
||||
} if name == &table
|
||||
&& crate::catalog::valid_generation(source)
|
||||
&& !target.is_empty()
|
||||
&& crate::catalog::valid_generation(target) =>
|
||||
{
|
||||
op
|
||||
}
|
||||
_ => return Err("invalid migration WAL".into()),
|
||||
}
|
||||
}
|
||||
OP_COMMIT => WalOp::Commit,
|
||||
OP_HIDE_COLUMN => {
|
||||
let value: serde_json::Value = serde_json::from_slice(payload)
|
||||
@@ -498,28 +566,61 @@ fn decode_body(body: &[u8]) -> Result<(u64, WalOp), String> {
|
||||
.map_err(|e| format!("bad hide_column payload: {e}"))?;
|
||||
(p.column, p.schema_version)
|
||||
};
|
||||
WalOp::HideColumn { table, column, schema_version }
|
||||
WalOp::HideColumn {
|
||||
table,
|
||||
column,
|
||||
schema_version,
|
||||
}
|
||||
}
|
||||
OP_COMPACT => {
|
||||
let rows: Vec<Row> = serde_json::from_slice(payload)
|
||||
.map_err(|e| format!("bad compact payload: {e}"))?;
|
||||
let rows: Vec<Row> =
|
||||
serde_json::from_slice(payload).map_err(|e| format!("bad compact payload: {e}"))?;
|
||||
WalOp::Compact { table, rows }
|
||||
}
|
||||
OP_ACCOUNTS => {
|
||||
if table != USERS_TABLE { return Err("invalid account WAL table".into()); }
|
||||
let value: serde_json::Value = serde_json::from_slice(payload).map_err(|e| format!("bad account payload: {e}"))?;
|
||||
if table != USERS_TABLE {
|
||||
return Err("invalid account WAL table".into());
|
||||
}
|
||||
let value: serde_json::Value =
|
||||
serde_json::from_slice(payload).map_err(|e| format!("bad account payload: {e}"))?;
|
||||
let (accounts, removed, migrate_roles) = if value.is_array() {
|
||||
(serde_json::from_value(value).map_err(|e| format!("bad account payload: {e}"))?, Vec::new(), false)
|
||||
(
|
||||
serde_json::from_value(value)
|
||||
.map_err(|e| format!("bad account payload: {e}"))?,
|
||||
Vec::new(),
|
||||
false,
|
||||
)
|
||||
} else {
|
||||
(serde_json::from_value(value.get("accounts").cloned().ok_or("missing account payload")?).map_err(|e| format!("bad account payload: {e}"))?,
|
||||
serde_json::from_value(value.get("removed").cloned().ok_or("missing identity cleanup")?).map_err(|e| format!("bad account cleanup: {e}"))?,
|
||||
value.get("migrate_roles").and_then(serde_json::Value::as_bool).ok_or("missing role migration flag")?)
|
||||
(
|
||||
serde_json::from_value(
|
||||
value
|
||||
.get("accounts")
|
||||
.cloned()
|
||||
.ok_or("missing account payload")?,
|
||||
)
|
||||
.map_err(|e| format!("bad account payload: {e}"))?,
|
||||
serde_json::from_value(
|
||||
value
|
||||
.get("removed")
|
||||
.cloned()
|
||||
.ok_or("missing identity cleanup")?,
|
||||
)
|
||||
.map_err(|e| format!("bad account cleanup: {e}"))?,
|
||||
value
|
||||
.get("migrate_roles")
|
||||
.and_then(serde_json::Value::as_bool)
|
||||
.ok_or("missing role migration flag")?,
|
||||
)
|
||||
};
|
||||
WalOp::Accounts { accounts, removed, migrate_roles }
|
||||
WalOp::Accounts {
|
||||
accounts,
|
||||
removed,
|
||||
migrate_roles,
|
||||
}
|
||||
}
|
||||
OP_INSERT => {
|
||||
let row: Row =
|
||||
serde_json::from_slice(payload).map_err(|e| format!("bad insert payload: {}", e))?;
|
||||
let row: Row = serde_json::from_slice(payload)
|
||||
.map_err(|e| format!("bad insert payload: {}", e))?;
|
||||
WalOp::Insert { table, row }
|
||||
}
|
||||
OP_INSERT_BATCH => {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use chusql_core_storage::btree::{fits_in_page, DiskBTree, MAX_KEY_BYTES};
|
||||
use chusql_core_storage::btree::{DiskBTree, MAX_KEY_BYTES, fits_in_page};
|
||||
use chusql_core_storage::config::{DEFAULT_BTREE_ORDER, DEFAULT_PAGE_SIZE};
|
||||
use chusql_core_storage::page::DEFAULT_POOL_SIZE;
|
||||
use std::io::{ErrorKind, Seek, SeekFrom, Write};
|
||||
@@ -16,20 +16,28 @@ fn malformed_node_fields_return_invalid_data() {
|
||||
tree.insert(b"y", 2).unwrap();
|
||||
}
|
||||
let original = std::fs::read(&path).unwrap();
|
||||
let cell = u16::from_le_bytes([original[DEFAULT_PAGE_SIZE + 11], original[DEFAULT_PAGE_SIZE + 12]]) as usize;
|
||||
let cell = u16::from_le_bytes([
|
||||
original[DEFAULT_PAGE_SIZE + 11],
|
||||
original[DEFAULT_PAGE_SIZE + 12],
|
||||
]) as usize;
|
||||
for (name, offset, bytes) in [
|
||||
("slot count", 1, vec![255, 255]),
|
||||
("node kind", 0, vec![2]),
|
||||
("cell overlaps slots", 11, vec![0, 0]),
|
||||
("cell beyond page", 11, vec![255, 15]),
|
||||
("overlapping cells", 13, (cell as u16).to_le_bytes().to_vec()),
|
||||
(
|
||||
"overlapping cells",
|
||||
13,
|
||||
(cell as u16).to_le_bytes().to_vec(),
|
||||
),
|
||||
("key length", cell, vec![255, 255]),
|
||||
("key order", cell + 2, vec![b'z']),
|
||||
("zero child", 0, vec![1]),
|
||||
] {
|
||||
std::fs::write(&path, &original).unwrap();
|
||||
let mut file = std::fs::OpenOptions::new().write(true).open(&path).unwrap();
|
||||
file.seek(SeekFrom::Start((DEFAULT_PAGE_SIZE + offset) as u64)).unwrap();
|
||||
file.seek(SeekFrom::Start((DEFAULT_PAGE_SIZE + offset) as u64))
|
||||
.unwrap();
|
||||
file.write_all(&bytes).unwrap();
|
||||
drop(file);
|
||||
let mut tree = open(dir.path(), "corrupt.idx");
|
||||
@@ -42,7 +50,13 @@ fn malformed_node_fields_return_invalid_data() {
|
||||
|
||||
/// 按默认参数打开指定索引文件
|
||||
fn open(dir: &std::path::Path, name: &str) -> DiskBTree {
|
||||
DiskBTree::open(dir.join(name), DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).unwrap()
|
||||
DiskBTree::open(
|
||||
dir.join(name),
|
||||
DEFAULT_PAGE_SIZE,
|
||||
DEFAULT_BTREE_ORDER,
|
||||
DEFAULT_POOL_SIZE,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// 整数键编码成保序的 8 字节(与堆表一致)
|
||||
@@ -101,12 +115,24 @@ fn reopen_persists() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("t.idx");
|
||||
{
|
||||
let mut t = DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).unwrap();
|
||||
let mut t = DiskBTree::open(
|
||||
&path,
|
||||
DEFAULT_PAGE_SIZE,
|
||||
DEFAULT_BTREE_ORDER,
|
||||
DEFAULT_POOL_SIZE,
|
||||
)
|
||||
.unwrap();
|
||||
for i in 0..50 {
|
||||
t.insert(&key(i), i as u64).unwrap();
|
||||
}
|
||||
}
|
||||
let mut t = DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).unwrap();
|
||||
let mut t = DiskBTree::open(
|
||||
&path,
|
||||
DEFAULT_PAGE_SIZE,
|
||||
DEFAULT_BTREE_ORDER,
|
||||
DEFAULT_POOL_SIZE,
|
||||
)
|
||||
.unwrap();
|
||||
for i in 0..50 {
|
||||
assert_eq!(t.get(&key(i)).unwrap(), Some(i as u64));
|
||||
}
|
||||
@@ -148,12 +174,26 @@ fn config_mismatch_errors() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("t.idx");
|
||||
{
|
||||
let mut t = DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).unwrap();
|
||||
let mut t = DiskBTree::open(
|
||||
&path,
|
||||
DEFAULT_PAGE_SIZE,
|
||||
DEFAULT_BTREE_ORDER,
|
||||
DEFAULT_POOL_SIZE,
|
||||
)
|
||||
.unwrap();
|
||||
t.insert(&key(1), 1).unwrap();
|
||||
}
|
||||
assert!(DiskBTree::open(&path, DEFAULT_PAGE_SIZE, 8, DEFAULT_POOL_SIZE).is_err());
|
||||
assert!(DiskBTree::open(&path, 8192, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).is_err());
|
||||
assert!(DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).is_ok());
|
||||
assert!(
|
||||
DiskBTree::open(
|
||||
&path,
|
||||
DEFAULT_PAGE_SIZE,
|
||||
DEFAULT_BTREE_ORDER,
|
||||
DEFAULT_POOL_SIZE
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
/// 不是索引文件要报错
|
||||
@@ -162,7 +202,15 @@ fn not_an_index_file_errors() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("junk.idx");
|
||||
std::fs::write(&path, vec![7u8; DEFAULT_PAGE_SIZE]).unwrap();
|
||||
assert!(DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, DEFAULT_POOL_SIZE).is_err());
|
||||
assert!(
|
||||
DiskBTree::open(
|
||||
&path,
|
||||
DEFAULT_PAGE_SIZE,
|
||||
DEFAULT_BTREE_ORDER,
|
||||
DEFAULT_POOL_SIZE
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
/// order 放不进一页要报错
|
||||
@@ -182,7 +230,10 @@ fn delete_removes_only_that_key() {
|
||||
t.insert(&key(i), i as u64 + 1).unwrap();
|
||||
}
|
||||
|
||||
assert!(t.delete(&key(50), 51).unwrap(), "deleting an existing entry should return true");
|
||||
assert!(
|
||||
t.delete(&key(50), 51).unwrap(),
|
||||
"deleting an existing entry should return true"
|
||||
);
|
||||
assert_eq!(t.get(&key(50)).unwrap(), None);
|
||||
assert_eq!(t.get(&key(49)).unwrap(), Some(50));
|
||||
assert_eq!(t.get(&key(51)).unwrap(), Some(52));
|
||||
@@ -262,7 +313,11 @@ fn splits_keep_keys_sorted() {
|
||||
let all: Vec<Vec<u8>> = t.iter_all().unwrap().into_iter().map(|(k, _)| k).collect();
|
||||
assert_eq!(all, expected);
|
||||
for k in &expected {
|
||||
assert!(t.get(k).unwrap().is_some(), "missing {:?}", String::from_utf8_lossy(k));
|
||||
assert!(
|
||||
t.get(k).unwrap().is_some(),
|
||||
"missing {:?}",
|
||||
String::from_utf8_lossy(k)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -314,3 +369,61 @@ fn clear_resets_the_tree() {
|
||||
t.insert(b"c", 3).unwrap();
|
||||
assert_eq!(t.get(b"c").unwrap(), Some(3));
|
||||
}
|
||||
|
||||
/// 长键跨页后可重启查询并保留旧配置
|
||||
#[test]
|
||||
fn long_keys_split_reopen_and_preserve_dense_configuration() {
|
||||
use chusql_core_storage::btree::key_limit;
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("long.idx");
|
||||
let mut tree = DiskBTree::open(&path, 512, 4, DEFAULT_POOL_SIZE).unwrap();
|
||||
for n in 0..80_u64 {
|
||||
let key = format!("{n:03}{}", "a".repeat(125));
|
||||
tree.insert(key.as_bytes(), n).unwrap();
|
||||
}
|
||||
drop(tree);
|
||||
let mut tree = DiskBTree::open(&path, 512, 4, DEFAULT_POOL_SIZE).unwrap();
|
||||
assert_eq!(tree.scan(None, None).unwrap().len(), 80);
|
||||
for n in 0..80_u64 {
|
||||
let key = format!("{n:03}{}", "a".repeat(125));
|
||||
assert_eq!(tree.get(key.as_bytes()).unwrap(), Some(n));
|
||||
}
|
||||
let dense_path = dir.path().join("dense.idx");
|
||||
let mut dense = DiskBTree::open(&dense_path, 512, 7, DEFAULT_POOL_SIZE).unwrap();
|
||||
let limit = key_limit(512, 7);
|
||||
assert!((64..MAX_KEY_BYTES).contains(&limit));
|
||||
dense.insert(&vec![b'a'; limit], 1).unwrap();
|
||||
assert_eq!(
|
||||
dense.insert(&vec![b'b'; limit + 1], 2).unwrap_err().kind(),
|
||||
ErrorKind::InvalidInput
|
||||
);
|
||||
assert_eq!(dense.get(&vec![b'a'; limit]).unwrap(), Some(1));
|
||||
}
|
||||
|
||||
/// 旧文件头可无损打开并首次写入长键
|
||||
#[test]
|
||||
fn legacy_key_header_upgrades_only_when_long_keys_are_written() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("legacy.idx");
|
||||
let mut tree = open(dir.path(), "legacy.idx");
|
||||
tree.insert(b"old", 1).unwrap();
|
||||
drop(tree);
|
||||
let mut bytes = std::fs::read(&path).unwrap();
|
||||
bytes[8..10].copy_from_slice(&64_u16.to_le_bytes());
|
||||
std::fs::write(&path, &bytes).unwrap();
|
||||
let mut tree = open(dir.path(), "legacy.idx");
|
||||
assert_eq!(tree.get(b"old").unwrap(), Some(1));
|
||||
drop(tree);
|
||||
assert_eq!(std::fs::read(&path).unwrap(), bytes);
|
||||
let mut tree = open(dir.path(), "legacy.idx");
|
||||
let key = "author/".to_string() + &"p".repeat(80);
|
||||
tree.insert(key.as_bytes(), 2).unwrap();
|
||||
drop(tree);
|
||||
let mut tree = open(dir.path(), "legacy.idx");
|
||||
assert_eq!(tree.get(key.as_bytes()).unwrap(), Some(2));
|
||||
assert_eq!(tree.get(b"old").unwrap(), Some(1));
|
||||
assert_eq!(
|
||||
&std::fs::read(&path).unwrap()[8..10],
|
||||
&128_u16.to_le_bytes()
|
||||
);
|
||||
}
|
||||
@@ -33,9 +33,14 @@ fn schema_versions_track_structure_and_survive_reload() {
|
||||
fn legacy_schema_version_remains_unknown_until_change() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("catalog.json");
|
||||
std::fs::write(&path, json!({"tables": {
|
||||
"t": {"object_id": 42, "columns": []}
|
||||
}}).to_string()).unwrap();
|
||||
std::fs::write(
|
||||
&path,
|
||||
json!({"tables": {
|
||||
"t": {"object_id": 42, "columns": []}
|
||||
}})
|
||||
.to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
let mut c = Catalog::load(&path).unwrap();
|
||||
assert_eq!(c.describe("t").unwrap().schema_version, 0);
|
||||
c.save(&path).unwrap();
|
||||
@@ -51,10 +56,15 @@ fn legacy_schema_version_remains_unknown_until_change() {
|
||||
fn exhausted_schema_version_preserves_structure() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("catalog.json");
|
||||
std::fs::write(&path, json!({"tables": {
|
||||
"t": {"object_id": 42, "schema_version": u64::MAX,
|
||||
"columns": [{"name": "id", "ty": "int"}]}
|
||||
}}).to_string()).unwrap();
|
||||
std::fs::write(
|
||||
&path,
|
||||
json!({"tables": {
|
||||
"t": {"object_id": 42, "schema_version": u64::MAX,
|
||||
"columns": [{"name": "id", "ty": "int"}]}
|
||||
}})
|
||||
.to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
let mut c = Catalog::load(&path).unwrap();
|
||||
assert!(c.remove_column("t", "id").is_err());
|
||||
assert!(c.set_columns("t", vec![]).is_err());
|
||||
@@ -90,20 +100,42 @@ fn object_ids_survive_reload_and_are_not_reused() {
|
||||
fn legacy_object_ids_upgrade_without_collisions() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("catalog.json");
|
||||
std::fs::write(&path, json!({"tables": {
|
||||
"a": {"columns": [], "object_id": 42},
|
||||
"b": {"columns": []}
|
||||
}}).to_string()).unwrap();
|
||||
std::fs::write(
|
||||
&path,
|
||||
json!({"tables": {
|
||||
"a": {"columns": [], "object_id": 42},
|
||||
"b": {"columns": []}
|
||||
}})
|
||||
.to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
let catalog = Catalog::load(&path).unwrap();
|
||||
assert_eq!(catalog.describe("a").unwrap().object_id, 42);
|
||||
assert_eq!(catalog.describe("b").unwrap().object_id, 43);
|
||||
catalog.save(&path).unwrap();
|
||||
assert_eq!(Catalog::load(&path).unwrap().describe("b").unwrap().object_id, 43);
|
||||
std::fs::write(&path, json!({"tables": {
|
||||
"a": {"columns": [], "object_id": 42},
|
||||
"b": {"columns": [], "object_id": 42}
|
||||
}}).to_string()).unwrap();
|
||||
assert!(Catalog::load(&path).unwrap_err().to_string().contains("duplicate"));
|
||||
assert_eq!(
|
||||
Catalog::load(&path)
|
||||
.unwrap()
|
||||
.describe("b")
|
||||
.unwrap()
|
||||
.object_id,
|
||||
43
|
||||
);
|
||||
std::fs::write(
|
||||
&path,
|
||||
json!({"tables": {
|
||||
"a": {"columns": [], "object_id": 42},
|
||||
"b": {"columns": [], "object_id": 42}
|
||||
}})
|
||||
.to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
Catalog::load(&path)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("duplicate")
|
||||
);
|
||||
}
|
||||
|
||||
/// 编号耗尽明确失败且不建立对象
|
||||
@@ -115,8 +147,6 @@ fn exhausted_object_ids_fail_without_creating_table() {
|
||||
assert!(catalog.describe("t").is_none());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/// 用键值对构造一行
|
||||
fn row(pairs: &[(&str, serde_json::Value)]) -> Row {
|
||||
let mut m = Row::new();
|
||||
@@ -128,7 +158,8 @@ fn row(pairs: &[(&str, serde_json::Value)]) -> Row {
|
||||
|
||||
/// 空文件得到空字典,未知表返回 None
|
||||
#[test]
|
||||
fn empty_file_gives_empty_catalog_and_unknown_table_returns_none() { let dir = tempfile::tempdir().unwrap();
|
||||
fn empty_file_gives_empty_catalog_and_unknown_table_returns_none() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("catalog.json");
|
||||
let c = Catalog::load(&path).unwrap();
|
||||
assert_eq!(c.table_names().len(), 0);
|
||||
@@ -142,7 +173,8 @@ fn ensure_then_describe() {
|
||||
let path = dir.path().join("catalog.json");
|
||||
|
||||
let mut c = Catalog::load(&path).unwrap();
|
||||
c.ensure_columns("users", &row(&[("id", json!(1)), ("name", json!("Alice"))])).unwrap();
|
||||
c.ensure_columns("users", &row(&[("id", json!(1)), ("name", json!("Alice"))]))
|
||||
.unwrap();
|
||||
c.save(&path).unwrap();
|
||||
|
||||
let c2 = Catalog::load(&path).unwrap();
|
||||
@@ -164,8 +196,10 @@ fn second_insert_does_not_duplicate_columns() {
|
||||
let path = dir.path().join("catalog.json");
|
||||
|
||||
let mut c = Catalog::load(&path).unwrap();
|
||||
c.ensure_columns("users", &row(&[("id", json!(1))])).unwrap();
|
||||
c.ensure_columns("users", &row(&[("id", json!(2))])).unwrap();
|
||||
c.ensure_columns("users", &row(&[("id", json!(1))]))
|
||||
.unwrap();
|
||||
c.ensure_columns("users", &row(&[("id", json!(2))]))
|
||||
.unwrap();
|
||||
assert_eq!(c.describe("users").unwrap().columns.len(), 1);
|
||||
}
|
||||
|
||||
@@ -176,9 +210,11 @@ fn remove_column_drops_definition_stats_and_index() {
|
||||
let path = dir.path().join("catalog.json");
|
||||
|
||||
let mut c = Catalog::load(&path).unwrap();
|
||||
c.ensure_columns("t", &row(&[("id", json!(1)), ("code", json!(5))])).unwrap();
|
||||
c.ensure_columns("t", &row(&[("id", json!(1)), ("code", json!(5))]))
|
||||
.unwrap();
|
||||
c.add_index("t", "code").unwrap();
|
||||
c.rebuild_stats("t", &[row(&[("id", json!(1)), ("code", json!(5))])]).unwrap();
|
||||
c.rebuild_stats("t", &[row(&[("id", json!(1)), ("code", json!(5))])])
|
||||
.unwrap();
|
||||
|
||||
let removed = c.remove_column("t", "code").unwrap();
|
||||
assert!(removed, "first remove should hit the column");
|
||||
@@ -186,7 +222,10 @@ fn remove_column_drops_definition_stats_and_index() {
|
||||
assert_eq!(s.columns.len(), 1);
|
||||
assert_eq!(s.columns[0].name, "id");
|
||||
assert!(s.indexes.is_empty(), "index on code should be gone");
|
||||
assert!(!s.stats.contains_key("code"), "stats for code should be gone");
|
||||
assert!(
|
||||
!s.stats.contains_key("code"),
|
||||
"stats for code should be gone"
|
||||
);
|
||||
|
||||
let again = c.remove_column("t", "code").unwrap();
|
||||
assert!(!again);
|
||||
@@ -217,7 +256,8 @@ fn stats_track_a_histogram_for_numeric_columns() {
|
||||
assert!(name.hist.is_empty(), "字符串列不进直方图");
|
||||
assert_eq!(name.lo, None);
|
||||
|
||||
c.record_insert("t", &row(&[("id", json!(1000)), ("name", json!("x"))])).unwrap();
|
||||
c.record_insert("t", &row(&[("id", json!(1000)), ("name", json!("x"))]))
|
||||
.unwrap();
|
||||
let s = c.describe("t").unwrap();
|
||||
let id = s.stats.get("id").unwrap();
|
||||
assert_eq!(id.lo, Some(0.0));
|
||||
@@ -240,7 +280,9 @@ fn stats_track_a_histogram_for_numeric_columns() {
|
||||
fn duplicate_deletion_keeps_distinct_until_last_occurrence() {
|
||||
let mut catalog = Catalog::default();
|
||||
let duplicate = row(&[("id", json!(1)), ("v", json!(7))]);
|
||||
catalog.rebuild_stats("t", &[duplicate.clone(), duplicate.clone()]).unwrap();
|
||||
catalog
|
||||
.rebuild_stats("t", &[duplicate.clone(), duplicate.clone()])
|
||||
.unwrap();
|
||||
catalog.record_delete("t", std::slice::from_ref(&duplicate));
|
||||
assert_eq!(catalog.describe("t").unwrap().stats["v"].distinct, 1);
|
||||
catalog.record_delete("t", &[duplicate]);
|
||||
@@ -251,7 +293,9 @@ fn duplicate_deletion_keeps_distinct_until_last_occurrence() {
|
||||
#[test]
|
||||
fn capped_distinct_keeps_histogram_and_other_columns_accurate() {
|
||||
let mut catalog = Catalog::default();
|
||||
let rows: Vec<Row> = (0..5000).map(|n| row(&[("id", json!(n)), ("v", json!(n % 2))])).collect();
|
||||
let rows: Vec<Row> = (0..5000)
|
||||
.map(|n| row(&[("id", json!(n)), ("v", json!(n % 2))]))
|
||||
.collect();
|
||||
catalog.rebuild_stats("t", &rows).unwrap();
|
||||
let schema = catalog.describe("t").unwrap();
|
||||
assert!(schema.stats["id"].capped);
|
||||
@@ -277,7 +321,9 @@ fn rebuilt_histogram_has_exact_buckets_in_any_order() {
|
||||
let stat = &forward.describe("t").unwrap().stats["v"];
|
||||
assert_eq!(stat.hist, vec![10; 16]);
|
||||
assert_eq!(stat.hist, reverse.describe("t").unwrap().stats["v"].hist);
|
||||
forward.record_insert("t", &row(&[("v", json!(10000))])).unwrap();
|
||||
forward
|
||||
.record_insert("t", &row(&[("v", json!(10000))]))
|
||||
.unwrap();
|
||||
forward.record_delete("t", &[row(&[("v", json!(10000))])]);
|
||||
forward.rebuild_stats("t", &rows[80..]).unwrap();
|
||||
let repaired = &forward.describe("t").unwrap().stats["v"];
|
||||
@@ -316,7 +362,13 @@ fn legacy_catalog_loads_and_normalizes() {
|
||||
assert_eq!(s.columns.len(), 2);
|
||||
assert!(s.columns[0].nullable, "老列默认可空");
|
||||
assert!(c.normalize_types().unwrap());
|
||||
let types: Vec<&str> = c.describe("legacy").unwrap().columns.iter().map(|col| col.ty.as_str()).collect();
|
||||
let types: Vec<&str> = c
|
||||
.describe("legacy")
|
||||
.unwrap()
|
||||
.columns
|
||||
.iter()
|
||||
.map(|col| col.ty.as_str())
|
||||
.collect();
|
||||
assert_eq!(types, vec!["int", "str"]);
|
||||
assert!(!c.normalize_types().unwrap(), "第二次没有可改的");
|
||||
}
|
||||
@@ -36,8 +36,7 @@ fn defaults_without_file() {
|
||||
fn file_overrides_only_written_keys() {
|
||||
let _guard = env_guard();
|
||||
let path = PathBuf::from("t.toml");
|
||||
let loaded = config::resolve(Some("[page]\nsize = 8192\n"), Some(path.clone()))
|
||||
.unwrap();
|
||||
let loaded = config::resolve(Some("[page]\nsize = 8192\n"), Some(path.clone())).unwrap();
|
||||
|
||||
assert_eq!(loaded.config.page_size, 8192);
|
||||
assert_eq!(loaded.config.btree_order, DEFAULT_BTREE_ORDER);
|
||||
@@ -45,9 +44,17 @@ fn file_overrides_only_written_keys() {
|
||||
assert_eq!(loaded.config.log_files, PathBuf::from(DEFAULT_LOG_FILES));
|
||||
assert_eq!(loaded.config_path, Some(path.clone()));
|
||||
|
||||
let page = loaded.origins.iter().find(|(n, _, _)| *n == "page.size").unwrap();
|
||||
let page = loaded
|
||||
.origins
|
||||
.iter()
|
||||
.find(|(n, _, _)| *n == "page.size")
|
||||
.unwrap();
|
||||
assert_eq!(page.2, Origin::File(path));
|
||||
let order = loaded.origins.iter().find(|(n, _, _)| *n == "btree.order").unwrap();
|
||||
let order = loaded
|
||||
.origins
|
||||
.iter()
|
||||
.find(|(n, _, _)| *n == "btree.order")
|
||||
.unwrap();
|
||||
assert_eq!(order.2, Origin::Default);
|
||||
}
|
||||
|
||||
@@ -110,7 +117,11 @@ fn default_config_path_follows_system_convention() {
|
||||
let path = config::default_config_path();
|
||||
assert_eq!(path.file_name().unwrap(), "settings.toml");
|
||||
assert_eq!(path.parent().unwrap().file_name().unwrap(), "ChuSQL");
|
||||
assert!(path.ends_with(format!("ChuSQL{}settings.toml", std::path::MAIN_SEPARATOR)), "got {}", path.display());
|
||||
assert!(
|
||||
path.ends_with(format!("ChuSQL{}settings.toml", std::path::MAIN_SEPARATOR)),
|
||||
"got {}",
|
||||
path.display()
|
||||
);
|
||||
|
||||
// 本机 APPDATA 存在时必须是 %APPDATA%\ChuSQL\settings.toml
|
||||
#[cfg(windows)]
|
||||
@@ -168,7 +179,11 @@ fn explicit_missing_path_is_an_error() {
|
||||
"got {}",
|
||||
err
|
||||
);
|
||||
assert!(err.contains("chusql-no-such-config-9f21.toml"), "got {}", err);
|
||||
assert!(
|
||||
err.contains("chusql-no-such-config-9f21.toml"),
|
||||
"got {}",
|
||||
err
|
||||
);
|
||||
}
|
||||
|
||||
/// 不给 --config 且无配置文件时用内置默认
|
||||
|
||||
@@ -26,16 +26,40 @@ fn maintenance_rejects_invalid_arguments_and_resets_outputs() {
|
||||
let mut len = 99;
|
||||
// SAFETY: 输出均可写;无效输入应在任何读取之前被拒绝。
|
||||
unsafe {
|
||||
assert_eq!(chusql_storage_maintenance(ptr::null(), ptr::null(), 0, &mut out, &mut len), -1);
|
||||
assert_eq!(
|
||||
chusql_storage_maintenance(ptr::null(), ptr::null(), 0, &mut out, &mut len),
|
||||
-1
|
||||
);
|
||||
assert!(out.is_null());
|
||||
assert_eq!(len, 0);
|
||||
len = 99;
|
||||
assert_eq!(chusql_storage_maintenance(ptr::null(), input.as_ptr(), input.len(), ptr::null_mut(), &mut len), -1);
|
||||
assert_eq!(
|
||||
chusql_storage_maintenance(
|
||||
ptr::null(),
|
||||
input.as_ptr(),
|
||||
input.len(),
|
||||
ptr::null_mut(),
|
||||
&mut len
|
||||
),
|
||||
-1
|
||||
);
|
||||
assert_eq!(len, 0);
|
||||
out = input.as_ptr() as *mut u8;
|
||||
assert_eq!(chusql_storage_maintenance(ptr::null(), input.as_ptr(), input.len(), &mut out, ptr::null_mut()), -1);
|
||||
assert_eq!(
|
||||
chusql_storage_maintenance(
|
||||
ptr::null(),
|
||||
input.as_ptr(),
|
||||
input.len(),
|
||||
&mut out,
|
||||
ptr::null_mut()
|
||||
),
|
||||
-1
|
||||
);
|
||||
assert!(out.is_null());
|
||||
assert_eq!(chusql_storage_maintenance(ptr::null(), input.as_ptr(), usize::MAX, &mut out, &mut len), -1);
|
||||
assert_eq!(
|
||||
chusql_storage_maintenance(ptr::null(), input.as_ptr(), usize::MAX, &mut out, &mut len),
|
||||
-1
|
||||
);
|
||||
}
|
||||
assert!(out.is_null());
|
||||
assert_eq!(len, 0);
|
||||
@@ -47,20 +71,36 @@ fn maintenance_rejects_invalid_arguments_and_resets_outputs() {
|
||||
fn request_validates_input_and_returns_owned_response() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let config = temp.path().join("settings.toml");
|
||||
std::fs::write(&config, format!("[storage]\ndata_dir = '{}'\nlog_files = '{}'\n[log]\nlevel = 'off'\n", temp.path().join("data").display(), temp.path().join("logs").display()))?;
|
||||
std::fs::write(
|
||||
&config,
|
||||
format!(
|
||||
"[storage]\ndata_dir = '{}'\nlog_files = '{}'\n[log]\nlevel = 'off'\n",
|
||||
temp.path().join("data").display(),
|
||||
temp.path().join("logs").display()
|
||||
),
|
||||
)?;
|
||||
let storage = Storage::open(config.to_str())?;
|
||||
let input = br#"{"method":"ping"}"#;
|
||||
let mut out = ptr::null_mut();
|
||||
let mut len = 0;
|
||||
// SAFETY: 句柄和实际请求有效;超长长度必须在构造切片前被拒绝。
|
||||
unsafe {
|
||||
assert_eq!(chusql_storage_request(&storage, input.as_ptr(), usize::MAX, &mut out, &mut len), -1);
|
||||
assert_eq!(
|
||||
chusql_storage_request(&storage, input.as_ptr(), usize::MAX, &mut out, &mut len),
|
||||
-1
|
||||
);
|
||||
assert!(out.is_null());
|
||||
assert_eq!(len, 0);
|
||||
assert_eq!(last_error(), "invalid argument");
|
||||
assert_eq!(chusql_storage_request(&storage, [0xff].as_ptr(), 1, &mut out, &mut len), -1);
|
||||
assert_eq!(
|
||||
chusql_storage_request(&storage, [0xff].as_ptr(), 1, &mut out, &mut len),
|
||||
-1
|
||||
);
|
||||
assert!(last_error().starts_with("request is not valid UTF-8:"));
|
||||
assert_eq!(chusql_storage_request(&storage, input.as_ptr(), input.len(), &mut out, &mut len), 0);
|
||||
assert_eq!(
|
||||
chusql_storage_request(&storage, input.as_ptr(), input.len(), &mut out, &mut len),
|
||||
0
|
||||
);
|
||||
assert!(!out.is_null());
|
||||
let response = slice::from_raw_parts(out, len).to_vec();
|
||||
chusql_storage_free(out, len);
|
||||
@@ -71,7 +111,10 @@ fn request_validates_input_and_returns_owned_response() -> Result<(), Box<dyn st
|
||||
let config = CString::new(config.to_str().unwrap())?;
|
||||
// SAFETY: 路径与输出有效;失败不产生需要释放的缓冲。
|
||||
unsafe {
|
||||
assert_eq!(chusql_storage_maintenance(config.as_ptr(), [0xff].as_ptr(), 1, &mut out, &mut len), -1);
|
||||
assert_eq!(
|
||||
chusql_storage_maintenance(config.as_ptr(), [0xff].as_ptr(), 1, &mut out, &mut len),
|
||||
-1
|
||||
);
|
||||
}
|
||||
assert!(out.is_null());
|
||||
assert_eq!(len, 0);
|
||||
@@ -79,7 +122,18 @@ fn request_validates_input_and_returns_owned_response() -> Result<(), Box<dyn st
|
||||
let maintenance = br#"{"mode":"reset","user":"root","password_hash":"hash","types":[{"name":"int","base_type":"int","parameterized":false}]}"#;
|
||||
// SAFETY: 维护仅使用本测试临时目录;成功响应复制后由配套函数释放。
|
||||
unsafe {
|
||||
assert_eq!(chusql_storage_maintenance(config.as_ptr(), maintenance.as_ptr(), maintenance.len(), &mut out, &mut len), 0, "{}", last_error());
|
||||
assert_eq!(
|
||||
chusql_storage_maintenance(
|
||||
config.as_ptr(),
|
||||
maintenance.as_ptr(),
|
||||
maintenance.len(),
|
||||
&mut out,
|
||||
&mut len
|
||||
),
|
||||
0,
|
||||
"{}",
|
||||
last_error()
|
||||
);
|
||||
assert!(!out.is_null());
|
||||
let response = slice::from_raw_parts(out, len).to_vec();
|
||||
chusql_storage_free(out, len);
|
||||
|
||||
@@ -25,20 +25,61 @@ fn delete_propagates_index_read_errors() {
|
||||
let original = row(&[("id", json!(1))]);
|
||||
t.insert_row(&original).unwrap();
|
||||
t.flush().unwrap();
|
||||
let file = std::fs::OpenOptions::new().write(true).open(&index).unwrap();
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.open(&index)
|
||||
.unwrap();
|
||||
file.set_len(DEFAULT_PAGE_SIZE as u64).unwrap();
|
||||
let err = t.delete_by_keys(&[1]).unwrap_err();
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
|
||||
assert_eq!(t.scan().unwrap(), vec![original]);
|
||||
}
|
||||
|
||||
/// 索引键失败不得留下堆行或清空旧表
|
||||
#[test]
|
||||
fn invalid_index_keys_leave_heap_and_indexes_unchanged() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut table = HeapTable::open(dir.path().join("t.db"), DEFAULT_PAGE_SIZE, 1).unwrap();
|
||||
let original = row(&[("id", json!(1)), ("name", json!("old"))]);
|
||||
table.insert_row(&original).unwrap();
|
||||
table
|
||||
.build_index(
|
||||
"name",
|
||||
&dir.path().join("t.name.idx"),
|
||||
DEFAULT_PAGE_SIZE,
|
||||
4,
|
||||
1,
|
||||
)
|
||||
.unwrap();
|
||||
let invalid = row(&[("id", json!(2)), ("name", json!("x".repeat(129)))]);
|
||||
assert!(table.insert_row(&invalid).is_err());
|
||||
assert_eq!(table.scan().unwrap(), vec![original.clone()]);
|
||||
assert!(table.replace_all(&[invalid]).is_err());
|
||||
assert_eq!(table.scan().unwrap(), vec![original.clone()]);
|
||||
assert_eq!(
|
||||
table.get_by_string_key("name", "old").unwrap(),
|
||||
Some(original)
|
||||
);
|
||||
assert!(
|
||||
table
|
||||
.replace_all(&[row(&[("name", json!("x".repeat(10_000)))])])
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(table.scan().unwrap().len(), 1);
|
||||
}
|
||||
|
||||
/// 跨页投影与分片保持行序及隐藏列
|
||||
#[test]
|
||||
fn projected_shards_preserve_order_and_hidden_columns() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut t = HeapTable::open(dir.path().join("t.db"), 256, 1).unwrap();
|
||||
for i in 0..40 {
|
||||
t.insert_row(&row(&[("id", json!(i)), ("name", json!("alice")), ("secret", json!(i))])).unwrap();
|
||||
t.insert_row(&row(&[
|
||||
("id", json!(i)),
|
||||
("name", json!("alice")),
|
||||
("secret", json!(i)),
|
||||
]))
|
||||
.unwrap();
|
||||
}
|
||||
t.set_hidden_columns(["secret".to_string()].into_iter().collect());
|
||||
let columns = vec!["id".to_string(), "secret".to_string()];
|
||||
@@ -47,9 +88,15 @@ fn projected_shards_preserve_order_and_hidden_columns() {
|
||||
let pages = t.num_pages().unwrap();
|
||||
assert!(pages > 1);
|
||||
let mut shards = t.scan_columns_pages(Some(&columns), 0, pages / 2).unwrap();
|
||||
shards.extend(t.scan_columns_pages(Some(&columns), pages / 2, pages + 1).unwrap());
|
||||
shards.extend(
|
||||
t.scan_columns_pages(Some(&columns), pages / 2, pages + 1)
|
||||
.unwrap(),
|
||||
);
|
||||
assert_eq!(shards, expected);
|
||||
assert_eq!(t.read_at(0, 0).unwrap().unwrap(), row(&[("id", json!(0)), ("name", json!("alice"))]));
|
||||
assert_eq!(
|
||||
t.read_at(0, 0).unwrap().unwrap(),
|
||||
row(&[("id", json!(0)), ("name", json!("alice"))])
|
||||
);
|
||||
}
|
||||
|
||||
/// 扫描与点读均返回行解码错误
|
||||
@@ -78,15 +125,20 @@ fn empty_index_file_is_rebuilt_from_rows() {
|
||||
let data = dir.path().join("t.db");
|
||||
let index = dir.path().join("t.idx");
|
||||
{
|
||||
let mut t = HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
|
||||
let mut t = HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE)
|
||||
.unwrap();
|
||||
for i in 1..=5 {
|
||||
t.insert_row(&row(&[("id", json!(i))])).unwrap();
|
||||
}
|
||||
t.flush().unwrap();
|
||||
}
|
||||
std::fs::write(&index, b"").unwrap();
|
||||
let mut t = HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
|
||||
assert!(t.get_by_column_key("id", 3).unwrap().is_some(), "空索引文件要从堆数据重建");
|
||||
let mut t =
|
||||
HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
|
||||
assert!(
|
||||
t.get_by_column_key("id", 3).unwrap().is_some(),
|
||||
"空索引文件要从堆数据重建"
|
||||
);
|
||||
assert!(t.get_by_column_key("id", 9).unwrap().is_none());
|
||||
}
|
||||
|
||||
@@ -97,14 +149,24 @@ fn string_index_looks_up_by_value() {
|
||||
let data = dir.path().join("s.db");
|
||||
let index = dir.path().join("s.idx");
|
||||
let mut t = HeapTable::open(&data, DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap();
|
||||
t.insert_row(&row(&[("id", json!(1)), ("name", json!("alice"))])).unwrap();
|
||||
t.insert_row(&row(&[("id", json!(2)), ("name", json!("bob"))])).unwrap();
|
||||
t.insert_row(&row(&[("id", json!(3)), ("name", json!(null))])).unwrap();
|
||||
t.build_index("name", &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
|
||||
t.insert_row(&row(&[("id", json!(1)), ("name", json!("alice"))]))
|
||||
.unwrap();
|
||||
t.insert_row(&row(&[("id", json!(2)), ("name", json!("bob"))]))
|
||||
.unwrap();
|
||||
t.insert_row(&row(&[("id", json!(3)), ("name", json!(null))]))
|
||||
.unwrap();
|
||||
t.build_index("name", &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(t.get_by_string_key("name", "bob").unwrap().unwrap()["id"], json!(2));
|
||||
assert_eq!(
|
||||
t.get_by_string_key("name", "bob").unwrap().unwrap()["id"],
|
||||
json!(2)
|
||||
);
|
||||
assert!(t.get_by_string_key("name", "dave").unwrap().is_none());
|
||||
assert!(t.get_by_string_key("name", "").unwrap().is_none(), "NULL 不入索引");
|
||||
assert!(
|
||||
t.get_by_string_key("name", "").unwrap().is_none(),
|
||||
"NULL 不入索引"
|
||||
);
|
||||
}
|
||||
|
||||
/// 允许重复值建索引:点查把重复的行都取回来
|
||||
@@ -114,9 +176,12 @@ fn duplicate_values_are_indexed_and_all_returned() {
|
||||
let data = dir.path().join("d.db");
|
||||
let index = dir.path().join("d.idx");
|
||||
let mut t = HeapTable::open(&data, DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap();
|
||||
t.insert_row(&row(&[("id", json!(1)), ("name", json!("alice"))])).unwrap();
|
||||
t.insert_row(&row(&[("id", json!(2)), ("name", json!("alice"))])).unwrap();
|
||||
t.build_index("name", &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
|
||||
t.insert_row(&row(&[("id", json!(1)), ("name", json!("alice"))]))
|
||||
.unwrap();
|
||||
t.insert_row(&row(&[("id", json!(2)), ("name", json!("alice"))]))
|
||||
.unwrap();
|
||||
t.build_index("name", &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE)
|
||||
.unwrap();
|
||||
|
||||
let mut ids: Vec<i64> = t
|
||||
.get_all_by_string_key("name", "alice")
|
||||
@@ -218,9 +283,15 @@ fn replace_all_rebuilds_index() {
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
assert!(t.get_by_key(1).unwrap().is_none(), "old key must be invalidated");
|
||||
assert!(
|
||||
t.get_by_key(1).unwrap().is_none(),
|
||||
"old key must be invalidated"
|
||||
);
|
||||
assert!(t.get_by_key(2).unwrap().is_none());
|
||||
let got = t.get_by_key(7).unwrap().expect("new row should be indexed by id");
|
||||
let got = t
|
||||
.get_by_key(7)
|
||||
.unwrap()
|
||||
.expect("new row should be indexed by id");
|
||||
assert_eq!(got["name"], json!("Zoe"));
|
||||
assert_eq!(t.scan().unwrap().len(), 2);
|
||||
}
|
||||
@@ -239,8 +310,11 @@ fn delete_by_keys_removes_rows() {
|
||||
.unwrap();
|
||||
|
||||
for i in 1..=5 {
|
||||
t.insert_row(&row(&[("id", json!(i)), ("name", json!(format!("u{}", i)))]))
|
||||
.unwrap();
|
||||
t.insert_row(&row(&[
|
||||
("id", json!(i)),
|
||||
("name", json!(format!("u{}", i))),
|
||||
]))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let deleted = t.delete_by_keys(&[2, 4, 99]).unwrap();
|
||||
@@ -254,7 +328,10 @@ fn delete_by_keys_removes_rows() {
|
||||
.collect();
|
||||
assert_eq!(ids, vec![1, 3, 5], "remaining rows must keep their order");
|
||||
|
||||
assert!(t.get_by_key(2).unwrap().is_none(), "index entry should be deleted too");
|
||||
assert!(
|
||||
t.get_by_key(2).unwrap().is_none(),
|
||||
"index entry should be deleted too"
|
||||
);
|
||||
assert!(t.get_by_key(4).unwrap().is_none());
|
||||
assert!(t.get_by_key(3).unwrap().is_some());
|
||||
|
||||
@@ -284,10 +361,7 @@ fn insert_rows_batch() {
|
||||
assert_eq!(t.scan().unwrap().len(), 20);
|
||||
assert_eq!(t.get_by_key(20).unwrap().unwrap()["name"], json!("u20"));
|
||||
|
||||
let dup = vec![
|
||||
row(&[("id", json!(100))]),
|
||||
row(&[("id", json!(100))]),
|
||||
];
|
||||
let dup = vec![row(&[("id", json!(100))]), row(&[("id", json!(100))])];
|
||||
assert!(t.insert_rows(&dup).is_err());
|
||||
}
|
||||
|
||||
@@ -320,16 +394,23 @@ fn secondary_index_is_maintained() {
|
||||
)
|
||||
.unwrap();
|
||||
assert!(t.has_index("code"));
|
||||
assert_eq!(t.get_by_column_key("code", 7002).unwrap().unwrap()["id"], json!(2));
|
||||
assert_eq!(
|
||||
t.get_by_column_key("code", 7002).unwrap().unwrap()["id"],
|
||||
json!(2)
|
||||
);
|
||||
|
||||
t.insert_row(&row(&[("id", json!(3)), ("code", json!(7003))]))
|
||||
.unwrap();
|
||||
assert_eq!(t.get_by_column_key("code", 7003).unwrap().unwrap()["id"], json!(3));
|
||||
assert_eq!(
|
||||
t.get_by_column_key("code", 7003).unwrap().unwrap()["id"],
|
||||
json!(3)
|
||||
);
|
||||
|
||||
t.delete_by_keys(&[2]).unwrap();
|
||||
assert!(t.get_by_column_key("code", 7002).unwrap().is_none());
|
||||
|
||||
t.insert_row(&row(&[("id", json!(4)), ("code", json!(7001))])).unwrap();
|
||||
t.insert_row(&row(&[("id", json!(4)), ("code", json!(7001))]))
|
||||
.unwrap();
|
||||
let mut dup_ids: Vec<i64> = t
|
||||
.get_all_by_column_key("code", 7001)
|
||||
.unwrap()
|
||||
@@ -339,7 +420,12 @@ fn secondary_index_is_maintained() {
|
||||
dup_ids.sort_unstable();
|
||||
assert_eq!(dup_ids, vec![1, 4], "duplicate index values keep both rows");
|
||||
|
||||
let mut t2 = HeapTable::open(dir.path().join("d.db"), DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap();
|
||||
let mut t2 = HeapTable::open(
|
||||
dir.path().join("d.db"),
|
||||
DEFAULT_PAGE_SIZE,
|
||||
DEFAULT_POOL_SIZE,
|
||||
)
|
||||
.unwrap();
|
||||
t2.insert_rows(&[
|
||||
row(&[("id", json!(1)), ("age", json!(30))]),
|
||||
row(&[("id", json!(2)), ("age", json!(30))]),
|
||||
@@ -360,7 +446,6 @@ fn secondary_index_is_maintained() {
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
/// 整表整理要收回删行留下的页内空洞
|
||||
#[test]
|
||||
fn compact_pages_reclaims_dead_space() {
|
||||
@@ -370,7 +455,8 @@ fn compact_pages_reclaims_dead_space() {
|
||||
|
||||
let pad = "x".repeat(200);
|
||||
for i in 0..10 {
|
||||
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap();
|
||||
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))]))
|
||||
.unwrap();
|
||||
}
|
||||
let pages = t.num_pages().unwrap();
|
||||
|
||||
@@ -395,7 +481,8 @@ fn deletes_compact_pages_so_space_is_reused() {
|
||||
|
||||
let pad = "y".repeat(80);
|
||||
for i in 0..30 {
|
||||
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap();
|
||||
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))]))
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(t.num_pages().unwrap(), 1, "30 行要先塞进一页");
|
||||
|
||||
@@ -404,7 +491,8 @@ fn deletes_compact_pages_so_space_is_reused() {
|
||||
t.delete_by_keys(&keys).unwrap();
|
||||
|
||||
for i in 100..119 {
|
||||
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap();
|
||||
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))]))
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(t.num_pages().unwrap(), 1, "删掉的空间要能被复用");
|
||||
assert_eq!(t.scan().unwrap().len(), 30);
|
||||
@@ -416,11 +504,13 @@ fn compact_pages_keeps_index_positions_valid() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let data = dir.path().join("t.db");
|
||||
let index = dir.path().join("t.idx");
|
||||
let mut t = HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
|
||||
let mut t =
|
||||
HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
|
||||
|
||||
let pad = "z".repeat(80);
|
||||
for i in 0..30 {
|
||||
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap();
|
||||
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))]))
|
||||
.unwrap();
|
||||
}
|
||||
let keys: Vec<i64> = (5..25).collect();
|
||||
t.delete_by_keys(&keys).unwrap();
|
||||
@@ -428,9 +518,17 @@ fn compact_pages_keeps_index_positions_valid() {
|
||||
|
||||
for i in [0, 1, 2, 3, 4, 25, 26, 27, 28, 29] {
|
||||
let found = t.get_by_column_key("id", i).unwrap();
|
||||
assert_eq!(found.as_ref().map(|r| r["pad"].clone()), Some(json!(pad)), "整理后 id={} 仍要能点查", i);
|
||||
assert_eq!(
|
||||
found.as_ref().map(|r| r["pad"].clone()),
|
||||
Some(json!(pad)),
|
||||
"整理后 id={} 仍要能点查",
|
||||
i
|
||||
);
|
||||
}
|
||||
for i in 5..25 {
|
||||
assert!(t.get_by_column_key("id", i).unwrap().is_none(), "删掉的行不该再查出来");
|
||||
assert!(
|
||||
t.get_by_column_key("id", i).unwrap().is_none(),
|
||||
"删掉的行不该再查出来"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -142,3 +142,147 @@ fn dirty_page_flushed_on_drop() {
|
||||
let p = pf2.read_page(0).unwrap();
|
||||
assert_eq!(p.data[0], 42);
|
||||
}
|
||||
|
||||
/// 已同步镜像恢复撕裂页且页 LSN 持久保留
|
||||
#[test]
|
||||
fn full_page_image_repairs_torn_write_before_read() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("torn.db");
|
||||
let metadata = dir.path().join("torn.db.pages");
|
||||
let journal = dir.path().join("torn.db.page-wal");
|
||||
let mut file = PageFile::with_options(&path, 512, 1).unwrap();
|
||||
file.set_lsn(7);
|
||||
file.write_page(&Page::new(0, 512)).unwrap();
|
||||
file.flush().unwrap();
|
||||
let old_metadata = std::fs::read(&metadata).unwrap();
|
||||
file.set_lsn(13);
|
||||
let mut changed = Page::new(0, 512);
|
||||
changed.data.fill(0x5a);
|
||||
file.write_page(&changed).unwrap();
|
||||
file.append_page().unwrap(); // 淘汰页零前镜像已同步
|
||||
let image = std::fs::read(&journal).unwrap();
|
||||
assert!(!image.is_empty());
|
||||
drop(file);
|
||||
std::fs::write(&metadata, old_metadata).unwrap();
|
||||
std::fs::write(&journal, &image).unwrap();
|
||||
std::fs::write(&path, [0x11; 213]).unwrap();
|
||||
let mut reopened = PageFile::open(&path, 512).unwrap();
|
||||
assert_eq!(reopened.read_page(0).unwrap().data, changed.data);
|
||||
assert_eq!(reopened.page_lsn(0), Some(13));
|
||||
drop(reopened);
|
||||
assert_eq!(std::fs::metadata(&journal).unwrap().len(), 0);
|
||||
let mut reopened = PageFile::open(&path, 512).unwrap();
|
||||
assert_eq!(reopened.read_page(0).unwrap().data, changed.data);
|
||||
assert_eq!(reopened.page_lsn(0), Some(13));
|
||||
}
|
||||
|
||||
/// 撕裂日志尾不发布,完整损坏帧明确拒绝
|
||||
#[test]
|
||||
fn page_image_tail_and_corruption_are_distinguished() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tail.db");
|
||||
let metadata = dir.path().join("tail.db.pages");
|
||||
let journal = dir.path().join("tail.db.page-wal");
|
||||
let mut file = PageFile::with_options(&path, 512, 1).unwrap();
|
||||
file.write_page(&Page::new(0, 512)).unwrap();
|
||||
file.flush().unwrap();
|
||||
let old_metadata = std::fs::read(&metadata).unwrap();
|
||||
let old_data = std::fs::read(&path).unwrap();
|
||||
file.set_lsn(20);
|
||||
let mut changed = Page::new(0, 512);
|
||||
changed.data.fill(9);
|
||||
file.write_page(&changed).unwrap();
|
||||
file.append_page().unwrap();
|
||||
let image = std::fs::read(&journal).unwrap();
|
||||
drop(file);
|
||||
for end in [1, 39, image.len() - 1] {
|
||||
std::fs::write(&metadata, &old_metadata).unwrap();
|
||||
std::fs::write(&path, &old_data).unwrap();
|
||||
std::fs::write(&journal, &image[..end]).unwrap();
|
||||
let mut reopened = PageFile::open(&path, 512).unwrap();
|
||||
assert_eq!(reopened.read_page(0).unwrap().data, vec![0; 512]);
|
||||
}
|
||||
let mut corrupt = image;
|
||||
corrupt[40] ^= 1;
|
||||
std::fs::write(&metadata, &old_metadata).unwrap();
|
||||
std::fs::write(&journal, &corrupt).unwrap();
|
||||
assert!(
|
||||
PageFile::open(&path, 512)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("checksum")
|
||||
);
|
||||
assert_eq!(std::fs::read(&journal).unwrap(), corrupt);
|
||||
}
|
||||
|
||||
/// 截断标记阻止旧页重放后复活
|
||||
#[test]
|
||||
fn truncate_clears_page_versions_and_survives_reopen() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("reset.db");
|
||||
let mut file = PageFile::open(&path, 512).unwrap();
|
||||
file.set_lsn(5);
|
||||
for _ in 0..3 {
|
||||
file.append_page().unwrap();
|
||||
}
|
||||
file.flush().unwrap();
|
||||
file.set_lsn(8);
|
||||
file.truncate().unwrap();
|
||||
file.append_page().unwrap();
|
||||
file.flush().unwrap();
|
||||
assert_eq!(file.page_lsn(0), Some(8));
|
||||
assert_eq!(file.page_lsn(2), None);
|
||||
drop(file);
|
||||
let mut file = PageFile::open(&path, 512).unwrap();
|
||||
assert_eq!(file.num_pages().unwrap(), 1);
|
||||
assert_eq!(file.page_lsn(0), Some(8));
|
||||
assert_eq!(file.page_lsn(2), None);
|
||||
}
|
||||
|
||||
/// 镜像写入失败后禁止析构或重试绕过恢复
|
||||
#[test]
|
||||
fn page_journal_error_fences_further_flushes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("fenced.db");
|
||||
let journal = dir.path().join("fenced.db.page-wal");
|
||||
std::fs::write(&path, [0; 512]).unwrap();
|
||||
let mut file = PageFile::open(&path, 512).unwrap();
|
||||
let mut page = Page::new(0, 512);
|
||||
page.data.fill(7);
|
||||
file.write_page(&page).unwrap();
|
||||
std::fs::create_dir(&journal).unwrap();
|
||||
assert!(file.flush().is_err());
|
||||
std::fs::remove_dir(&journal).unwrap();
|
||||
assert!(
|
||||
file.flush()
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("requires recovery")
|
||||
);
|
||||
drop(file);
|
||||
let mut file = PageFile::open(&path, 512).unwrap();
|
||||
assert_eq!(file.read_page(0).unwrap().data, vec![0; 512]);
|
||||
}
|
||||
|
||||
/// 小缓存淘汰不会反复记录同一批未改页
|
||||
#[test]
|
||||
fn eviction_batches_images_without_pool_sized_write_amplification() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("bounded.db");
|
||||
let journal = dir.path().join("bounded.db.page-wal");
|
||||
let mut file = PageFile::with_options(&path, 512, 2).unwrap();
|
||||
file.set_lsn(9);
|
||||
for id in 0..100 {
|
||||
let mut page = Page::new(id, 512);
|
||||
page.data.fill(id as u8);
|
||||
file.write_page(&page).unwrap();
|
||||
}
|
||||
assert!(std::fs::metadata(&journal).unwrap().len() <= 100 * (512 + 40));
|
||||
file.flush().unwrap();
|
||||
drop(file);
|
||||
let mut file = PageFile::open(&path, 512).unwrap();
|
||||
for id in 0..100 {
|
||||
assert_eq!(file.read_page(id).unwrap().data, vec![id as u8; 512]);
|
||||
assert_eq!(file.page_lsn(id), Some(9));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,5 +1,5 @@
|
||||
use chusql_core_storage::protocol::Row;
|
||||
use chusql_core_storage::wal::{committed_ops, records_after, Wal, WalOp};
|
||||
use chusql_core_storage::wal::{Wal, WalOp, committed_ops, records_after};
|
||||
use serde_json::json;
|
||||
|
||||
// WAL 测试:追加、读取、清空、截断尾巴、LSN 与提交标记。
|
||||
@@ -11,12 +11,32 @@ fn schema_versions_round_trip_and_legacy_hide_decodes() {
|
||||
let path = dir.path().join("wal.log");
|
||||
let wal = Wal::new(&path);
|
||||
let ops = [
|
||||
WalOp::HideColumn { table: "t".into(), column: "name".into(), schema_version: Some(u64::MAX) },
|
||||
WalOp::DropColumn { table: "t".into(), column: "name".into(), rows: vec![], schema_version: Some(3) },
|
||||
WalOp::ReplaceSchema { table: "t".into(), columns: vec![], rows: vec![], schema_version: Some(4) },
|
||||
WalOp::HideColumn {
|
||||
table: "t".into(),
|
||||
column: "name".into(),
|
||||
schema_version: Some(u64::MAX),
|
||||
},
|
||||
WalOp::DropColumn {
|
||||
table: "t".into(),
|
||||
column: "name".into(),
|
||||
rows: vec![],
|
||||
schema_version: Some(3),
|
||||
},
|
||||
WalOp::ReplaceSchema {
|
||||
table: "t".into(),
|
||||
columns: vec![],
|
||||
rows: vec![],
|
||||
schema_version: Some(4),
|
||||
},
|
||||
];
|
||||
wal.append_group(&ops).unwrap();
|
||||
assert_eq!(committed_ops(&wal.read_all().unwrap().records).into_iter().map(|(_, op)| op).collect::<Vec<_>>(), ops);
|
||||
assert_eq!(
|
||||
committed_ops(&wal.read_all().unwrap().records)
|
||||
.into_iter()
|
||||
.map(|(_, op)| op)
|
||||
.collect::<Vec<_>>(),
|
||||
ops
|
||||
);
|
||||
drop(wal);
|
||||
let mut body = 1_u64.to_le_bytes().to_vec();
|
||||
body.push(8);
|
||||
@@ -26,9 +46,14 @@ fn schema_versions_round_trip_and_legacy_hide_decodes() {
|
||||
let mut frame = (body.len() as u32).to_le_bytes().to_vec();
|
||||
frame.extend_from_slice(&body);
|
||||
std::fs::write(&path, frame).unwrap();
|
||||
assert_eq!(Wal::new(&path).read().unwrap(), Some(WalOp::HideColumn {
|
||||
table: "t".into(), column: "name".into(), schema_version: None,
|
||||
}));
|
||||
assert_eq!(
|
||||
Wal::new(&path).read().unwrap(),
|
||||
Some(WalOp::HideColumn {
|
||||
table: "t".into(),
|
||||
column: "name".into(),
|
||||
schema_version: None,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
/// 建表编号完整往返且兼容旧载荷
|
||||
@@ -37,7 +62,11 @@ fn create_table_ids_round_trip_and_legacy_payloads_decode() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("wal.log");
|
||||
let wal = Wal::new(&path);
|
||||
let op = WalOp::CreateTable { table: "t".into(), columns: vec![], object_id: Some(42) };
|
||||
let op = WalOp::CreateTable {
|
||||
table: "t".into(),
|
||||
columns: vec![],
|
||||
object_id: Some(42),
|
||||
};
|
||||
wal.append(&op).unwrap();
|
||||
assert_eq!(wal.read_all().unwrap().records[0].1, op);
|
||||
drop(wal);
|
||||
@@ -50,11 +79,16 @@ fn create_table_ids_round_trip_and_legacy_payloads_decode() {
|
||||
frame.extend_from_slice(&body);
|
||||
std::fs::write(&path, frame).unwrap();
|
||||
let wal = Wal::new(&path);
|
||||
assert_eq!(wal.read_all().unwrap().records[0].1,
|
||||
WalOp::CreateTable { table: "t".into(), columns: vec![], object_id: None });
|
||||
assert_eq!(
|
||||
wal.read_all().unwrap().records[0].1,
|
||||
WalOp::CreateTable {
|
||||
table: "t".into(),
|
||||
columns: vec![],
|
||||
object_id: None
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
/// 用键值对构造一行
|
||||
fn row(pairs: &[(&str, serde_json::Value)]) -> Row {
|
||||
let mut m = Row::new();
|
||||
@@ -167,7 +201,10 @@ fn every_record_gets_an_increasing_lsn() {
|
||||
let scan = wal.read_all().unwrap();
|
||||
let lsns: Vec<u64> = scan.records.iter().map(|(lsn, _)| *lsn).collect();
|
||||
assert_eq!(lsns, vec![1, 2, 3]);
|
||||
assert_eq!(scan.good_len as u64, std::fs::metadata(&path).unwrap().len());
|
||||
assert_eq!(
|
||||
scan.good_len as u64,
|
||||
std::fs::metadata(&path).unwrap().len()
|
||||
);
|
||||
|
||||
let reopened = Wal::new(&path);
|
||||
reopened
|
||||
@@ -254,7 +291,10 @@ fn trailing_torn_frame_can_be_truncated() {
|
||||
assert!(scan.good_len < full.len() - 3);
|
||||
|
||||
wal.truncate_to(scan.good_len).unwrap();
|
||||
assert_eq!(std::fs::metadata(&path).unwrap().len(), scan.good_len as u64);
|
||||
assert_eq!(
|
||||
std::fs::metadata(&path).unwrap().len(),
|
||||
scan.good_len as u64
|
||||
);
|
||||
assert_eq!(wal.read_all().unwrap().records.len(), 1);
|
||||
}
|
||||
|
||||
@@ -360,9 +400,21 @@ fn lsn_continues_from_the_checkpoint() {
|
||||
#[test]
|
||||
fn records_after_keeps_only_the_tail() {
|
||||
let records = vec![
|
||||
(1u64, WalOp::Insert { table: "t".into(), row: row(&[("id", json!(1))]) }),
|
||||
(
|
||||
1u64,
|
||||
WalOp::Insert {
|
||||
table: "t".into(),
|
||||
row: row(&[("id", json!(1))]),
|
||||
},
|
||||
),
|
||||
(2, WalOp::Commit),
|
||||
(3, WalOp::Insert { table: "t".into(), row: row(&[("id", json!(2))]) }),
|
||||
(
|
||||
3,
|
||||
WalOp::Insert {
|
||||
table: "t".into(),
|
||||
row: row(&[("id", json!(2))]),
|
||||
},
|
||||
),
|
||||
(4, WalOp::Commit),
|
||||
];
|
||||
let kept = records_after(&records, 2);
|
||||
@@ -371,4 +423,3 @@ fn records_after_keeps_only_the_tail() {
|
||||
assert_eq!(records_after(&records, 4).len(), 0);
|
||||
assert_eq!(records_after(&records, 0).len(), 4);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user