修复错误,优化性能,实现类型系统,优化安装卸载过程

This commit is contained in:
LIm1tZz7 committed 2026-10-05 00:15:01 +08:00
1 parent fd5f320360
commit 48eec484b7
104 files changed
+4127 -845

No files matched your search

@@ -1,7 +1,7 @@
# ChuSQL 存储服务配置示例
#
# 用法:安装时把本文件放到固定的用户配置目录(Windows 是 %APPDATA%\ChuSQL\chusql.toml,
# 其它平台是 ${XDG_CONFIG_HOME:-$HOME/.config}/ChuSQL/chusql.toml);
# 用法:安装时把本文件放到固定的用户配置目录(Windows 是 %APPDATA%\ChuSQL\settings.toml,
# 其它平台是 ${XDG_CONFIG_HOME:-$HOME/.config}/ChuSQL/settings.toml);
# 临时换一份就用命令行参数 --config <path>(没有环境变量通道了)。
#
# 每一项都可以删掉:没写的项回退到内置默认值(下面注释里的值就是默认值)。
@@ -28,6 +28,9 @@ pool_size = 1024
# Unix ${XDG_DATA_HOME:-$HOME/.local/share}/chusql/data,跟安装脚本装的位置一致);
# 要固定就写绝对路径,例如:
# data_dir = "/srv/chusql/data"
#
# 日志输出目录:启动器(csql-web)与安装脚本按它决定日志写哪;相对路径以启动目录为基准。
# log_files = "./logs"
[log]
# 日志等级:off / error / warn / info / debug。debug 会多打每个请求和连接。
+22 -9
View File
@@ -1,5 +1,5 @@
use std::collections::hash_map::DefaultHasher;
use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::hash::{Hash, Hasher};
use std::io;
use std::io::Write;
@@ -51,9 +51,11 @@ pub struct TableSchema {
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct Catalog {
#[serde(default)]
pub identity_high_water: i64,
tables: BTreeMap<String, TableSchema>,
#[serde(skip)]
seen: BTreeMap<String, HashSet<u64>>,
seen: BTreeMap<String, HashMap<u64, u64>>,
}
impl Catalog {
@@ -153,17 +155,25 @@ impl Catalog {
for r in rows {
for (name, v) in r {
let stat = entry.stats.entry(name.clone()).or_default();
hist_remove(stat, v);
if stat.capped {
continue;
}
if seen.remove(&fold(name, value_hash(v))) {
stat.distinct = stat.distinct.saturating_sub(1);
let key = fold(name, value_hash(v));
if let Some(count) = seen.get_mut(&key) {
*count = count.saturating_sub(1);
if *count == 0 {
seen.remove(&key);
stat.distinct = stat.distinct.saturating_sub(1);
}
}
hist_remove(stat, v);
}
}
if entry.row_count == 0 {
seen.clear();
for stat in entry.stats.values_mut() {
stat.distinct = 0;
stat.capped = false;
stat.lo = None;
stat.hi = None;
stat.hist.clear();
@@ -192,17 +202,18 @@ impl Catalog {
let seen = self.seen.entry(table.to_string()).or_default();
for (name, v) in row {
let stat = entry.stats.entry(name.clone()).or_default();
hist_add(stat, v);
if stat.capped {
continue;
}
if seen.insert(fold(name, value_hash(v))) {
let count = seen.entry(fold(name, value_hash(v))).or_insert(0);
*count += 1;
if *count == 1 {
stat.distinct += 1;
if stat.distinct >= STATS_DISTINCT_CAP {
stat.capped = true;
seen.clear();
}
}
hist_add(stat, v);
}
}
@@ -476,5 +487,7 @@ fn hist_remove(stat: &mut ColumnStat, v: &serde_json::Value) {
return;
}
let index = hist_index(lo, hi, x);
stat.hist[index] = stat.hist[index].saturating_sub(1);
if let Some(bucket) = (0..stat.hist.len()).filter(|&i| stat.hist[i] > 0).min_by_key(|&i| i.abs_diff(index)) {
stat.hist[bucket] -= 1;
}
}
+19 -4
View File
@@ -8,7 +8,7 @@ use crate::log::Level;
// 存储相关分区由本模块解析校验,其它层的分区一概不碰。
/// 配置文件名
pub const CONFIG_FILE_NAME: &str = "chusql.toml";
pub const CONFIG_FILE_NAME: &str = "settings.toml";
/// 配置目录名(随系统惯例放在用户配置目录下)
pub const APP_DIR_NAME: &str = "ChuSQL";
/// Unix 下的数据目录名(XDG 惯例小写,跟安装脚本的默认安装目录一致)
@@ -20,6 +20,8 @@ pub const DEFAULT_POOL_SIZE: usize = 1024;
pub const DEFAULT_PAGE_SIZE: usize = 4096;
pub const DEFAULT_BTREE_ORDER: usize = 4;
pub const DEFAULT_LOG_LEVEL: &str = "info";
/// 默认日志输出目录(相对启动目录)
pub const DEFAULT_LOG_FILES: &str = "./logs";
const MIN_PAGE_SIZE: usize = 512;
const MAX_PAGE_SIZE: usize = 65536;
@@ -91,6 +93,7 @@ pub struct Config {
pub btree_order: usize,
pub pool_size: usize,
pub data_dir: PathBuf,
pub log_files: PathBuf,
pub log_level: Level,
}
@@ -102,6 +105,7 @@ impl Default for Config {
btree_order: DEFAULT_BTREE_ORDER,
pool_size: DEFAULT_POOL_SIZE,
data_dir: default_data_dir(),
log_files: PathBuf::from(DEFAULT_LOG_FILES),
log_level: Level::Info,
}
}
@@ -161,6 +165,7 @@ struct FileBtree {
#[serde(deny_unknown_fields)]
struct FileStorage {
data_dir: Option<String>,
log_files: Option<String>,
}
#[derive(Debug, Default, Deserialize)]
@@ -229,6 +234,11 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
default_data_dir().to_string_lossy().into_owned(),
&file_origin,
);
let (log_files, log_files_origin) = pick(
file.storage.as_ref().and_then(|s| s.log_files.clone()),
DEFAULT_LOG_FILES.to_string(),
&file_origin,
);
// 管名已经没有了:存储不再是独立进程,管道/套接字那一层随 FFI 一起删掉
let (log_level, log_origin) = pick(
file.log.and_then(|l| l.level),
@@ -237,6 +247,7 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
);
let data_dir = data_dir.trim().to_string();
let log_files = log_files.trim().to_string();
let log_level = log_level.trim().to_string();
let log_level = Level::parse(&log_level)
@@ -245,6 +256,9 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
if data_dir.is_empty() {
return Err("storage.data_dir must not be empty".to_string());
}
if log_files.is_empty() {
return Err("storage.log_files must not be empty".to_string());
}
validate_layout(page_size, btree_order)?;
if pool_size == 0 {
return Err("buffer.pool_size must be at least 1".to_string());
@@ -254,6 +268,7 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
("btree.order", btree_order.to_string(), order_origin),
("buffer.pool_size", pool_size.to_string(), pool_origin),
("storage.data_dir", data_dir.clone(), dir_origin),
("storage.log_files", log_files.clone(), log_files_origin),
("log.level", log_level.name().to_string(), log_origin),
];
@@ -263,6 +278,7 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
btree_order,
pool_size,
data_dir: PathBuf::from(data_dir),
log_files: PathBuf::from(log_files),
log_level,
},
config_path: if text.is_some() { path } else { None },
@@ -318,15 +334,14 @@ size = 8192
data_dir = "../data"
[server]
host = "127.0.0.1"
listen_host = "127.0.0.1"
port = 7777
max_message = 1048576
[web]
port = 7778
user = "root"
"#;
let loaded = resolve(Some(text), Some(PathBuf::from("chusql.toml"))).unwrap();
let loaded = resolve(Some(text), Some(PathBuf::from("settings.toml"))).unwrap();
assert_eq!(loaded.config.page_size, 8192);
assert_eq!(loaded.config.data_dir, PathBuf::from("../data"));
}
+27
View File
@@ -71,6 +71,33 @@ pub unsafe extern "C" fn chusql_storage_close(handle: *mut Storage) {
let _ = catch_unwind(AssertUnwindSafe(|| unsafe { drop(Box::from_raw(handle)) }));
}
/// 离线维护系统目录并返回 JSON
///
/// # Safety
/// 输入为有效字符串;输出指针可写,响应使用 free 释放。
#[unsafe(no_mangle)]
pub unsafe extern "C" fn chusql_storage_maintenance(
config_path: *const c_char, request: *const u8, request_len: usize,
out: *mut *mut u8, out_len: *mut usize,
) -> i32 {
if request.is_null() || out.is_null() || out_len.is_null() { set_error("invalid maintenance argument"); return ERR; }
unsafe { *out = ptr::null_mut(); *out_len = 0; }
let result = catch_unwind(AssertUnwindSafe(|| {
let path = if config_path.is_null() { None } else { Some(unsafe { CStr::from_ptr(config_path) }.to_str().map_err(|e| e.to_string())?) };
let text = std::str::from_utf8(unsafe { slice::from_raw_parts(request, request_len) }).map_err(|e| e.to_string())?;
Storage::maintenance(path, text)
}));
let response = match result {
Ok(Ok(response)) => response,
Ok(Err(err)) => { set_error(err); return ERR; }
Err(_) => { set_error("panic in system maintenance"); return ERR; }
};
let mut buffer = response.into_bytes().into_boxed_slice();
unsafe { *out = buffer.as_mut_ptr(); *out_len = buffer.len(); }
std::mem::forget(buffer);
OK
}
/// 处理一条请求,响应缓冲须归还
///
/// # Safety
+22 -2
View File
@@ -6,6 +6,8 @@ use std::collections::HashMap;
pub type Row = HashMap<String, serde_json::Value>;
pub const USERS_TABLE: &str = "__system_users";
pub const IDENTITIES_TABLE: &str = "__system_identities";
pub const TYPES_TABLE: &str = "__system_types";
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Account {
@@ -17,6 +19,16 @@ pub struct Account {
pub registered_at: String,
#[serde(default)]
pub last_login_at: Option<String>,
#[serde(default = "default_true")]
pub can_login: bool,
#[serde(default)]
pub is_superuser: bool,
#[serde(default)]
pub system_catalog_manager: bool,
#[serde(default = "default_true")]
pub enabled: bool,
#[serde(default)]
pub identity_version: u8,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
@@ -80,7 +92,11 @@ pub struct IndexWire {
#[derive(Debug, Serialize, Deserialize)]
#[serde(tag = "method", rename_all = "snake_case")]
pub enum Request {
CatalogManage { command: Box<Request> },
AccountsList,
IdentityInitialize { administrator: String },
RoleCreate { user: String },
IdentityAlter { user: String, can_login: Option<bool>, is_superuser: Option<bool>, enabled: Option<bool>, system_catalog_manager: Option<bool> },
AccountCreate { user: String, password_hash: String },
AccountReset { user: String, password_hash: String },
AccountLogin {
@@ -97,6 +113,7 @@ pub enum Request {
password_hash: Option<String>,
},
SystemStatus,
BootstrapTypes { types: Vec<Row> },
Ping,
Scan {
table: String,
@@ -222,9 +239,11 @@ impl Request {
| Self::DropIndex { table, .. } | Self::DropColumn { table, .. }
| Self::Compact { table }
| Self::ReplaceSchema { table, .. } => Some(table),
Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList
Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList | Self::CatalogManage { .. }
| Self::IdentityInitialize { .. } | Self::RoleCreate { .. } | Self::IdentityAlter { .. }
| Self::AccountCreate { .. } | Self::AccountReset { .. } | Self::AccountLogin { .. }
| Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus
| Self::BootstrapTypes { .. }
| Self::ApplyTransaction { .. } => None,
}
}
@@ -245,7 +264,8 @@ pub const PRIVILEGE_TABLES: [&str; 4] = [
/// 是不是服务自己经请求通道读写的系统表
pub fn internal_table(table: &str) -> bool {
PRIVILEGE_TABLES.contains(&table.to_ascii_lowercase().as_str())
let name = table.to_ascii_lowercase();
PRIVILEGE_TABLES.contains(&name.as_str()) || name.starts_with("__system_domain_")
}
/// 是不是请求通道要拒绝的系统表
+529 -30
View File
@@ -8,8 +8,8 @@ use crate::config::{self, Config, Loaded};
use crate::heap::{scalar_int, HeapTable};
use crate::log;
use crate::protocol::{
blocked_table, reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row,
SchemaColumn, StorageOp, TableSchemaWire, USERS_TABLE,
blocked_table, reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row, IDENTITIES_TABLE,
SchemaColumn, StorageOp, TableSchemaWire, USERS_TABLE, TYPES_TABLE,
};
use crate::wal::{committed_ops, records_after, Wal, WalOp};
use crate::{log_debug, log_error, log_info, log_warn};
@@ -222,15 +222,41 @@ impl Server {
c.rebuild_stats(table, rows);
c.save(self.catalog_path())
}
WalOp::Accounts { accounts } => {
WalOp::Accounts { accounts, removed, migrate_roles } => {
let rows = account_rows(accounts)?;
self.tables.lock().map_err(|_| std::io::Error::other("table lock poisoned"))?.remove(USERS_TABLE);
for path in [self.table_path(USERS_TABLE), self.index_path(USERS_TABLE, "id")] {
std::fs::File::create(path)?.sync_all()?;
let identities = identity_rows(accounts)?;
for (table, records) in [(USERS_TABLE, &rows), (IDENTITIES_TABLE, &identities)] {
self.tables.lock().map_err(|_| std::io::Error::other("table lock poisoned"))?.remove(table);
for path in [self.table_path(table), self.index_path(table, "id")] {
std::fs::File::create(path)?.sync_all()?;
}
self.with_table(table, |t| { t.replace_all(records)?; t.flush() })?;
}
let legacy_roles = "__system_roles";
let has_roles = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?.describe(legacy_roles).is_some();
if has_roles && *migrate_roles {
self.with_existing_table(legacy_roles, |t| { t.replace_all(&[])?; t.flush() })?;
}
let mut cleaned = Vec::new();
for table in ["__system_grants", "__system_grant_options", "__system_members"] {
let exists = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?.describe(table).is_some();
if exists && !removed.is_empty() {
let rows = self.with_existing_table(table, |t| t.scan())?.into_iter().filter(|r|
!["role", "member"].iter().any(|field| r.get(*field).and_then(serde_json::Value::as_str).is_some_and(|name| removed.iter().any(|old| old == name)))).collect::<Vec<_>>();
self.with_existing_table(table, |t| { t.replace_all(&rows)?; t.flush() })?;
cleaned.push((table, rows));
}
}
self.with_table(USERS_TABLE, |t| { t.replace_all(&rows)?; t.flush() })?;
let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
declare_system_schema(&mut c, &rows)?;
c.rebuild_stats(IDENTITIES_TABLE, &identities);
c.set_columns(IDENTITIES_TABLE, identity_columns())?;
c.rebuild_stats(IDENTITIES_TABLE, &identities);
c.mark_system(IDENTITIES_TABLE)?;
c.add_index(IDENTITIES_TABLE, "user")?;
c.identity_high_water = c.identity_high_water.max(accounts.iter().map(|a| a.id).max().unwrap_or(0));
if has_roles && *migrate_roles { c.rebuild_stats(legacy_roles, &[]); }
for (table, records) in cleaned { c.rebuild_stats(table, &records); }
c.save(self.catalog_path())
}
WalOp::Insert { table, row } => {
@@ -284,6 +310,7 @@ impl Server {
self.with_table(table, |_t| Ok(()))?;
let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
c.create_table(table, columns.clone())?;
if table == TYPES_TABLE { c.mark_system(table)?; }
c.save(self.catalog_path())
}
WalOp::DropTable { table } => {
@@ -342,8 +369,67 @@ impl Server {
if self.recovery_required.load(Ordering::Acquire) {
return Err(std::io::Error::other("storage recovery required"));
}
self.validate_op(op)?;
self.group_via_wal(std::slice::from_ref(op))
let op = self.normalize_decimal_write(op)?;
self.validate_op(&op)?;
self.validate_domains(&op)?;
self.group_via_wal(std::slice::from_ref(&op))
}
/// 写日志前规范化定点数值
fn normalize_decimal_write(&self, op: &WalOp) -> std::io::Result<WalOp> {
let mut normalized = op.clone();
let catalog = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
let (table, rows): (&str, Vec<&mut Row>) = match &mut normalized {
WalOp::Insert { table, row } => (table, vec![row]),
WalOp::InsertBatch { table, rows } | WalOp::ReplaceAll { table, rows } => (table, rows.iter_mut().collect()),
WalOp::CreateTable { columns, .. } => {
for column in columns { decimal_parameters(&column.ty)?; }
return Ok(normalized);
}
WalOp::ReplaceSchema { columns, rows, .. } => {
for column in columns.iter() { decimal_parameters(&column.ty)?; }
for row in rows { normalize_decimal_row(row, columns)?; }
return Ok(normalized);
}
_ => return Ok(normalized),
};
if let Some(schema) = catalog.describe(table) {
for row in rows {
normalize_decimal_row(row, &schema.columns)?;
}
}
Ok(normalized)
}
/// 校验命名类型引用与删除依赖
fn validate_domains(&self, op: &WalOp) -> std::io::Result<()> {
let catalog = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
match op {
WalOp::CreateTable { columns, .. } | WalOp::ReplaceSchema { columns, .. } => {
for column in columns {
let mut ty = column.ty.as_str();
while let Some(body) = ty.strip_prefix("domain(").and_then(|s| s.strip_suffix(')')) {
let (name, base) = body.split_once(',').ok_or_else(|| std::io::Error::other("invalid domain type"))?;
let key = format!("__system_domain_{name}");
let definition = catalog.describe(&key).ok_or_else(|| std::io::Error::other(format!("unknown domain: {name}")))?;
if !definition.columns.iter().any(|c| c.name == "base" && c.ty == base) {
return Err(std::io::Error::other(format!("domain base type mismatch: {name}")));
}
ty = base;
}
}
}
WalOp::DropTable { table } => {
if let Some(name) = table.strip_prefix("__system_domain_") {
let prefix = format!("domain({name},");
if catalog.all_tables().iter().any(|(_, schema)| schema.columns.iter().any(|c| !schema.dropped_columns.contains(&c.name) && c.ty.contains(&prefix))) {
return Err(std::io::Error::other(format!("domain is still used by a column: {name}")));
}
}
}
_ => {}
}
Ok(())
}
/// 校验一条操作:命中被删的列就拒绝
@@ -406,6 +492,7 @@ impl Server {
rows: rows.clone(),
},
};
let wal_op = self.normalize_decimal_write(&wal_op)?;
self.validate_op(&wal_op)?;
wal_ops.push(wal_op);
}
@@ -420,13 +507,9 @@ impl Server {
std::fs::create_dir_all(&self.cfg.data_dir)?;
let wal = &self.wal;
let commit_lsn = wal.append_group(ops)?;
self.recovery_required.store(true, Ordering::Release);
for op in ops {
let result = self.apply_op(op);
if result.is_err() && matches!(op, WalOp::HideColumn { .. }) {
self.recovery_required.store(true, Ordering::Release);
return result;
}
result?;
self.apply_op(op)?;
}
for op in ops {
self.flush_op_tables(op)?;
@@ -434,6 +517,7 @@ impl Server {
self.flush_all_tables()?;
wal.write_checkpoint(commit_lsn)?;
wal.truncate_before(commit_lsn)?;
self.recovery_required.store(false, Ordering::Release);
log_debug!(core, "WAL committed group at LSN {}", commit_lsn);
Ok(())
}
@@ -479,15 +563,15 @@ impl Server {
fn recover_wal(&self) -> std::io::Result<()> {
std::fs::create_dir_all(&self.cfg.data_dir)?;
let wal = &self.wal;
let checkpoint = wal.read_checkpoint()?.unwrap_or(0);
let scan = wal.read_all()?;
if scan.records.is_empty() {
if scan.good_len > 0 {
if wal.path().exists() {
wal.truncate_to(scan.good_len)?;
}
return Ok(());
}
// 检查点以内的记录已经落盘,直接按边界截掉
let checkpoint = wal.read_checkpoint()?.unwrap_or(0);
let pending = records_after(&scan.records, checkpoint);
let records = committed_ops(&pending);
if records.is_empty() {
@@ -859,6 +943,12 @@ impl Server {
use std::io::{Error, ErrorKind};
let registered = {
let c = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?;
match c.describe(IDENTITIES_TABLE) {
Some(schema) if !schema.system => return Err(Error::new(ErrorKind::AlreadyExists, "reserved identity table collision")),
None if self.table_path(IDENTITIES_TABLE).exists() || self.index_path(IDENTITIES_TABLE, "id").exists() =>
return Err(Error::new(ErrorKind::AlreadyExists, "reserved identity file collision")),
_ => {}
}
c.describe(USERS_TABLE).map(|schema| schema.system)
};
match registered {
@@ -893,6 +983,21 @@ impl Server {
if !self.table_path(USERS_TABLE).exists() {
return Err(Error::other("missing account data"));
}
let unified = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe(IDENTITIES_TABLE).is_some();
if unified {
let credentials = self.with_existing_table(USERS_TABLE, |t| t.scan())?;
let identities = self.with_existing_table(IDENTITIES_TABLE, |t| t.scan())?;
return identities.into_iter().map(|mut row| {
for field in ["can_login", "is_superuser", "enabled"] {
if row.get(field).and_then(serde_json::Value::as_bool).is_none() { return Err(Error::other("invalid identity attributes")); }
}
let id = row.get("id").and_then(serde_json::Value::as_i64).ok_or_else(|| Error::other("invalid identity record"))?;
let credential = credentials.iter().find(|r| r.get("id").and_then(serde_json::Value::as_i64) == Some(id))
.ok_or_else(|| Error::other("missing identity credential"))?;
row.insert("password_hash".into(), credential.get("password_hash").cloned().ok_or_else(|| Error::other("invalid identity credential"))?);
decode_account_row(&row)
}).collect();
}
let mut accounts: Vec<Account> = self
.with_existing_table(USERS_TABLE, |t| t.scan())?
.iter()
@@ -919,9 +1024,11 @@ impl Server {
// 表里已经有行就不再补人:引导程序只在空表上播种,
// 免得有人拿 storage 透传绕过账号接口往里塞一个已知口令的账号
if accounts.is_empty() && !accounts.iter().any(|a| a.user == name) {
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0)
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0).max(self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.identity_high_water)
.checked_add(1).ok_or_else(|| Error::other("account id exhausted"))?;
accounts.push(new_account(id, &name, password_hash.unwrap_or_default().to_string())?);
let mut administrator = new_account(id, &name, password_hash.unwrap_or_default().to_string())?;
administrator.is_superuser = true;
accounts.push(administrator);
changed = true;
}
}
@@ -931,7 +1038,7 @@ impl Server {
for row in account_rows(&accounts)? {
HeapTable::encode_row(&row, self.cfg.page_size)?;
}
let op = WalOp::Accounts { accounts: accounts.clone() };
let op = WalOp::Accounts { accounts: accounts.clone(), removed: Vec::new(), migrate_roles: false };
self.recovery_required.store(true, Ordering::Release);
self.wal.truncate()?;
self.wal.append(&op)?;
@@ -952,17 +1059,93 @@ impl Server {
return Err(Error::other("system catalog is not initialized; run csql-bootstrap"));
}
let mut accounts = self.load_accounts()?;
let request = match request {
Request::CatalogManage { command } => {
let target = match command.as_ref() {
Request::AccountCreate { user, .. } | Request::AccountReset { user, .. } | Request::AccountDrop { user }
| Request::IdentityAlter { user, is_superuser: None, system_catalog_manager: None, .. } => Some(account_name(user)?),
Request::AccountsList => None,
_ => return Err(Error::other("superuser required for this catalog operation")),
};
if accounts.iter().any(|a| target.as_ref() == Some(&a.user) && (a.is_superuser || a.system_catalog_manager)) {
return Err(Error::other("superuser required to manage privileged identities"));
}
*command
}
other => other,
};
let mut names = std::collections::HashSet::new();
let mut ids = std::collections::HashSet::new();
for account in &mut accounts {
let name = account_name(&account.user)?;
if !names.insert(name.clone()) || account.id < 1 || !ids.insert(account.id) {
return Err(Error::other("invalid or duplicate identity record"));
}
if account.identity_version == 0 { account.user = name; }
}
let previous = accounts.clone();
let migrate_roles = matches!(request, Request::IdentityInitialize { .. });
if !migrate_roles && !matches!(request, Request::AccountsList) {
let legacy_exists = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe("__system_roles").is_some();
if legacy_exists && !self.with_existing_table("__system_roles", |t| t.scan())?.is_empty() {
return Err(Error::other("identity migration required"));
}
}
let had_superuser = accounts.iter().any(|a| a.enabled && a.can_login && a.is_superuser);
match request {
Request::AccountsList => return Ok(accounts),
Request::IdentityInitialize { administrator } => {
for account in &mut accounts {
if account.identity_version == 0 {
account.is_superuser = account.user == administrator.to_ascii_lowercase();
account.identity_version = 1;
}
}
let legacy_roles = "__system_roles";
let exists = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe(legacy_roles).is_some();
if exists {
for row in self.with_existing_table(legacy_roles, |t| t.scan())? {
let name = account_name(row.get("name").and_then(serde_json::Value::as_str).ok_or_else(|| Error::other("invalid legacy role"))?)?;
if accounts.iter().any(|a| a.user == name) { return Err(Error::other(format!("identity name collision: {name}"))); }
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0).max(self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.identity_high_water)
.checked_add(1).ok_or_else(|| Error::other("account id exhausted"))?;
let mut role = new_account(id, &name, String::new())?;
role.can_login = false;
accounts.push(role);
}
}
let unified = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe(IDENTITIES_TABLE).is_some();
if !accounts.is_empty() { require_superuser(&accounts)?; }
if unified && previous == accounts { return Ok(accounts); }
}
Request::AccountCreate { user, password_hash } => {
let name = account_name(&user)?;
if accounts.iter().any(|a| a.user == name) {
return Err(Error::new(ErrorKind::AlreadyExists, "account already exists"));
}
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0)
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0).max(self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.identity_high_water)
.checked_add(1).ok_or_else(|| Error::other("account id exhausted"))?;
accounts.push(new_account(id, &name, password_hash)?);
}
Request::RoleCreate { user } => {
let name = account_name(&user)?;
if accounts.iter().any(|a| a.user == name) { return Err(Error::new(ErrorKind::AlreadyExists, "identity already exists")); }
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0).max(self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.identity_high_water)
.checked_add(1).ok_or_else(|| Error::other("account id exhausted"))?;
let mut role = new_account(id, &name, String::new())?;
role.can_login = false;
accounts.push(role);
}
Request::IdentityAlter { user, can_login, is_superuser, enabled, system_catalog_manager } => {
let name = account_name(&user)?;
let account = accounts.iter_mut().find(|a| a.user == name).ok_or_else(|| Error::new(ErrorKind::NotFound, "unknown account"))?;
account.can_login = can_login.unwrap_or(account.can_login);
account.is_superuser = is_superuser.unwrap_or(account.is_superuser);
account.enabled = enabled.unwrap_or(account.enabled);
account.system_catalog_manager = system_catalog_manager.unwrap_or(account.system_catalog_manager);
account.revision = account.revision.checked_add(1).ok_or_else(|| Error::other("account revision exhausted"))?;
if had_superuser { require_superuser(&accounts)?; }
}
Request::AccountReset { user, password_hash } => {
let name = account_name(&user)?;
validate_account_hash(&password_hash)?;
@@ -976,6 +1159,7 @@ impl Server {
let stamp = at.unwrap_or_else(now_stamp);
let account = accounts.iter_mut().find(|a| a.user == name)
.ok_or_else(|| Error::new(ErrorKind::NotFound, "unknown account"))?;
if !account.enabled || !account.can_login { return Err(Error::other("identity cannot login")); }
account.last_login_at = Some(stamp);
}
Request::AccountDrop { user } => {
@@ -985,13 +1169,15 @@ impl Server {
if accounts.len() == before {
return Err(Error::new(ErrorKind::NotFound, "unknown account"));
}
if had_superuser { require_superuser(&accounts)?; }
}
_ => return Err(Error::other("unexpected account request")),
}
for row in account_rows(&accounts)? {
for row in account_rows(&accounts)?.into_iter().chain(identity_rows(&accounts)?) {
HeapTable::encode_row(&row, self.cfg.page_size)?;
}
let op = WalOp::Accounts { accounts: accounts.clone() };
let removed = previous.iter().filter(|old| !accounts.iter().any(|a| a.user == old.user)).map(|a| a.user.clone()).collect();
let op = WalOp::Accounts { accounts: accounts.clone(), removed, migrate_roles };
self.recovery_required.store(true, Ordering::Release);
self.wal.truncate()?;
self.wal.append(&op)?;
@@ -1001,6 +1187,53 @@ impl Server {
Ok(accounts)
}
/// 初始化并校验内置类型目录
fn bootstrap_types(&self, types: Vec<Row>) -> std::io::Result<Vec<Row>> {
use std::io::Error;
let _guard = self.write_lock.lock().map_err(|_| Error::other("write lock poisoned"))?;
if !self.account_table_ready()? { return Err(Error::other("system catalog is not initialized; run csql-bootstrap")); }
require_superuser(&self.load_accounts()?)?;
let columns = types_columns();
let existing = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe(TYPES_TABLE).cloned();
if existing.as_ref().is_some_and(|s| !s.system || s.columns != columns) { return Err(Error::other("reserved type catalog collision")); }
if existing.is_none() && (self.table_path(TYPES_TABLE).exists() || self.index_path(TYPES_TABLE, "id").exists()) {
return Err(Error::other("reserved type file collision"));
}
let mut rows = if existing.is_some() { self.with_existing_table(TYPES_TABLE, |t| t.scan())? } else { Vec::new() };
let mut names = std::collections::HashSet::new();
let mut ids = std::collections::HashSet::new();
for row in &rows {
let name = type_definition_name(row)?;
let id = row.get("id").and_then(serde_json::Value::as_i64).ok_or_else(|| Error::other("invalid type id"))?;
if id < 1 || !ids.insert(id) || !names.insert(name.to_string()) { return Err(Error::other("duplicate type catalog record")); }
}
let original = rows.clone();
let mut requested = std::collections::HashSet::new();
for mut definition in types {
let name = type_definition_name(&definition)?.to_string();
if definition.len() != 3 || !requested.insert(name.clone()) { return Err(Error::other("invalid or duplicate preinstalled type")); }
if let Some(saved) = rows.iter().find(|row| row.get("name").and_then(serde_json::Value::as_str) == Some(&name)) {
if saved.get("base_type") != definition.get("base_type") || saved.get("parameterized") != definition.get("parameterized") {
return Err(Error::other(format!("preinstalled type definition conflict: {name}")));
}
} else {
let id = rows.iter().filter_map(|r| r.get("id").and_then(serde_json::Value::as_i64)).max().unwrap_or(0)
.checked_add(1).ok_or_else(|| Error::other("type id exhausted"))?;
definition.insert("id".into(), serde_json::json!(id));
HeapTable::encode_row(&definition, self.cfg.page_size)?;
rows.push(definition);
}
}
if existing.is_some() && original == rows { return Ok(rows); }
let mut ops = Vec::new();
if existing.is_none() { ops.push(WalOp::CreateTable { table: TYPES_TABLE.into(), columns }); }
ops.push(WalOp::ReplaceAll { table: TYPES_TABLE.into(), rows: rows.clone() });
self.recovery_required.store(true, Ordering::Release);
self.group_via_wal(&ops)?;
self.recovery_required.store(false, Ordering::Release);
Ok(rows)
}
/// 一条请求翻成一条响应:先过读/写门再分发
fn handle_request(&self, req: Request) -> Response {
if concurrent_read(&req) {
@@ -1037,7 +1270,12 @@ impl Server {
}
}
match req {
request @ (Request::AccountsList | Request::AccountCreate { .. }
Request::BootstrapTypes { types } => match self.bootstrap_types(types) {
Ok(rows) => Response::Rows { rows },
Err(e) => Response::Error { message: e.to_string() },
},
request @ (Request::AccountsList | Request::CatalogManage { .. } | Request::AccountCreate { .. }
| Request::IdentityInitialize { .. } | Request::RoleCreate { .. } | Request::IdentityAlter { .. }
| Request::AccountReset { .. } | Request::AccountLogin { .. }
| Request::AccountDrop { .. }) => {
match self.accounts_request(request) {
@@ -1458,12 +1696,45 @@ impl Server {
}
}
/// 构造内置类型目录列定义
fn types_columns() -> Vec<SchemaColumn> {
vec![
SchemaColumn { name: "id".into(), ty: "int".into(), nullable: false, primary_key: true, ..Default::default() },
SchemaColumn { name: "name".into(), ty: "varchar(64)".into(), nullable: false, unique: true, ..Default::default() },
SchemaColumn { name: "base_type".into(), ty: "varchar(64)".into(), nullable: false, ..Default::default() },
SchemaColumn { name: "parameterized".into(), ty: "bool".into(), nullable: false, ..Default::default() },
]
}
/// 校验类型定义并读取名称
fn type_definition_name(row: &Row) -> std::io::Result<&str> {
let name = row.get("name").and_then(serde_json::Value::as_str).ok_or_else(|| std::io::Error::other("invalid type name"))?;
let base = row.get("base_type").and_then(serde_json::Value::as_str).ok_or_else(|| std::io::Error::other("invalid type definition"))?;
if name.is_empty() || name.len() > 48 || !name.bytes().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == b'_')
|| base.is_empty() || base.len() > 64 || row.get("parameterized").and_then(serde_json::Value::as_bool).is_none() {
return Err(std::io::Error::other("invalid type definition"));
}
Ok(name)
}
/// 账号列表转成行
fn account_rows(accounts: &[Account]) -> std::io::Result<Vec<Row>> {
accounts.iter().map(|a| Ok(HashMap::from([
("id".into(), serde_json::json!(a.id)),
("user".into(), serde_json::Value::String(a.user.clone())),
("password_hash".into(), serde_json::Value::String(a.password_hash.clone())),
]))).collect()
}
/// 身份元数据转成行
fn identity_rows(accounts: &[Account]) -> std::io::Result<Vec<Row>> {
accounts.iter().map(|a| Ok(HashMap::from([
("id".into(), serde_json::json!(a.id)),
("user".into(), serde_json::Value::String(a.user.clone())),
("can_login".into(), serde_json::json!(a.can_login)),
("is_superuser".into(), serde_json::json!(a.is_superuser)),
("system_catalog_manager".into(), serde_json::json!(a.system_catalog_manager)),
("enabled".into(), serde_json::json!(a.enabled)),
("identity_version".into(), serde_json::json!(a.identity_version)),
("registered_at".into(), serde_json::Value::String(a.registered_at.clone())),
("last_login_at".into(), match &a.last_login_at {
Some(stamp) => serde_json::Value::String(stamp.clone()),
@@ -1478,10 +1749,23 @@ fn users_columns() -> Vec<SchemaColumn> {
vec![
SchemaColumn { name: "id".into(), ty: "int".into(), nullable: false,
auto_increment: true, primary_key: true, ..Default::default() },
SchemaColumn { name: "user".into(), ty: "varchar(64)".into(), nullable: false,
unique: true, ..Default::default() },
SchemaColumn { name: "password_hash".into(), ty: "varchar(256)".into(),
nullable: false, ..Default::default() },
]
}
/// 系统身份目录结构
fn identity_columns() -> Vec<SchemaColumn> {
vec![
SchemaColumn { name: "id".into(), ty: "int".into(), nullable: false,
primary_key: true, ..Default::default() },
SchemaColumn { name: "user".into(), ty: "varchar(64)".into(), nullable: false,
unique: true, ..Default::default() },
SchemaColumn { name: "can_login".into(), ty: "bool".into(), nullable: false, ..Default::default() },
SchemaColumn { name: "is_superuser".into(), ty: "bool".into(), nullable: false, ..Default::default() },
SchemaColumn { name: "system_catalog_manager".into(), ty: "bool".into(), nullable: false, ..Default::default() },
SchemaColumn { name: "enabled".into(), ty: "bool".into(), nullable: false, ..Default::default() },
SchemaColumn { name: "identity_version".into(), ty: "int".into(), nullable: false, ..Default::default() },
SchemaColumn { name: "registered_at".into(), ty: "timestamp".into(),
nullable: false, ..Default::default() },
SchemaColumn { name: "last_login_at".into(), ty: "timestamp".into(),
@@ -1497,7 +1781,6 @@ fn declare_system_schema(catalog: &mut Catalog, rows: &[Row]) -> std::io::Result
catalog.set_columns(USERS_TABLE, users_columns())?;
catalog.rebuild_stats(USERS_TABLE, rows);
catalog.mark_system(USERS_TABLE)?;
catalog.add_index(USERS_TABLE, "user")?;
Ok(())
}
@@ -1519,6 +1802,11 @@ fn decode_account_row(row: &Row) -> std::io::Result<Account> {
revision: row.get("revision").and_then(serde_json::Value::as_u64).unwrap_or(1),
registered_at: row.get("registered_at").and_then(serde_json::Value::as_str).unwrap_or("").to_string(),
last_login_at: row.get("last_login_at").and_then(serde_json::Value::as_str).map(str::to_string),
can_login: row.get("can_login").and_then(serde_json::Value::as_bool).unwrap_or(true),
is_superuser: row.get("is_superuser").and_then(serde_json::Value::as_bool).unwrap_or(false),
system_catalog_manager: row.get("system_catalog_manager").and_then(serde_json::Value::as_bool).unwrap_or(false),
enabled: row.get("enabled").and_then(serde_json::Value::as_bool).unwrap_or(true),
identity_version: row.get("identity_version").and_then(serde_json::Value::as_u64).unwrap_or(0) as u8,
})
}
@@ -1589,9 +1877,20 @@ fn new_account(id: i64, user: &str, password_hash: String) -> std::io::Result<Ac
revision: 1,
registered_at: now_stamp(),
last_login_at: None,
can_login: true,
is_superuser: false,
system_catalog_manager: false,
enabled: true,
identity_version: 1,
})
}
/// 保留至少一个启用的登录管理员
fn require_superuser(accounts: &[Account]) -> std::io::Result<()> {
if accounts.iter().any(|a| a.can_login && a.enabled && a.is_superuser) { Ok(()) }
else { Err(std::io::Error::other("the last enabled login superuser cannot be removed")) }
}
/// 一张表的索引定义转线上格式
fn index_wires(schema: &crate::catalog::TableSchema) -> Vec<IndexWire> {
let mut out = vec![IndexWire {
@@ -1632,6 +1931,40 @@ impl Drop for Server {
}
}
/// 按列精度规范化一行定点数
fn normalize_decimal_row(row: &mut Row, columns: &[SchemaColumn]) -> std::io::Result<()> {
for column in columns {
if let Some((precision, scale)) = decimal_parameters(&column.ty)?
&& let Some(value) = row.get_mut(&column.name).filter(|v| !v.is_null()) {
let number = value.as_f64().ok_or_else(|| std::io::Error::other("invalid decimal value"))?;
let factor = 10f64.powi(scale);
let scaled = number * factor;
if !scaled.is_finite() { return Err(std::io::Error::other("decimal value is not finite or exceeds precision")); }
let rounded = scaled.round_ties_even() / factor;
if rounded.abs() >= 10f64.powi(precision - scale) { return Err(std::io::Error::other("decimal value exceeds precision")); }
*value = serde_json::json!(rounded);
}
}
Ok(())
}
/// 解析基础或命名定点类型参数
fn decimal_parameters(ty: &str) -> std::io::Result<Option<(i32, i32)>> {
let mut base = ty.trim();
while let Some(body) = base.strip_prefix("domain(").and_then(|s| s.strip_suffix(')')) {
base = body.split_once(',').ok_or_else(|| std::io::Error::other("invalid domain type"))?.1.trim();
}
let normalized = crate::catalog::normalize_type(base);
if normalized == "decimal" { return Ok(Some((10, 0))); }
if !normalized.starts_with("decimal(") { return Ok(None); }
let error = || std::io::Error::other("invalid decimal precision or scale");
let body = normalized.strip_prefix("decimal(").and_then(|s| s.strip_suffix(')')).ok_or_else(error)?;
let values = body.split(',').map(|s| s.trim().parse::<i32>()).collect::<Result<Vec<_>, _>>().map_err(|_| error())?;
let (precision, scale) = match values.as_slice() { [p] => (*p, 0), [p, s] => (*p, *s), _ => return Err(error()) };
if !(1..=308).contains(&precision) || scale < 0 || scale > precision { return Err(error()); }
Ok(Some((precision, scale)))
}
/// 系统库名:账号表住在这里,其中的表都是系统表。服务启动时只建它一个,
/// 别的库一律要显式建(没有默认工作库)。
const SYSTEM_DATABASE: &str = "system";
@@ -1793,9 +2126,11 @@ impl Databases {
value["table"] = serde_json::Value::String(table.to_string());
}
let request: Request = serde_json::from_value(value).map_err(Error::other)?;
let global = matches!(request, Request::AccountsList | Request::AccountCreate { .. }
let global = matches!(request, Request::AccountsList | Request::CatalogManage { .. } | Request::AccountCreate { .. }
| Request::IdentityInitialize { .. } | Request::RoleCreate { .. } | Request::IdentityAlter { .. }
| Request::AccountReset { .. } | Request::AccountLogin { .. }
| Request::AccountDrop { .. } | Request::BootstrapSystem { .. }
| Request::BootstrapTypes { .. }
| Request::SystemStatus | Request::Ping);
if !global && selected.is_empty() { return Err(Error::other("no database selected")); }
let server = if global || selected == SYSTEM_DATABASE { Arc::clone(&self.system) }
@@ -1838,21 +2173,185 @@ fn log_config(loaded: &Loaded) {
}
}
/// 获取数据目录的跨进程独占锁
fn lock_data_directory(cfg: &Config) -> std::io::Result<std::fs::File> {
std::fs::create_dir_all(&cfg.data_dir)?;
let file = std::fs::OpenOptions::new().read(true).write(true).create(true).truncate(false)
.open(cfg.data_dir.join(".chusql.lock"))?;
file.try_lock().map_err(|e| std::io::Error::other(format!("data directory is in use; stop all database processes: {e}")))?;
Ok(file)
}
/// 复制系统目录并拒绝链接文件
fn copy_system_directory(source: &std::path::Path, target: &std::path::Path) -> std::io::Result<()> {
std::fs::create_dir_all(target)?;
for entry in std::fs::read_dir(source)? {
let entry = entry?;
let ty = entry.file_type()?;
if ty.is_symlink() { return Err(std::io::Error::other("system directory contains a symbolic link")); }
if ty.is_dir() { copy_system_directory(&entry.path(), &target.join(entry.file_name()))?; }
else if ty.is_file() { std::fs::copy(entry.path(), target.join(entry.file_name()))?; }
else { return Err(std::io::Error::other("unsupported system directory entry")); }
}
Ok(())
}
/// 构造可重建的系统表结构
fn maintenance_schemas() -> Vec<(&'static str, Vec<SchemaColumn>)> {
let text = |name: &str, size: usize| SchemaColumn { name: name.into(), ty: format!("varchar({size})"), ..Default::default() };
vec![(USERS_TABLE, users_columns()), (IDENTITIES_TABLE, identity_columns()), (TYPES_TABLE, types_columns()),
("__system_roles", vec![text("name", 64)]),
("__system_grants", vec![text("role", 64), text("privilege", 16), text("object", 128)]),
("__system_grant_options", vec![text("role", 64), text("privilege", 16), text("object", 128)]),
("__system_members", vec![text("role", 64), text("member", 64)])]
}
/// 验证隔离副本并执行所选恢复策略
fn maintenance_candidate(cfg: Config, mode: &str, user: &str, hash: &str, types: Vec<Row>) -> Result<(), String> {
if mode == "reset" {
let seed = Server::new(cfg.clone()).map_err(|e| e.to_string())?;
seed.bootstrap_system(Some(user), Some(hash)).map_err(|e| e.to_string())?;
seed.bootstrap_types(types.clone()).map_err(|e| e.to_string())?;
}
let repair = mode == "repair";
let catalog_path = cfg.data_dir.join("catalog.json");
let schemas = maintenance_schemas();
let mut missing_data = Vec::new();
let mut catalog = match Catalog::load(&catalog_path) {
Ok(catalog) => catalog,
Err(_) => {
for entry in std::fs::read_dir(&cfg.data_dir).map_err(|e| e.to_string())? {
let entry = entry.map_err(|e| e.to_string())?;
if entry.path().extension().and_then(|s| s.to_str()) == Some("db") {
let name = entry.path().file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default();
if !schemas.iter().any(|(known, _)| *known == name) { return Err(format!("cannot reconstruct unknown system table schema: {name}")); }
}
}
Catalog::default()
}
};
if catalog.all_tables().is_empty() {
for entry in std::fs::read_dir(&cfg.data_dir).map_err(|e| e.to_string())? {
let path = entry.map_err(|e| e.to_string())?.path();
if path.extension().and_then(|s| s.to_str()) == Some("db") {
let name = path.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default();
if !schemas.iter().any(|(known, _)| *known == name) { return Err(format!("cannot reconstruct unknown system table schema: {name}")); }
}
}
}
for (name, columns) in &schemas {
let file = cfg.data_dir.join(format!("{name}.db"));
if let Some(schema) = catalog.describe(name).filter(|s| !file.exists() && s.row_count > 0) {
missing_data.push((*name, schema.row_count));
}
if repair && !file.exists() && catalog.describe(name).is_none_or(|s| s.row_count != 0) {
return Err(format!("repair would require initializing or recovering data in {name}"));
}
if catalog.describe(name).is_none() { catalog.create_table(name, columns.clone()).map_err(|e| e.to_string())?; }
else { catalog.set_columns(name, columns.clone()).map_err(|e| e.to_string())?; }
catalog.mark_system(name).map_err(|e| e.to_string())?;
}
catalog.save(&catalog_path).map_err(|e| e.to_string())?;
let server = Server::new(cfg).map_err(|e| e.to_string())?;
if repair {
let scan = server.wal.read_all().map_err(|e| e.to_string())?;
let checkpoint = server.wal.read_checkpoint().map_err(|e| e.to_string())?.unwrap_or(0);
if !committed_ops(&records_after(&scan.records, checkpoint)).is_empty() { return Err("repair would require replaying WAL data".into()); }
} else {
server.recover_wal().map_err(|e| e.to_string())?;
}
for (name, _) in &schemas { server.with_table(name, |t| t.flush()).map_err(|e| e.to_string())?; }
if !repair {
if mode == "recover" && server.load_accounts().map_err(|e| e.to_string())?.is_empty() && hash.is_empty() {
return Err("recover requires --password-stdin to initialize a missing administrator".into());
}
server.bootstrap_system(Some(user), Some(hash)).map_err(|e| e.to_string())?;
server.accounts_request(Request::IdentityInitialize { administrator: user.into() }).map_err(|e| e.to_string())?;
server.bootstrap_types(types.clone()).map_err(|e| e.to_string())?;
}
let accounts = server.load_accounts().map_err(|e| e.to_string())?;
require_superuser(&accounts).map_err(|e| e.to_string())?;
for (name, expected) in missing_data {
let rows = server.with_existing_table(name, |t| t.scan()).map_err(|e| e.to_string())?;
if (rows.len() as u64) < expected { return Err(format!("WAL cannot reconstruct all missing data in {name}; restore a backup or use reset")); }
}
if repair {
let saved = server.with_existing_table(TYPES_TABLE, |t| t.scan()).map_err(|e| e.to_string())?;
if types.iter().any(|wanted| !saved.iter().any(|row| wanted.iter().all(|(key, value)| row.get(key) == Some(value)))) {
return Err("repair would require changing preinstalled type data".into());
}
}
for (name, _) in &schemas { server.with_existing_table(name, |t| t.scan()).map_err(|e| e.to_string())?; }
{
let mut catalog = server.catalog.lock().map_err(|_| "catalog lock poisoned")?;
catalog.identity_high_water = catalog.identity_high_water.max(accounts.iter().map(|a| a.id).max().unwrap_or(0));
catalog.save(server.catalog_path()).map_err(|e| e.to_string())?;
}
server.rebuild_stats().map_err(|e| e.to_string())?;
server.flush_all_tables().map_err(|e| e.to_string())?;
Ok(())
}
/// 进程内存储句柄:一个数据目录一份状态,跨线程共享(内部各自加锁)。
pub struct Storage {
databases: Databases,
_directory_lock: std::fs::File,
}
impl Storage {
/// 离线修复或重建系统目录
pub fn maintenance(config_path: Option<&str>, request: &str) -> Result<String, String> {
let loaded = config::load(config_path)?;
let cfg = loaded.config;
let _lock = lock_data_directory(&cfg).map_err(|e| e.to_string())?;
let input: serde_json::Value = serde_json::from_str(request).map_err(|e| e.to_string())?;
let mode = input.get("mode").and_then(|v| v.as_str()).ok_or("missing maintenance mode")?;
if !["repair", "recover", "reset"].contains(&mode) { return Err("invalid maintenance mode".into()); }
let user = input.get("user").and_then(|v| v.as_str()).unwrap_or("root");
let hash = input.get("password_hash").and_then(|v| v.as_str()).unwrap_or("");
let types: Vec<Row> = serde_json::from_value(input.get("types").cloned().ok_or("missing type manifest")?).map_err(|e| e.to_string())?;
if types.is_empty() { return Err("type manifest must not be empty".into()); }
for row in &types { type_definition_name(row).map_err(|e| e.to_string())?; }
let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_err(|e| e.to_string())?.as_nanos();
let system = cfg.data_dir.join("system");
if system.is_symlink() { return Err("system directory cannot be a symbolic link".into()); }
let work = cfg.data_dir.join(format!("bootstrap-work-{stamp}"));
let backup = cfg.data_dir.join(format!("system-backup-{stamp}"));
let staged = work.join("system");
std::fs::create_dir_all(&staged).map_err(|e| e.to_string())?;
if mode != "reset" && system.exists() { copy_system_directory(&system, &staged).map_err(|e| e.to_string())?; }
let mut candidate_cfg = cfg.clone();
candidate_cfg.data_dir = staged.clone();
let result = maintenance_candidate(candidate_cfg, mode, user, hash, types);
if let Err(err) = result { return Err(format!("{err}; original system directory was not changed; candidate: {}. Try csql-bootstrap recover or reset", work.display())); }
if mode == "repair" && system.exists() {
for entry in std::fs::read_dir(&system).map_err(|e| e.to_string())? {
let entry = entry.map_err(|e| e.to_string())?;
if entry.path().extension().and_then(|s| s.to_str()) == Some("db")
&& std::fs::read(entry.path()).map_err(|e| e.to_string())? != std::fs::read(staged.join(entry.file_name())).map_err(|e| e.to_string())? {
return Err("repair would modify table data; original system directory was not changed. Use recover or reset".into());
}
}
}
if system.exists() { std::fs::rename(&system, &backup).map_err(|e| e.to_string())?; }
if let Err(err) = std::fs::rename(&staged, &system) {
if backup.exists() { std::fs::rename(&backup, &system).map_err(|rollback| format!("publish failed: {err}; rollback failed: {rollback}; backup: {}", backup.display()))?; }
return Err(format!("cannot publish repaired system directory: {err}"));
}
serde_json::to_string(&serde_json::json!({"status":"maintenance", "mode":mode, "backup":backup, "candidate":work,
"message":"System maintenance completed. Business databases were preserved."})).map_err(|e| e.to_string())
}
/// 打开数据目录并初始化存储
pub fn open(config_path: Option<&str>) -> Result<Storage, String> {
let loaded = config::load(config_path)?;
let directory_lock = lock_data_directory(&loaded.config).map_err(|e| e.to_string())?;
log::init();
log::set_level(loaded.config.log_level);
log_info!(core, "chusql-core-storage {} starting", env!("CARGO_PKG_VERSION"));
log_config(&loaded);
let databases = Databases::new(loaded.config).map_err(|e| e.to_string())?;
Ok(Storage { databases })
Ok(Storage { databases, _directory_lock: directory_lock })
}
/// 一行请求 JSON 换一行响应 JSON
+23 -13
View File
@@ -31,7 +31,7 @@ const OP_DROP_INDEX: u8 = 14;
pub enum WalOp {
HideColumn { table: String, column: String },
Compact { table: String, rows: Vec<Row> },
Accounts { accounts: Vec<Account> },
Accounts { accounts: Vec<Account>, removed: Vec<String>, migrate_roles: bool },
Insert {
table: String,
row: Row,
@@ -283,7 +283,7 @@ impl Wal {
frames.push((lsn, op, offset, end));
offset = end;
}
Err(_) => break,
Err(error) => return Err(io::Error::new(io::ErrorKind::InvalidData, error)),
}
}
Ok((frames, offset))
@@ -291,9 +291,13 @@ impl Wal {
/// 记录检查点:到此为止的 LSN 都已落盘
pub fn write_checkpoint(&self, lsn: u64) -> io::Result<()> {
let mut f = File::create(self.checkpoint_path())?;
let checkpoint = self.checkpoint_path();
let pending = checkpoint.with_extension("checkpoint.pending");
let mut f = File::create(&pending)?;
f.write_all(&lsn.to_le_bytes())?;
f.sync_all()
f.sync_all()?;
drop(f);
std::fs::rename(pending, checkpoint)
}
/// 读检查点 LSN
@@ -303,8 +307,8 @@ impl Wal {
return Ok(None);
}
let bytes = std::fs::read(&path)?;
if bytes.len() < 8 {
return Ok(None);
if bytes.len() != 8 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid WAL checkpoint length"));
}
let mut raw = [0u8; 8];
raw.copy_from_slice(&bytes[..8]);
@@ -322,10 +326,10 @@ impl Wal {
if let Some((lsn, _, _, _)) = frames.last() {
base = base.max(*lsn);
}
*slot = base + 1;
*slot = base.checked_add(1).ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
}
let lsn = *slot;
*slot += 1;
*slot = slot.checked_add(1).ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
Ok(lsn)
}
@@ -378,8 +382,8 @@ fn encode(op: &WalOp, lsn: u64) -> io::Result<Vec<u8>> {
let p = serde_json::to_vec(rows).map_err(io::Error::other)?;
(OP_COMPACT, table.clone(), p)
}
WalOp::Accounts { accounts } => (OP_ACCOUNTS, USERS_TABLE.to_string(),
serde_json::to_vec(accounts).map_err(io::Error::other)?),
WalOp::Accounts { accounts, removed, migrate_roles } => (OP_ACCOUNTS, USERS_TABLE.to_string(),
serde_json::to_vec(&serde_json::json!({"accounts":accounts,"removed":removed,"migrate_roles":migrate_roles})).map_err(io::Error::other)?),
WalOp::Insert { table, row } => {
let p = serde_json::to_vec(row).map_err(io::Error::other)?;
(OP_INSERT, table.clone(), p)
@@ -478,9 +482,15 @@ fn decode_body(body: &[u8]) -> Result<(u64, WalOp), String> {
}
OP_ACCOUNTS => {
if table != USERS_TABLE { return Err("invalid account WAL table".into()); }
let accounts = serde_json::from_slice(payload)
.map_err(|e| format!("bad account payload: {e}"))?;
WalOp::Accounts { accounts }
let value: serde_json::Value = serde_json::from_slice(payload).map_err(|e| format!("bad account payload: {e}"))?;
let (accounts, removed, migrate_roles) = if value.is_array() {
(serde_json::from_value(value).map_err(|e| format!("bad account payload: {e}"))?, Vec::new(), false)
} else {
(serde_json::from_value(value.get("accounts").cloned().ok_or("missing account payload")?).map_err(|e| format!("bad account payload: {e}"))?,
serde_json::from_value(value.get("removed").cloned().ok_or("missing identity cleanup")?).map_err(|e| format!("bad account cleanup: {e}"))?,
value.get("migrate_roles").and_then(serde_json::Value::as_bool).ok_or("missing role migration flag")?)
};
WalOp::Accounts { accounts, removed, migrate_roles }
}
OP_INSERT => {
let row: Row =
+30
View File
@@ -124,6 +124,36 @@ fn stats_track_a_histogram_for_numeric_columns() {
assert_eq!(id.hist.iter().sum::<u64>(), 0);
}
/// 重复值删除保留仍存在的不同值计数
#[test]
fn duplicate_deletion_keeps_distinct_until_last_occurrence() {
let mut catalog = Catalog::default();
let duplicate = row(&[("id", json!(1)), ("v", json!(7))]);
catalog.rebuild_stats("t", &[duplicate.clone(), duplicate.clone()]);
catalog.record_delete("t", std::slice::from_ref(&duplicate));
assert_eq!(catalog.describe("t").unwrap().stats["v"].distinct, 1);
catalog.record_delete("t", &[duplicate]);
assert_eq!(catalog.describe("t").unwrap().stats["v"].distinct, 0);
}
/// 去重封顶不影响直方图和其他列
#[test]
fn capped_distinct_keeps_histogram_and_other_columns_accurate() {
let mut catalog = Catalog::default();
let rows: Vec<Row> = (0..5000).map(|n| row(&[("id", json!(n)), ("v", json!(n % 2))])).collect();
catalog.rebuild_stats("t", &rows);
let schema = catalog.describe("t").unwrap();
assert!(schema.stats["id"].capped);
assert_eq!(schema.stats["id"].hist.iter().sum::<u64>(), 5000);
assert_eq!(schema.stats["v"].distinct, 2);
catalog.record_delete("t", &rows[..1000]);
let schema = catalog.describe("t").unwrap();
assert_eq!(schema.stats["id"].hist.iter().sum::<u64>(), 4000);
assert_eq!(schema.stats["v"].distinct, 2);
catalog.rebuild_stats("t", &rows[1000..]);
assert_eq!(catalog.describe("t").unwrap().stats["id"].distinct, 4000);
}
/// 老类型名并到新写法,参数保留,未知类型原样
#[test]
fn normalize_type_maps_legacy_names() {
+21 -8
View File
@@ -1,7 +1,9 @@
use std::path::PathBuf;
use std::sync::Mutex;
use chusql_core_storage::config::{self, Config, DEFAULT_BTREE_ORDER, DEFAULT_PAGE_SIZE, Origin};
use chusql_core_storage::config::{
self, Config, DEFAULT_BTREE_ORDER, DEFAULT_LOG_FILES, DEFAULT_PAGE_SIZE, Origin,
};
use chusql_core_storage::log::Level;
// 配置测试:默认值、文件覆盖、环境变量彻底无效、非法值报错。
@@ -25,6 +27,7 @@ fn defaults_without_file() {
assert_eq!(loaded.config.page_size, DEFAULT_PAGE_SIZE);
assert_eq!(loaded.config.btree_order, DEFAULT_BTREE_ORDER);
assert_eq!(loaded.config.data_dir, config::default_data_dir());
assert_eq!(loaded.config.log_files, PathBuf::from(DEFAULT_LOG_FILES));
assert_eq!(loaded.config.log_level, Level::Info);
}
@@ -39,6 +42,7 @@ fn file_overrides_only_written_keys() {
assert_eq!(loaded.config.page_size, 8192);
assert_eq!(loaded.config.btree_order, DEFAULT_BTREE_ORDER);
assert_eq!(loaded.config.data_dir, config::default_data_dir());
assert_eq!(loaded.config.log_files, PathBuf::from(DEFAULT_LOG_FILES));
assert_eq!(loaded.config_path, Some(path.clone()));
let page = loaded.origins.iter().find(|(n, _, _)| *n == "page.size").unwrap();
@@ -47,11 +51,11 @@ fn file_overrides_only_written_keys() {
assert_eq!(order.2, Origin::Default);
}
/// 五项全写都生效
/// 全部配置项及其来源生效
#[test]
fn file_all_keys() {
let _guard = env_guard();
let text = "[page]\nsize = 8192\n[btree]\norder = 32\n[buffer]\npool_size = 128\n[storage]\ndata_dir = \"mydata\"\n[log]\nlevel = \"debug\"\n";
let text = "[page]\nsize = 8192\n[btree]\norder = 32\n[buffer]\npool_size = 128\n[storage]\ndata_dir = \"mydata\"\nlog_files = \"mylogs\"\n[log]\nlevel = \"debug\"\n";
let path = PathBuf::from("t.toml");
let loaded = config::resolve(Some(text), Some(path.clone())).unwrap();
@@ -59,6 +63,7 @@ fn file_all_keys() {
assert_eq!(loaded.config.btree_order, 32);
assert_eq!(loaded.config.pool_size, 128);
assert_eq!(loaded.config.data_dir, PathBuf::from("mydata"));
assert_eq!(loaded.config.log_files, PathBuf::from("mylogs"));
assert_eq!(loaded.config.log_level, Level::Debug);
assert!(
loaded
@@ -103,16 +108,16 @@ fn env_vars_are_ignored_entirely() {
fn default_config_path_follows_system_convention() {
let _guard = env_guard();
let path = config::default_config_path();
assert_eq!(path.file_name().unwrap(), "chusql.toml");
assert_eq!(path.file_name().unwrap(), "settings.toml");
assert_eq!(path.parent().unwrap().file_name().unwrap(), "ChuSQL");
assert!(path.ends_with(format!("ChuSQL{}chusql.toml", std::path::MAIN_SEPARATOR)), "got {}", path.display());
assert!(path.ends_with(format!("ChuSQL{}settings.toml", std::path::MAIN_SEPARATOR)), "got {}", path.display());
// 本机 APPDATA 存在时必须是 %APPDATA%\ChuSQL\chusql.toml
// 本机 APPDATA 存在时必须是 %APPDATA%\ChuSQL\settings.toml
#[cfg(windows)]
if let Some(appdata) = std::env::var_os("APPDATA") {
assert_eq!(
path,
PathBuf::from(appdata).join("ChuSQL").join("chusql.toml")
PathBuf::from(appdata).join("ChuSQL").join("settings.toml")
);
}
}
@@ -186,7 +191,7 @@ fn load_without_explicit_path_uses_builtin_defaults() {
#[test]
fn server_section_belongs_to_another_layer() {
let _guard = env_guard();
let db_server = "[server]\nhost = \"0.0.0.0\"\nport = 7778\nmax_rows = 10\n";
let db_server = "[server]\nlisten_host = \"0.0.0.0\"\nport = 7778\nmax_rows = 10\n";
let loaded = config::resolve(Some(db_server), None).unwrap();
assert_eq!(loaded.config.data_dir, config::default_data_dir());
@@ -226,6 +231,14 @@ fn rejects_unknown_key() {
assert!(err.contains("bad config"), "got {}", err);
}
/// 空的日志目录要报错(否则日志会落到当前目录)
#[test]
fn rejects_empty_log_files() {
let _guard = env_guard();
let err = config::resolve(Some("[storage]\nlog_files = \" \"\n"), None).unwrap_err();
assert!(err.contains("storage.log_files"), "got {}", err);
}
/// 坏的页面尺寸只能来自文件
#[test]
fn rejects_bad_page_size_from_file() {
+434 -5
View File
@@ -10,6 +10,433 @@ thread_local! {
static CURRENT: RefCell<Option<Arc<Storage>>> = const { RefCell::new(None) };
}
/// 定点写入检查舍入和整批原子拒绝
#[test]
fn decimal_writes_enforce_precision_before_wal() -> Result<(), Box<dyn std::error::Error>> {
let (_server, data) = start_server();
let mut c = connect()?;
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"amounts","columns":[{"name":"id","ty":"int"},{"name":"value","ty":"decimal(4,2)"}]}))?;
request_ok(&mut c, serde_json::json!({"method":"insert","table":"amounts","row":{"id":1,"value":99.994}}))?;
let snapshot = send(&mut c, r#"{"method":"scan","table":"amounts"}"#);
let rows: serde_json::Value = serde_json::from_str(&snapshot)?;
assert_eq!(rows["rows"][0]["value"], serde_json::json!(99.99));
let checkpoint = std::fs::read(data.path().join("databases/main/wal.checkpoint"))?;
for value in [99.995, -99.995, 100.0] {
let response = send(&mut c, &serde_json::json!({"method":"insert_batch","table":"amounts","rows":[{"id":2,"value":1.2},{"id":3,"value":value}]}).to_string());
assert!(response.contains("exceeds precision"), "{response}");
let remaining: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"amounts"}"#))?;
assert_eq!(remaining, rows);
assert_eq!(std::fs::read(data.path().join("databases/main/wal.checkpoint"))?, checkpoint);
}
let invalid = send(&mut c, &serde_json::json!({"method":"create_table","table":"invalid","columns":[{"name":"value","ty":"decimal(2,3)"}]}).to_string());
assert!(invalid.contains("invalid decimal"), "{invalid}");
Ok(())
}
/// 检查点失败封闭请求并在重启恢复
#[test]
fn checkpoint_failure_requires_recovery_and_preserves_committed_rows() -> Result<(), Box<dyn std::error::Error>> {
let (server, data) = start_server();
let mut c = connect()?;
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"checkpointed","columns":[{"name":"id","ty":"int"}]}))?;
let pending = data.path().join("databases/main/wal.checkpoint.pending");
std::fs::create_dir(&pending)?;
let failed = send(&mut c, r#"{"method":"insert","table":"checkpointed","row":{"id":1}}"#);
assert!(failed.contains("error"), "{failed}");
assert!(send(&mut c, r#"{"method":"scan","table":"checkpointed"}"#).contains("recovery required"));
assert!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len() > 0);
drop(c);
drop(server);
std::fs::remove_dir(pending)?;
let _restarted = restart_at(data.path())?;
let mut c = connect()?;
let rows: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"checkpointed"}"#))?;
assert_eq!(rows["rows"], serde_json::json!([{"id":1}]));
Ok(())
}
/// 应用失败封闭读写并重放完整提交组
#[test]
fn committed_apply_failures_require_recovery() -> Result<(), Box<dyn std::error::Error>> {
for operation in [
serde_json::json!({"method":"create_table","table":"created","columns":[{"name":"id","ty":"int"}]}),
serde_json::json!({"method":"replace_all","table":"items","rows":[{"id":2}]}),
serde_json::json!({"method":"apply_transaction","ops":[{"op":"delete","table":"items","ids":[1]},{"op":"replace","table":"items","rows":[{"id":2}]}]}),
] {
let (server, data) = start_server();
let mut c = connect()?;
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"items","columns":[{"name":"id","ty":"int"}]}))?;
request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":1}}))?;
let directory = data.path().join("databases/main");
let checkpoint = std::fs::read(directory.join("wal.checkpoint"))?;
let pending = directory.join("catalog.json.pending");
std::fs::create_dir(&pending)?;
let failed = send(&mut c, &operation.to_string());
assert!(failed.contains("error"), "{operation}: {failed}");
for request in [
r#"{"method":"scan","table":"items"}"#,
r#"{"method":"insert","table":"items","row":{"id":3}}"#,
] {
let blocked = send(&mut c, request);
assert!(blocked.contains("recovery required"), "{blocked}");
}
assert_eq!(std::fs::read(directory.join("wal.checkpoint"))?, checkpoint);
assert!(std::fs::metadata(directory.join("wal.log"))?.len() > 0);
drop(c);
drop(server);
std::fs::remove_dir(pending)?;
let restarted = restart_at(data.path())?;
let mut c = connect()?;
let table = if operation["method"] == "create_table" { "created" } else { "items" };
let result: serde_json::Value = serde_json::from_str(&send(&mut c, &serde_json::json!({"method":"scan","table":table}).to_string()))?;
let expected = if table == "created" { serde_json::json!([]) } else { serde_json::json!([{"id":2}]) };
assert_eq!(result["rows"], expected, "{operation}");
assert_eq!(std::fs::metadata(directory.join("wal.log"))?.len(), 0);
drop(c);
drop(restarted);
}
Ok(())
}
/// 首帧撕裂恢复后新日志仍可重放
#[test]
fn torn_first_frame_is_removed_before_new_writes() -> Result<(), Box<dyn std::error::Error>> {
let (server, data) = start_server();
let mut c = connect()?;
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"torn","columns":[{"name":"id","ty":"int"}]}))?;
drop(c);
drop(server);
std::fs::write(data.path().join("databases/main/wal.log"), [20, 0, 0, 0, 1])?;
let restarted = restart_at(data.path())?;
assert_eq!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len(), 0);
let mut c = connect()?;
request_ok(&mut c, serde_json::json!({"method":"insert","table":"torn","row":{"id":1}}))?;
drop(c);
drop(restarted);
let _again = restart_at(data.path())?;
let mut c = connect()?;
let rows: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"torn"}"#))?;
assert_eq!(rows["rows"], serde_json::json!([{"id":1}]));
Ok(())
}
/// 校验预装类型幂等与冲突保护
#[test]
fn preinstalled_types_are_idempotent_hidden_and_conflict_safe() -> Result<(), Box<dyn std::error::Error>> {
let (_server, data) = start_server();
let mut c = connect()?;
let premature = send(&mut c, &type_seed_request().to_string());
assert!(premature.contains("login superuser"), "{premature}");
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"original-hash"}"#);
let accounts = send(&mut c, r#"{"method":"accounts_list"}"#);
let request = type_seed_request();
let seeded: serde_json::Value = serde_json::from_str(&send(&mut c, &request.to_string()))?;
assert_eq!(seeded["status"], "rows", "{seeded}");
assert_eq!(seeded["rows"].as_array().map(Vec::len), Some(2));
assert_eq!(serde_json::from_str::<serde_json::Value>(&send(&mut c, &request.to_string()))?, seeded);
let mut conflict = request;
conflict["types"][1]["base_type"] = "int".into();
let rejected = send(&mut c, &conflict.to_string());
assert!(rejected.contains("definition conflict: text"), "{rejected}");
assert_eq!(serde_json::from_str::<serde_json::Value>(&send(&mut c, &type_seed_request().to_string()))?, seeded);
assert_eq!(send(&mut c, r#"{"method":"accounts_list"}"#), accounts);
let catalog: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(data.path().join("system/catalog.json"))?)?;
assert_eq!(catalog["tables"]["__system_types"]["system"], true);
assert_eq!(catalog["tables"]["__system_types"]["row_count"], 2);
let denied = send(&mut c, r#"{"method":"scan","database":"system","table":"__system_types"}"#);
assert!(denied.contains("reserved system table"), "{denied}");
Ok(())
}
/// 类型预装失败保留日志并在重启后恢复
#[test]
fn preinstalled_types_recover_after_catalog_write_failure() -> Result<(), Box<dyn std::error::Error>> {
let (server, data) = start_server();
let mut c = connect()?;
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
std::fs::create_dir(data.path().join("system/catalog.json.pending"))?;
let failed = send(&mut c, &type_seed_request().to_string());
assert!(failed.contains("error"), "{failed}");
assert!(std::fs::metadata(data.path().join("system/wal.log"))?.len() > 0);
assert!(send(&mut c, &type_seed_request().to_string()).contains("recovery required"));
drop(c);
drop(server);
std::fs::remove_dir(data.path().join("system/catalog.json.pending"))?;
let _restarted = restart_at(data.path())?;
let mut c = connect()?;
let restored: serde_json::Value = serde_json::from_str(&send(&mut c, &type_seed_request().to_string()))?;
assert_eq!(restored["rows"].as_array().map(Vec::len), Some(2), "{restored}");
Ok(())
}
/// 类型目录不覆盖未登记的现有文件
#[test]
fn preinstalled_types_reject_reserved_file_collision() -> Result<(), Box<dyn std::error::Error>> {
let (_server, data) = start_server();
let mut c = connect()?;
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
let path = data.path().join("system/__system_types.db");
std::fs::write(&path, b"preserved")?;
let rejected = send(&mut c, &type_seed_request().to_string());
assert!(rejected.contains("reserved type file collision"), "{rejected}");
assert_eq!(std::fs::read(path)?, b"preserved");
Ok(())
}
/// 生成两项预装类型定义
fn type_seed_request() -> serde_json::Value {
serde_json::json!({"method":"bootstrap_types","types":[
{"name":"int","base_type":"int","parameterized":false},
{"name":"text","base_type":"str","parameterized":false}
]})
}
/// 引导保留已用身份编号
#[test]
fn bootstrap_respects_previous_identity_ids() -> Result<(), Box<dyn std::error::Error>> {
let (_server, _data) = start_server();
let mut c = connect()?;
send(&mut c, r#"{"method":"account_create","user":"alice","password_hash":"hash"}"#);
let deleted: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"account_drop","user":"alice"}"#))?;
assert_eq!(deleted["accounts"], serde_json::json!([]));
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
let seeded: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"accounts_list"}"#))?;
assert_eq!(seeded["accounts"][0]["id"], 2);
Ok(())
}
/// 旧账号日志迁移只提升初始管理员
#[test]
fn legacy_account_wal_migrates_without_promoting_other_names() -> Result<(), Box<dyn std::error::Error>> {
use chusql_core_storage::wal::Wal;
let (server, data) = start_server();
let mut c = connect()?;
request_ok(&mut c, serde_json::json!({"method":"create_table","database":"system","table":"__system_roles","columns":[{"name":"name","ty":"str"}]}))?;
request_ok(&mut c, serde_json::json!({"method":"insert","database":"system","table":"__system_roles","row":{"name":"reader"}}))?;
drop(c);
drop(server);
let path = data.path().join("system/wal.log");
let lsn = Wal::new(&path).read_checkpoint()?.unwrap_or(0) + 1;
let legacy = serde_json::json!([
{"id":7,"user":"owner","password_hash":"owner-hash","revision":3,"registered_at":"2020-01-01 00:00:00","last_login_at":null},
{"id":9,"user":"alice","password_hash":"alice-hash","revision":2,"registered_at":"2020-01-02 00:00:00","last_login_at":null}
]);
let table = b"__system_users";
let mut body = lsn.to_le_bytes().to_vec();
body.push(6);
body.extend_from_slice(&(table.len() as u16).to_le_bytes());
body.extend_from_slice(table);
body.extend_from_slice(&serde_json::to_vec(&legacy)?);
let mut frame = (body.len() as u32).to_le_bytes().to_vec();
frame.extend_from_slice(&body);
std::fs::write(&path, frame)?;
let _restarted = restart_at(data.path())?;
let mut c = connect()?;
let wrong = send(&mut c, r#"{"method":"identity_initialize","administrator":"missing"}"#);
assert!(wrong.contains("last enabled login superuser"), "{wrong}");
let migrated: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"identity_initialize","administrator":"owner"}"#))?;
assert_eq!(migrated["status"], "accounts", "{migrated}");
assert_eq!(migrated["accounts"][0]["id"], 7);
assert_eq!(migrated["accounts"][0]["is_superuser"], true);
assert_eq!(migrated["accounts"][0]["password_hash"], "owner-hash");
assert_eq!(migrated["accounts"][1]["is_superuser"], false);
assert_eq!(migrated["accounts"][2]["user"], "reader");
assert_eq!(migrated["accounts"][2]["id"], 10);
assert_eq!(serde_json::from_str::<serde_json::Value>(&send(&mut c, r#"{"method":"identity_initialize","administrator":"alice"}"#))?, migrated);
Ok(())
}
/// 身份属性重启保留且禁止删除最后管理员
#[test]
fn identity_attributes_survive_restart_and_protect_superuser() -> Result<(), Box<dyn std::error::Error>> {
let (server, data) = start_server();
let mut c = connect()?;
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
let created: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"role_create","user":"Reader"}"#))?;
assert_eq!(created["status"], "accounts", "{created}");
assert_eq!(created["accounts"][1]["can_login"], false);
assert_eq!(created["accounts"][1]["enabled"], true);
for change in [serde_json::json!({"enabled":false}), serde_json::json!({"can_login":false}), serde_json::json!({"is_superuser":false})] {
let mut request = change;
request["method"] = "identity_alter".into();
request["user"] = "root".into();
let denied = send(&mut c, &request.to_string());
assert!(denied.contains("last enabled login superuser"), "{denied}");
}
let collision = send(&mut c, r#"{"method":"account_create","user":"READER","password_hash":"hash"}"#);
assert!(collision.contains("already exists"), "{collision}");
let changed: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"identity_alter","user":"reader","can_login":true,"is_superuser":true}"#))?;
assert_eq!(changed["accounts"][1]["revision"], 2);
drop(c);
drop(server);
let _restarted = restart_at(data.path())?;
let mut c = connect()?;
let restored: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"accounts_list"}"#))?;
assert_eq!(restored, changed);
let denied = send(&mut c, r#"{"method":"scan","database":"system","table":"__system_identities"}"#);
assert!(denied.contains("reserved system table"), "{denied}");
Ok(())
}
/// 旧角色迁移与重名拒绝不破坏原数据
#[test]
fn identity_migration_preserves_roles_and_rejects_collisions() -> Result<(), Box<dyn std::error::Error>> {
let (_server, _data) = start_server();
let mut c = connect()?;
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
request_ok(&mut c, serde_json::json!({"method":"create_table","database":"system","table":"__system_roles","columns":[{"name":"name","ty":"str"}]}))?;
request_ok(&mut c, serde_json::json!({"method":"insert","database":"system","table":"__system_roles","row":{"name":"root"}}))?;
let before = send(&mut c, r#"{"method":"accounts_list"}"#);
let rejected = send(&mut c, r#"{"method":"identity_initialize","administrator":"root"}"#);
assert!(rejected.contains("identity name collision: root"), "{rejected}");
assert_eq!(send(&mut c, r#"{"method":"accounts_list"}"#), before);
request_ok(&mut c, serde_json::json!({"method":"replace_all","database":"system","table":"__system_roles","rows":[{"name":"reader"}]}))?;
let migrated: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"identity_initialize","administrator":"root"}"#))?;
assert_eq!(migrated["accounts"][1]["user"], "reader");
assert_eq!(migrated["accounts"][1]["can_login"], false);
assert_eq!(serde_json::from_str::<serde_json::Value>(&send(&mut c, r#"{"method":"identity_initialize","administrator":"reader"}"#))?, migrated);
Ok(())
}
/// 删除身份清理授权并在恢复后保留编号上界
#[test]
fn identity_drop_cleanup_replays_without_reusing_ids() -> Result<(), Box<dyn std::error::Error>> {
let (server, data) = start_server();
let mut c = connect()?;
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
send(&mut c, r#"{"method":"role_create","user":"reader"}"#);
for table in ["__system_grants", "__system_grant_options", "__system_members"] {
request_ok(&mut c, serde_json::json!({"method":"create_table","database":"system","table":table,"columns":[{"name":"role","ty":"str"},{"name":"member","ty":"str"}]}))?;
request_ok(&mut c, serde_json::json!({"method":"insert","database":"system","table":table,"row":{"role":"reader","member":"root"}}))?;
}
std::fs::create_dir(data.path().join("system/catalog.json.pending"))?;
let failed = send(&mut c, r#"{"method":"account_drop","user":"reader"}"#);
assert!(failed.contains("error"), "{failed}");
drop(c);
drop(server);
std::fs::remove_dir(data.path().join("system/catalog.json.pending"))?;
let _restarted = restart_at(data.path())?;
let mut c = connect()?;
for table in ["__system_grants", "__system_grant_options", "__system_members"] {
let rows: serde_json::Value = serde_json::from_str(&send(&mut c, &serde_json::json!({"method":"scan","database":"system","table":table}).to_string()))?;
assert_eq!(rows["rows"], serde_json::json!([]));
}
let recreated: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"role_create","user":"reader"}"#))?;
assert_eq!(recreated["accounts"][1]["id"], 3);
Ok(())
}
/// 命名类型引用与删除依赖在写入时校验
#[test]
fn domain_dependencies_are_checked_before_wal() -> Result<(), Box<dyn std::error::Error>> {
let (_srv, _data) = start_server();
let mut connection = connect()?;
let columns = serde_json::json!([{"name":"code","ty":"domain(code,int)"}]);
let missing = serde_json::json!({"method":"create_table","table":"typed","columns":columns});
let response: serde_json::Value = serde_json::from_str(&send(&mut connection, &missing.to_string()))?;
assert_eq!(response["status"], "error");
request_ok(&mut connection, serde_json::json!({"method":"create_table","table":"__system_domain_code","columns":[{"name":"base","ty":"int"}]}))?;
let forged = serde_json::json!({"method":"create_table","table":"forged","columns":[{"name":"code","ty":"domain(code,str)"}]});
let response: serde_json::Value = serde_json::from_str(&send(&mut connection, &forged.to_string()))?;
assert_eq!(response["status"], "error");
request_ok(&mut connection, missing)?;
let response: serde_json::Value = serde_json::from_str(&send(&mut connection, r#"{"method":"drop_table","table":"__system_domain_code"}"#))?;
assert_eq!(response["status"], "error");
request_ok(&mut connection, serde_json::json!({"method":"drop_table","table":"typed"}))?;
request_ok(&mut connection, serde_json::json!({"method":"drop_table","table":"__system_domain_code"}))?;
let response: serde_json::Value = serde_json::from_str(&send(&mut connection, r#"{"method":"create_table","table":"stale","columns":[{"name":"code","ty":"domain(code,int)"}]}"#))?;
assert_eq!(response["status"], "error");
Ok(())
}
/// 嵌套类型依赖与重启持久化
#[test]
fn nested_domains_preserve_dependencies_across_restart() -> Result<(), Box<dyn std::error::Error>> {
let (_srv, data) = start_server();
let mut connection = connect()?;
request_ok(&mut connection, serde_json::json!({"method":"create_table","table":"__system_domain_root","columns":[{"name":"base","ty":"int"}]}))?;
request_ok(&mut connection, serde_json::json!({"method":"create_table","table":"__system_domain_child","columns":[{"name":"base","ty":"domain(root,int)"}]}))?;
let rejected: serde_json::Value = serde_json::from_str(&send(&mut connection, r#"{"method":"drop_table","table":"__system_domain_root"}"#))?;
assert_eq!(rejected["status"], "error");
request_ok(&mut connection, serde_json::json!({"method":"create_table","table":"typed","columns":[{"name":"id","ty":"domain(child,domain(root,int))"}]}))?;
let forged: serde_json::Value = serde_json::from_str(&send(&mut connection, r#"{"method":"create_table","table":"forged","columns":[{"name":"id","ty":"domain(child,domain(root,str))"}]}"#))?;
assert_eq!(forged["status"], "error");
drop(connection);
CURRENT.with(|c| *c.borrow_mut() = None);
drop(_srv);
let _reopened = start_server_in(data.path());
let mut connection = connect()?;
let schema: serde_json::Value = serde_json::from_str(&send(&mut connection, r#"{"method":"describe_table","table":"typed"}"#))?;
assert_eq!(schema["status"], "schema");
assert_eq!(schema["columns"][0]["ty"], "domain(child,domain(root,int))");
request_ok(&mut connection, serde_json::json!({"method":"drop_table","table":"typed"}))?;
request_ok(&mut connection, serde_json::json!({"method":"drop_table","table":"__system_domain_child"}))?;
request_ok(&mut connection, serde_json::json!({"method":"drop_table","table":"__system_domain_root"}))?;
Ok(())
}
/// 重置与结构修复保留业务文件及系统数据
#[test]
fn offline_maintenance_preserves_data_and_rejects_live_storage() -> Result<(), Box<dyn std::error::Error>> {
let data = tempfile::tempdir()?;
let config = write_config(data.path());
let business = data.path().join("databases/business");
std::fs::create_dir_all(&business)?;
std::fs::write(business.join("sentinel"), b"business data")?;
let types = type_seed_request()["types"].clone();
let request = serde_json::json!({"mode":"reset","user":"root","password_hash":"original-hash","types":types});
Storage::maintenance(config.to_str(), &request.to_string())?;
let system = data.path().join("system");
let credentials = std::fs::read(system.join("__system_users.db"))?;
let mut catalog: serde_json::Value = serde_json::from_slice(&std::fs::read(system.join("catalog.json"))?)?;
catalog["tables"]["__system_identities"]["columns"] = serde_json::json!([]);
std::fs::write(system.join("catalog.json"), serde_json::to_vec(&catalog)?)?;
let repair = serde_json::json!({"mode":"repair","types":types});
Storage::maintenance(config.to_str(), &repair.to_string())?;
assert_eq!(std::fs::read(system.join("__system_users.db"))?, credentials);
assert_eq!(std::fs::read(business.join("sentinel"))?, b"business data");
let opened = Storage::open(config.to_str())?;
assert!(Storage::maintenance(config.to_str(), &request.to_string()).unwrap_err().contains("data directory is in use"));
drop(opened);
std::fs::remove_file(system.join("__system_users.db"))?;
let before = std::fs::read(system.join("catalog.json"))?;
let failure = Storage::maintenance(config.to_str(), &repair.to_string()).unwrap_err();
assert!(failure.contains("repair would require"));
assert_eq!(std::fs::read(system.join("catalog.json"))?, before);
assert!(!system.join("__system_users.db").exists());
Ok(())
}
/// 从账号 WAL 恢复缺失凭据文件
#[test]
fn offline_recovery_replays_account_wal() -> Result<(), Box<dyn std::error::Error>> {
let data = tempfile::tempdir()?;
let config = write_config(data.path());
let types = type_seed_request()["types"].clone();
Storage::maintenance(config.to_str(), &serde_json::json!({"mode":"reset","user":"root","password_hash":"original-hash","types":types}).to_string())?;
let storage = Storage::open(config.to_str())?;
let accounts: serde_json::Value = serde_json::from_str(&storage.request_line(r#"{"method":"accounts_list"}"#))?;
let saved = serde_json::from_value(accounts["accounts"].clone())?;
drop(storage);
let system = data.path().join("system");
let wal = chusql_core_storage::wal::Wal::new(system.join("wal.log"));
wal.append(&chusql_core_storage::wal::WalOp::Accounts { accounts: saved, removed: Vec::new(), migrate_roles: false })?;
wal.append(&chusql_core_storage::wal::WalOp::Commit)?;
drop(wal);
std::fs::remove_file(system.join("__system_users.db"))?;
Storage::maintenance(config.to_str(), &serde_json::json!({"mode":"recover","types":types}).to_string())?;
let storage = Storage::open(config.to_str())?;
let restored: serde_json::Value = serde_json::from_str(&storage.request_line(r#"{"method":"accounts_list"}"#))?;
assert_eq!(restored["accounts"], accounts["accounts"]);
drop(storage);
std::fs::write(system.join("catalog.json"), b"corrupted catalog")?;
Storage::maintenance(config.to_str(), &serde_json::json!({"mode":"recover","types":types}).to_string())?;
let storage = Storage::open(config.to_str())?;
let reconstructed: serde_json::Value = serde_json::from_str(&storage.request_line(r#"{"method":"accounts_list"}"#))?;
assert_eq!(reconstructed["accounts"], accounts["accounts"]);
Ok(())
}
/// 一次会话就是当前线程的存储句柄
struct Conn(Arc<Storage>);
@@ -800,7 +1227,7 @@ fn account_snapshot_replays_twice_without_duplicating_accounts() -> Result<(), B
for _ in 0..2 {
let wal = Wal::new(data.path().join("system/wal.log"));
wal.truncate()?;
wal.append(&WalOp::Accounts { accounts: accounts.clone() })?;
wal.append(&WalOp::Accounts { accounts: accounts.clone(), removed: Vec::new(), migrate_roles: false })?;
drop(wal);
std::fs::write(data.path().join("system/__system_users.idx"), b"torn index")?;
std::fs::write(data.path().join("system/__system_users.db"), b"torn heap")?;
@@ -1595,7 +2022,7 @@ fn stats_report_distinct_values() {
send(&mut c, r#"{"method":"delete_keys","table":"stat_t","keys":[1]}"#);
let r = send(&mut c, r#"{"method":"describe_table","table":"stat_t"}"#);
assert!(r.contains(r#""row_count":2"#), "after delete: {}", r);
assert!(r.contains(r#""name":"age","distinct":1"#), "after delete: {}", r);
assert!(r.contains(r#""name":"age","distinct":2"#), "remaining ages are still 30 and 41: {}", r);
}
/// 重开服务后索引仍在
@@ -1832,12 +2259,14 @@ fn account_table_uses_typed_columns_and_stamps_login() -> Result<(), Box<dyn std
assert!(account["last_login_at"].is_null(), "还没登录过: {}", account);
let catalog: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(data.path().join("system/catalog.json"))?)?;
let entry = &catalog["tables"]["__system_users"];
let credential_columns = &catalog["tables"]["__system_users"]["columns"];
assert_eq!(credential_columns.as_array().ok_or("missing credentials")?.iter().map(|c| c["name"].as_str().unwrap_or("")).collect::<Vec<_>>(), vec!["id", "password_hash"]);
let entry = &catalog["tables"]["__system_identities"];
let columns = entry["columns"].as_array().ok_or("missing columns")?;
let names: Vec<&str> = columns.iter().map(|col| col["name"].as_str().unwrap_or("")).collect();
assert_eq!(names, vec!["id", "user", "password_hash", "registered_at", "last_login_at", "revision"]);
assert_eq!(names, vec!["id", "user", "can_login", "is_superuser", "system_catalog_manager", "enabled", "identity_version", "registered_at", "last_login_at", "revision"]);
let types: Vec<&str> = columns.iter().map(|col| col["ty"].as_str().unwrap_or("")).collect();
assert_eq!(types, vec!["int", "varchar(64)", "varchar(256)", "timestamp", "timestamp", "int"]);
assert_eq!(types, vec!["int", "varchar(64)", "bool", "bool", "bool", "bool", "int", "timestamp", "timestamp", "int"]);
assert_eq!(columns[0]["primary_key"], true, "id 是主键");
assert_eq!(columns[0]["nullable"], false);
assert_eq!(columns[1]["unique"], true, "用户名唯一");
+25
View File
@@ -207,6 +207,31 @@ fn trailing_torn_frame_can_be_truncated() {
assert_eq!(wal.read_all().unwrap().records.len(), 1);
}
/// 完整损坏帧返回错误而非截断成功
#[test]
fn complete_corrupt_frame_is_rejected() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("wal.log");
std::fs::write(&path, [1, 0, 0, 0, 255]).unwrap();
assert!(Wal::new(&path).read_all().is_err());
assert_eq!(std::fs::read(path).unwrap(), [1, 0, 0, 0, 255]);
}
/// 检查点损坏与替换失败不丢旧值
#[test]
fn checkpoint_corruption_and_write_failure_are_explicit() {
let dir = tempfile::tempdir().unwrap();
let wal = Wal::new(dir.path().join("wal.log"));
for bytes in [vec![], vec![1; 7], vec![1; 9]] {
std::fs::write(dir.path().join("wal.checkpoint"), bytes).unwrap();
assert!(wal.read_checkpoint().is_err());
}
wal.write_checkpoint(7).unwrap();
std::fs::create_dir(dir.path().join("wal.checkpoint.pending")).unwrap();
assert!(wal.write_checkpoint(9).is_err());
assert_eq!(wal.read_checkpoint().unwrap(), Some(7));
}
/// 检查点写读一致
#[test]
fn checkpoint_round_trips() {