修复错误,优化性能,实现类型系统,优化安装卸载过程
This commit is contained in:
1 parent
fd5f320360
commit
48eec484b7
104 files changed
+4127
-845
No files matched your search
@@ -1,7 +1,7 @@
|
||||
# ChuSQL 存储服务配置示例
|
||||
#
|
||||
# 用法:安装时把本文件放到固定的用户配置目录(Windows 是 %APPDATA%\ChuSQL\chusql.toml,
|
||||
# 其它平台是 ${XDG_CONFIG_HOME:-$HOME/.config}/ChuSQL/chusql.toml);
|
||||
# 用法:安装时把本文件放到固定的用户配置目录(Windows 是 %APPDATA%\ChuSQL\settings.toml,
|
||||
# 其它平台是 ${XDG_CONFIG_HOME:-$HOME/.config}/ChuSQL/settings.toml);
|
||||
# 临时换一份就用命令行参数 --config <path>(没有环境变量通道了)。
|
||||
#
|
||||
# 每一项都可以删掉:没写的项回退到内置默认值(下面注释里的值就是默认值)。
|
||||
@@ -28,6 +28,9 @@ pool_size = 1024
|
||||
# Unix ${XDG_DATA_HOME:-$HOME/.local/share}/chusql/data,跟安装脚本装的位置一致);
|
||||
# 要固定就写绝对路径,例如:
|
||||
# data_dir = "/srv/chusql/data"
|
||||
#
|
||||
# 日志输出目录:启动器(csql-web)与安装脚本按它决定日志写哪;相对路径以启动目录为基准。
|
||||
# log_files = "./logs"
|
||||
|
||||
[log]
|
||||
# 日志等级:off / error / warn / info / debug。debug 会多打每个请求和连接。
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::collections::{BTreeMap, BTreeSet, HashSet};
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::io;
|
||||
use std::io::Write;
|
||||
@@ -51,9 +51,11 @@ pub struct TableSchema {
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
pub struct Catalog {
|
||||
#[serde(default)]
|
||||
pub identity_high_water: i64,
|
||||
tables: BTreeMap<String, TableSchema>,
|
||||
#[serde(skip)]
|
||||
seen: BTreeMap<String, HashSet<u64>>,
|
||||
seen: BTreeMap<String, HashMap<u64, u64>>,
|
||||
}
|
||||
|
||||
impl Catalog {
|
||||
@@ -153,17 +155,25 @@ impl Catalog {
|
||||
for r in rows {
|
||||
for (name, v) in r {
|
||||
let stat = entry.stats.entry(name.clone()).or_default();
|
||||
hist_remove(stat, v);
|
||||
if stat.capped {
|
||||
continue;
|
||||
}
|
||||
if seen.remove(&fold(name, value_hash(v))) {
|
||||
stat.distinct = stat.distinct.saturating_sub(1);
|
||||
let key = fold(name, value_hash(v));
|
||||
if let Some(count) = seen.get_mut(&key) {
|
||||
*count = count.saturating_sub(1);
|
||||
if *count == 0 {
|
||||
seen.remove(&key);
|
||||
stat.distinct = stat.distinct.saturating_sub(1);
|
||||
}
|
||||
}
|
||||
hist_remove(stat, v);
|
||||
}
|
||||
}
|
||||
if entry.row_count == 0 {
|
||||
seen.clear();
|
||||
for stat in entry.stats.values_mut() {
|
||||
stat.distinct = 0;
|
||||
stat.capped = false;
|
||||
stat.lo = None;
|
||||
stat.hi = None;
|
||||
stat.hist.clear();
|
||||
@@ -192,17 +202,18 @@ impl Catalog {
|
||||
let seen = self.seen.entry(table.to_string()).or_default();
|
||||
for (name, v) in row {
|
||||
let stat = entry.stats.entry(name.clone()).or_default();
|
||||
hist_add(stat, v);
|
||||
if stat.capped {
|
||||
continue;
|
||||
}
|
||||
if seen.insert(fold(name, value_hash(v))) {
|
||||
let count = seen.entry(fold(name, value_hash(v))).or_insert(0);
|
||||
*count += 1;
|
||||
if *count == 1 {
|
||||
stat.distinct += 1;
|
||||
if stat.distinct >= STATS_DISTINCT_CAP {
|
||||
stat.capped = true;
|
||||
seen.clear();
|
||||
}
|
||||
}
|
||||
hist_add(stat, v);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -476,5 +487,7 @@ fn hist_remove(stat: &mut ColumnStat, v: &serde_json::Value) {
|
||||
return;
|
||||
}
|
||||
let index = hist_index(lo, hi, x);
|
||||
stat.hist[index] = stat.hist[index].saturating_sub(1);
|
||||
if let Some(bucket) = (0..stat.hist.len()).filter(|&i| stat.hist[i] > 0).min_by_key(|&i| i.abs_diff(index)) {
|
||||
stat.hist[bucket] -= 1;
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ use crate::log::Level;
|
||||
// 存储相关分区由本模块解析校验,其它层的分区一概不碰。
|
||||
|
||||
/// 配置文件名
|
||||
pub const CONFIG_FILE_NAME: &str = "chusql.toml";
|
||||
pub const CONFIG_FILE_NAME: &str = "settings.toml";
|
||||
/// 配置目录名(随系统惯例放在用户配置目录下)
|
||||
pub const APP_DIR_NAME: &str = "ChuSQL";
|
||||
/// Unix 下的数据目录名(XDG 惯例小写,跟安装脚本的默认安装目录一致)
|
||||
@@ -20,6 +20,8 @@ pub const DEFAULT_POOL_SIZE: usize = 1024;
|
||||
pub const DEFAULT_PAGE_SIZE: usize = 4096;
|
||||
pub const DEFAULT_BTREE_ORDER: usize = 4;
|
||||
pub const DEFAULT_LOG_LEVEL: &str = "info";
|
||||
/// 默认日志输出目录(相对启动目录)
|
||||
pub const DEFAULT_LOG_FILES: &str = "./logs";
|
||||
|
||||
const MIN_PAGE_SIZE: usize = 512;
|
||||
const MAX_PAGE_SIZE: usize = 65536;
|
||||
@@ -91,6 +93,7 @@ pub struct Config {
|
||||
pub btree_order: usize,
|
||||
pub pool_size: usize,
|
||||
pub data_dir: PathBuf,
|
||||
pub log_files: PathBuf,
|
||||
pub log_level: Level,
|
||||
}
|
||||
|
||||
@@ -102,6 +105,7 @@ impl Default for Config {
|
||||
btree_order: DEFAULT_BTREE_ORDER,
|
||||
pool_size: DEFAULT_POOL_SIZE,
|
||||
data_dir: default_data_dir(),
|
||||
log_files: PathBuf::from(DEFAULT_LOG_FILES),
|
||||
log_level: Level::Info,
|
||||
}
|
||||
}
|
||||
@@ -161,6 +165,7 @@ struct FileBtree {
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct FileStorage {
|
||||
data_dir: Option<String>,
|
||||
log_files: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
@@ -229,6 +234,11 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
|
||||
default_data_dir().to_string_lossy().into_owned(),
|
||||
&file_origin,
|
||||
);
|
||||
let (log_files, log_files_origin) = pick(
|
||||
file.storage.as_ref().and_then(|s| s.log_files.clone()),
|
||||
DEFAULT_LOG_FILES.to_string(),
|
||||
&file_origin,
|
||||
);
|
||||
// 管名已经没有了:存储不再是独立进程,管道/套接字那一层随 FFI 一起删掉
|
||||
let (log_level, log_origin) = pick(
|
||||
file.log.and_then(|l| l.level),
|
||||
@@ -237,6 +247,7 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
|
||||
);
|
||||
|
||||
let data_dir = data_dir.trim().to_string();
|
||||
let log_files = log_files.trim().to_string();
|
||||
let log_level = log_level.trim().to_string();
|
||||
|
||||
let log_level = Level::parse(&log_level)
|
||||
@@ -245,6 +256,9 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
|
||||
if data_dir.is_empty() {
|
||||
return Err("storage.data_dir must not be empty".to_string());
|
||||
}
|
||||
if log_files.is_empty() {
|
||||
return Err("storage.log_files must not be empty".to_string());
|
||||
}
|
||||
validate_layout(page_size, btree_order)?;
|
||||
if pool_size == 0 {
|
||||
return Err("buffer.pool_size must be at least 1".to_string());
|
||||
@@ -254,6 +268,7 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
|
||||
("btree.order", btree_order.to_string(), order_origin),
|
||||
("buffer.pool_size", pool_size.to_string(), pool_origin),
|
||||
("storage.data_dir", data_dir.clone(), dir_origin),
|
||||
("storage.log_files", log_files.clone(), log_files_origin),
|
||||
("log.level", log_level.name().to_string(), log_origin),
|
||||
];
|
||||
|
||||
@@ -263,6 +278,7 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
|
||||
btree_order,
|
||||
pool_size,
|
||||
data_dir: PathBuf::from(data_dir),
|
||||
log_files: PathBuf::from(log_files),
|
||||
log_level,
|
||||
},
|
||||
config_path: if text.is_some() { path } else { None },
|
||||
@@ -318,15 +334,14 @@ size = 8192
|
||||
data_dir = "../data"
|
||||
|
||||
[server]
|
||||
host = "127.0.0.1"
|
||||
listen_host = "127.0.0.1"
|
||||
port = 7777
|
||||
max_message = 1048576
|
||||
|
||||
[web]
|
||||
port = 7778
|
||||
user = "root"
|
||||
"#;
|
||||
let loaded = resolve(Some(text), Some(PathBuf::from("chusql.toml"))).unwrap();
|
||||
let loaded = resolve(Some(text), Some(PathBuf::from("settings.toml"))).unwrap();
|
||||
assert_eq!(loaded.config.page_size, 8192);
|
||||
assert_eq!(loaded.config.data_dir, PathBuf::from("../data"));
|
||||
}
|
||||
|
||||
@@ -71,6 +71,33 @@ pub unsafe extern "C" fn chusql_storage_close(handle: *mut Storage) {
|
||||
let _ = catch_unwind(AssertUnwindSafe(|| unsafe { drop(Box::from_raw(handle)) }));
|
||||
}
|
||||
|
||||
/// 离线维护系统目录并返回 JSON
|
||||
///
|
||||
/// # Safety
|
||||
/// 输入为有效字符串;输出指针可写,响应使用 free 释放。
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn chusql_storage_maintenance(
|
||||
config_path: *const c_char, request: *const u8, request_len: usize,
|
||||
out: *mut *mut u8, out_len: *mut usize,
|
||||
) -> i32 {
|
||||
if request.is_null() || out.is_null() || out_len.is_null() { set_error("invalid maintenance argument"); return ERR; }
|
||||
unsafe { *out = ptr::null_mut(); *out_len = 0; }
|
||||
let result = catch_unwind(AssertUnwindSafe(|| {
|
||||
let path = if config_path.is_null() { None } else { Some(unsafe { CStr::from_ptr(config_path) }.to_str().map_err(|e| e.to_string())?) };
|
||||
let text = std::str::from_utf8(unsafe { slice::from_raw_parts(request, request_len) }).map_err(|e| e.to_string())?;
|
||||
Storage::maintenance(path, text)
|
||||
}));
|
||||
let response = match result {
|
||||
Ok(Ok(response)) => response,
|
||||
Ok(Err(err)) => { set_error(err); return ERR; }
|
||||
Err(_) => { set_error("panic in system maintenance"); return ERR; }
|
||||
};
|
||||
let mut buffer = response.into_bytes().into_boxed_slice();
|
||||
unsafe { *out = buffer.as_mut_ptr(); *out_len = buffer.len(); }
|
||||
std::mem::forget(buffer);
|
||||
OK
|
||||
}
|
||||
|
||||
/// 处理一条请求,响应缓冲须归还
|
||||
///
|
||||
/// # Safety
|
||||
|
||||
@@ -6,6 +6,8 @@ use std::collections::HashMap;
|
||||
pub type Row = HashMap<String, serde_json::Value>;
|
||||
|
||||
pub const USERS_TABLE: &str = "__system_users";
|
||||
pub const IDENTITIES_TABLE: &str = "__system_identities";
|
||||
pub const TYPES_TABLE: &str = "__system_types";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct Account {
|
||||
@@ -17,6 +19,16 @@ pub struct Account {
|
||||
pub registered_at: String,
|
||||
#[serde(default)]
|
||||
pub last_login_at: Option<String>,
|
||||
#[serde(default = "default_true")]
|
||||
pub can_login: bool,
|
||||
#[serde(default)]
|
||||
pub is_superuser: bool,
|
||||
#[serde(default)]
|
||||
pub system_catalog_manager: bool,
|
||||
#[serde(default = "default_true")]
|
||||
pub enabled: bool,
|
||||
#[serde(default)]
|
||||
pub identity_version: u8,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
@@ -80,7 +92,11 @@ pub struct IndexWire {
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "method", rename_all = "snake_case")]
|
||||
pub enum Request {
|
||||
CatalogManage { command: Box<Request> },
|
||||
AccountsList,
|
||||
IdentityInitialize { administrator: String },
|
||||
RoleCreate { user: String },
|
||||
IdentityAlter { user: String, can_login: Option<bool>, is_superuser: Option<bool>, enabled: Option<bool>, system_catalog_manager: Option<bool> },
|
||||
AccountCreate { user: String, password_hash: String },
|
||||
AccountReset { user: String, password_hash: String },
|
||||
AccountLogin {
|
||||
@@ -97,6 +113,7 @@ pub enum Request {
|
||||
password_hash: Option<String>,
|
||||
},
|
||||
SystemStatus,
|
||||
BootstrapTypes { types: Vec<Row> },
|
||||
Ping,
|
||||
Scan {
|
||||
table: String,
|
||||
@@ -222,9 +239,11 @@ impl Request {
|
||||
| Self::DropIndex { table, .. } | Self::DropColumn { table, .. }
|
||||
| Self::Compact { table }
|
||||
| Self::ReplaceSchema { table, .. } => Some(table),
|
||||
Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList
|
||||
Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList | Self::CatalogManage { .. }
|
||||
| Self::IdentityInitialize { .. } | Self::RoleCreate { .. } | Self::IdentityAlter { .. }
|
||||
| Self::AccountCreate { .. } | Self::AccountReset { .. } | Self::AccountLogin { .. }
|
||||
| Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus
|
||||
| Self::BootstrapTypes { .. }
|
||||
| Self::ApplyTransaction { .. } => None,
|
||||
}
|
||||
}
|
||||
@@ -245,7 +264,8 @@ pub const PRIVILEGE_TABLES: [&str; 4] = [
|
||||
|
||||
/// 是不是服务自己经请求通道读写的系统表
|
||||
pub fn internal_table(table: &str) -> bool {
|
||||
PRIVILEGE_TABLES.contains(&table.to_ascii_lowercase().as_str())
|
||||
let name = table.to_ascii_lowercase();
|
||||
PRIVILEGE_TABLES.contains(&name.as_str()) || name.starts_with("__system_domain_")
|
||||
}
|
||||
|
||||
/// 是不是请求通道要拒绝的系统表
|
||||
|
||||
@@ -8,8 +8,8 @@ use crate::config::{self, Config, Loaded};
|
||||
use crate::heap::{scalar_int, HeapTable};
|
||||
use crate::log;
|
||||
use crate::protocol::{
|
||||
blocked_table, reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row,
|
||||
SchemaColumn, StorageOp, TableSchemaWire, USERS_TABLE,
|
||||
blocked_table, reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row, IDENTITIES_TABLE,
|
||||
SchemaColumn, StorageOp, TableSchemaWire, USERS_TABLE, TYPES_TABLE,
|
||||
};
|
||||
use crate::wal::{committed_ops, records_after, Wal, WalOp};
|
||||
use crate::{log_debug, log_error, log_info, log_warn};
|
||||
@@ -222,15 +222,41 @@ impl Server {
|
||||
c.rebuild_stats(table, rows);
|
||||
c.save(self.catalog_path())
|
||||
}
|
||||
WalOp::Accounts { accounts } => {
|
||||
WalOp::Accounts { accounts, removed, migrate_roles } => {
|
||||
let rows = account_rows(accounts)?;
|
||||
self.tables.lock().map_err(|_| std::io::Error::other("table lock poisoned"))?.remove(USERS_TABLE);
|
||||
for path in [self.table_path(USERS_TABLE), self.index_path(USERS_TABLE, "id")] {
|
||||
std::fs::File::create(path)?.sync_all()?;
|
||||
let identities = identity_rows(accounts)?;
|
||||
for (table, records) in [(USERS_TABLE, &rows), (IDENTITIES_TABLE, &identities)] {
|
||||
self.tables.lock().map_err(|_| std::io::Error::other("table lock poisoned"))?.remove(table);
|
||||
for path in [self.table_path(table), self.index_path(table, "id")] {
|
||||
std::fs::File::create(path)?.sync_all()?;
|
||||
}
|
||||
self.with_table(table, |t| { t.replace_all(records)?; t.flush() })?;
|
||||
}
|
||||
let legacy_roles = "__system_roles";
|
||||
let has_roles = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?.describe(legacy_roles).is_some();
|
||||
if has_roles && *migrate_roles {
|
||||
self.with_existing_table(legacy_roles, |t| { t.replace_all(&[])?; t.flush() })?;
|
||||
}
|
||||
let mut cleaned = Vec::new();
|
||||
for table in ["__system_grants", "__system_grant_options", "__system_members"] {
|
||||
let exists = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?.describe(table).is_some();
|
||||
if exists && !removed.is_empty() {
|
||||
let rows = self.with_existing_table(table, |t| t.scan())?.into_iter().filter(|r|
|
||||
!["role", "member"].iter().any(|field| r.get(*field).and_then(serde_json::Value::as_str).is_some_and(|name| removed.iter().any(|old| old == name)))).collect::<Vec<_>>();
|
||||
self.with_existing_table(table, |t| { t.replace_all(&rows)?; t.flush() })?;
|
||||
cleaned.push((table, rows));
|
||||
}
|
||||
}
|
||||
self.with_table(USERS_TABLE, |t| { t.replace_all(&rows)?; t.flush() })?;
|
||||
let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||
declare_system_schema(&mut c, &rows)?;
|
||||
c.rebuild_stats(IDENTITIES_TABLE, &identities);
|
||||
c.set_columns(IDENTITIES_TABLE, identity_columns())?;
|
||||
c.rebuild_stats(IDENTITIES_TABLE, &identities);
|
||||
c.mark_system(IDENTITIES_TABLE)?;
|
||||
c.add_index(IDENTITIES_TABLE, "user")?;
|
||||
c.identity_high_water = c.identity_high_water.max(accounts.iter().map(|a| a.id).max().unwrap_or(0));
|
||||
if has_roles && *migrate_roles { c.rebuild_stats(legacy_roles, &[]); }
|
||||
for (table, records) in cleaned { c.rebuild_stats(table, &records); }
|
||||
c.save(self.catalog_path())
|
||||
}
|
||||
WalOp::Insert { table, row } => {
|
||||
@@ -284,6 +310,7 @@ impl Server {
|
||||
self.with_table(table, |_t| Ok(()))?;
|
||||
let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||
c.create_table(table, columns.clone())?;
|
||||
if table == TYPES_TABLE { c.mark_system(table)?; }
|
||||
c.save(self.catalog_path())
|
||||
}
|
||||
WalOp::DropTable { table } => {
|
||||
@@ -342,8 +369,67 @@ impl Server {
|
||||
if self.recovery_required.load(Ordering::Acquire) {
|
||||
return Err(std::io::Error::other("storage recovery required"));
|
||||
}
|
||||
self.validate_op(op)?;
|
||||
self.group_via_wal(std::slice::from_ref(op))
|
||||
let op = self.normalize_decimal_write(op)?;
|
||||
self.validate_op(&op)?;
|
||||
self.validate_domains(&op)?;
|
||||
self.group_via_wal(std::slice::from_ref(&op))
|
||||
}
|
||||
|
||||
/// 写日志前规范化定点数值
|
||||
fn normalize_decimal_write(&self, op: &WalOp) -> std::io::Result<WalOp> {
|
||||
let mut normalized = op.clone();
|
||||
let catalog = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||
let (table, rows): (&str, Vec<&mut Row>) = match &mut normalized {
|
||||
WalOp::Insert { table, row } => (table, vec![row]),
|
||||
WalOp::InsertBatch { table, rows } | WalOp::ReplaceAll { table, rows } => (table, rows.iter_mut().collect()),
|
||||
WalOp::CreateTable { columns, .. } => {
|
||||
for column in columns { decimal_parameters(&column.ty)?; }
|
||||
return Ok(normalized);
|
||||
}
|
||||
WalOp::ReplaceSchema { columns, rows, .. } => {
|
||||
for column in columns.iter() { decimal_parameters(&column.ty)?; }
|
||||
for row in rows { normalize_decimal_row(row, columns)?; }
|
||||
return Ok(normalized);
|
||||
}
|
||||
_ => return Ok(normalized),
|
||||
};
|
||||
if let Some(schema) = catalog.describe(table) {
|
||||
for row in rows {
|
||||
normalize_decimal_row(row, &schema.columns)?;
|
||||
}
|
||||
}
|
||||
Ok(normalized)
|
||||
}
|
||||
|
||||
/// 校验命名类型引用与删除依赖
|
||||
fn validate_domains(&self, op: &WalOp) -> std::io::Result<()> {
|
||||
let catalog = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||
match op {
|
||||
WalOp::CreateTable { columns, .. } | WalOp::ReplaceSchema { columns, .. } => {
|
||||
for column in columns {
|
||||
let mut ty = column.ty.as_str();
|
||||
while let Some(body) = ty.strip_prefix("domain(").and_then(|s| s.strip_suffix(')')) {
|
||||
let (name, base) = body.split_once(',').ok_or_else(|| std::io::Error::other("invalid domain type"))?;
|
||||
let key = format!("__system_domain_{name}");
|
||||
let definition = catalog.describe(&key).ok_or_else(|| std::io::Error::other(format!("unknown domain: {name}")))?;
|
||||
if !definition.columns.iter().any(|c| c.name == "base" && c.ty == base) {
|
||||
return Err(std::io::Error::other(format!("domain base type mismatch: {name}")));
|
||||
}
|
||||
ty = base;
|
||||
}
|
||||
}
|
||||
}
|
||||
WalOp::DropTable { table } => {
|
||||
if let Some(name) = table.strip_prefix("__system_domain_") {
|
||||
let prefix = format!("domain({name},");
|
||||
if catalog.all_tables().iter().any(|(_, schema)| schema.columns.iter().any(|c| !schema.dropped_columns.contains(&c.name) && c.ty.contains(&prefix))) {
|
||||
return Err(std::io::Error::other(format!("domain is still used by a column: {name}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 校验一条操作:命中被删的列就拒绝
|
||||
@@ -406,6 +492,7 @@ impl Server {
|
||||
rows: rows.clone(),
|
||||
},
|
||||
};
|
||||
let wal_op = self.normalize_decimal_write(&wal_op)?;
|
||||
self.validate_op(&wal_op)?;
|
||||
wal_ops.push(wal_op);
|
||||
}
|
||||
@@ -420,13 +507,9 @@ impl Server {
|
||||
std::fs::create_dir_all(&self.cfg.data_dir)?;
|
||||
let wal = &self.wal;
|
||||
let commit_lsn = wal.append_group(ops)?;
|
||||
self.recovery_required.store(true, Ordering::Release);
|
||||
for op in ops {
|
||||
let result = self.apply_op(op);
|
||||
if result.is_err() && matches!(op, WalOp::HideColumn { .. }) {
|
||||
self.recovery_required.store(true, Ordering::Release);
|
||||
return result;
|
||||
}
|
||||
result?;
|
||||
self.apply_op(op)?;
|
||||
}
|
||||
for op in ops {
|
||||
self.flush_op_tables(op)?;
|
||||
@@ -434,6 +517,7 @@ impl Server {
|
||||
self.flush_all_tables()?;
|
||||
wal.write_checkpoint(commit_lsn)?;
|
||||
wal.truncate_before(commit_lsn)?;
|
||||
self.recovery_required.store(false, Ordering::Release);
|
||||
log_debug!(core, "WAL committed group at LSN {}", commit_lsn);
|
||||
Ok(())
|
||||
}
|
||||
@@ -479,15 +563,15 @@ impl Server {
|
||||
fn recover_wal(&self) -> std::io::Result<()> {
|
||||
std::fs::create_dir_all(&self.cfg.data_dir)?;
|
||||
let wal = &self.wal;
|
||||
let checkpoint = wal.read_checkpoint()?.unwrap_or(0);
|
||||
let scan = wal.read_all()?;
|
||||
if scan.records.is_empty() {
|
||||
if scan.good_len > 0 {
|
||||
if wal.path().exists() {
|
||||
wal.truncate_to(scan.good_len)?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
// 检查点以内的记录已经落盘,直接按边界截掉
|
||||
let checkpoint = wal.read_checkpoint()?.unwrap_or(0);
|
||||
let pending = records_after(&scan.records, checkpoint);
|
||||
let records = committed_ops(&pending);
|
||||
if records.is_empty() {
|
||||
@@ -859,6 +943,12 @@ impl Server {
|
||||
use std::io::{Error, ErrorKind};
|
||||
let registered = {
|
||||
let c = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?;
|
||||
match c.describe(IDENTITIES_TABLE) {
|
||||
Some(schema) if !schema.system => return Err(Error::new(ErrorKind::AlreadyExists, "reserved identity table collision")),
|
||||
None if self.table_path(IDENTITIES_TABLE).exists() || self.index_path(IDENTITIES_TABLE, "id").exists() =>
|
||||
return Err(Error::new(ErrorKind::AlreadyExists, "reserved identity file collision")),
|
||||
_ => {}
|
||||
}
|
||||
c.describe(USERS_TABLE).map(|schema| schema.system)
|
||||
};
|
||||
match registered {
|
||||
@@ -893,6 +983,21 @@ impl Server {
|
||||
if !self.table_path(USERS_TABLE).exists() {
|
||||
return Err(Error::other("missing account data"));
|
||||
}
|
||||
let unified = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe(IDENTITIES_TABLE).is_some();
|
||||
if unified {
|
||||
let credentials = self.with_existing_table(USERS_TABLE, |t| t.scan())?;
|
||||
let identities = self.with_existing_table(IDENTITIES_TABLE, |t| t.scan())?;
|
||||
return identities.into_iter().map(|mut row| {
|
||||
for field in ["can_login", "is_superuser", "enabled"] {
|
||||
if row.get(field).and_then(serde_json::Value::as_bool).is_none() { return Err(Error::other("invalid identity attributes")); }
|
||||
}
|
||||
let id = row.get("id").and_then(serde_json::Value::as_i64).ok_or_else(|| Error::other("invalid identity record"))?;
|
||||
let credential = credentials.iter().find(|r| r.get("id").and_then(serde_json::Value::as_i64) == Some(id))
|
||||
.ok_or_else(|| Error::other("missing identity credential"))?;
|
||||
row.insert("password_hash".into(), credential.get("password_hash").cloned().ok_or_else(|| Error::other("invalid identity credential"))?);
|
||||
decode_account_row(&row)
|
||||
}).collect();
|
||||
}
|
||||
let mut accounts: Vec<Account> = self
|
||||
.with_existing_table(USERS_TABLE, |t| t.scan())?
|
||||
.iter()
|
||||
@@ -919,9 +1024,11 @@ impl Server {
|
||||
// 表里已经有行就不再补人:引导程序只在空表上播种,
|
||||
// 免得有人拿 storage 透传绕过账号接口往里塞一个已知口令的账号
|
||||
if accounts.is_empty() && !accounts.iter().any(|a| a.user == name) {
|
||||
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0)
|
||||
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0).max(self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.identity_high_water)
|
||||
.checked_add(1).ok_or_else(|| Error::other("account id exhausted"))?;
|
||||
accounts.push(new_account(id, &name, password_hash.unwrap_or_default().to_string())?);
|
||||
let mut administrator = new_account(id, &name, password_hash.unwrap_or_default().to_string())?;
|
||||
administrator.is_superuser = true;
|
||||
accounts.push(administrator);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
@@ -931,7 +1038,7 @@ impl Server {
|
||||
for row in account_rows(&accounts)? {
|
||||
HeapTable::encode_row(&row, self.cfg.page_size)?;
|
||||
}
|
||||
let op = WalOp::Accounts { accounts: accounts.clone() };
|
||||
let op = WalOp::Accounts { accounts: accounts.clone(), removed: Vec::new(), migrate_roles: false };
|
||||
self.recovery_required.store(true, Ordering::Release);
|
||||
self.wal.truncate()?;
|
||||
self.wal.append(&op)?;
|
||||
@@ -952,17 +1059,93 @@ impl Server {
|
||||
return Err(Error::other("system catalog is not initialized; run csql-bootstrap"));
|
||||
}
|
||||
let mut accounts = self.load_accounts()?;
|
||||
let request = match request {
|
||||
Request::CatalogManage { command } => {
|
||||
let target = match command.as_ref() {
|
||||
Request::AccountCreate { user, .. } | Request::AccountReset { user, .. } | Request::AccountDrop { user }
|
||||
| Request::IdentityAlter { user, is_superuser: None, system_catalog_manager: None, .. } => Some(account_name(user)?),
|
||||
Request::AccountsList => None,
|
||||
_ => return Err(Error::other("superuser required for this catalog operation")),
|
||||
};
|
||||
if accounts.iter().any(|a| target.as_ref() == Some(&a.user) && (a.is_superuser || a.system_catalog_manager)) {
|
||||
return Err(Error::other("superuser required to manage privileged identities"));
|
||||
}
|
||||
*command
|
||||
}
|
||||
other => other,
|
||||
};
|
||||
let mut names = std::collections::HashSet::new();
|
||||
let mut ids = std::collections::HashSet::new();
|
||||
for account in &mut accounts {
|
||||
let name = account_name(&account.user)?;
|
||||
if !names.insert(name.clone()) || account.id < 1 || !ids.insert(account.id) {
|
||||
return Err(Error::other("invalid or duplicate identity record"));
|
||||
}
|
||||
if account.identity_version == 0 { account.user = name; }
|
||||
}
|
||||
let previous = accounts.clone();
|
||||
let migrate_roles = matches!(request, Request::IdentityInitialize { .. });
|
||||
if !migrate_roles && !matches!(request, Request::AccountsList) {
|
||||
let legacy_exists = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe("__system_roles").is_some();
|
||||
if legacy_exists && !self.with_existing_table("__system_roles", |t| t.scan())?.is_empty() {
|
||||
return Err(Error::other("identity migration required"));
|
||||
}
|
||||
}
|
||||
let had_superuser = accounts.iter().any(|a| a.enabled && a.can_login && a.is_superuser);
|
||||
match request {
|
||||
Request::AccountsList => return Ok(accounts),
|
||||
Request::IdentityInitialize { administrator } => {
|
||||
for account in &mut accounts {
|
||||
if account.identity_version == 0 {
|
||||
account.is_superuser = account.user == administrator.to_ascii_lowercase();
|
||||
account.identity_version = 1;
|
||||
}
|
||||
}
|
||||
let legacy_roles = "__system_roles";
|
||||
let exists = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe(legacy_roles).is_some();
|
||||
if exists {
|
||||
for row in self.with_existing_table(legacy_roles, |t| t.scan())? {
|
||||
let name = account_name(row.get("name").and_then(serde_json::Value::as_str).ok_or_else(|| Error::other("invalid legacy role"))?)?;
|
||||
if accounts.iter().any(|a| a.user == name) { return Err(Error::other(format!("identity name collision: {name}"))); }
|
||||
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0).max(self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.identity_high_water)
|
||||
.checked_add(1).ok_or_else(|| Error::other("account id exhausted"))?;
|
||||
let mut role = new_account(id, &name, String::new())?;
|
||||
role.can_login = false;
|
||||
accounts.push(role);
|
||||
}
|
||||
}
|
||||
let unified = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe(IDENTITIES_TABLE).is_some();
|
||||
if !accounts.is_empty() { require_superuser(&accounts)?; }
|
||||
if unified && previous == accounts { return Ok(accounts); }
|
||||
}
|
||||
Request::AccountCreate { user, password_hash } => {
|
||||
let name = account_name(&user)?;
|
||||
if accounts.iter().any(|a| a.user == name) {
|
||||
return Err(Error::new(ErrorKind::AlreadyExists, "account already exists"));
|
||||
}
|
||||
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0)
|
||||
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0).max(self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.identity_high_water)
|
||||
.checked_add(1).ok_or_else(|| Error::other("account id exhausted"))?;
|
||||
accounts.push(new_account(id, &name, password_hash)?);
|
||||
}
|
||||
Request::RoleCreate { user } => {
|
||||
let name = account_name(&user)?;
|
||||
if accounts.iter().any(|a| a.user == name) { return Err(Error::new(ErrorKind::AlreadyExists, "identity already exists")); }
|
||||
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0).max(self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.identity_high_water)
|
||||
.checked_add(1).ok_or_else(|| Error::other("account id exhausted"))?;
|
||||
let mut role = new_account(id, &name, String::new())?;
|
||||
role.can_login = false;
|
||||
accounts.push(role);
|
||||
}
|
||||
Request::IdentityAlter { user, can_login, is_superuser, enabled, system_catalog_manager } => {
|
||||
let name = account_name(&user)?;
|
||||
let account = accounts.iter_mut().find(|a| a.user == name).ok_or_else(|| Error::new(ErrorKind::NotFound, "unknown account"))?;
|
||||
account.can_login = can_login.unwrap_or(account.can_login);
|
||||
account.is_superuser = is_superuser.unwrap_or(account.is_superuser);
|
||||
account.enabled = enabled.unwrap_or(account.enabled);
|
||||
account.system_catalog_manager = system_catalog_manager.unwrap_or(account.system_catalog_manager);
|
||||
account.revision = account.revision.checked_add(1).ok_or_else(|| Error::other("account revision exhausted"))?;
|
||||
if had_superuser { require_superuser(&accounts)?; }
|
||||
}
|
||||
Request::AccountReset { user, password_hash } => {
|
||||
let name = account_name(&user)?;
|
||||
validate_account_hash(&password_hash)?;
|
||||
@@ -976,6 +1159,7 @@ impl Server {
|
||||
let stamp = at.unwrap_or_else(now_stamp);
|
||||
let account = accounts.iter_mut().find(|a| a.user == name)
|
||||
.ok_or_else(|| Error::new(ErrorKind::NotFound, "unknown account"))?;
|
||||
if !account.enabled || !account.can_login { return Err(Error::other("identity cannot login")); }
|
||||
account.last_login_at = Some(stamp);
|
||||
}
|
||||
Request::AccountDrop { user } => {
|
||||
@@ -985,13 +1169,15 @@ impl Server {
|
||||
if accounts.len() == before {
|
||||
return Err(Error::new(ErrorKind::NotFound, "unknown account"));
|
||||
}
|
||||
if had_superuser { require_superuser(&accounts)?; }
|
||||
}
|
||||
_ => return Err(Error::other("unexpected account request")),
|
||||
}
|
||||
for row in account_rows(&accounts)? {
|
||||
for row in account_rows(&accounts)?.into_iter().chain(identity_rows(&accounts)?) {
|
||||
HeapTable::encode_row(&row, self.cfg.page_size)?;
|
||||
}
|
||||
let op = WalOp::Accounts { accounts: accounts.clone() };
|
||||
let removed = previous.iter().filter(|old| !accounts.iter().any(|a| a.user == old.user)).map(|a| a.user.clone()).collect();
|
||||
let op = WalOp::Accounts { accounts: accounts.clone(), removed, migrate_roles };
|
||||
self.recovery_required.store(true, Ordering::Release);
|
||||
self.wal.truncate()?;
|
||||
self.wal.append(&op)?;
|
||||
@@ -1001,6 +1187,53 @@ impl Server {
|
||||
Ok(accounts)
|
||||
}
|
||||
|
||||
/// 初始化并校验内置类型目录
|
||||
fn bootstrap_types(&self, types: Vec<Row>) -> std::io::Result<Vec<Row>> {
|
||||
use std::io::Error;
|
||||
let _guard = self.write_lock.lock().map_err(|_| Error::other("write lock poisoned"))?;
|
||||
if !self.account_table_ready()? { return Err(Error::other("system catalog is not initialized; run csql-bootstrap")); }
|
||||
require_superuser(&self.load_accounts()?)?;
|
||||
let columns = types_columns();
|
||||
let existing = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe(TYPES_TABLE).cloned();
|
||||
if existing.as_ref().is_some_and(|s| !s.system || s.columns != columns) { return Err(Error::other("reserved type catalog collision")); }
|
||||
if existing.is_none() && (self.table_path(TYPES_TABLE).exists() || self.index_path(TYPES_TABLE, "id").exists()) {
|
||||
return Err(Error::other("reserved type file collision"));
|
||||
}
|
||||
let mut rows = if existing.is_some() { self.with_existing_table(TYPES_TABLE, |t| t.scan())? } else { Vec::new() };
|
||||
let mut names = std::collections::HashSet::new();
|
||||
let mut ids = std::collections::HashSet::new();
|
||||
for row in &rows {
|
||||
let name = type_definition_name(row)?;
|
||||
let id = row.get("id").and_then(serde_json::Value::as_i64).ok_or_else(|| Error::other("invalid type id"))?;
|
||||
if id < 1 || !ids.insert(id) || !names.insert(name.to_string()) { return Err(Error::other("duplicate type catalog record")); }
|
||||
}
|
||||
let original = rows.clone();
|
||||
let mut requested = std::collections::HashSet::new();
|
||||
for mut definition in types {
|
||||
let name = type_definition_name(&definition)?.to_string();
|
||||
if definition.len() != 3 || !requested.insert(name.clone()) { return Err(Error::other("invalid or duplicate preinstalled type")); }
|
||||
if let Some(saved) = rows.iter().find(|row| row.get("name").and_then(serde_json::Value::as_str) == Some(&name)) {
|
||||
if saved.get("base_type") != definition.get("base_type") || saved.get("parameterized") != definition.get("parameterized") {
|
||||
return Err(Error::other(format!("preinstalled type definition conflict: {name}")));
|
||||
}
|
||||
} else {
|
||||
let id = rows.iter().filter_map(|r| r.get("id").and_then(serde_json::Value::as_i64)).max().unwrap_or(0)
|
||||
.checked_add(1).ok_or_else(|| Error::other("type id exhausted"))?;
|
||||
definition.insert("id".into(), serde_json::json!(id));
|
||||
HeapTable::encode_row(&definition, self.cfg.page_size)?;
|
||||
rows.push(definition);
|
||||
}
|
||||
}
|
||||
if existing.is_some() && original == rows { return Ok(rows); }
|
||||
let mut ops = Vec::new();
|
||||
if existing.is_none() { ops.push(WalOp::CreateTable { table: TYPES_TABLE.into(), columns }); }
|
||||
ops.push(WalOp::ReplaceAll { table: TYPES_TABLE.into(), rows: rows.clone() });
|
||||
self.recovery_required.store(true, Ordering::Release);
|
||||
self.group_via_wal(&ops)?;
|
||||
self.recovery_required.store(false, Ordering::Release);
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// 一条请求翻成一条响应:先过读/写门再分发
|
||||
fn handle_request(&self, req: Request) -> Response {
|
||||
if concurrent_read(&req) {
|
||||
@@ -1037,7 +1270,12 @@ impl Server {
|
||||
}
|
||||
}
|
||||
match req {
|
||||
request @ (Request::AccountsList | Request::AccountCreate { .. }
|
||||
Request::BootstrapTypes { types } => match self.bootstrap_types(types) {
|
||||
Ok(rows) => Response::Rows { rows },
|
||||
Err(e) => Response::Error { message: e.to_string() },
|
||||
},
|
||||
request @ (Request::AccountsList | Request::CatalogManage { .. } | Request::AccountCreate { .. }
|
||||
| Request::IdentityInitialize { .. } | Request::RoleCreate { .. } | Request::IdentityAlter { .. }
|
||||
| Request::AccountReset { .. } | Request::AccountLogin { .. }
|
||||
| Request::AccountDrop { .. }) => {
|
||||
match self.accounts_request(request) {
|
||||
@@ -1458,12 +1696,45 @@ impl Server {
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造内置类型目录列定义
|
||||
fn types_columns() -> Vec<SchemaColumn> {
|
||||
vec![
|
||||
SchemaColumn { name: "id".into(), ty: "int".into(), nullable: false, primary_key: true, ..Default::default() },
|
||||
SchemaColumn { name: "name".into(), ty: "varchar(64)".into(), nullable: false, unique: true, ..Default::default() },
|
||||
SchemaColumn { name: "base_type".into(), ty: "varchar(64)".into(), nullable: false, ..Default::default() },
|
||||
SchemaColumn { name: "parameterized".into(), ty: "bool".into(), nullable: false, ..Default::default() },
|
||||
]
|
||||
}
|
||||
|
||||
/// 校验类型定义并读取名称
|
||||
fn type_definition_name(row: &Row) -> std::io::Result<&str> {
|
||||
let name = row.get("name").and_then(serde_json::Value::as_str).ok_or_else(|| std::io::Error::other("invalid type name"))?;
|
||||
let base = row.get("base_type").and_then(serde_json::Value::as_str).ok_or_else(|| std::io::Error::other("invalid type definition"))?;
|
||||
if name.is_empty() || name.len() > 48 || !name.bytes().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == b'_')
|
||||
|| base.is_empty() || base.len() > 64 || row.get("parameterized").and_then(serde_json::Value::as_bool).is_none() {
|
||||
return Err(std::io::Error::other("invalid type definition"));
|
||||
}
|
||||
Ok(name)
|
||||
}
|
||||
|
||||
/// 账号列表转成行
|
||||
fn account_rows(accounts: &[Account]) -> std::io::Result<Vec<Row>> {
|
||||
accounts.iter().map(|a| Ok(HashMap::from([
|
||||
("id".into(), serde_json::json!(a.id)),
|
||||
("user".into(), serde_json::Value::String(a.user.clone())),
|
||||
("password_hash".into(), serde_json::Value::String(a.password_hash.clone())),
|
||||
]))).collect()
|
||||
}
|
||||
|
||||
/// 身份元数据转成行
|
||||
fn identity_rows(accounts: &[Account]) -> std::io::Result<Vec<Row>> {
|
||||
accounts.iter().map(|a| Ok(HashMap::from([
|
||||
("id".into(), serde_json::json!(a.id)),
|
||||
("user".into(), serde_json::Value::String(a.user.clone())),
|
||||
("can_login".into(), serde_json::json!(a.can_login)),
|
||||
("is_superuser".into(), serde_json::json!(a.is_superuser)),
|
||||
("system_catalog_manager".into(), serde_json::json!(a.system_catalog_manager)),
|
||||
("enabled".into(), serde_json::json!(a.enabled)),
|
||||
("identity_version".into(), serde_json::json!(a.identity_version)),
|
||||
("registered_at".into(), serde_json::Value::String(a.registered_at.clone())),
|
||||
("last_login_at".into(), match &a.last_login_at {
|
||||
Some(stamp) => serde_json::Value::String(stamp.clone()),
|
||||
@@ -1478,10 +1749,23 @@ fn users_columns() -> Vec<SchemaColumn> {
|
||||
vec![
|
||||
SchemaColumn { name: "id".into(), ty: "int".into(), nullable: false,
|
||||
auto_increment: true, primary_key: true, ..Default::default() },
|
||||
SchemaColumn { name: "user".into(), ty: "varchar(64)".into(), nullable: false,
|
||||
unique: true, ..Default::default() },
|
||||
SchemaColumn { name: "password_hash".into(), ty: "varchar(256)".into(),
|
||||
nullable: false, ..Default::default() },
|
||||
]
|
||||
}
|
||||
|
||||
/// 系统身份目录结构
|
||||
fn identity_columns() -> Vec<SchemaColumn> {
|
||||
vec![
|
||||
SchemaColumn { name: "id".into(), ty: "int".into(), nullable: false,
|
||||
primary_key: true, ..Default::default() },
|
||||
SchemaColumn { name: "user".into(), ty: "varchar(64)".into(), nullable: false,
|
||||
unique: true, ..Default::default() },
|
||||
SchemaColumn { name: "can_login".into(), ty: "bool".into(), nullable: false, ..Default::default() },
|
||||
SchemaColumn { name: "is_superuser".into(), ty: "bool".into(), nullable: false, ..Default::default() },
|
||||
SchemaColumn { name: "system_catalog_manager".into(), ty: "bool".into(), nullable: false, ..Default::default() },
|
||||
SchemaColumn { name: "enabled".into(), ty: "bool".into(), nullable: false, ..Default::default() },
|
||||
SchemaColumn { name: "identity_version".into(), ty: "int".into(), nullable: false, ..Default::default() },
|
||||
SchemaColumn { name: "registered_at".into(), ty: "timestamp".into(),
|
||||
nullable: false, ..Default::default() },
|
||||
SchemaColumn { name: "last_login_at".into(), ty: "timestamp".into(),
|
||||
@@ -1497,7 +1781,6 @@ fn declare_system_schema(catalog: &mut Catalog, rows: &[Row]) -> std::io::Result
|
||||
catalog.set_columns(USERS_TABLE, users_columns())?;
|
||||
catalog.rebuild_stats(USERS_TABLE, rows);
|
||||
catalog.mark_system(USERS_TABLE)?;
|
||||
catalog.add_index(USERS_TABLE, "user")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1519,6 +1802,11 @@ fn decode_account_row(row: &Row) -> std::io::Result<Account> {
|
||||
revision: row.get("revision").and_then(serde_json::Value::as_u64).unwrap_or(1),
|
||||
registered_at: row.get("registered_at").and_then(serde_json::Value::as_str).unwrap_or("").to_string(),
|
||||
last_login_at: row.get("last_login_at").and_then(serde_json::Value::as_str).map(str::to_string),
|
||||
can_login: row.get("can_login").and_then(serde_json::Value::as_bool).unwrap_or(true),
|
||||
is_superuser: row.get("is_superuser").and_then(serde_json::Value::as_bool).unwrap_or(false),
|
||||
system_catalog_manager: row.get("system_catalog_manager").and_then(serde_json::Value::as_bool).unwrap_or(false),
|
||||
enabled: row.get("enabled").and_then(serde_json::Value::as_bool).unwrap_or(true),
|
||||
identity_version: row.get("identity_version").and_then(serde_json::Value::as_u64).unwrap_or(0) as u8,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1589,9 +1877,20 @@ fn new_account(id: i64, user: &str, password_hash: String) -> std::io::Result<Ac
|
||||
revision: 1,
|
||||
registered_at: now_stamp(),
|
||||
last_login_at: None,
|
||||
can_login: true,
|
||||
is_superuser: false,
|
||||
system_catalog_manager: false,
|
||||
enabled: true,
|
||||
identity_version: 1,
|
||||
})
|
||||
}
|
||||
|
||||
/// 保留至少一个启用的登录管理员
|
||||
fn require_superuser(accounts: &[Account]) -> std::io::Result<()> {
|
||||
if accounts.iter().any(|a| a.can_login && a.enabled && a.is_superuser) { Ok(()) }
|
||||
else { Err(std::io::Error::other("the last enabled login superuser cannot be removed")) }
|
||||
}
|
||||
|
||||
/// 一张表的索引定义转线上格式
|
||||
fn index_wires(schema: &crate::catalog::TableSchema) -> Vec<IndexWire> {
|
||||
let mut out = vec![IndexWire {
|
||||
@@ -1632,6 +1931,40 @@ impl Drop for Server {
|
||||
}
|
||||
}
|
||||
|
||||
/// 按列精度规范化一行定点数
|
||||
fn normalize_decimal_row(row: &mut Row, columns: &[SchemaColumn]) -> std::io::Result<()> {
|
||||
for column in columns {
|
||||
if let Some((precision, scale)) = decimal_parameters(&column.ty)?
|
||||
&& let Some(value) = row.get_mut(&column.name).filter(|v| !v.is_null()) {
|
||||
let number = value.as_f64().ok_or_else(|| std::io::Error::other("invalid decimal value"))?;
|
||||
let factor = 10f64.powi(scale);
|
||||
let scaled = number * factor;
|
||||
if !scaled.is_finite() { return Err(std::io::Error::other("decimal value is not finite or exceeds precision")); }
|
||||
let rounded = scaled.round_ties_even() / factor;
|
||||
if rounded.abs() >= 10f64.powi(precision - scale) { return Err(std::io::Error::other("decimal value exceeds precision")); }
|
||||
*value = serde_json::json!(rounded);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 解析基础或命名定点类型参数
|
||||
fn decimal_parameters(ty: &str) -> std::io::Result<Option<(i32, i32)>> {
|
||||
let mut base = ty.trim();
|
||||
while let Some(body) = base.strip_prefix("domain(").and_then(|s| s.strip_suffix(')')) {
|
||||
base = body.split_once(',').ok_or_else(|| std::io::Error::other("invalid domain type"))?.1.trim();
|
||||
}
|
||||
let normalized = crate::catalog::normalize_type(base);
|
||||
if normalized == "decimal" { return Ok(Some((10, 0))); }
|
||||
if !normalized.starts_with("decimal(") { return Ok(None); }
|
||||
let error = || std::io::Error::other("invalid decimal precision or scale");
|
||||
let body = normalized.strip_prefix("decimal(").and_then(|s| s.strip_suffix(')')).ok_or_else(error)?;
|
||||
let values = body.split(',').map(|s| s.trim().parse::<i32>()).collect::<Result<Vec<_>, _>>().map_err(|_| error())?;
|
||||
let (precision, scale) = match values.as_slice() { [p] => (*p, 0), [p, s] => (*p, *s), _ => return Err(error()) };
|
||||
if !(1..=308).contains(&precision) || scale < 0 || scale > precision { return Err(error()); }
|
||||
Ok(Some((precision, scale)))
|
||||
}
|
||||
|
||||
/// 系统库名:账号表住在这里,其中的表都是系统表。服务启动时只建它一个,
|
||||
/// 别的库一律要显式建(没有默认工作库)。
|
||||
const SYSTEM_DATABASE: &str = "system";
|
||||
@@ -1793,9 +2126,11 @@ impl Databases {
|
||||
value["table"] = serde_json::Value::String(table.to_string());
|
||||
}
|
||||
let request: Request = serde_json::from_value(value).map_err(Error::other)?;
|
||||
let global = matches!(request, Request::AccountsList | Request::AccountCreate { .. }
|
||||
let global = matches!(request, Request::AccountsList | Request::CatalogManage { .. } | Request::AccountCreate { .. }
|
||||
| Request::IdentityInitialize { .. } | Request::RoleCreate { .. } | Request::IdentityAlter { .. }
|
||||
| Request::AccountReset { .. } | Request::AccountLogin { .. }
|
||||
| Request::AccountDrop { .. } | Request::BootstrapSystem { .. }
|
||||
| Request::BootstrapTypes { .. }
|
||||
| Request::SystemStatus | Request::Ping);
|
||||
if !global && selected.is_empty() { return Err(Error::other("no database selected")); }
|
||||
let server = if global || selected == SYSTEM_DATABASE { Arc::clone(&self.system) }
|
||||
@@ -1838,21 +2173,185 @@ fn log_config(loaded: &Loaded) {
|
||||
}
|
||||
}
|
||||
|
||||
/// 获取数据目录的跨进程独占锁
|
||||
fn lock_data_directory(cfg: &Config) -> std::io::Result<std::fs::File> {
|
||||
std::fs::create_dir_all(&cfg.data_dir)?;
|
||||
let file = std::fs::OpenOptions::new().read(true).write(true).create(true).truncate(false)
|
||||
.open(cfg.data_dir.join(".chusql.lock"))?;
|
||||
file.try_lock().map_err(|e| std::io::Error::other(format!("data directory is in use; stop all database processes: {e}")))?;
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
/// 复制系统目录并拒绝链接文件
|
||||
fn copy_system_directory(source: &std::path::Path, target: &std::path::Path) -> std::io::Result<()> {
|
||||
std::fs::create_dir_all(target)?;
|
||||
for entry in std::fs::read_dir(source)? {
|
||||
let entry = entry?;
|
||||
let ty = entry.file_type()?;
|
||||
if ty.is_symlink() { return Err(std::io::Error::other("system directory contains a symbolic link")); }
|
||||
if ty.is_dir() { copy_system_directory(&entry.path(), &target.join(entry.file_name()))?; }
|
||||
else if ty.is_file() { std::fs::copy(entry.path(), target.join(entry.file_name()))?; }
|
||||
else { return Err(std::io::Error::other("unsupported system directory entry")); }
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 构造可重建的系统表结构
|
||||
fn maintenance_schemas() -> Vec<(&'static str, Vec<SchemaColumn>)> {
|
||||
let text = |name: &str, size: usize| SchemaColumn { name: name.into(), ty: format!("varchar({size})"), ..Default::default() };
|
||||
vec![(USERS_TABLE, users_columns()), (IDENTITIES_TABLE, identity_columns()), (TYPES_TABLE, types_columns()),
|
||||
("__system_roles", vec![text("name", 64)]),
|
||||
("__system_grants", vec![text("role", 64), text("privilege", 16), text("object", 128)]),
|
||||
("__system_grant_options", vec![text("role", 64), text("privilege", 16), text("object", 128)]),
|
||||
("__system_members", vec![text("role", 64), text("member", 64)])]
|
||||
}
|
||||
|
||||
/// 验证隔离副本并执行所选恢复策略
|
||||
fn maintenance_candidate(cfg: Config, mode: &str, user: &str, hash: &str, types: Vec<Row>) -> Result<(), String> {
|
||||
if mode == "reset" {
|
||||
let seed = Server::new(cfg.clone()).map_err(|e| e.to_string())?;
|
||||
seed.bootstrap_system(Some(user), Some(hash)).map_err(|e| e.to_string())?;
|
||||
seed.bootstrap_types(types.clone()).map_err(|e| e.to_string())?;
|
||||
}
|
||||
let repair = mode == "repair";
|
||||
let catalog_path = cfg.data_dir.join("catalog.json");
|
||||
let schemas = maintenance_schemas();
|
||||
let mut missing_data = Vec::new();
|
||||
let mut catalog = match Catalog::load(&catalog_path) {
|
||||
Ok(catalog) => catalog,
|
||||
Err(_) => {
|
||||
for entry in std::fs::read_dir(&cfg.data_dir).map_err(|e| e.to_string())? {
|
||||
let entry = entry.map_err(|e| e.to_string())?;
|
||||
if entry.path().extension().and_then(|s| s.to_str()) == Some("db") {
|
||||
let name = entry.path().file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default();
|
||||
if !schemas.iter().any(|(known, _)| *known == name) { return Err(format!("cannot reconstruct unknown system table schema: {name}")); }
|
||||
}
|
||||
}
|
||||
Catalog::default()
|
||||
}
|
||||
};
|
||||
if catalog.all_tables().is_empty() {
|
||||
for entry in std::fs::read_dir(&cfg.data_dir).map_err(|e| e.to_string())? {
|
||||
let path = entry.map_err(|e| e.to_string())?.path();
|
||||
if path.extension().and_then(|s| s.to_str()) == Some("db") {
|
||||
let name = path.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default();
|
||||
if !schemas.iter().any(|(known, _)| *known == name) { return Err(format!("cannot reconstruct unknown system table schema: {name}")); }
|
||||
}
|
||||
}
|
||||
}
|
||||
for (name, columns) in &schemas {
|
||||
let file = cfg.data_dir.join(format!("{name}.db"));
|
||||
if let Some(schema) = catalog.describe(name).filter(|s| !file.exists() && s.row_count > 0) {
|
||||
missing_data.push((*name, schema.row_count));
|
||||
}
|
||||
if repair && !file.exists() && catalog.describe(name).is_none_or(|s| s.row_count != 0) {
|
||||
return Err(format!("repair would require initializing or recovering data in {name}"));
|
||||
}
|
||||
if catalog.describe(name).is_none() { catalog.create_table(name, columns.clone()).map_err(|e| e.to_string())?; }
|
||||
else { catalog.set_columns(name, columns.clone()).map_err(|e| e.to_string())?; }
|
||||
catalog.mark_system(name).map_err(|e| e.to_string())?;
|
||||
}
|
||||
catalog.save(&catalog_path).map_err(|e| e.to_string())?;
|
||||
let server = Server::new(cfg).map_err(|e| e.to_string())?;
|
||||
if repair {
|
||||
let scan = server.wal.read_all().map_err(|e| e.to_string())?;
|
||||
let checkpoint = server.wal.read_checkpoint().map_err(|e| e.to_string())?.unwrap_or(0);
|
||||
if !committed_ops(&records_after(&scan.records, checkpoint)).is_empty() { return Err("repair would require replaying WAL data".into()); }
|
||||
} else {
|
||||
server.recover_wal().map_err(|e| e.to_string())?;
|
||||
}
|
||||
for (name, _) in &schemas { server.with_table(name, |t| t.flush()).map_err(|e| e.to_string())?; }
|
||||
if !repair {
|
||||
if mode == "recover" && server.load_accounts().map_err(|e| e.to_string())?.is_empty() && hash.is_empty() {
|
||||
return Err("recover requires --password-stdin to initialize a missing administrator".into());
|
||||
}
|
||||
server.bootstrap_system(Some(user), Some(hash)).map_err(|e| e.to_string())?;
|
||||
server.accounts_request(Request::IdentityInitialize { administrator: user.into() }).map_err(|e| e.to_string())?;
|
||||
server.bootstrap_types(types.clone()).map_err(|e| e.to_string())?;
|
||||
}
|
||||
let accounts = server.load_accounts().map_err(|e| e.to_string())?;
|
||||
require_superuser(&accounts).map_err(|e| e.to_string())?;
|
||||
for (name, expected) in missing_data {
|
||||
let rows = server.with_existing_table(name, |t| t.scan()).map_err(|e| e.to_string())?;
|
||||
if (rows.len() as u64) < expected { return Err(format!("WAL cannot reconstruct all missing data in {name}; restore a backup or use reset")); }
|
||||
}
|
||||
if repair {
|
||||
let saved = server.with_existing_table(TYPES_TABLE, |t| t.scan()).map_err(|e| e.to_string())?;
|
||||
if types.iter().any(|wanted| !saved.iter().any(|row| wanted.iter().all(|(key, value)| row.get(key) == Some(value)))) {
|
||||
return Err("repair would require changing preinstalled type data".into());
|
||||
}
|
||||
}
|
||||
for (name, _) in &schemas { server.with_existing_table(name, |t| t.scan()).map_err(|e| e.to_string())?; }
|
||||
{
|
||||
let mut catalog = server.catalog.lock().map_err(|_| "catalog lock poisoned")?;
|
||||
catalog.identity_high_water = catalog.identity_high_water.max(accounts.iter().map(|a| a.id).max().unwrap_or(0));
|
||||
catalog.save(server.catalog_path()).map_err(|e| e.to_string())?;
|
||||
}
|
||||
server.rebuild_stats().map_err(|e| e.to_string())?;
|
||||
server.flush_all_tables().map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 进程内存储句柄:一个数据目录一份状态,跨线程共享(内部各自加锁)。
|
||||
pub struct Storage {
|
||||
databases: Databases,
|
||||
_directory_lock: std::fs::File,
|
||||
}
|
||||
|
||||
impl Storage {
|
||||
/// 离线修复或重建系统目录
|
||||
pub fn maintenance(config_path: Option<&str>, request: &str) -> Result<String, String> {
|
||||
let loaded = config::load(config_path)?;
|
||||
let cfg = loaded.config;
|
||||
let _lock = lock_data_directory(&cfg).map_err(|e| e.to_string())?;
|
||||
let input: serde_json::Value = serde_json::from_str(request).map_err(|e| e.to_string())?;
|
||||
let mode = input.get("mode").and_then(|v| v.as_str()).ok_or("missing maintenance mode")?;
|
||||
if !["repair", "recover", "reset"].contains(&mode) { return Err("invalid maintenance mode".into()); }
|
||||
let user = input.get("user").and_then(|v| v.as_str()).unwrap_or("root");
|
||||
let hash = input.get("password_hash").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let types: Vec<Row> = serde_json::from_value(input.get("types").cloned().ok_or("missing type manifest")?).map_err(|e| e.to_string())?;
|
||||
if types.is_empty() { return Err("type manifest must not be empty".into()); }
|
||||
for row in &types { type_definition_name(row).map_err(|e| e.to_string())?; }
|
||||
let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_err(|e| e.to_string())?.as_nanos();
|
||||
let system = cfg.data_dir.join("system");
|
||||
if system.is_symlink() { return Err("system directory cannot be a symbolic link".into()); }
|
||||
let work = cfg.data_dir.join(format!("bootstrap-work-{stamp}"));
|
||||
let backup = cfg.data_dir.join(format!("system-backup-{stamp}"));
|
||||
let staged = work.join("system");
|
||||
std::fs::create_dir_all(&staged).map_err(|e| e.to_string())?;
|
||||
if mode != "reset" && system.exists() { copy_system_directory(&system, &staged).map_err(|e| e.to_string())?; }
|
||||
let mut candidate_cfg = cfg.clone();
|
||||
candidate_cfg.data_dir = staged.clone();
|
||||
let result = maintenance_candidate(candidate_cfg, mode, user, hash, types);
|
||||
if let Err(err) = result { return Err(format!("{err}; original system directory was not changed; candidate: {}. Try csql-bootstrap recover or reset", work.display())); }
|
||||
if mode == "repair" && system.exists() {
|
||||
for entry in std::fs::read_dir(&system).map_err(|e| e.to_string())? {
|
||||
let entry = entry.map_err(|e| e.to_string())?;
|
||||
if entry.path().extension().and_then(|s| s.to_str()) == Some("db")
|
||||
&& std::fs::read(entry.path()).map_err(|e| e.to_string())? != std::fs::read(staged.join(entry.file_name())).map_err(|e| e.to_string())? {
|
||||
return Err("repair would modify table data; original system directory was not changed. Use recover or reset".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
if system.exists() { std::fs::rename(&system, &backup).map_err(|e| e.to_string())?; }
|
||||
if let Err(err) = std::fs::rename(&staged, &system) {
|
||||
if backup.exists() { std::fs::rename(&backup, &system).map_err(|rollback| format!("publish failed: {err}; rollback failed: {rollback}; backup: {}", backup.display()))?; }
|
||||
return Err(format!("cannot publish repaired system directory: {err}"));
|
||||
}
|
||||
serde_json::to_string(&serde_json::json!({"status":"maintenance", "mode":mode, "backup":backup, "candidate":work,
|
||||
"message":"System maintenance completed. Business databases were preserved."})).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// 打开数据目录并初始化存储
|
||||
pub fn open(config_path: Option<&str>) -> Result<Storage, String> {
|
||||
let loaded = config::load(config_path)?;
|
||||
let directory_lock = lock_data_directory(&loaded.config).map_err(|e| e.to_string())?;
|
||||
log::init();
|
||||
log::set_level(loaded.config.log_level);
|
||||
log_info!(core, "chusql-core-storage {} starting", env!("CARGO_PKG_VERSION"));
|
||||
log_config(&loaded);
|
||||
let databases = Databases::new(loaded.config).map_err(|e| e.to_string())?;
|
||||
Ok(Storage { databases })
|
||||
Ok(Storage { databases, _directory_lock: directory_lock })
|
||||
}
|
||||
|
||||
/// 一行请求 JSON 换一行响应 JSON
|
||||
|
||||
@@ -31,7 +31,7 @@ const OP_DROP_INDEX: u8 = 14;
|
||||
pub enum WalOp {
|
||||
HideColumn { table: String, column: String },
|
||||
Compact { table: String, rows: Vec<Row> },
|
||||
Accounts { accounts: Vec<Account> },
|
||||
Accounts { accounts: Vec<Account>, removed: Vec<String>, migrate_roles: bool },
|
||||
Insert {
|
||||
table: String,
|
||||
row: Row,
|
||||
@@ -283,7 +283,7 @@ impl Wal {
|
||||
frames.push((lsn, op, offset, end));
|
||||
offset = end;
|
||||
}
|
||||
Err(_) => break,
|
||||
Err(error) => return Err(io::Error::new(io::ErrorKind::InvalidData, error)),
|
||||
}
|
||||
}
|
||||
Ok((frames, offset))
|
||||
@@ -291,9 +291,13 @@ impl Wal {
|
||||
|
||||
/// 记录检查点:到此为止的 LSN 都已落盘
|
||||
pub fn write_checkpoint(&self, lsn: u64) -> io::Result<()> {
|
||||
let mut f = File::create(self.checkpoint_path())?;
|
||||
let checkpoint = self.checkpoint_path();
|
||||
let pending = checkpoint.with_extension("checkpoint.pending");
|
||||
let mut f = File::create(&pending)?;
|
||||
f.write_all(&lsn.to_le_bytes())?;
|
||||
f.sync_all()
|
||||
f.sync_all()?;
|
||||
drop(f);
|
||||
std::fs::rename(pending, checkpoint)
|
||||
}
|
||||
|
||||
/// 读检查点 LSN
|
||||
@@ -303,8 +307,8 @@ impl Wal {
|
||||
return Ok(None);
|
||||
}
|
||||
let bytes = std::fs::read(&path)?;
|
||||
if bytes.len() < 8 {
|
||||
return Ok(None);
|
||||
if bytes.len() != 8 {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid WAL checkpoint length"));
|
||||
}
|
||||
let mut raw = [0u8; 8];
|
||||
raw.copy_from_slice(&bytes[..8]);
|
||||
@@ -322,10 +326,10 @@ impl Wal {
|
||||
if let Some((lsn, _, _, _)) = frames.last() {
|
||||
base = base.max(*lsn);
|
||||
}
|
||||
*slot = base + 1;
|
||||
*slot = base.checked_add(1).ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
|
||||
}
|
||||
let lsn = *slot;
|
||||
*slot += 1;
|
||||
*slot = slot.checked_add(1).ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
|
||||
Ok(lsn)
|
||||
}
|
||||
|
||||
@@ -378,8 +382,8 @@ fn encode(op: &WalOp, lsn: u64) -> io::Result<Vec<u8>> {
|
||||
let p = serde_json::to_vec(rows).map_err(io::Error::other)?;
|
||||
(OP_COMPACT, table.clone(), p)
|
||||
}
|
||||
WalOp::Accounts { accounts } => (OP_ACCOUNTS, USERS_TABLE.to_string(),
|
||||
serde_json::to_vec(accounts).map_err(io::Error::other)?),
|
||||
WalOp::Accounts { accounts, removed, migrate_roles } => (OP_ACCOUNTS, USERS_TABLE.to_string(),
|
||||
serde_json::to_vec(&serde_json::json!({"accounts":accounts,"removed":removed,"migrate_roles":migrate_roles})).map_err(io::Error::other)?),
|
||||
WalOp::Insert { table, row } => {
|
||||
let p = serde_json::to_vec(row).map_err(io::Error::other)?;
|
||||
(OP_INSERT, table.clone(), p)
|
||||
@@ -478,9 +482,15 @@ fn decode_body(body: &[u8]) -> Result<(u64, WalOp), String> {
|
||||
}
|
||||
OP_ACCOUNTS => {
|
||||
if table != USERS_TABLE { return Err("invalid account WAL table".into()); }
|
||||
let accounts = serde_json::from_slice(payload)
|
||||
.map_err(|e| format!("bad account payload: {e}"))?;
|
||||
WalOp::Accounts { accounts }
|
||||
let value: serde_json::Value = serde_json::from_slice(payload).map_err(|e| format!("bad account payload: {e}"))?;
|
||||
let (accounts, removed, migrate_roles) = if value.is_array() {
|
||||
(serde_json::from_value(value).map_err(|e| format!("bad account payload: {e}"))?, Vec::new(), false)
|
||||
} else {
|
||||
(serde_json::from_value(value.get("accounts").cloned().ok_or("missing account payload")?).map_err(|e| format!("bad account payload: {e}"))?,
|
||||
serde_json::from_value(value.get("removed").cloned().ok_or("missing identity cleanup")?).map_err(|e| format!("bad account cleanup: {e}"))?,
|
||||
value.get("migrate_roles").and_then(serde_json::Value::as_bool).ok_or("missing role migration flag")?)
|
||||
};
|
||||
WalOp::Accounts { accounts, removed, migrate_roles }
|
||||
}
|
||||
OP_INSERT => {
|
||||
let row: Row =
|
||||
|
||||
@@ -124,6 +124,36 @@ fn stats_track_a_histogram_for_numeric_columns() {
|
||||
assert_eq!(id.hist.iter().sum::<u64>(), 0);
|
||||
}
|
||||
|
||||
/// 重复值删除保留仍存在的不同值计数
|
||||
#[test]
|
||||
fn duplicate_deletion_keeps_distinct_until_last_occurrence() {
|
||||
let mut catalog = Catalog::default();
|
||||
let duplicate = row(&[("id", json!(1)), ("v", json!(7))]);
|
||||
catalog.rebuild_stats("t", &[duplicate.clone(), duplicate.clone()]);
|
||||
catalog.record_delete("t", std::slice::from_ref(&duplicate));
|
||||
assert_eq!(catalog.describe("t").unwrap().stats["v"].distinct, 1);
|
||||
catalog.record_delete("t", &[duplicate]);
|
||||
assert_eq!(catalog.describe("t").unwrap().stats["v"].distinct, 0);
|
||||
}
|
||||
|
||||
/// 去重封顶不影响直方图和其他列
|
||||
#[test]
|
||||
fn capped_distinct_keeps_histogram_and_other_columns_accurate() {
|
||||
let mut catalog = Catalog::default();
|
||||
let rows: Vec<Row> = (0..5000).map(|n| row(&[("id", json!(n)), ("v", json!(n % 2))])).collect();
|
||||
catalog.rebuild_stats("t", &rows);
|
||||
let schema = catalog.describe("t").unwrap();
|
||||
assert!(schema.stats["id"].capped);
|
||||
assert_eq!(schema.stats["id"].hist.iter().sum::<u64>(), 5000);
|
||||
assert_eq!(schema.stats["v"].distinct, 2);
|
||||
catalog.record_delete("t", &rows[..1000]);
|
||||
let schema = catalog.describe("t").unwrap();
|
||||
assert_eq!(schema.stats["id"].hist.iter().sum::<u64>(), 4000);
|
||||
assert_eq!(schema.stats["v"].distinct, 2);
|
||||
catalog.rebuild_stats("t", &rows[1000..]);
|
||||
assert_eq!(catalog.describe("t").unwrap().stats["id"].distinct, 4000);
|
||||
}
|
||||
|
||||
/// 老类型名并到新写法,参数保留,未知类型原样
|
||||
#[test]
|
||||
fn normalize_type_maps_legacy_names() {
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use chusql_core_storage::config::{self, Config, DEFAULT_BTREE_ORDER, DEFAULT_PAGE_SIZE, Origin};
|
||||
use chusql_core_storage::config::{
|
||||
self, Config, DEFAULT_BTREE_ORDER, DEFAULT_LOG_FILES, DEFAULT_PAGE_SIZE, Origin,
|
||||
};
|
||||
use chusql_core_storage::log::Level;
|
||||
|
||||
// 配置测试:默认值、文件覆盖、环境变量彻底无效、非法值报错。
|
||||
@@ -25,6 +27,7 @@ fn defaults_without_file() {
|
||||
assert_eq!(loaded.config.page_size, DEFAULT_PAGE_SIZE);
|
||||
assert_eq!(loaded.config.btree_order, DEFAULT_BTREE_ORDER);
|
||||
assert_eq!(loaded.config.data_dir, config::default_data_dir());
|
||||
assert_eq!(loaded.config.log_files, PathBuf::from(DEFAULT_LOG_FILES));
|
||||
assert_eq!(loaded.config.log_level, Level::Info);
|
||||
}
|
||||
|
||||
@@ -39,6 +42,7 @@ fn file_overrides_only_written_keys() {
|
||||
assert_eq!(loaded.config.page_size, 8192);
|
||||
assert_eq!(loaded.config.btree_order, DEFAULT_BTREE_ORDER);
|
||||
assert_eq!(loaded.config.data_dir, config::default_data_dir());
|
||||
assert_eq!(loaded.config.log_files, PathBuf::from(DEFAULT_LOG_FILES));
|
||||
assert_eq!(loaded.config_path, Some(path.clone()));
|
||||
|
||||
let page = loaded.origins.iter().find(|(n, _, _)| *n == "page.size").unwrap();
|
||||
@@ -47,11 +51,11 @@ fn file_overrides_only_written_keys() {
|
||||
assert_eq!(order.2, Origin::Default);
|
||||
}
|
||||
|
||||
/// 五项全写都生效
|
||||
/// 全部配置项及其来源生效
|
||||
#[test]
|
||||
fn file_all_keys() {
|
||||
let _guard = env_guard();
|
||||
let text = "[page]\nsize = 8192\n[btree]\norder = 32\n[buffer]\npool_size = 128\n[storage]\ndata_dir = \"mydata\"\n[log]\nlevel = \"debug\"\n";
|
||||
let text = "[page]\nsize = 8192\n[btree]\norder = 32\n[buffer]\npool_size = 128\n[storage]\ndata_dir = \"mydata\"\nlog_files = \"mylogs\"\n[log]\nlevel = \"debug\"\n";
|
||||
let path = PathBuf::from("t.toml");
|
||||
let loaded = config::resolve(Some(text), Some(path.clone())).unwrap();
|
||||
|
||||
@@ -59,6 +63,7 @@ fn file_all_keys() {
|
||||
assert_eq!(loaded.config.btree_order, 32);
|
||||
assert_eq!(loaded.config.pool_size, 128);
|
||||
assert_eq!(loaded.config.data_dir, PathBuf::from("mydata"));
|
||||
assert_eq!(loaded.config.log_files, PathBuf::from("mylogs"));
|
||||
assert_eq!(loaded.config.log_level, Level::Debug);
|
||||
assert!(
|
||||
loaded
|
||||
@@ -103,16 +108,16 @@ fn env_vars_are_ignored_entirely() {
|
||||
fn default_config_path_follows_system_convention() {
|
||||
let _guard = env_guard();
|
||||
let path = config::default_config_path();
|
||||
assert_eq!(path.file_name().unwrap(), "chusql.toml");
|
||||
assert_eq!(path.file_name().unwrap(), "settings.toml");
|
||||
assert_eq!(path.parent().unwrap().file_name().unwrap(), "ChuSQL");
|
||||
assert!(path.ends_with(format!("ChuSQL{}chusql.toml", std::path::MAIN_SEPARATOR)), "got {}", path.display());
|
||||
assert!(path.ends_with(format!("ChuSQL{}settings.toml", std::path::MAIN_SEPARATOR)), "got {}", path.display());
|
||||
|
||||
// 本机 APPDATA 存在时必须是 %APPDATA%\ChuSQL\chusql.toml
|
||||
// 本机 APPDATA 存在时必须是 %APPDATA%\ChuSQL\settings.toml
|
||||
#[cfg(windows)]
|
||||
if let Some(appdata) = std::env::var_os("APPDATA") {
|
||||
assert_eq!(
|
||||
path,
|
||||
PathBuf::from(appdata).join("ChuSQL").join("chusql.toml")
|
||||
PathBuf::from(appdata).join("ChuSQL").join("settings.toml")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -186,7 +191,7 @@ fn load_without_explicit_path_uses_builtin_defaults() {
|
||||
#[test]
|
||||
fn server_section_belongs_to_another_layer() {
|
||||
let _guard = env_guard();
|
||||
let db_server = "[server]\nhost = \"0.0.0.0\"\nport = 7778\nmax_rows = 10\n";
|
||||
let db_server = "[server]\nlisten_host = \"0.0.0.0\"\nport = 7778\nmax_rows = 10\n";
|
||||
let loaded = config::resolve(Some(db_server), None).unwrap();
|
||||
assert_eq!(loaded.config.data_dir, config::default_data_dir());
|
||||
|
||||
@@ -226,6 +231,14 @@ fn rejects_unknown_key() {
|
||||
assert!(err.contains("bad config"), "got {}", err);
|
||||
}
|
||||
|
||||
/// 空的日志目录要报错(否则日志会落到当前目录)
|
||||
#[test]
|
||||
fn rejects_empty_log_files() {
|
||||
let _guard = env_guard();
|
||||
let err = config::resolve(Some("[storage]\nlog_files = \" \"\n"), None).unwrap_err();
|
||||
assert!(err.contains("storage.log_files"), "got {}", err);
|
||||
}
|
||||
|
||||
/// 坏的页面尺寸只能来自文件
|
||||
#[test]
|
||||
fn rejects_bad_page_size_from_file() {
|
||||
|
||||
@@ -10,6 +10,433 @@ thread_local! {
|
||||
static CURRENT: RefCell<Option<Arc<Storage>>> = const { RefCell::new(None) };
|
||||
}
|
||||
|
||||
/// 定点写入检查舍入和整批原子拒绝
|
||||
#[test]
|
||||
fn decimal_writes_enforce_precision_before_wal() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let (_server, data) = start_server();
|
||||
let mut c = connect()?;
|
||||
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"amounts","columns":[{"name":"id","ty":"int"},{"name":"value","ty":"decimal(4,2)"}]}))?;
|
||||
request_ok(&mut c, serde_json::json!({"method":"insert","table":"amounts","row":{"id":1,"value":99.994}}))?;
|
||||
let snapshot = send(&mut c, r#"{"method":"scan","table":"amounts"}"#);
|
||||
let rows: serde_json::Value = serde_json::from_str(&snapshot)?;
|
||||
assert_eq!(rows["rows"][0]["value"], serde_json::json!(99.99));
|
||||
let checkpoint = std::fs::read(data.path().join("databases/main/wal.checkpoint"))?;
|
||||
for value in [99.995, -99.995, 100.0] {
|
||||
let response = send(&mut c, &serde_json::json!({"method":"insert_batch","table":"amounts","rows":[{"id":2,"value":1.2},{"id":3,"value":value}]}).to_string());
|
||||
assert!(response.contains("exceeds precision"), "{response}");
|
||||
let remaining: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"amounts"}"#))?;
|
||||
assert_eq!(remaining, rows);
|
||||
assert_eq!(std::fs::read(data.path().join("databases/main/wal.checkpoint"))?, checkpoint);
|
||||
}
|
||||
let invalid = send(&mut c, &serde_json::json!({"method":"create_table","table":"invalid","columns":[{"name":"value","ty":"decimal(2,3)"}]}).to_string());
|
||||
assert!(invalid.contains("invalid decimal"), "{invalid}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 检查点失败封闭请求并在重启恢复
|
||||
#[test]
|
||||
fn checkpoint_failure_requires_recovery_and_preserves_committed_rows() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let (server, data) = start_server();
|
||||
let mut c = connect()?;
|
||||
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"checkpointed","columns":[{"name":"id","ty":"int"}]}))?;
|
||||
let pending = data.path().join("databases/main/wal.checkpoint.pending");
|
||||
std::fs::create_dir(&pending)?;
|
||||
let failed = send(&mut c, r#"{"method":"insert","table":"checkpointed","row":{"id":1}}"#);
|
||||
assert!(failed.contains("error"), "{failed}");
|
||||
assert!(send(&mut c, r#"{"method":"scan","table":"checkpointed"}"#).contains("recovery required"));
|
||||
assert!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len() > 0);
|
||||
drop(c);
|
||||
drop(server);
|
||||
std::fs::remove_dir(pending)?;
|
||||
let _restarted = restart_at(data.path())?;
|
||||
let mut c = connect()?;
|
||||
let rows: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"checkpointed"}"#))?;
|
||||
assert_eq!(rows["rows"], serde_json::json!([{"id":1}]));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 应用失败封闭读写并重放完整提交组
|
||||
#[test]
|
||||
fn committed_apply_failures_require_recovery() -> Result<(), Box<dyn std::error::Error>> {
|
||||
for operation in [
|
||||
serde_json::json!({"method":"create_table","table":"created","columns":[{"name":"id","ty":"int"}]}),
|
||||
serde_json::json!({"method":"replace_all","table":"items","rows":[{"id":2}]}),
|
||||
serde_json::json!({"method":"apply_transaction","ops":[{"op":"delete","table":"items","ids":[1]},{"op":"replace","table":"items","rows":[{"id":2}]}]}),
|
||||
] {
|
||||
let (server, data) = start_server();
|
||||
let mut c = connect()?;
|
||||
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"items","columns":[{"name":"id","ty":"int"}]}))?;
|
||||
request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":1}}))?;
|
||||
let directory = data.path().join("databases/main");
|
||||
let checkpoint = std::fs::read(directory.join("wal.checkpoint"))?;
|
||||
let pending = directory.join("catalog.json.pending");
|
||||
std::fs::create_dir(&pending)?;
|
||||
let failed = send(&mut c, &operation.to_string());
|
||||
assert!(failed.contains("error"), "{operation}: {failed}");
|
||||
for request in [
|
||||
r#"{"method":"scan","table":"items"}"#,
|
||||
r#"{"method":"insert","table":"items","row":{"id":3}}"#,
|
||||
] {
|
||||
let blocked = send(&mut c, request);
|
||||
assert!(blocked.contains("recovery required"), "{blocked}");
|
||||
}
|
||||
assert_eq!(std::fs::read(directory.join("wal.checkpoint"))?, checkpoint);
|
||||
assert!(std::fs::metadata(directory.join("wal.log"))?.len() > 0);
|
||||
drop(c);
|
||||
drop(server);
|
||||
std::fs::remove_dir(pending)?;
|
||||
let restarted = restart_at(data.path())?;
|
||||
let mut c = connect()?;
|
||||
let table = if operation["method"] == "create_table" { "created" } else { "items" };
|
||||
let result: serde_json::Value = serde_json::from_str(&send(&mut c, &serde_json::json!({"method":"scan","table":table}).to_string()))?;
|
||||
let expected = if table == "created" { serde_json::json!([]) } else { serde_json::json!([{"id":2}]) };
|
||||
assert_eq!(result["rows"], expected, "{operation}");
|
||||
assert_eq!(std::fs::metadata(directory.join("wal.log"))?.len(), 0);
|
||||
drop(c);
|
||||
drop(restarted);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 首帧撕裂恢复后新日志仍可重放
|
||||
#[test]
|
||||
fn torn_first_frame_is_removed_before_new_writes() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let (server, data) = start_server();
|
||||
let mut c = connect()?;
|
||||
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"torn","columns":[{"name":"id","ty":"int"}]}))?;
|
||||
drop(c);
|
||||
drop(server);
|
||||
std::fs::write(data.path().join("databases/main/wal.log"), [20, 0, 0, 0, 1])?;
|
||||
let restarted = restart_at(data.path())?;
|
||||
assert_eq!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len(), 0);
|
||||
let mut c = connect()?;
|
||||
request_ok(&mut c, serde_json::json!({"method":"insert","table":"torn","row":{"id":1}}))?;
|
||||
drop(c);
|
||||
drop(restarted);
|
||||
let _again = restart_at(data.path())?;
|
||||
let mut c = connect()?;
|
||||
let rows: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"torn"}"#))?;
|
||||
assert_eq!(rows["rows"], serde_json::json!([{"id":1}]));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 校验预装类型幂等与冲突保护
|
||||
#[test]
|
||||
fn preinstalled_types_are_idempotent_hidden_and_conflict_safe() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let (_server, data) = start_server();
|
||||
let mut c = connect()?;
|
||||
let premature = send(&mut c, &type_seed_request().to_string());
|
||||
assert!(premature.contains("login superuser"), "{premature}");
|
||||
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"original-hash"}"#);
|
||||
let accounts = send(&mut c, r#"{"method":"accounts_list"}"#);
|
||||
let request = type_seed_request();
|
||||
let seeded: serde_json::Value = serde_json::from_str(&send(&mut c, &request.to_string()))?;
|
||||
assert_eq!(seeded["status"], "rows", "{seeded}");
|
||||
assert_eq!(seeded["rows"].as_array().map(Vec::len), Some(2));
|
||||
assert_eq!(serde_json::from_str::<serde_json::Value>(&send(&mut c, &request.to_string()))?, seeded);
|
||||
let mut conflict = request;
|
||||
conflict["types"][1]["base_type"] = "int".into();
|
||||
let rejected = send(&mut c, &conflict.to_string());
|
||||
assert!(rejected.contains("definition conflict: text"), "{rejected}");
|
||||
assert_eq!(serde_json::from_str::<serde_json::Value>(&send(&mut c, &type_seed_request().to_string()))?, seeded);
|
||||
assert_eq!(send(&mut c, r#"{"method":"accounts_list"}"#), accounts);
|
||||
let catalog: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(data.path().join("system/catalog.json"))?)?;
|
||||
assert_eq!(catalog["tables"]["__system_types"]["system"], true);
|
||||
assert_eq!(catalog["tables"]["__system_types"]["row_count"], 2);
|
||||
let denied = send(&mut c, r#"{"method":"scan","database":"system","table":"__system_types"}"#);
|
||||
assert!(denied.contains("reserved system table"), "{denied}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 类型预装失败保留日志并在重启后恢复
|
||||
#[test]
|
||||
fn preinstalled_types_recover_after_catalog_write_failure() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let (server, data) = start_server();
|
||||
let mut c = connect()?;
|
||||
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
|
||||
std::fs::create_dir(data.path().join("system/catalog.json.pending"))?;
|
||||
let failed = send(&mut c, &type_seed_request().to_string());
|
||||
assert!(failed.contains("error"), "{failed}");
|
||||
assert!(std::fs::metadata(data.path().join("system/wal.log"))?.len() > 0);
|
||||
assert!(send(&mut c, &type_seed_request().to_string()).contains("recovery required"));
|
||||
drop(c);
|
||||
drop(server);
|
||||
std::fs::remove_dir(data.path().join("system/catalog.json.pending"))?;
|
||||
let _restarted = restart_at(data.path())?;
|
||||
let mut c = connect()?;
|
||||
let restored: serde_json::Value = serde_json::from_str(&send(&mut c, &type_seed_request().to_string()))?;
|
||||
assert_eq!(restored["rows"].as_array().map(Vec::len), Some(2), "{restored}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 类型目录不覆盖未登记的现有文件
|
||||
#[test]
|
||||
fn preinstalled_types_reject_reserved_file_collision() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let (_server, data) = start_server();
|
||||
let mut c = connect()?;
|
||||
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
|
||||
let path = data.path().join("system/__system_types.db");
|
||||
std::fs::write(&path, b"preserved")?;
|
||||
let rejected = send(&mut c, &type_seed_request().to_string());
|
||||
assert!(rejected.contains("reserved type file collision"), "{rejected}");
|
||||
assert_eq!(std::fs::read(path)?, b"preserved");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 生成两项预装类型定义
|
||||
fn type_seed_request() -> serde_json::Value {
|
||||
serde_json::json!({"method":"bootstrap_types","types":[
|
||||
{"name":"int","base_type":"int","parameterized":false},
|
||||
{"name":"text","base_type":"str","parameterized":false}
|
||||
]})
|
||||
}
|
||||
|
||||
/// 引导保留已用身份编号
|
||||
#[test]
|
||||
fn bootstrap_respects_previous_identity_ids() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let (_server, _data) = start_server();
|
||||
let mut c = connect()?;
|
||||
send(&mut c, r#"{"method":"account_create","user":"alice","password_hash":"hash"}"#);
|
||||
let deleted: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"account_drop","user":"alice"}"#))?;
|
||||
assert_eq!(deleted["accounts"], serde_json::json!([]));
|
||||
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
|
||||
let seeded: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"accounts_list"}"#))?;
|
||||
assert_eq!(seeded["accounts"][0]["id"], 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 旧账号日志迁移只提升初始管理员
|
||||
#[test]
|
||||
fn legacy_account_wal_migrates_without_promoting_other_names() -> Result<(), Box<dyn std::error::Error>> {
|
||||
use chusql_core_storage::wal::Wal;
|
||||
let (server, data) = start_server();
|
||||
let mut c = connect()?;
|
||||
request_ok(&mut c, serde_json::json!({"method":"create_table","database":"system","table":"__system_roles","columns":[{"name":"name","ty":"str"}]}))?;
|
||||
request_ok(&mut c, serde_json::json!({"method":"insert","database":"system","table":"__system_roles","row":{"name":"reader"}}))?;
|
||||
drop(c);
|
||||
drop(server);
|
||||
let path = data.path().join("system/wal.log");
|
||||
let lsn = Wal::new(&path).read_checkpoint()?.unwrap_or(0) + 1;
|
||||
let legacy = serde_json::json!([
|
||||
{"id":7,"user":"owner","password_hash":"owner-hash","revision":3,"registered_at":"2020-01-01 00:00:00","last_login_at":null},
|
||||
{"id":9,"user":"alice","password_hash":"alice-hash","revision":2,"registered_at":"2020-01-02 00:00:00","last_login_at":null}
|
||||
]);
|
||||
let table = b"__system_users";
|
||||
let mut body = lsn.to_le_bytes().to_vec();
|
||||
body.push(6);
|
||||
body.extend_from_slice(&(table.len() as u16).to_le_bytes());
|
||||
body.extend_from_slice(table);
|
||||
body.extend_from_slice(&serde_json::to_vec(&legacy)?);
|
||||
let mut frame = (body.len() as u32).to_le_bytes().to_vec();
|
||||
frame.extend_from_slice(&body);
|
||||
std::fs::write(&path, frame)?;
|
||||
let _restarted = restart_at(data.path())?;
|
||||
let mut c = connect()?;
|
||||
let wrong = send(&mut c, r#"{"method":"identity_initialize","administrator":"missing"}"#);
|
||||
assert!(wrong.contains("last enabled login superuser"), "{wrong}");
|
||||
let migrated: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"identity_initialize","administrator":"owner"}"#))?;
|
||||
assert_eq!(migrated["status"], "accounts", "{migrated}");
|
||||
assert_eq!(migrated["accounts"][0]["id"], 7);
|
||||
assert_eq!(migrated["accounts"][0]["is_superuser"], true);
|
||||
assert_eq!(migrated["accounts"][0]["password_hash"], "owner-hash");
|
||||
assert_eq!(migrated["accounts"][1]["is_superuser"], false);
|
||||
assert_eq!(migrated["accounts"][2]["user"], "reader");
|
||||
assert_eq!(migrated["accounts"][2]["id"], 10);
|
||||
assert_eq!(serde_json::from_str::<serde_json::Value>(&send(&mut c, r#"{"method":"identity_initialize","administrator":"alice"}"#))?, migrated);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 身份属性重启保留且禁止删除最后管理员
|
||||
#[test]
|
||||
fn identity_attributes_survive_restart_and_protect_superuser() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let (server, data) = start_server();
|
||||
let mut c = connect()?;
|
||||
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
|
||||
let created: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"role_create","user":"Reader"}"#))?;
|
||||
assert_eq!(created["status"], "accounts", "{created}");
|
||||
assert_eq!(created["accounts"][1]["can_login"], false);
|
||||
assert_eq!(created["accounts"][1]["enabled"], true);
|
||||
for change in [serde_json::json!({"enabled":false}), serde_json::json!({"can_login":false}), serde_json::json!({"is_superuser":false})] {
|
||||
let mut request = change;
|
||||
request["method"] = "identity_alter".into();
|
||||
request["user"] = "root".into();
|
||||
let denied = send(&mut c, &request.to_string());
|
||||
assert!(denied.contains("last enabled login superuser"), "{denied}");
|
||||
}
|
||||
let collision = send(&mut c, r#"{"method":"account_create","user":"READER","password_hash":"hash"}"#);
|
||||
assert!(collision.contains("already exists"), "{collision}");
|
||||
let changed: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"identity_alter","user":"reader","can_login":true,"is_superuser":true}"#))?;
|
||||
assert_eq!(changed["accounts"][1]["revision"], 2);
|
||||
drop(c);
|
||||
drop(server);
|
||||
let _restarted = restart_at(data.path())?;
|
||||
let mut c = connect()?;
|
||||
let restored: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"accounts_list"}"#))?;
|
||||
assert_eq!(restored, changed);
|
||||
let denied = send(&mut c, r#"{"method":"scan","database":"system","table":"__system_identities"}"#);
|
||||
assert!(denied.contains("reserved system table"), "{denied}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 旧角色迁移与重名拒绝不破坏原数据
|
||||
#[test]
|
||||
fn identity_migration_preserves_roles_and_rejects_collisions() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let (_server, _data) = start_server();
|
||||
let mut c = connect()?;
|
||||
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
|
||||
request_ok(&mut c, serde_json::json!({"method":"create_table","database":"system","table":"__system_roles","columns":[{"name":"name","ty":"str"}]}))?;
|
||||
request_ok(&mut c, serde_json::json!({"method":"insert","database":"system","table":"__system_roles","row":{"name":"root"}}))?;
|
||||
let before = send(&mut c, r#"{"method":"accounts_list"}"#);
|
||||
let rejected = send(&mut c, r#"{"method":"identity_initialize","administrator":"root"}"#);
|
||||
assert!(rejected.contains("identity name collision: root"), "{rejected}");
|
||||
assert_eq!(send(&mut c, r#"{"method":"accounts_list"}"#), before);
|
||||
request_ok(&mut c, serde_json::json!({"method":"replace_all","database":"system","table":"__system_roles","rows":[{"name":"reader"}]}))?;
|
||||
let migrated: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"identity_initialize","administrator":"root"}"#))?;
|
||||
assert_eq!(migrated["accounts"][1]["user"], "reader");
|
||||
assert_eq!(migrated["accounts"][1]["can_login"], false);
|
||||
assert_eq!(serde_json::from_str::<serde_json::Value>(&send(&mut c, r#"{"method":"identity_initialize","administrator":"reader"}"#))?, migrated);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 删除身份清理授权并在恢复后保留编号上界
|
||||
#[test]
|
||||
fn identity_drop_cleanup_replays_without_reusing_ids() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let (server, data) = start_server();
|
||||
let mut c = connect()?;
|
||||
send(&mut c, r#"{"method":"bootstrap_system","user":"root","password_hash":"hash"}"#);
|
||||
send(&mut c, r#"{"method":"role_create","user":"reader"}"#);
|
||||
for table in ["__system_grants", "__system_grant_options", "__system_members"] {
|
||||
request_ok(&mut c, serde_json::json!({"method":"create_table","database":"system","table":table,"columns":[{"name":"role","ty":"str"},{"name":"member","ty":"str"}]}))?;
|
||||
request_ok(&mut c, serde_json::json!({"method":"insert","database":"system","table":table,"row":{"role":"reader","member":"root"}}))?;
|
||||
}
|
||||
std::fs::create_dir(data.path().join("system/catalog.json.pending"))?;
|
||||
let failed = send(&mut c, r#"{"method":"account_drop","user":"reader"}"#);
|
||||
assert!(failed.contains("error"), "{failed}");
|
||||
drop(c);
|
||||
drop(server);
|
||||
std::fs::remove_dir(data.path().join("system/catalog.json.pending"))?;
|
||||
let _restarted = restart_at(data.path())?;
|
||||
let mut c = connect()?;
|
||||
for table in ["__system_grants", "__system_grant_options", "__system_members"] {
|
||||
let rows: serde_json::Value = serde_json::from_str(&send(&mut c, &serde_json::json!({"method":"scan","database":"system","table":table}).to_string()))?;
|
||||
assert_eq!(rows["rows"], serde_json::json!([]));
|
||||
}
|
||||
let recreated: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"role_create","user":"reader"}"#))?;
|
||||
assert_eq!(recreated["accounts"][1]["id"], 3);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 命名类型引用与删除依赖在写入时校验
|
||||
#[test]
|
||||
fn domain_dependencies_are_checked_before_wal() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let (_srv, _data) = start_server();
|
||||
let mut connection = connect()?;
|
||||
let columns = serde_json::json!([{"name":"code","ty":"domain(code,int)"}]);
|
||||
let missing = serde_json::json!({"method":"create_table","table":"typed","columns":columns});
|
||||
let response: serde_json::Value = serde_json::from_str(&send(&mut connection, &missing.to_string()))?;
|
||||
assert_eq!(response["status"], "error");
|
||||
request_ok(&mut connection, serde_json::json!({"method":"create_table","table":"__system_domain_code","columns":[{"name":"base","ty":"int"}]}))?;
|
||||
let forged = serde_json::json!({"method":"create_table","table":"forged","columns":[{"name":"code","ty":"domain(code,str)"}]});
|
||||
let response: serde_json::Value = serde_json::from_str(&send(&mut connection, &forged.to_string()))?;
|
||||
assert_eq!(response["status"], "error");
|
||||
request_ok(&mut connection, missing)?;
|
||||
let response: serde_json::Value = serde_json::from_str(&send(&mut connection, r#"{"method":"drop_table","table":"__system_domain_code"}"#))?;
|
||||
assert_eq!(response["status"], "error");
|
||||
request_ok(&mut connection, serde_json::json!({"method":"drop_table","table":"typed"}))?;
|
||||
request_ok(&mut connection, serde_json::json!({"method":"drop_table","table":"__system_domain_code"}))?;
|
||||
let response: serde_json::Value = serde_json::from_str(&send(&mut connection, r#"{"method":"create_table","table":"stale","columns":[{"name":"code","ty":"domain(code,int)"}]}"#))?;
|
||||
assert_eq!(response["status"], "error");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 嵌套类型依赖与重启持久化
|
||||
#[test]
|
||||
fn nested_domains_preserve_dependencies_across_restart() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let (_srv, data) = start_server();
|
||||
let mut connection = connect()?;
|
||||
request_ok(&mut connection, serde_json::json!({"method":"create_table","table":"__system_domain_root","columns":[{"name":"base","ty":"int"}]}))?;
|
||||
request_ok(&mut connection, serde_json::json!({"method":"create_table","table":"__system_domain_child","columns":[{"name":"base","ty":"domain(root,int)"}]}))?;
|
||||
let rejected: serde_json::Value = serde_json::from_str(&send(&mut connection, r#"{"method":"drop_table","table":"__system_domain_root"}"#))?;
|
||||
assert_eq!(rejected["status"], "error");
|
||||
request_ok(&mut connection, serde_json::json!({"method":"create_table","table":"typed","columns":[{"name":"id","ty":"domain(child,domain(root,int))"}]}))?;
|
||||
let forged: serde_json::Value = serde_json::from_str(&send(&mut connection, r#"{"method":"create_table","table":"forged","columns":[{"name":"id","ty":"domain(child,domain(root,str))"}]}"#))?;
|
||||
assert_eq!(forged["status"], "error");
|
||||
drop(connection);
|
||||
CURRENT.with(|c| *c.borrow_mut() = None);
|
||||
drop(_srv);
|
||||
let _reopened = start_server_in(data.path());
|
||||
let mut connection = connect()?;
|
||||
let schema: serde_json::Value = serde_json::from_str(&send(&mut connection, r#"{"method":"describe_table","table":"typed"}"#))?;
|
||||
assert_eq!(schema["status"], "schema");
|
||||
assert_eq!(schema["columns"][0]["ty"], "domain(child,domain(root,int))");
|
||||
request_ok(&mut connection, serde_json::json!({"method":"drop_table","table":"typed"}))?;
|
||||
request_ok(&mut connection, serde_json::json!({"method":"drop_table","table":"__system_domain_child"}))?;
|
||||
request_ok(&mut connection, serde_json::json!({"method":"drop_table","table":"__system_domain_root"}))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 重置与结构修复保留业务文件及系统数据
|
||||
#[test]
|
||||
fn offline_maintenance_preserves_data_and_rejects_live_storage() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let data = tempfile::tempdir()?;
|
||||
let config = write_config(data.path());
|
||||
let business = data.path().join("databases/business");
|
||||
std::fs::create_dir_all(&business)?;
|
||||
std::fs::write(business.join("sentinel"), b"business data")?;
|
||||
let types = type_seed_request()["types"].clone();
|
||||
let request = serde_json::json!({"mode":"reset","user":"root","password_hash":"original-hash","types":types});
|
||||
Storage::maintenance(config.to_str(), &request.to_string())?;
|
||||
let system = data.path().join("system");
|
||||
let credentials = std::fs::read(system.join("__system_users.db"))?;
|
||||
let mut catalog: serde_json::Value = serde_json::from_slice(&std::fs::read(system.join("catalog.json"))?)?;
|
||||
catalog["tables"]["__system_identities"]["columns"] = serde_json::json!([]);
|
||||
std::fs::write(system.join("catalog.json"), serde_json::to_vec(&catalog)?)?;
|
||||
let repair = serde_json::json!({"mode":"repair","types":types});
|
||||
Storage::maintenance(config.to_str(), &repair.to_string())?;
|
||||
assert_eq!(std::fs::read(system.join("__system_users.db"))?, credentials);
|
||||
assert_eq!(std::fs::read(business.join("sentinel"))?, b"business data");
|
||||
let opened = Storage::open(config.to_str())?;
|
||||
assert!(Storage::maintenance(config.to_str(), &request.to_string()).unwrap_err().contains("data directory is in use"));
|
||||
drop(opened);
|
||||
std::fs::remove_file(system.join("__system_users.db"))?;
|
||||
let before = std::fs::read(system.join("catalog.json"))?;
|
||||
let failure = Storage::maintenance(config.to_str(), &repair.to_string()).unwrap_err();
|
||||
assert!(failure.contains("repair would require"));
|
||||
assert_eq!(std::fs::read(system.join("catalog.json"))?, before);
|
||||
assert!(!system.join("__system_users.db").exists());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 从账号 WAL 恢复缺失凭据文件
|
||||
#[test]
|
||||
fn offline_recovery_replays_account_wal() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let data = tempfile::tempdir()?;
|
||||
let config = write_config(data.path());
|
||||
let types = type_seed_request()["types"].clone();
|
||||
Storage::maintenance(config.to_str(), &serde_json::json!({"mode":"reset","user":"root","password_hash":"original-hash","types":types}).to_string())?;
|
||||
let storage = Storage::open(config.to_str())?;
|
||||
let accounts: serde_json::Value = serde_json::from_str(&storage.request_line(r#"{"method":"accounts_list"}"#))?;
|
||||
let saved = serde_json::from_value(accounts["accounts"].clone())?;
|
||||
drop(storage);
|
||||
let system = data.path().join("system");
|
||||
let wal = chusql_core_storage::wal::Wal::new(system.join("wal.log"));
|
||||
wal.append(&chusql_core_storage::wal::WalOp::Accounts { accounts: saved, removed: Vec::new(), migrate_roles: false })?;
|
||||
wal.append(&chusql_core_storage::wal::WalOp::Commit)?;
|
||||
drop(wal);
|
||||
std::fs::remove_file(system.join("__system_users.db"))?;
|
||||
Storage::maintenance(config.to_str(), &serde_json::json!({"mode":"recover","types":types}).to_string())?;
|
||||
let storage = Storage::open(config.to_str())?;
|
||||
let restored: serde_json::Value = serde_json::from_str(&storage.request_line(r#"{"method":"accounts_list"}"#))?;
|
||||
assert_eq!(restored["accounts"], accounts["accounts"]);
|
||||
drop(storage);
|
||||
std::fs::write(system.join("catalog.json"), b"corrupted catalog")?;
|
||||
Storage::maintenance(config.to_str(), &serde_json::json!({"mode":"recover","types":types}).to_string())?;
|
||||
let storage = Storage::open(config.to_str())?;
|
||||
let reconstructed: serde_json::Value = serde_json::from_str(&storage.request_line(r#"{"method":"accounts_list"}"#))?;
|
||||
assert_eq!(reconstructed["accounts"], accounts["accounts"]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 一次会话就是当前线程的存储句柄
|
||||
struct Conn(Arc<Storage>);
|
||||
|
||||
@@ -800,7 +1227,7 @@ fn account_snapshot_replays_twice_without_duplicating_accounts() -> Result<(), B
|
||||
for _ in 0..2 {
|
||||
let wal = Wal::new(data.path().join("system/wal.log"));
|
||||
wal.truncate()?;
|
||||
wal.append(&WalOp::Accounts { accounts: accounts.clone() })?;
|
||||
wal.append(&WalOp::Accounts { accounts: accounts.clone(), removed: Vec::new(), migrate_roles: false })?;
|
||||
drop(wal);
|
||||
std::fs::write(data.path().join("system/__system_users.idx"), b"torn index")?;
|
||||
std::fs::write(data.path().join("system/__system_users.db"), b"torn heap")?;
|
||||
@@ -1595,7 +2022,7 @@ fn stats_report_distinct_values() {
|
||||
send(&mut c, r#"{"method":"delete_keys","table":"stat_t","keys":[1]}"#);
|
||||
let r = send(&mut c, r#"{"method":"describe_table","table":"stat_t"}"#);
|
||||
assert!(r.contains(r#""row_count":2"#), "after delete: {}", r);
|
||||
assert!(r.contains(r#""name":"age","distinct":1"#), "after delete: {}", r);
|
||||
assert!(r.contains(r#""name":"age","distinct":2"#), "remaining ages are still 30 and 41: {}", r);
|
||||
}
|
||||
|
||||
/// 重开服务后索引仍在
|
||||
@@ -1832,12 +2259,14 @@ fn account_table_uses_typed_columns_and_stamps_login() -> Result<(), Box<dyn std
|
||||
assert!(account["last_login_at"].is_null(), "还没登录过: {}", account);
|
||||
|
||||
let catalog: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(data.path().join("system/catalog.json"))?)?;
|
||||
let entry = &catalog["tables"]["__system_users"];
|
||||
let credential_columns = &catalog["tables"]["__system_users"]["columns"];
|
||||
assert_eq!(credential_columns.as_array().ok_or("missing credentials")?.iter().map(|c| c["name"].as_str().unwrap_or("")).collect::<Vec<_>>(), vec!["id", "password_hash"]);
|
||||
let entry = &catalog["tables"]["__system_identities"];
|
||||
let columns = entry["columns"].as_array().ok_or("missing columns")?;
|
||||
let names: Vec<&str> = columns.iter().map(|col| col["name"].as_str().unwrap_or("")).collect();
|
||||
assert_eq!(names, vec!["id", "user", "password_hash", "registered_at", "last_login_at", "revision"]);
|
||||
assert_eq!(names, vec!["id", "user", "can_login", "is_superuser", "system_catalog_manager", "enabled", "identity_version", "registered_at", "last_login_at", "revision"]);
|
||||
let types: Vec<&str> = columns.iter().map(|col| col["ty"].as_str().unwrap_or("")).collect();
|
||||
assert_eq!(types, vec!["int", "varchar(64)", "varchar(256)", "timestamp", "timestamp", "int"]);
|
||||
assert_eq!(types, vec!["int", "varchar(64)", "bool", "bool", "bool", "bool", "int", "timestamp", "timestamp", "int"]);
|
||||
assert_eq!(columns[0]["primary_key"], true, "id 是主键");
|
||||
assert_eq!(columns[0]["nullable"], false);
|
||||
assert_eq!(columns[1]["unique"], true, "用户名唯一");
|
||||
|
||||
@@ -207,6 +207,31 @@ fn trailing_torn_frame_can_be_truncated() {
|
||||
assert_eq!(wal.read_all().unwrap().records.len(), 1);
|
||||
}
|
||||
|
||||
/// 完整损坏帧返回错误而非截断成功
|
||||
#[test]
|
||||
fn complete_corrupt_frame_is_rejected() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("wal.log");
|
||||
std::fs::write(&path, [1, 0, 0, 0, 255]).unwrap();
|
||||
assert!(Wal::new(&path).read_all().is_err());
|
||||
assert_eq!(std::fs::read(path).unwrap(), [1, 0, 0, 0, 255]);
|
||||
}
|
||||
|
||||
/// 检查点损坏与替换失败不丢旧值
|
||||
#[test]
|
||||
fn checkpoint_corruption_and_write_failure_are_explicit() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let wal = Wal::new(dir.path().join("wal.log"));
|
||||
for bytes in [vec![], vec![1; 7], vec![1; 9]] {
|
||||
std::fs::write(dir.path().join("wal.checkpoint"), bytes).unwrap();
|
||||
assert!(wal.read_checkpoint().is_err());
|
||||
}
|
||||
wal.write_checkpoint(7).unwrap();
|
||||
std::fs::create_dir(dir.path().join("wal.checkpoint.pending")).unwrap();
|
||||
assert!(wal.write_checkpoint(9).is_err());
|
||||
assert_eq!(wal.read_checkpoint().unwrap(), Some(7));
|
||||
}
|
||||
|
||||
/// 检查点写读一致
|
||||
#[test]
|
||||
fn checkpoint_round_trips() {
|
||||
|
||||
Reference in new issue
Block a user