修复错误,优化性能,实现类型系统,优化安装卸载过程
This commit is contained in:
1 parent
fd5f320360
commit
48eec484b7
104 files changed
+4127
-845
No files matched your search
@@ -1,5 +1,5 @@
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::collections::{BTreeMap, BTreeSet, HashSet};
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::io;
|
||||
use std::io::Write;
|
||||
@@ -51,9 +51,11 @@ pub struct TableSchema {
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
pub struct Catalog {
|
||||
#[serde(default)]
|
||||
pub identity_high_water: i64,
|
||||
tables: BTreeMap<String, TableSchema>,
|
||||
#[serde(skip)]
|
||||
seen: BTreeMap<String, HashSet<u64>>,
|
||||
seen: BTreeMap<String, HashMap<u64, u64>>,
|
||||
}
|
||||
|
||||
impl Catalog {
|
||||
@@ -153,17 +155,25 @@ impl Catalog {
|
||||
for r in rows {
|
||||
for (name, v) in r {
|
||||
let stat = entry.stats.entry(name.clone()).or_default();
|
||||
hist_remove(stat, v);
|
||||
if stat.capped {
|
||||
continue;
|
||||
}
|
||||
if seen.remove(&fold(name, value_hash(v))) {
|
||||
stat.distinct = stat.distinct.saturating_sub(1);
|
||||
let key = fold(name, value_hash(v));
|
||||
if let Some(count) = seen.get_mut(&key) {
|
||||
*count = count.saturating_sub(1);
|
||||
if *count == 0 {
|
||||
seen.remove(&key);
|
||||
stat.distinct = stat.distinct.saturating_sub(1);
|
||||
}
|
||||
}
|
||||
hist_remove(stat, v);
|
||||
}
|
||||
}
|
||||
if entry.row_count == 0 {
|
||||
seen.clear();
|
||||
for stat in entry.stats.values_mut() {
|
||||
stat.distinct = 0;
|
||||
stat.capped = false;
|
||||
stat.lo = None;
|
||||
stat.hi = None;
|
||||
stat.hist.clear();
|
||||
@@ -192,17 +202,18 @@ impl Catalog {
|
||||
let seen = self.seen.entry(table.to_string()).or_default();
|
||||
for (name, v) in row {
|
||||
let stat = entry.stats.entry(name.clone()).or_default();
|
||||
hist_add(stat, v);
|
||||
if stat.capped {
|
||||
continue;
|
||||
}
|
||||
if seen.insert(fold(name, value_hash(v))) {
|
||||
let count = seen.entry(fold(name, value_hash(v))).or_insert(0);
|
||||
*count += 1;
|
||||
if *count == 1 {
|
||||
stat.distinct += 1;
|
||||
if stat.distinct >= STATS_DISTINCT_CAP {
|
||||
stat.capped = true;
|
||||
seen.clear();
|
||||
}
|
||||
}
|
||||
hist_add(stat, v);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -476,5 +487,7 @@ fn hist_remove(stat: &mut ColumnStat, v: &serde_json::Value) {
|
||||
return;
|
||||
}
|
||||
let index = hist_index(lo, hi, x);
|
||||
stat.hist[index] = stat.hist[index].saturating_sub(1);
|
||||
if let Some(bucket) = (0..stat.hist.len()).filter(|&i| stat.hist[i] > 0).min_by_key(|&i| i.abs_diff(index)) {
|
||||
stat.hist[bucket] -= 1;
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ use crate::log::Level;
|
||||
// 存储相关分区由本模块解析校验,其它层的分区一概不碰。
|
||||
|
||||
/// 配置文件名
|
||||
pub const CONFIG_FILE_NAME: &str = "chusql.toml";
|
||||
pub const CONFIG_FILE_NAME: &str = "settings.toml";
|
||||
/// 配置目录名(随系统惯例放在用户配置目录下)
|
||||
pub const APP_DIR_NAME: &str = "ChuSQL";
|
||||
/// Unix 下的数据目录名(XDG 惯例小写,跟安装脚本的默认安装目录一致)
|
||||
@@ -20,6 +20,8 @@ pub const DEFAULT_POOL_SIZE: usize = 1024;
|
||||
pub const DEFAULT_PAGE_SIZE: usize = 4096;
|
||||
pub const DEFAULT_BTREE_ORDER: usize = 4;
|
||||
pub const DEFAULT_LOG_LEVEL: &str = "info";
|
||||
/// 默认日志输出目录(相对启动目录)
|
||||
pub const DEFAULT_LOG_FILES: &str = "./logs";
|
||||
|
||||
const MIN_PAGE_SIZE: usize = 512;
|
||||
const MAX_PAGE_SIZE: usize = 65536;
|
||||
@@ -91,6 +93,7 @@ pub struct Config {
|
||||
pub btree_order: usize,
|
||||
pub pool_size: usize,
|
||||
pub data_dir: PathBuf,
|
||||
pub log_files: PathBuf,
|
||||
pub log_level: Level,
|
||||
}
|
||||
|
||||
@@ -102,6 +105,7 @@ impl Default for Config {
|
||||
btree_order: DEFAULT_BTREE_ORDER,
|
||||
pool_size: DEFAULT_POOL_SIZE,
|
||||
data_dir: default_data_dir(),
|
||||
log_files: PathBuf::from(DEFAULT_LOG_FILES),
|
||||
log_level: Level::Info,
|
||||
}
|
||||
}
|
||||
@@ -161,6 +165,7 @@ struct FileBtree {
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct FileStorage {
|
||||
data_dir: Option<String>,
|
||||
log_files: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
@@ -229,6 +234,11 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
|
||||
default_data_dir().to_string_lossy().into_owned(),
|
||||
&file_origin,
|
||||
);
|
||||
let (log_files, log_files_origin) = pick(
|
||||
file.storage.as_ref().and_then(|s| s.log_files.clone()),
|
||||
DEFAULT_LOG_FILES.to_string(),
|
||||
&file_origin,
|
||||
);
|
||||
// 管名已经没有了:存储不再是独立进程,管道/套接字那一层随 FFI 一起删掉
|
||||
let (log_level, log_origin) = pick(
|
||||
file.log.and_then(|l| l.level),
|
||||
@@ -237,6 +247,7 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
|
||||
);
|
||||
|
||||
let data_dir = data_dir.trim().to_string();
|
||||
let log_files = log_files.trim().to_string();
|
||||
let log_level = log_level.trim().to_string();
|
||||
|
||||
let log_level = Level::parse(&log_level)
|
||||
@@ -245,6 +256,9 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
|
||||
if data_dir.is_empty() {
|
||||
return Err("storage.data_dir must not be empty".to_string());
|
||||
}
|
||||
if log_files.is_empty() {
|
||||
return Err("storage.log_files must not be empty".to_string());
|
||||
}
|
||||
validate_layout(page_size, btree_order)?;
|
||||
if pool_size == 0 {
|
||||
return Err("buffer.pool_size must be at least 1".to_string());
|
||||
@@ -254,6 +268,7 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
|
||||
("btree.order", btree_order.to_string(), order_origin),
|
||||
("buffer.pool_size", pool_size.to_string(), pool_origin),
|
||||
("storage.data_dir", data_dir.clone(), dir_origin),
|
||||
("storage.log_files", log_files.clone(), log_files_origin),
|
||||
("log.level", log_level.name().to_string(), log_origin),
|
||||
];
|
||||
|
||||
@@ -263,6 +278,7 @@ pub fn resolve(text: Option<&str>, path: Option<PathBuf>) -> Result<Loaded, Stri
|
||||
btree_order,
|
||||
pool_size,
|
||||
data_dir: PathBuf::from(data_dir),
|
||||
log_files: PathBuf::from(log_files),
|
||||
log_level,
|
||||
},
|
||||
config_path: if text.is_some() { path } else { None },
|
||||
@@ -318,15 +334,14 @@ size = 8192
|
||||
data_dir = "../data"
|
||||
|
||||
[server]
|
||||
host = "127.0.0.1"
|
||||
listen_host = "127.0.0.1"
|
||||
port = 7777
|
||||
max_message = 1048576
|
||||
|
||||
[web]
|
||||
port = 7778
|
||||
user = "root"
|
||||
"#;
|
||||
let loaded = resolve(Some(text), Some(PathBuf::from("chusql.toml"))).unwrap();
|
||||
let loaded = resolve(Some(text), Some(PathBuf::from("settings.toml"))).unwrap();
|
||||
assert_eq!(loaded.config.page_size, 8192);
|
||||
assert_eq!(loaded.config.data_dir, PathBuf::from("../data"));
|
||||
}
|
||||
|
||||
@@ -71,6 +71,33 @@ pub unsafe extern "C" fn chusql_storage_close(handle: *mut Storage) {
|
||||
let _ = catch_unwind(AssertUnwindSafe(|| unsafe { drop(Box::from_raw(handle)) }));
|
||||
}
|
||||
|
||||
/// 离线维护系统目录并返回 JSON
|
||||
///
|
||||
/// # Safety
|
||||
/// 输入为有效字符串;输出指针可写,响应使用 free 释放。
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn chusql_storage_maintenance(
|
||||
config_path: *const c_char, request: *const u8, request_len: usize,
|
||||
out: *mut *mut u8, out_len: *mut usize,
|
||||
) -> i32 {
|
||||
if request.is_null() || out.is_null() || out_len.is_null() { set_error("invalid maintenance argument"); return ERR; }
|
||||
unsafe { *out = ptr::null_mut(); *out_len = 0; }
|
||||
let result = catch_unwind(AssertUnwindSafe(|| {
|
||||
let path = if config_path.is_null() { None } else { Some(unsafe { CStr::from_ptr(config_path) }.to_str().map_err(|e| e.to_string())?) };
|
||||
let text = std::str::from_utf8(unsafe { slice::from_raw_parts(request, request_len) }).map_err(|e| e.to_string())?;
|
||||
Storage::maintenance(path, text)
|
||||
}));
|
||||
let response = match result {
|
||||
Ok(Ok(response)) => response,
|
||||
Ok(Err(err)) => { set_error(err); return ERR; }
|
||||
Err(_) => { set_error("panic in system maintenance"); return ERR; }
|
||||
};
|
||||
let mut buffer = response.into_bytes().into_boxed_slice();
|
||||
unsafe { *out = buffer.as_mut_ptr(); *out_len = buffer.len(); }
|
||||
std::mem::forget(buffer);
|
||||
OK
|
||||
}
|
||||
|
||||
/// 处理一条请求,响应缓冲须归还
|
||||
///
|
||||
/// # Safety
|
||||
|
||||
@@ -6,6 +6,8 @@ use std::collections::HashMap;
|
||||
pub type Row = HashMap<String, serde_json::Value>;
|
||||
|
||||
pub const USERS_TABLE: &str = "__system_users";
|
||||
pub const IDENTITIES_TABLE: &str = "__system_identities";
|
||||
pub const TYPES_TABLE: &str = "__system_types";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct Account {
|
||||
@@ -17,6 +19,16 @@ pub struct Account {
|
||||
pub registered_at: String,
|
||||
#[serde(default)]
|
||||
pub last_login_at: Option<String>,
|
||||
#[serde(default = "default_true")]
|
||||
pub can_login: bool,
|
||||
#[serde(default)]
|
||||
pub is_superuser: bool,
|
||||
#[serde(default)]
|
||||
pub system_catalog_manager: bool,
|
||||
#[serde(default = "default_true")]
|
||||
pub enabled: bool,
|
||||
#[serde(default)]
|
||||
pub identity_version: u8,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
@@ -80,7 +92,11 @@ pub struct IndexWire {
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "method", rename_all = "snake_case")]
|
||||
pub enum Request {
|
||||
CatalogManage { command: Box<Request> },
|
||||
AccountsList,
|
||||
IdentityInitialize { administrator: String },
|
||||
RoleCreate { user: String },
|
||||
IdentityAlter { user: String, can_login: Option<bool>, is_superuser: Option<bool>, enabled: Option<bool>, system_catalog_manager: Option<bool> },
|
||||
AccountCreate { user: String, password_hash: String },
|
||||
AccountReset { user: String, password_hash: String },
|
||||
AccountLogin {
|
||||
@@ -97,6 +113,7 @@ pub enum Request {
|
||||
password_hash: Option<String>,
|
||||
},
|
||||
SystemStatus,
|
||||
BootstrapTypes { types: Vec<Row> },
|
||||
Ping,
|
||||
Scan {
|
||||
table: String,
|
||||
@@ -222,9 +239,11 @@ impl Request {
|
||||
| Self::DropIndex { table, .. } | Self::DropColumn { table, .. }
|
||||
| Self::Compact { table }
|
||||
| Self::ReplaceSchema { table, .. } => Some(table),
|
||||
Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList
|
||||
Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList | Self::CatalogManage { .. }
|
||||
| Self::IdentityInitialize { .. } | Self::RoleCreate { .. } | Self::IdentityAlter { .. }
|
||||
| Self::AccountCreate { .. } | Self::AccountReset { .. } | Self::AccountLogin { .. }
|
||||
| Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus
|
||||
| Self::BootstrapTypes { .. }
|
||||
| Self::ApplyTransaction { .. } => None,
|
||||
}
|
||||
}
|
||||
@@ -245,7 +264,8 @@ pub const PRIVILEGE_TABLES: [&str; 4] = [
|
||||
|
||||
/// 是不是服务自己经请求通道读写的系统表
|
||||
pub fn internal_table(table: &str) -> bool {
|
||||
PRIVILEGE_TABLES.contains(&table.to_ascii_lowercase().as_str())
|
||||
let name = table.to_ascii_lowercase();
|
||||
PRIVILEGE_TABLES.contains(&name.as_str()) || name.starts_with("__system_domain_")
|
||||
}
|
||||
|
||||
/// 是不是请求通道要拒绝的系统表
|
||||
|
||||
@@ -8,8 +8,8 @@ use crate::config::{self, Config, Loaded};
|
||||
use crate::heap::{scalar_int, HeapTable};
|
||||
use crate::log;
|
||||
use crate::protocol::{
|
||||
blocked_table, reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row,
|
||||
SchemaColumn, StorageOp, TableSchemaWire, USERS_TABLE,
|
||||
blocked_table, reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row, IDENTITIES_TABLE,
|
||||
SchemaColumn, StorageOp, TableSchemaWire, USERS_TABLE, TYPES_TABLE,
|
||||
};
|
||||
use crate::wal::{committed_ops, records_after, Wal, WalOp};
|
||||
use crate::{log_debug, log_error, log_info, log_warn};
|
||||
@@ -222,15 +222,41 @@ impl Server {
|
||||
c.rebuild_stats(table, rows);
|
||||
c.save(self.catalog_path())
|
||||
}
|
||||
WalOp::Accounts { accounts } => {
|
||||
WalOp::Accounts { accounts, removed, migrate_roles } => {
|
||||
let rows = account_rows(accounts)?;
|
||||
self.tables.lock().map_err(|_| std::io::Error::other("table lock poisoned"))?.remove(USERS_TABLE);
|
||||
for path in [self.table_path(USERS_TABLE), self.index_path(USERS_TABLE, "id")] {
|
||||
std::fs::File::create(path)?.sync_all()?;
|
||||
let identities = identity_rows(accounts)?;
|
||||
for (table, records) in [(USERS_TABLE, &rows), (IDENTITIES_TABLE, &identities)] {
|
||||
self.tables.lock().map_err(|_| std::io::Error::other("table lock poisoned"))?.remove(table);
|
||||
for path in [self.table_path(table), self.index_path(table, "id")] {
|
||||
std::fs::File::create(path)?.sync_all()?;
|
||||
}
|
||||
self.with_table(table, |t| { t.replace_all(records)?; t.flush() })?;
|
||||
}
|
||||
let legacy_roles = "__system_roles";
|
||||
let has_roles = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?.describe(legacy_roles).is_some();
|
||||
if has_roles && *migrate_roles {
|
||||
self.with_existing_table(legacy_roles, |t| { t.replace_all(&[])?; t.flush() })?;
|
||||
}
|
||||
let mut cleaned = Vec::new();
|
||||
for table in ["__system_grants", "__system_grant_options", "__system_members"] {
|
||||
let exists = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?.describe(table).is_some();
|
||||
if exists && !removed.is_empty() {
|
||||
let rows = self.with_existing_table(table, |t| t.scan())?.into_iter().filter(|r|
|
||||
!["role", "member"].iter().any(|field| r.get(*field).and_then(serde_json::Value::as_str).is_some_and(|name| removed.iter().any(|old| old == name)))).collect::<Vec<_>>();
|
||||
self.with_existing_table(table, |t| { t.replace_all(&rows)?; t.flush() })?;
|
||||
cleaned.push((table, rows));
|
||||
}
|
||||
}
|
||||
self.with_table(USERS_TABLE, |t| { t.replace_all(&rows)?; t.flush() })?;
|
||||
let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||
declare_system_schema(&mut c, &rows)?;
|
||||
c.rebuild_stats(IDENTITIES_TABLE, &identities);
|
||||
c.set_columns(IDENTITIES_TABLE, identity_columns())?;
|
||||
c.rebuild_stats(IDENTITIES_TABLE, &identities);
|
||||
c.mark_system(IDENTITIES_TABLE)?;
|
||||
c.add_index(IDENTITIES_TABLE, "user")?;
|
||||
c.identity_high_water = c.identity_high_water.max(accounts.iter().map(|a| a.id).max().unwrap_or(0));
|
||||
if has_roles && *migrate_roles { c.rebuild_stats(legacy_roles, &[]); }
|
||||
for (table, records) in cleaned { c.rebuild_stats(table, &records); }
|
||||
c.save(self.catalog_path())
|
||||
}
|
||||
WalOp::Insert { table, row } => {
|
||||
@@ -284,6 +310,7 @@ impl Server {
|
||||
self.with_table(table, |_t| Ok(()))?;
|
||||
let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||
c.create_table(table, columns.clone())?;
|
||||
if table == TYPES_TABLE { c.mark_system(table)?; }
|
||||
c.save(self.catalog_path())
|
||||
}
|
||||
WalOp::DropTable { table } => {
|
||||
@@ -342,8 +369,67 @@ impl Server {
|
||||
if self.recovery_required.load(Ordering::Acquire) {
|
||||
return Err(std::io::Error::other("storage recovery required"));
|
||||
}
|
||||
self.validate_op(op)?;
|
||||
self.group_via_wal(std::slice::from_ref(op))
|
||||
let op = self.normalize_decimal_write(op)?;
|
||||
self.validate_op(&op)?;
|
||||
self.validate_domains(&op)?;
|
||||
self.group_via_wal(std::slice::from_ref(&op))
|
||||
}
|
||||
|
||||
/// 写日志前规范化定点数值
|
||||
fn normalize_decimal_write(&self, op: &WalOp) -> std::io::Result<WalOp> {
|
||||
let mut normalized = op.clone();
|
||||
let catalog = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||
let (table, rows): (&str, Vec<&mut Row>) = match &mut normalized {
|
||||
WalOp::Insert { table, row } => (table, vec![row]),
|
||||
WalOp::InsertBatch { table, rows } | WalOp::ReplaceAll { table, rows } => (table, rows.iter_mut().collect()),
|
||||
WalOp::CreateTable { columns, .. } => {
|
||||
for column in columns { decimal_parameters(&column.ty)?; }
|
||||
return Ok(normalized);
|
||||
}
|
||||
WalOp::ReplaceSchema { columns, rows, .. } => {
|
||||
for column in columns.iter() { decimal_parameters(&column.ty)?; }
|
||||
for row in rows { normalize_decimal_row(row, columns)?; }
|
||||
return Ok(normalized);
|
||||
}
|
||||
_ => return Ok(normalized),
|
||||
};
|
||||
if let Some(schema) = catalog.describe(table) {
|
||||
for row in rows {
|
||||
normalize_decimal_row(row, &schema.columns)?;
|
||||
}
|
||||
}
|
||||
Ok(normalized)
|
||||
}
|
||||
|
||||
/// 校验命名类型引用与删除依赖
|
||||
fn validate_domains(&self, op: &WalOp) -> std::io::Result<()> {
|
||||
let catalog = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||
match op {
|
||||
WalOp::CreateTable { columns, .. } | WalOp::ReplaceSchema { columns, .. } => {
|
||||
for column in columns {
|
||||
let mut ty = column.ty.as_str();
|
||||
while let Some(body) = ty.strip_prefix("domain(").and_then(|s| s.strip_suffix(')')) {
|
||||
let (name, base) = body.split_once(',').ok_or_else(|| std::io::Error::other("invalid domain type"))?;
|
||||
let key = format!("__system_domain_{name}");
|
||||
let definition = catalog.describe(&key).ok_or_else(|| std::io::Error::other(format!("unknown domain: {name}")))?;
|
||||
if !definition.columns.iter().any(|c| c.name == "base" && c.ty == base) {
|
||||
return Err(std::io::Error::other(format!("domain base type mismatch: {name}")));
|
||||
}
|
||||
ty = base;
|
||||
}
|
||||
}
|
||||
}
|
||||
WalOp::DropTable { table } => {
|
||||
if let Some(name) = table.strip_prefix("__system_domain_") {
|
||||
let prefix = format!("domain({name},");
|
||||
if catalog.all_tables().iter().any(|(_, schema)| schema.columns.iter().any(|c| !schema.dropped_columns.contains(&c.name) && c.ty.contains(&prefix))) {
|
||||
return Err(std::io::Error::other(format!("domain is still used by a column: {name}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 校验一条操作:命中被删的列就拒绝
|
||||
@@ -406,6 +492,7 @@ impl Server {
|
||||
rows: rows.clone(),
|
||||
},
|
||||
};
|
||||
let wal_op = self.normalize_decimal_write(&wal_op)?;
|
||||
self.validate_op(&wal_op)?;
|
||||
wal_ops.push(wal_op);
|
||||
}
|
||||
@@ -420,13 +507,9 @@ impl Server {
|
||||
std::fs::create_dir_all(&self.cfg.data_dir)?;
|
||||
let wal = &self.wal;
|
||||
let commit_lsn = wal.append_group(ops)?;
|
||||
self.recovery_required.store(true, Ordering::Release);
|
||||
for op in ops {
|
||||
let result = self.apply_op(op);
|
||||
if result.is_err() && matches!(op, WalOp::HideColumn { .. }) {
|
||||
self.recovery_required.store(true, Ordering::Release);
|
||||
return result;
|
||||
}
|
||||
result?;
|
||||
self.apply_op(op)?;
|
||||
}
|
||||
for op in ops {
|
||||
self.flush_op_tables(op)?;
|
||||
@@ -434,6 +517,7 @@ impl Server {
|
||||
self.flush_all_tables()?;
|
||||
wal.write_checkpoint(commit_lsn)?;
|
||||
wal.truncate_before(commit_lsn)?;
|
||||
self.recovery_required.store(false, Ordering::Release);
|
||||
log_debug!(core, "WAL committed group at LSN {}", commit_lsn);
|
||||
Ok(())
|
||||
}
|
||||
@@ -479,15 +563,15 @@ impl Server {
|
||||
fn recover_wal(&self) -> std::io::Result<()> {
|
||||
std::fs::create_dir_all(&self.cfg.data_dir)?;
|
||||
let wal = &self.wal;
|
||||
let checkpoint = wal.read_checkpoint()?.unwrap_or(0);
|
||||
let scan = wal.read_all()?;
|
||||
if scan.records.is_empty() {
|
||||
if scan.good_len > 0 {
|
||||
if wal.path().exists() {
|
||||
wal.truncate_to(scan.good_len)?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
// 检查点以内的记录已经落盘,直接按边界截掉
|
||||
let checkpoint = wal.read_checkpoint()?.unwrap_or(0);
|
||||
let pending = records_after(&scan.records, checkpoint);
|
||||
let records = committed_ops(&pending);
|
||||
if records.is_empty() {
|
||||
@@ -859,6 +943,12 @@ impl Server {
|
||||
use std::io::{Error, ErrorKind};
|
||||
let registered = {
|
||||
let c = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?;
|
||||
match c.describe(IDENTITIES_TABLE) {
|
||||
Some(schema) if !schema.system => return Err(Error::new(ErrorKind::AlreadyExists, "reserved identity table collision")),
|
||||
None if self.table_path(IDENTITIES_TABLE).exists() || self.index_path(IDENTITIES_TABLE, "id").exists() =>
|
||||
return Err(Error::new(ErrorKind::AlreadyExists, "reserved identity file collision")),
|
||||
_ => {}
|
||||
}
|
||||
c.describe(USERS_TABLE).map(|schema| schema.system)
|
||||
};
|
||||
match registered {
|
||||
@@ -893,6 +983,21 @@ impl Server {
|
||||
if !self.table_path(USERS_TABLE).exists() {
|
||||
return Err(Error::other("missing account data"));
|
||||
}
|
||||
let unified = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe(IDENTITIES_TABLE).is_some();
|
||||
if unified {
|
||||
let credentials = self.with_existing_table(USERS_TABLE, |t| t.scan())?;
|
||||
let identities = self.with_existing_table(IDENTITIES_TABLE, |t| t.scan())?;
|
||||
return identities.into_iter().map(|mut row| {
|
||||
for field in ["can_login", "is_superuser", "enabled"] {
|
||||
if row.get(field).and_then(serde_json::Value::as_bool).is_none() { return Err(Error::other("invalid identity attributes")); }
|
||||
}
|
||||
let id = row.get("id").and_then(serde_json::Value::as_i64).ok_or_else(|| Error::other("invalid identity record"))?;
|
||||
let credential = credentials.iter().find(|r| r.get("id").and_then(serde_json::Value::as_i64) == Some(id))
|
||||
.ok_or_else(|| Error::other("missing identity credential"))?;
|
||||
row.insert("password_hash".into(), credential.get("password_hash").cloned().ok_or_else(|| Error::other("invalid identity credential"))?);
|
||||
decode_account_row(&row)
|
||||
}).collect();
|
||||
}
|
||||
let mut accounts: Vec<Account> = self
|
||||
.with_existing_table(USERS_TABLE, |t| t.scan())?
|
||||
.iter()
|
||||
@@ -919,9 +1024,11 @@ impl Server {
|
||||
// 表里已经有行就不再补人:引导程序只在空表上播种,
|
||||
// 免得有人拿 storage 透传绕过账号接口往里塞一个已知口令的账号
|
||||
if accounts.is_empty() && !accounts.iter().any(|a| a.user == name) {
|
||||
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0)
|
||||
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0).max(self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.identity_high_water)
|
||||
.checked_add(1).ok_or_else(|| Error::other("account id exhausted"))?;
|
||||
accounts.push(new_account(id, &name, password_hash.unwrap_or_default().to_string())?);
|
||||
let mut administrator = new_account(id, &name, password_hash.unwrap_or_default().to_string())?;
|
||||
administrator.is_superuser = true;
|
||||
accounts.push(administrator);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
@@ -931,7 +1038,7 @@ impl Server {
|
||||
for row in account_rows(&accounts)? {
|
||||
HeapTable::encode_row(&row, self.cfg.page_size)?;
|
||||
}
|
||||
let op = WalOp::Accounts { accounts: accounts.clone() };
|
||||
let op = WalOp::Accounts { accounts: accounts.clone(), removed: Vec::new(), migrate_roles: false };
|
||||
self.recovery_required.store(true, Ordering::Release);
|
||||
self.wal.truncate()?;
|
||||
self.wal.append(&op)?;
|
||||
@@ -952,17 +1059,93 @@ impl Server {
|
||||
return Err(Error::other("system catalog is not initialized; run csql-bootstrap"));
|
||||
}
|
||||
let mut accounts = self.load_accounts()?;
|
||||
let request = match request {
|
||||
Request::CatalogManage { command } => {
|
||||
let target = match command.as_ref() {
|
||||
Request::AccountCreate { user, .. } | Request::AccountReset { user, .. } | Request::AccountDrop { user }
|
||||
| Request::IdentityAlter { user, is_superuser: None, system_catalog_manager: None, .. } => Some(account_name(user)?),
|
||||
Request::AccountsList => None,
|
||||
_ => return Err(Error::other("superuser required for this catalog operation")),
|
||||
};
|
||||
if accounts.iter().any(|a| target.as_ref() == Some(&a.user) && (a.is_superuser || a.system_catalog_manager)) {
|
||||
return Err(Error::other("superuser required to manage privileged identities"));
|
||||
}
|
||||
*command
|
||||
}
|
||||
other => other,
|
||||
};
|
||||
let mut names = std::collections::HashSet::new();
|
||||
let mut ids = std::collections::HashSet::new();
|
||||
for account in &mut accounts {
|
||||
let name = account_name(&account.user)?;
|
||||
if !names.insert(name.clone()) || account.id < 1 || !ids.insert(account.id) {
|
||||
return Err(Error::other("invalid or duplicate identity record"));
|
||||
}
|
||||
if account.identity_version == 0 { account.user = name; }
|
||||
}
|
||||
let previous = accounts.clone();
|
||||
let migrate_roles = matches!(request, Request::IdentityInitialize { .. });
|
||||
if !migrate_roles && !matches!(request, Request::AccountsList) {
|
||||
let legacy_exists = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe("__system_roles").is_some();
|
||||
if legacy_exists && !self.with_existing_table("__system_roles", |t| t.scan())?.is_empty() {
|
||||
return Err(Error::other("identity migration required"));
|
||||
}
|
||||
}
|
||||
let had_superuser = accounts.iter().any(|a| a.enabled && a.can_login && a.is_superuser);
|
||||
match request {
|
||||
Request::AccountsList => return Ok(accounts),
|
||||
Request::IdentityInitialize { administrator } => {
|
||||
for account in &mut accounts {
|
||||
if account.identity_version == 0 {
|
||||
account.is_superuser = account.user == administrator.to_ascii_lowercase();
|
||||
account.identity_version = 1;
|
||||
}
|
||||
}
|
||||
let legacy_roles = "__system_roles";
|
||||
let exists = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe(legacy_roles).is_some();
|
||||
if exists {
|
||||
for row in self.with_existing_table(legacy_roles, |t| t.scan())? {
|
||||
let name = account_name(row.get("name").and_then(serde_json::Value::as_str).ok_or_else(|| Error::other("invalid legacy role"))?)?;
|
||||
if accounts.iter().any(|a| a.user == name) { return Err(Error::other(format!("identity name collision: {name}"))); }
|
||||
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0).max(self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.identity_high_water)
|
||||
.checked_add(1).ok_or_else(|| Error::other("account id exhausted"))?;
|
||||
let mut role = new_account(id, &name, String::new())?;
|
||||
role.can_login = false;
|
||||
accounts.push(role);
|
||||
}
|
||||
}
|
||||
let unified = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe(IDENTITIES_TABLE).is_some();
|
||||
if !accounts.is_empty() { require_superuser(&accounts)?; }
|
||||
if unified && previous == accounts { return Ok(accounts); }
|
||||
}
|
||||
Request::AccountCreate { user, password_hash } => {
|
||||
let name = account_name(&user)?;
|
||||
if accounts.iter().any(|a| a.user == name) {
|
||||
return Err(Error::new(ErrorKind::AlreadyExists, "account already exists"));
|
||||
}
|
||||
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0)
|
||||
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0).max(self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.identity_high_water)
|
||||
.checked_add(1).ok_or_else(|| Error::other("account id exhausted"))?;
|
||||
accounts.push(new_account(id, &name, password_hash)?);
|
||||
}
|
||||
Request::RoleCreate { user } => {
|
||||
let name = account_name(&user)?;
|
||||
if accounts.iter().any(|a| a.user == name) { return Err(Error::new(ErrorKind::AlreadyExists, "identity already exists")); }
|
||||
let id = accounts.iter().map(|a| a.id).max().unwrap_or(0).max(self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.identity_high_water)
|
||||
.checked_add(1).ok_or_else(|| Error::other("account id exhausted"))?;
|
||||
let mut role = new_account(id, &name, String::new())?;
|
||||
role.can_login = false;
|
||||
accounts.push(role);
|
||||
}
|
||||
Request::IdentityAlter { user, can_login, is_superuser, enabled, system_catalog_manager } => {
|
||||
let name = account_name(&user)?;
|
||||
let account = accounts.iter_mut().find(|a| a.user == name).ok_or_else(|| Error::new(ErrorKind::NotFound, "unknown account"))?;
|
||||
account.can_login = can_login.unwrap_or(account.can_login);
|
||||
account.is_superuser = is_superuser.unwrap_or(account.is_superuser);
|
||||
account.enabled = enabled.unwrap_or(account.enabled);
|
||||
account.system_catalog_manager = system_catalog_manager.unwrap_or(account.system_catalog_manager);
|
||||
account.revision = account.revision.checked_add(1).ok_or_else(|| Error::other("account revision exhausted"))?;
|
||||
if had_superuser { require_superuser(&accounts)?; }
|
||||
}
|
||||
Request::AccountReset { user, password_hash } => {
|
||||
let name = account_name(&user)?;
|
||||
validate_account_hash(&password_hash)?;
|
||||
@@ -976,6 +1159,7 @@ impl Server {
|
||||
let stamp = at.unwrap_or_else(now_stamp);
|
||||
let account = accounts.iter_mut().find(|a| a.user == name)
|
||||
.ok_or_else(|| Error::new(ErrorKind::NotFound, "unknown account"))?;
|
||||
if !account.enabled || !account.can_login { return Err(Error::other("identity cannot login")); }
|
||||
account.last_login_at = Some(stamp);
|
||||
}
|
||||
Request::AccountDrop { user } => {
|
||||
@@ -985,13 +1169,15 @@ impl Server {
|
||||
if accounts.len() == before {
|
||||
return Err(Error::new(ErrorKind::NotFound, "unknown account"));
|
||||
}
|
||||
if had_superuser { require_superuser(&accounts)?; }
|
||||
}
|
||||
_ => return Err(Error::other("unexpected account request")),
|
||||
}
|
||||
for row in account_rows(&accounts)? {
|
||||
for row in account_rows(&accounts)?.into_iter().chain(identity_rows(&accounts)?) {
|
||||
HeapTable::encode_row(&row, self.cfg.page_size)?;
|
||||
}
|
||||
let op = WalOp::Accounts { accounts: accounts.clone() };
|
||||
let removed = previous.iter().filter(|old| !accounts.iter().any(|a| a.user == old.user)).map(|a| a.user.clone()).collect();
|
||||
let op = WalOp::Accounts { accounts: accounts.clone(), removed, migrate_roles };
|
||||
self.recovery_required.store(true, Ordering::Release);
|
||||
self.wal.truncate()?;
|
||||
self.wal.append(&op)?;
|
||||
@@ -1001,6 +1187,53 @@ impl Server {
|
||||
Ok(accounts)
|
||||
}
|
||||
|
||||
/// 初始化并校验内置类型目录
|
||||
fn bootstrap_types(&self, types: Vec<Row>) -> std::io::Result<Vec<Row>> {
|
||||
use std::io::Error;
|
||||
let _guard = self.write_lock.lock().map_err(|_| Error::other("write lock poisoned"))?;
|
||||
if !self.account_table_ready()? { return Err(Error::other("system catalog is not initialized; run csql-bootstrap")); }
|
||||
require_superuser(&self.load_accounts()?)?;
|
||||
let columns = types_columns();
|
||||
let existing = self.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.describe(TYPES_TABLE).cloned();
|
||||
if existing.as_ref().is_some_and(|s| !s.system || s.columns != columns) { return Err(Error::other("reserved type catalog collision")); }
|
||||
if existing.is_none() && (self.table_path(TYPES_TABLE).exists() || self.index_path(TYPES_TABLE, "id").exists()) {
|
||||
return Err(Error::other("reserved type file collision"));
|
||||
}
|
||||
let mut rows = if existing.is_some() { self.with_existing_table(TYPES_TABLE, |t| t.scan())? } else { Vec::new() };
|
||||
let mut names = std::collections::HashSet::new();
|
||||
let mut ids = std::collections::HashSet::new();
|
||||
for row in &rows {
|
||||
let name = type_definition_name(row)?;
|
||||
let id = row.get("id").and_then(serde_json::Value::as_i64).ok_or_else(|| Error::other("invalid type id"))?;
|
||||
if id < 1 || !ids.insert(id) || !names.insert(name.to_string()) { return Err(Error::other("duplicate type catalog record")); }
|
||||
}
|
||||
let original = rows.clone();
|
||||
let mut requested = std::collections::HashSet::new();
|
||||
for mut definition in types {
|
||||
let name = type_definition_name(&definition)?.to_string();
|
||||
if definition.len() != 3 || !requested.insert(name.clone()) { return Err(Error::other("invalid or duplicate preinstalled type")); }
|
||||
if let Some(saved) = rows.iter().find(|row| row.get("name").and_then(serde_json::Value::as_str) == Some(&name)) {
|
||||
if saved.get("base_type") != definition.get("base_type") || saved.get("parameterized") != definition.get("parameterized") {
|
||||
return Err(Error::other(format!("preinstalled type definition conflict: {name}")));
|
||||
}
|
||||
} else {
|
||||
let id = rows.iter().filter_map(|r| r.get("id").and_then(serde_json::Value::as_i64)).max().unwrap_or(0)
|
||||
.checked_add(1).ok_or_else(|| Error::other("type id exhausted"))?;
|
||||
definition.insert("id".into(), serde_json::json!(id));
|
||||
HeapTable::encode_row(&definition, self.cfg.page_size)?;
|
||||
rows.push(definition);
|
||||
}
|
||||
}
|
||||
if existing.is_some() && original == rows { return Ok(rows); }
|
||||
let mut ops = Vec::new();
|
||||
if existing.is_none() { ops.push(WalOp::CreateTable { table: TYPES_TABLE.into(), columns }); }
|
||||
ops.push(WalOp::ReplaceAll { table: TYPES_TABLE.into(), rows: rows.clone() });
|
||||
self.recovery_required.store(true, Ordering::Release);
|
||||
self.group_via_wal(&ops)?;
|
||||
self.recovery_required.store(false, Ordering::Release);
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// 一条请求翻成一条响应:先过读/写门再分发
|
||||
fn handle_request(&self, req: Request) -> Response {
|
||||
if concurrent_read(&req) {
|
||||
@@ -1037,7 +1270,12 @@ impl Server {
|
||||
}
|
||||
}
|
||||
match req {
|
||||
request @ (Request::AccountsList | Request::AccountCreate { .. }
|
||||
Request::BootstrapTypes { types } => match self.bootstrap_types(types) {
|
||||
Ok(rows) => Response::Rows { rows },
|
||||
Err(e) => Response::Error { message: e.to_string() },
|
||||
},
|
||||
request @ (Request::AccountsList | Request::CatalogManage { .. } | Request::AccountCreate { .. }
|
||||
| Request::IdentityInitialize { .. } | Request::RoleCreate { .. } | Request::IdentityAlter { .. }
|
||||
| Request::AccountReset { .. } | Request::AccountLogin { .. }
|
||||
| Request::AccountDrop { .. }) => {
|
||||
match self.accounts_request(request) {
|
||||
@@ -1458,12 +1696,45 @@ impl Server {
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造内置类型目录列定义
|
||||
fn types_columns() -> Vec<SchemaColumn> {
|
||||
vec![
|
||||
SchemaColumn { name: "id".into(), ty: "int".into(), nullable: false, primary_key: true, ..Default::default() },
|
||||
SchemaColumn { name: "name".into(), ty: "varchar(64)".into(), nullable: false, unique: true, ..Default::default() },
|
||||
SchemaColumn { name: "base_type".into(), ty: "varchar(64)".into(), nullable: false, ..Default::default() },
|
||||
SchemaColumn { name: "parameterized".into(), ty: "bool".into(), nullable: false, ..Default::default() },
|
||||
]
|
||||
}
|
||||
|
||||
/// 校验类型定义并读取名称
|
||||
fn type_definition_name(row: &Row) -> std::io::Result<&str> {
|
||||
let name = row.get("name").and_then(serde_json::Value::as_str).ok_or_else(|| std::io::Error::other("invalid type name"))?;
|
||||
let base = row.get("base_type").and_then(serde_json::Value::as_str).ok_or_else(|| std::io::Error::other("invalid type definition"))?;
|
||||
if name.is_empty() || name.len() > 48 || !name.bytes().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == b'_')
|
||||
|| base.is_empty() || base.len() > 64 || row.get("parameterized").and_then(serde_json::Value::as_bool).is_none() {
|
||||
return Err(std::io::Error::other("invalid type definition"));
|
||||
}
|
||||
Ok(name)
|
||||
}
|
||||
|
||||
/// 账号列表转成行
|
||||
fn account_rows(accounts: &[Account]) -> std::io::Result<Vec<Row>> {
|
||||
accounts.iter().map(|a| Ok(HashMap::from([
|
||||
("id".into(), serde_json::json!(a.id)),
|
||||
("user".into(), serde_json::Value::String(a.user.clone())),
|
||||
("password_hash".into(), serde_json::Value::String(a.password_hash.clone())),
|
||||
]))).collect()
|
||||
}
|
||||
|
||||
/// 身份元数据转成行
|
||||
fn identity_rows(accounts: &[Account]) -> std::io::Result<Vec<Row>> {
|
||||
accounts.iter().map(|a| Ok(HashMap::from([
|
||||
("id".into(), serde_json::json!(a.id)),
|
||||
("user".into(), serde_json::Value::String(a.user.clone())),
|
||||
("can_login".into(), serde_json::json!(a.can_login)),
|
||||
("is_superuser".into(), serde_json::json!(a.is_superuser)),
|
||||
("system_catalog_manager".into(), serde_json::json!(a.system_catalog_manager)),
|
||||
("enabled".into(), serde_json::json!(a.enabled)),
|
||||
("identity_version".into(), serde_json::json!(a.identity_version)),
|
||||
("registered_at".into(), serde_json::Value::String(a.registered_at.clone())),
|
||||
("last_login_at".into(), match &a.last_login_at {
|
||||
Some(stamp) => serde_json::Value::String(stamp.clone()),
|
||||
@@ -1478,10 +1749,23 @@ fn users_columns() -> Vec<SchemaColumn> {
|
||||
vec![
|
||||
SchemaColumn { name: "id".into(), ty: "int".into(), nullable: false,
|
||||
auto_increment: true, primary_key: true, ..Default::default() },
|
||||
SchemaColumn { name: "user".into(), ty: "varchar(64)".into(), nullable: false,
|
||||
unique: true, ..Default::default() },
|
||||
SchemaColumn { name: "password_hash".into(), ty: "varchar(256)".into(),
|
||||
nullable: false, ..Default::default() },
|
||||
]
|
||||
}
|
||||
|
||||
/// 系统身份目录结构
|
||||
fn identity_columns() -> Vec<SchemaColumn> {
|
||||
vec![
|
||||
SchemaColumn { name: "id".into(), ty: "int".into(), nullable: false,
|
||||
primary_key: true, ..Default::default() },
|
||||
SchemaColumn { name: "user".into(), ty: "varchar(64)".into(), nullable: false,
|
||||
unique: true, ..Default::default() },
|
||||
SchemaColumn { name: "can_login".into(), ty: "bool".into(), nullable: false, ..Default::default() },
|
||||
SchemaColumn { name: "is_superuser".into(), ty: "bool".into(), nullable: false, ..Default::default() },
|
||||
SchemaColumn { name: "system_catalog_manager".into(), ty: "bool".into(), nullable: false, ..Default::default() },
|
||||
SchemaColumn { name: "enabled".into(), ty: "bool".into(), nullable: false, ..Default::default() },
|
||||
SchemaColumn { name: "identity_version".into(), ty: "int".into(), nullable: false, ..Default::default() },
|
||||
SchemaColumn { name: "registered_at".into(), ty: "timestamp".into(),
|
||||
nullable: false, ..Default::default() },
|
||||
SchemaColumn { name: "last_login_at".into(), ty: "timestamp".into(),
|
||||
@@ -1497,7 +1781,6 @@ fn declare_system_schema(catalog: &mut Catalog, rows: &[Row]) -> std::io::Result
|
||||
catalog.set_columns(USERS_TABLE, users_columns())?;
|
||||
catalog.rebuild_stats(USERS_TABLE, rows);
|
||||
catalog.mark_system(USERS_TABLE)?;
|
||||
catalog.add_index(USERS_TABLE, "user")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1519,6 +1802,11 @@ fn decode_account_row(row: &Row) -> std::io::Result<Account> {
|
||||
revision: row.get("revision").and_then(serde_json::Value::as_u64).unwrap_or(1),
|
||||
registered_at: row.get("registered_at").and_then(serde_json::Value::as_str).unwrap_or("").to_string(),
|
||||
last_login_at: row.get("last_login_at").and_then(serde_json::Value::as_str).map(str::to_string),
|
||||
can_login: row.get("can_login").and_then(serde_json::Value::as_bool).unwrap_or(true),
|
||||
is_superuser: row.get("is_superuser").and_then(serde_json::Value::as_bool).unwrap_or(false),
|
||||
system_catalog_manager: row.get("system_catalog_manager").and_then(serde_json::Value::as_bool).unwrap_or(false),
|
||||
enabled: row.get("enabled").and_then(serde_json::Value::as_bool).unwrap_or(true),
|
||||
identity_version: row.get("identity_version").and_then(serde_json::Value::as_u64).unwrap_or(0) as u8,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1589,9 +1877,20 @@ fn new_account(id: i64, user: &str, password_hash: String) -> std::io::Result<Ac
|
||||
revision: 1,
|
||||
registered_at: now_stamp(),
|
||||
last_login_at: None,
|
||||
can_login: true,
|
||||
is_superuser: false,
|
||||
system_catalog_manager: false,
|
||||
enabled: true,
|
||||
identity_version: 1,
|
||||
})
|
||||
}
|
||||
|
||||
/// 保留至少一个启用的登录管理员
|
||||
fn require_superuser(accounts: &[Account]) -> std::io::Result<()> {
|
||||
if accounts.iter().any(|a| a.can_login && a.enabled && a.is_superuser) { Ok(()) }
|
||||
else { Err(std::io::Error::other("the last enabled login superuser cannot be removed")) }
|
||||
}
|
||||
|
||||
/// 一张表的索引定义转线上格式
|
||||
fn index_wires(schema: &crate::catalog::TableSchema) -> Vec<IndexWire> {
|
||||
let mut out = vec![IndexWire {
|
||||
@@ -1632,6 +1931,40 @@ impl Drop for Server {
|
||||
}
|
||||
}
|
||||
|
||||
/// 按列精度规范化一行定点数
|
||||
fn normalize_decimal_row(row: &mut Row, columns: &[SchemaColumn]) -> std::io::Result<()> {
|
||||
for column in columns {
|
||||
if let Some((precision, scale)) = decimal_parameters(&column.ty)?
|
||||
&& let Some(value) = row.get_mut(&column.name).filter(|v| !v.is_null()) {
|
||||
let number = value.as_f64().ok_or_else(|| std::io::Error::other("invalid decimal value"))?;
|
||||
let factor = 10f64.powi(scale);
|
||||
let scaled = number * factor;
|
||||
if !scaled.is_finite() { return Err(std::io::Error::other("decimal value is not finite or exceeds precision")); }
|
||||
let rounded = scaled.round_ties_even() / factor;
|
||||
if rounded.abs() >= 10f64.powi(precision - scale) { return Err(std::io::Error::other("decimal value exceeds precision")); }
|
||||
*value = serde_json::json!(rounded);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 解析基础或命名定点类型参数
|
||||
fn decimal_parameters(ty: &str) -> std::io::Result<Option<(i32, i32)>> {
|
||||
let mut base = ty.trim();
|
||||
while let Some(body) = base.strip_prefix("domain(").and_then(|s| s.strip_suffix(')')) {
|
||||
base = body.split_once(',').ok_or_else(|| std::io::Error::other("invalid domain type"))?.1.trim();
|
||||
}
|
||||
let normalized = crate::catalog::normalize_type(base);
|
||||
if normalized == "decimal" { return Ok(Some((10, 0))); }
|
||||
if !normalized.starts_with("decimal(") { return Ok(None); }
|
||||
let error = || std::io::Error::other("invalid decimal precision or scale");
|
||||
let body = normalized.strip_prefix("decimal(").and_then(|s| s.strip_suffix(')')).ok_or_else(error)?;
|
||||
let values = body.split(',').map(|s| s.trim().parse::<i32>()).collect::<Result<Vec<_>, _>>().map_err(|_| error())?;
|
||||
let (precision, scale) = match values.as_slice() { [p] => (*p, 0), [p, s] => (*p, *s), _ => return Err(error()) };
|
||||
if !(1..=308).contains(&precision) || scale < 0 || scale > precision { return Err(error()); }
|
||||
Ok(Some((precision, scale)))
|
||||
}
|
||||
|
||||
/// 系统库名:账号表住在这里,其中的表都是系统表。服务启动时只建它一个,
|
||||
/// 别的库一律要显式建(没有默认工作库)。
|
||||
const SYSTEM_DATABASE: &str = "system";
|
||||
@@ -1793,9 +2126,11 @@ impl Databases {
|
||||
value["table"] = serde_json::Value::String(table.to_string());
|
||||
}
|
||||
let request: Request = serde_json::from_value(value).map_err(Error::other)?;
|
||||
let global = matches!(request, Request::AccountsList | Request::AccountCreate { .. }
|
||||
let global = matches!(request, Request::AccountsList | Request::CatalogManage { .. } | Request::AccountCreate { .. }
|
||||
| Request::IdentityInitialize { .. } | Request::RoleCreate { .. } | Request::IdentityAlter { .. }
|
||||
| Request::AccountReset { .. } | Request::AccountLogin { .. }
|
||||
| Request::AccountDrop { .. } | Request::BootstrapSystem { .. }
|
||||
| Request::BootstrapTypes { .. }
|
||||
| Request::SystemStatus | Request::Ping);
|
||||
if !global && selected.is_empty() { return Err(Error::other("no database selected")); }
|
||||
let server = if global || selected == SYSTEM_DATABASE { Arc::clone(&self.system) }
|
||||
@@ -1838,21 +2173,185 @@ fn log_config(loaded: &Loaded) {
|
||||
}
|
||||
}
|
||||
|
||||
/// 获取数据目录的跨进程独占锁
|
||||
fn lock_data_directory(cfg: &Config) -> std::io::Result<std::fs::File> {
|
||||
std::fs::create_dir_all(&cfg.data_dir)?;
|
||||
let file = std::fs::OpenOptions::new().read(true).write(true).create(true).truncate(false)
|
||||
.open(cfg.data_dir.join(".chusql.lock"))?;
|
||||
file.try_lock().map_err(|e| std::io::Error::other(format!("data directory is in use; stop all database processes: {e}")))?;
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
/// 复制系统目录并拒绝链接文件
|
||||
fn copy_system_directory(source: &std::path::Path, target: &std::path::Path) -> std::io::Result<()> {
|
||||
std::fs::create_dir_all(target)?;
|
||||
for entry in std::fs::read_dir(source)? {
|
||||
let entry = entry?;
|
||||
let ty = entry.file_type()?;
|
||||
if ty.is_symlink() { return Err(std::io::Error::other("system directory contains a symbolic link")); }
|
||||
if ty.is_dir() { copy_system_directory(&entry.path(), &target.join(entry.file_name()))?; }
|
||||
else if ty.is_file() { std::fs::copy(entry.path(), target.join(entry.file_name()))?; }
|
||||
else { return Err(std::io::Error::other("unsupported system directory entry")); }
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 构造可重建的系统表结构
|
||||
fn maintenance_schemas() -> Vec<(&'static str, Vec<SchemaColumn>)> {
|
||||
let text = |name: &str, size: usize| SchemaColumn { name: name.into(), ty: format!("varchar({size})"), ..Default::default() };
|
||||
vec![(USERS_TABLE, users_columns()), (IDENTITIES_TABLE, identity_columns()), (TYPES_TABLE, types_columns()),
|
||||
("__system_roles", vec![text("name", 64)]),
|
||||
("__system_grants", vec![text("role", 64), text("privilege", 16), text("object", 128)]),
|
||||
("__system_grant_options", vec![text("role", 64), text("privilege", 16), text("object", 128)]),
|
||||
("__system_members", vec![text("role", 64), text("member", 64)])]
|
||||
}
|
||||
|
||||
/// 验证隔离副本并执行所选恢复策略
|
||||
fn maintenance_candidate(cfg: Config, mode: &str, user: &str, hash: &str, types: Vec<Row>) -> Result<(), String> {
|
||||
if mode == "reset" {
|
||||
let seed = Server::new(cfg.clone()).map_err(|e| e.to_string())?;
|
||||
seed.bootstrap_system(Some(user), Some(hash)).map_err(|e| e.to_string())?;
|
||||
seed.bootstrap_types(types.clone()).map_err(|e| e.to_string())?;
|
||||
}
|
||||
let repair = mode == "repair";
|
||||
let catalog_path = cfg.data_dir.join("catalog.json");
|
||||
let schemas = maintenance_schemas();
|
||||
let mut missing_data = Vec::new();
|
||||
let mut catalog = match Catalog::load(&catalog_path) {
|
||||
Ok(catalog) => catalog,
|
||||
Err(_) => {
|
||||
for entry in std::fs::read_dir(&cfg.data_dir).map_err(|e| e.to_string())? {
|
||||
let entry = entry.map_err(|e| e.to_string())?;
|
||||
if entry.path().extension().and_then(|s| s.to_str()) == Some("db") {
|
||||
let name = entry.path().file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default();
|
||||
if !schemas.iter().any(|(known, _)| *known == name) { return Err(format!("cannot reconstruct unknown system table schema: {name}")); }
|
||||
}
|
||||
}
|
||||
Catalog::default()
|
||||
}
|
||||
};
|
||||
if catalog.all_tables().is_empty() {
|
||||
for entry in std::fs::read_dir(&cfg.data_dir).map_err(|e| e.to_string())? {
|
||||
let path = entry.map_err(|e| e.to_string())?.path();
|
||||
if path.extension().and_then(|s| s.to_str()) == Some("db") {
|
||||
let name = path.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default();
|
||||
if !schemas.iter().any(|(known, _)| *known == name) { return Err(format!("cannot reconstruct unknown system table schema: {name}")); }
|
||||
}
|
||||
}
|
||||
}
|
||||
for (name, columns) in &schemas {
|
||||
let file = cfg.data_dir.join(format!("{name}.db"));
|
||||
if let Some(schema) = catalog.describe(name).filter(|s| !file.exists() && s.row_count > 0) {
|
||||
missing_data.push((*name, schema.row_count));
|
||||
}
|
||||
if repair && !file.exists() && catalog.describe(name).is_none_or(|s| s.row_count != 0) {
|
||||
return Err(format!("repair would require initializing or recovering data in {name}"));
|
||||
}
|
||||
if catalog.describe(name).is_none() { catalog.create_table(name, columns.clone()).map_err(|e| e.to_string())?; }
|
||||
else { catalog.set_columns(name, columns.clone()).map_err(|e| e.to_string())?; }
|
||||
catalog.mark_system(name).map_err(|e| e.to_string())?;
|
||||
}
|
||||
catalog.save(&catalog_path).map_err(|e| e.to_string())?;
|
||||
let server = Server::new(cfg).map_err(|e| e.to_string())?;
|
||||
if repair {
|
||||
let scan = server.wal.read_all().map_err(|e| e.to_string())?;
|
||||
let checkpoint = server.wal.read_checkpoint().map_err(|e| e.to_string())?.unwrap_or(0);
|
||||
if !committed_ops(&records_after(&scan.records, checkpoint)).is_empty() { return Err("repair would require replaying WAL data".into()); }
|
||||
} else {
|
||||
server.recover_wal().map_err(|e| e.to_string())?;
|
||||
}
|
||||
for (name, _) in &schemas { server.with_table(name, |t| t.flush()).map_err(|e| e.to_string())?; }
|
||||
if !repair {
|
||||
if mode == "recover" && server.load_accounts().map_err(|e| e.to_string())?.is_empty() && hash.is_empty() {
|
||||
return Err("recover requires --password-stdin to initialize a missing administrator".into());
|
||||
}
|
||||
server.bootstrap_system(Some(user), Some(hash)).map_err(|e| e.to_string())?;
|
||||
server.accounts_request(Request::IdentityInitialize { administrator: user.into() }).map_err(|e| e.to_string())?;
|
||||
server.bootstrap_types(types.clone()).map_err(|e| e.to_string())?;
|
||||
}
|
||||
let accounts = server.load_accounts().map_err(|e| e.to_string())?;
|
||||
require_superuser(&accounts).map_err(|e| e.to_string())?;
|
||||
for (name, expected) in missing_data {
|
||||
let rows = server.with_existing_table(name, |t| t.scan()).map_err(|e| e.to_string())?;
|
||||
if (rows.len() as u64) < expected { return Err(format!("WAL cannot reconstruct all missing data in {name}; restore a backup or use reset")); }
|
||||
}
|
||||
if repair {
|
||||
let saved = server.with_existing_table(TYPES_TABLE, |t| t.scan()).map_err(|e| e.to_string())?;
|
||||
if types.iter().any(|wanted| !saved.iter().any(|row| wanted.iter().all(|(key, value)| row.get(key) == Some(value)))) {
|
||||
return Err("repair would require changing preinstalled type data".into());
|
||||
}
|
||||
}
|
||||
for (name, _) in &schemas { server.with_existing_table(name, |t| t.scan()).map_err(|e| e.to_string())?; }
|
||||
{
|
||||
let mut catalog = server.catalog.lock().map_err(|_| "catalog lock poisoned")?;
|
||||
catalog.identity_high_water = catalog.identity_high_water.max(accounts.iter().map(|a| a.id).max().unwrap_or(0));
|
||||
catalog.save(server.catalog_path()).map_err(|e| e.to_string())?;
|
||||
}
|
||||
server.rebuild_stats().map_err(|e| e.to_string())?;
|
||||
server.flush_all_tables().map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 进程内存储句柄:一个数据目录一份状态,跨线程共享(内部各自加锁)。
|
||||
pub struct Storage {
|
||||
databases: Databases,
|
||||
_directory_lock: std::fs::File,
|
||||
}
|
||||
|
||||
impl Storage {
|
||||
/// 离线修复或重建系统目录
|
||||
pub fn maintenance(config_path: Option<&str>, request: &str) -> Result<String, String> {
|
||||
let loaded = config::load(config_path)?;
|
||||
let cfg = loaded.config;
|
||||
let _lock = lock_data_directory(&cfg).map_err(|e| e.to_string())?;
|
||||
let input: serde_json::Value = serde_json::from_str(request).map_err(|e| e.to_string())?;
|
||||
let mode = input.get("mode").and_then(|v| v.as_str()).ok_or("missing maintenance mode")?;
|
||||
if !["repair", "recover", "reset"].contains(&mode) { return Err("invalid maintenance mode".into()); }
|
||||
let user = input.get("user").and_then(|v| v.as_str()).unwrap_or("root");
|
||||
let hash = input.get("password_hash").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let types: Vec<Row> = serde_json::from_value(input.get("types").cloned().ok_or("missing type manifest")?).map_err(|e| e.to_string())?;
|
||||
if types.is_empty() { return Err("type manifest must not be empty".into()); }
|
||||
for row in &types { type_definition_name(row).map_err(|e| e.to_string())?; }
|
||||
let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_err(|e| e.to_string())?.as_nanos();
|
||||
let system = cfg.data_dir.join("system");
|
||||
if system.is_symlink() { return Err("system directory cannot be a symbolic link".into()); }
|
||||
let work = cfg.data_dir.join(format!("bootstrap-work-{stamp}"));
|
||||
let backup = cfg.data_dir.join(format!("system-backup-{stamp}"));
|
||||
let staged = work.join("system");
|
||||
std::fs::create_dir_all(&staged).map_err(|e| e.to_string())?;
|
||||
if mode != "reset" && system.exists() { copy_system_directory(&system, &staged).map_err(|e| e.to_string())?; }
|
||||
let mut candidate_cfg = cfg.clone();
|
||||
candidate_cfg.data_dir = staged.clone();
|
||||
let result = maintenance_candidate(candidate_cfg, mode, user, hash, types);
|
||||
if let Err(err) = result { return Err(format!("{err}; original system directory was not changed; candidate: {}. Try csql-bootstrap recover or reset", work.display())); }
|
||||
if mode == "repair" && system.exists() {
|
||||
for entry in std::fs::read_dir(&system).map_err(|e| e.to_string())? {
|
||||
let entry = entry.map_err(|e| e.to_string())?;
|
||||
if entry.path().extension().and_then(|s| s.to_str()) == Some("db")
|
||||
&& std::fs::read(entry.path()).map_err(|e| e.to_string())? != std::fs::read(staged.join(entry.file_name())).map_err(|e| e.to_string())? {
|
||||
return Err("repair would modify table data; original system directory was not changed. Use recover or reset".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
if system.exists() { std::fs::rename(&system, &backup).map_err(|e| e.to_string())?; }
|
||||
if let Err(err) = std::fs::rename(&staged, &system) {
|
||||
if backup.exists() { std::fs::rename(&backup, &system).map_err(|rollback| format!("publish failed: {err}; rollback failed: {rollback}; backup: {}", backup.display()))?; }
|
||||
return Err(format!("cannot publish repaired system directory: {err}"));
|
||||
}
|
||||
serde_json::to_string(&serde_json::json!({"status":"maintenance", "mode":mode, "backup":backup, "candidate":work,
|
||||
"message":"System maintenance completed. Business databases were preserved."})).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// 打开数据目录并初始化存储
|
||||
pub fn open(config_path: Option<&str>) -> Result<Storage, String> {
|
||||
let loaded = config::load(config_path)?;
|
||||
let directory_lock = lock_data_directory(&loaded.config).map_err(|e| e.to_string())?;
|
||||
log::init();
|
||||
log::set_level(loaded.config.log_level);
|
||||
log_info!(core, "chusql-core-storage {} starting", env!("CARGO_PKG_VERSION"));
|
||||
log_config(&loaded);
|
||||
let databases = Databases::new(loaded.config).map_err(|e| e.to_string())?;
|
||||
Ok(Storage { databases })
|
||||
Ok(Storage { databases, _directory_lock: directory_lock })
|
||||
}
|
||||
|
||||
/// 一行请求 JSON 换一行响应 JSON
|
||||
|
||||
@@ -31,7 +31,7 @@ const OP_DROP_INDEX: u8 = 14;
|
||||
pub enum WalOp {
|
||||
HideColumn { table: String, column: String },
|
||||
Compact { table: String, rows: Vec<Row> },
|
||||
Accounts { accounts: Vec<Account> },
|
||||
Accounts { accounts: Vec<Account>, removed: Vec<String>, migrate_roles: bool },
|
||||
Insert {
|
||||
table: String,
|
||||
row: Row,
|
||||
@@ -283,7 +283,7 @@ impl Wal {
|
||||
frames.push((lsn, op, offset, end));
|
||||
offset = end;
|
||||
}
|
||||
Err(_) => break,
|
||||
Err(error) => return Err(io::Error::new(io::ErrorKind::InvalidData, error)),
|
||||
}
|
||||
}
|
||||
Ok((frames, offset))
|
||||
@@ -291,9 +291,13 @@ impl Wal {
|
||||
|
||||
/// 记录检查点:到此为止的 LSN 都已落盘
|
||||
pub fn write_checkpoint(&self, lsn: u64) -> io::Result<()> {
|
||||
let mut f = File::create(self.checkpoint_path())?;
|
||||
let checkpoint = self.checkpoint_path();
|
||||
let pending = checkpoint.with_extension("checkpoint.pending");
|
||||
let mut f = File::create(&pending)?;
|
||||
f.write_all(&lsn.to_le_bytes())?;
|
||||
f.sync_all()
|
||||
f.sync_all()?;
|
||||
drop(f);
|
||||
std::fs::rename(pending, checkpoint)
|
||||
}
|
||||
|
||||
/// 读检查点 LSN
|
||||
@@ -303,8 +307,8 @@ impl Wal {
|
||||
return Ok(None);
|
||||
}
|
||||
let bytes = std::fs::read(&path)?;
|
||||
if bytes.len() < 8 {
|
||||
return Ok(None);
|
||||
if bytes.len() != 8 {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid WAL checkpoint length"));
|
||||
}
|
||||
let mut raw = [0u8; 8];
|
||||
raw.copy_from_slice(&bytes[..8]);
|
||||
@@ -322,10 +326,10 @@ impl Wal {
|
||||
if let Some((lsn, _, _, _)) = frames.last() {
|
||||
base = base.max(*lsn);
|
||||
}
|
||||
*slot = base + 1;
|
||||
*slot = base.checked_add(1).ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
|
||||
}
|
||||
let lsn = *slot;
|
||||
*slot += 1;
|
||||
*slot = slot.checked_add(1).ok_or_else(|| io::Error::other("WAL LSN exhausted"))?;
|
||||
Ok(lsn)
|
||||
}
|
||||
|
||||
@@ -378,8 +382,8 @@ fn encode(op: &WalOp, lsn: u64) -> io::Result<Vec<u8>> {
|
||||
let p = serde_json::to_vec(rows).map_err(io::Error::other)?;
|
||||
(OP_COMPACT, table.clone(), p)
|
||||
}
|
||||
WalOp::Accounts { accounts } => (OP_ACCOUNTS, USERS_TABLE.to_string(),
|
||||
serde_json::to_vec(accounts).map_err(io::Error::other)?),
|
||||
WalOp::Accounts { accounts, removed, migrate_roles } => (OP_ACCOUNTS, USERS_TABLE.to_string(),
|
||||
serde_json::to_vec(&serde_json::json!({"accounts":accounts,"removed":removed,"migrate_roles":migrate_roles})).map_err(io::Error::other)?),
|
||||
WalOp::Insert { table, row } => {
|
||||
let p = serde_json::to_vec(row).map_err(io::Error::other)?;
|
||||
(OP_INSERT, table.clone(), p)
|
||||
@@ -478,9 +482,15 @@ fn decode_body(body: &[u8]) -> Result<(u64, WalOp), String> {
|
||||
}
|
||||
OP_ACCOUNTS => {
|
||||
if table != USERS_TABLE { return Err("invalid account WAL table".into()); }
|
||||
let accounts = serde_json::from_slice(payload)
|
||||
.map_err(|e| format!("bad account payload: {e}"))?;
|
||||
WalOp::Accounts { accounts }
|
||||
let value: serde_json::Value = serde_json::from_slice(payload).map_err(|e| format!("bad account payload: {e}"))?;
|
||||
let (accounts, removed, migrate_roles) = if value.is_array() {
|
||||
(serde_json::from_value(value).map_err(|e| format!("bad account payload: {e}"))?, Vec::new(), false)
|
||||
} else {
|
||||
(serde_json::from_value(value.get("accounts").cloned().ok_or("missing account payload")?).map_err(|e| format!("bad account payload: {e}"))?,
|
||||
serde_json::from_value(value.get("removed").cloned().ok_or("missing identity cleanup")?).map_err(|e| format!("bad account cleanup: {e}"))?,
|
||||
value.get("migrate_roles").and_then(serde_json::Value::as_bool).ok_or("missing role migration flag")?)
|
||||
};
|
||||
WalOp::Accounts { accounts, removed, migrate_roles }
|
||||
}
|
||||
OP_INSERT => {
|
||||
let row: Row =
|
||||
|
||||
Reference in new issue
Block a user