事务模型

chusql-core/storage/src/protocol.rs

@@ -171,6 +171,22 @@ pub enum Request {
         columns: Vec<SchemaColumn>,
         rows: Vec<Row>,
     },
+    /// 事务提交：把一批写操作当成一个请求原子应用
+    ApplyTransaction {
+        ops: Vec<StorageOp>,
+    },
+}
+
+/// 事务提交里的一条写操作
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(tag = "op", rename_all = "snake_case")]
+pub enum StorageOp {
+    /// 追加写入的行（提交端已先删掉要覆盖的行）
+    Upsert { table: String, rows: Vec<Row> },
+    /// 按行 id 删除
+    Delete { table: String, ids: Vec<i64> },
+    /// 整表替换（没有整数 id 的表只能这么提交）
+    Replace { table: String, rows: Vec<Row> },
 }
 
 impl Request {
@@ -202,7 +218,8 @@ impl Request {
             | Self::ReplaceSchema { table, .. } => Some(table),
             Self::Ping | Self::ListTables | Self::ListCatalog | Self::AccountsList
             | Self::AccountCreate { .. } | Self::AccountReset { .. } | Self::AccountLogin { .. }
-            | Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus => None,
+            | Self::AccountDrop { .. } | Self::BootstrapSystem { .. } | Self::SystemStatus
+            | Self::ApplyTransaction { .. } => None,
         }
     }
 }
@@ -237,7 +254,11 @@ pub enum Response {
     NoIndex,
     Ok,
     /// 系统目录状态
-    System { initialized: bool },
+    System {
+        initialized: bool,
+        #[serde(default)]
+        last_lsn: u64,
+    },
     Error { message: String },
     Catalog { schemas: Vec<TableSchemaWire> },
 }

chusql-core/storage/src/runtime.rs

@@ -8,9 +8,10 @@ use crate::config::{self, Config, Loaded};
 use crate::heap::{scalar_int, HeapTable};
 use crate::log;
 use crate::protocol::{
-    reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row, SchemaColumn, TableSchemaWire, USERS_TABLE,
+    reserved_table, Account, ColumnStatWire, IndexWire, Request, Response, Row, SchemaColumn, StorageOp,
+    TableSchemaWire, USERS_TABLE,
 };
-use crate::wal::{Wal, WalOp};
+use crate::wal::{committed_ops, Wal, WalOp};
 use crate::{log_debug, log_error, log_info, log_warn};
 
 // 进程内存储运行时：Haskell 侧经 ffi.rs 调进来。
@@ -196,6 +197,7 @@ impl Server {
     /// 把一条 WAL 操作落到磁盘
     fn apply_op(&self, op: &WalOp) -> std::io::Result<()> {
         match op {
+            WalOp::Commit => Ok(()),
             WalOp::HideColumn { table, column } => {
                 let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
                 c.remove_column(table, column)?;
@@ -311,52 +313,119 @@ impl Server {
         if self.recovery_required.load(Ordering::Acquire) {
             return Err(std::io::Error::other("storage recovery required"));
         }
-        {
-            let c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
-            let validate = |table: &str, rows: &[Row]| -> std::io::Result<()> {
-                let Some(schema) = c.describe(table) else { return Ok(()); };
-                if let Some(name) = rows.iter().flat_map(|r| r.keys()).find(|k| schema.dropped_columns.contains(*k)) {
-                    return Err(std::io::Error::other(format!("unknown column: {name}")));
-                }
-                Ok(())
-            };
-            match op {
-                WalOp::Insert { table, row } => validate(table, std::slice::from_ref(row))?,
-                WalOp::InsertBatch { table, rows } | WalOp::ReplaceAll { table, rows } => validate(table, rows)?,
-                WalOp::HideColumn { table, column }
-                    if !c.describe(table).is_some_and(|s| s.columns.iter().any(|col| &col.name == column)) =>
-                {
-                    return Err(std::io::Error::other(format!("unknown column: {column}")));
-                }
-                _ => {}
+        self.validate_op(op)?;
+        self.group_via_wal(std::slice::from_ref(op))
+    }
+
+    /// 校验一条操作：命中被删的列就拒绝
+    fn validate_op(&self, op: &WalOp) -> std::io::Result<()> {
+        let c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
+        let validate = |table: &str, rows: &[Row]| -> std::io::Result<()> {
+            let Some(schema) = c.describe(table) else { return Ok(()); };
+            if let Some(name) = rows.iter().flat_map(|r| r.keys()).find(|k| schema.dropped_columns.contains(*k)) {
+                return Err(std::io::Error::other(format!("unknown column: {name}")));
             }
+            Ok(())
+        };
+        match op {
+            WalOp::Insert { table, row } => validate(table, std::slice::from_ref(row))?,
+            WalOp::InsertBatch { table, rows } | WalOp::ReplaceAll { table, rows } => validate(table, rows)?,
+            WalOp::HideColumn { table, column }
+                if !c.describe(table).is_some_and(|s| s.columns.iter().any(|col| &col.name == column)) =>
+            {
+                return Err(std::io::Error::other(format!("unknown column: {column}")));
+            }
+            _ => {}
         }
+        Ok(())
+    }
+
+    /// 事务提交：一组写操作加一个提交标记，一次落盘
+    ///
+    /// 组里每条都过 `validate_op`，跟单条写一致；调用方 `dispatch` 已经持有
+    /// `gate.write`，因此整批对外原子可见。崩在提交标记之前，这组记录在恢复
+    /// 时整组丢弃；崩在标记之后，逐条按「是否已生效」补写。
+    fn apply_transaction(&self, ops: &[StorageOp]) -> std::io::Result<()> {
+        let _guard = self.write_lock.lock().unwrap();
+        if self.recovery_required.load(Ordering::Acquire) {
+            return Err(std::io::Error::other("storage recovery required"));
+        }
+        let mut wal_ops = Vec::with_capacity(ops.len());
+        for op in ops {
+            let table = match op {
+                StorageOp::Upsert { table, .. }
+                | StorageOp::Delete { table, .. }
+                | StorageOp::Replace { table, .. } => table,
+            };
+            if reserved_table(table) {
+                return Err(std::io::Error::other("reserved system table"));
+            }
+            if table.is_empty() || !table.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') {
+                return Err(std::io::Error::other("invalid table name"));
+            }
+            let wal_op = match op {
+                StorageOp::Upsert { table, rows } => WalOp::InsertBatch {
+                    table: table.clone(),
+                    rows: rows.clone(),
+                },
+                StorageOp::Delete { table, ids } => WalOp::DeleteKeys {
+                    table: table.clone(),
+                    keys: ids.clone(),
+                },
+                StorageOp::Replace { table, rows } => WalOp::ReplaceAll {
+                    table: table.clone(),
+                    rows: rows.clone(),
+                },
+            };
+            self.validate_op(&wal_op)?;
+            wal_ops.push(wal_op);
+        }
+        self.group_via_wal(&wal_ops)
+    }
+
+    /// 一组操作加提交标记落盘，成功后清空日志
+    fn group_via_wal(&self, ops: &[WalOp]) -> std::io::Result<()> {
         std::fs::create_dir_all(&self.cfg.data_dir)?;
         let wal = &self.wal;
-
         wal.truncate()?;
-
-        wal.append(op)?;
-
-        let result = self.apply_op(op);
-
-        if result.is_err() && matches!(op, WalOp::HideColumn { .. }) {
-            self.recovery_required.store(true, Ordering::Release);
-            return result;
+        let commit_lsn = wal.append_group(ops)?;
+        for op in ops {
+            let result = self.apply_op(op);
+            if result.is_err() && matches!(op, WalOp::HideColumn { .. }) {
+                self.recovery_required.store(true, Ordering::Release);
+                return result;
+            }
+            result?;
         }
-
-        if result.is_ok() {
+        for op in ops {
             self.flush_op_tables(op)?;
         }
-
+        self.flush_all_tables()?;
         wal.clear()?;
-        result
+        log_debug!(core, "WAL committed group at LSN {}", commit_lsn);
+        Ok(())
+    }
+
+    /// 把打开的表都 fsync 一遍
+    fn flush_all_tables(&self) -> std::io::Result<()> {
+        let handles: Vec<Arc<Mutex<HeapTable>>> = self
+            .tables
+            .lock()
+            .map_err(|_| std::io::Error::other("table lock poisoned"))?
+            .values()
+            .cloned()
+            .collect();
+        for handle in handles {
+            handle.lock().map_err(|_| std::io::Error::other("heap lock poisoned"))?.flush()?;
+        }
+        Ok(())
     }
 
     /// 把涉及的表写回磁盘
     fn flush_op_tables(&self, op: &WalOp) -> std::io::Result<()> {
         let table = match op {
             WalOp::HideColumn { .. } => return Ok(()),
+            WalOp::Commit => return Ok(()),
             WalOp::Accounts { .. } => USERS_TABLE,
             WalOp::Insert { table, .. }
             | WalOp::InsertBatch { table, .. }
@@ -369,21 +438,97 @@ impl Server {
         self.with_table(table, |t| t.flush())
     }
 
-    /// 启动时重放残留 WAL
+    /// 重放残留 WAL：已提交组重放，未提交组丢弃
     fn recover_wal(&self) -> std::io::Result<()> {
         std::fs::create_dir_all(&self.cfg.data_dir)?;
         let wal = &self.wal;
-        match wal.read()? {
-            None => Ok(()),
-            Some(op) => {
-                log_warn!(core, "replaying incomplete WAL entry");
-                self.apply_op(&op)?;
-                self.flush_op_tables(&op)?;
-                wal.clear()?;
-                log_info!(core, "WAL replay complete");
-                Ok(())
+        let scan = wal.read_all()?;
+        if scan.records.is_empty() {
+            if scan.good_len > 0 {
+                wal.truncate_to(scan.good_len)?;
             }
+            return Ok(());
         }
+        let records = committed_ops(&scan.records);
+        if records.is_empty() {
+            log_warn!(core, "discarding uncommitted WAL records");
+            wal.clear()?;
+            return Ok(());
+        }
+        let max_lsn = scan
+            .records
+            .iter()
+            .rev()
+            .find(|(_, op)| *op == WalOp::Commit)
+            .map(|(lsn, _)| *lsn)
+            .or_else(|| records.last().map(|(lsn, _)| *lsn))
+            .unwrap_or(0);
+        log_warn!(core, "replaying {} committed WAL records", records.len());
+        for (_, op) in &records {
+            self.replay_op(op)?;
+            self.flush_op_tables(op)?;
+        }
+        self.flush_all_tables()?;
+        wal.write_checkpoint(max_lsn)?;
+        wal.clear()?;
+        log_info!(core, "WAL replay complete");
+        Ok(())
+    }
+
+    /// 重放一条操作：数据已经生效的不再重复写
+    fn replay_op(&self, op: &WalOp) -> std::io::Result<()> {
+        if self.op_already_applied(op)? {
+            log_debug!(core, "WAL record already applied, skipping");
+            return Ok(());
+        }
+        self.apply_op(op)
+    }
+
+    /// 这条操作对应的数据是不是已经落盘了
+    fn op_already_applied(&self, op: &WalOp) -> std::io::Result<bool> {
+        match op {
+            WalOp::Insert { table, row } => self.applied_rows(table, std::slice::from_ref(row)),
+            WalOp::InsertBatch { table, rows } => self.applied_rows(table, rows),
+            WalOp::ReplaceAll { table, rows } => {
+                if self.catalog.lock().unwrap().describe(table).is_none() {
+                    return Ok(false);
+                }
+                let stored = self.with_table(table, |t| t.scan())?;
+                Ok(stored == *rows)
+            }
+            WalOp::DeleteKeys { table, keys } => {
+                if keys.is_empty() {
+                    return Ok(true);
+                }
+                if self.catalog.lock().unwrap().describe(table).is_none() {
+                    return Ok(false);
+                }
+                let rows = self.with_table(table, |t| t.scan())?;
+                let present = rows.iter().any(|row| {
+                    row.get("id")
+                        .and_then(scalar_int)
+                        .is_some_and(|id| keys.contains(&id))
+                });
+                Ok(!present)
+            }
+            WalOp::HideColumn { table, column } => {
+                let c = self.catalog.lock().unwrap();
+                Ok(c.describe(table).is_some_and(|s| !s.columns.iter().any(|col| &col.name == column)))
+            }
+            _ => Ok(false),
+        }
+    }
+
+    /// 这些行是不是都已经在表里
+    fn applied_rows(&self, table: &str, rows: &[Row]) -> std::io::Result<bool> {
+        if rows.is_empty() {
+            return Ok(true);
+        }
+        if self.catalog.lock().unwrap().describe(table).is_none() {
+            return Ok(false);
+        }
+        let stored = self.with_table(table, |t| t.scan())?;
+        Ok(rows.iter().all(|row| stored.iter().any(|one| one == row)))
     }
 
     /// 启动时重建列统计
@@ -607,6 +752,11 @@ impl Server {
             .unwrap_or(false)
     }
 
+    /// 最近一次落盘的检查点 LSN
+    fn last_checkpoint(&self) -> u64 {
+        self.wal.read_checkpoint().ok().flatten().unwrap_or(0)
+    }
+
     /// 读账号表；表没登记或数据文件缺失都算错
     fn load_accounts(&self) -> std::io::Result<Vec<Account>> {
         use std::io::Error;
@@ -774,16 +924,29 @@ impl Server {
             }
             Request::BootstrapSystem { user, password_hash } => {
                 match self.bootstrap_system(user.as_deref(), password_hash.as_deref()) {
-                    Ok(_) => Response::System { initialized: true },
+                    Ok(_) => Response::System { initialized: true, last_lsn: self.last_checkpoint() },
                     Err(e) => Response::Error { message: e.to_string() },
                 }
             }
-            Request::SystemStatus => Response::System { initialized: self.system_initialized() },
+            Request::SystemStatus => Response::System { initialized: self.system_initialized(), last_lsn: self.last_checkpoint() },
             Request::Ping => {
                 log_debug!(request, "ping");
                 Response::Pong
             }
 
+            Request::ApplyTransaction { ops } => {
+                log_debug!(request, "apply_transaction ops={}", ops.len());
+                match self.apply_transaction(&ops) {
+                    Ok(()) => Response::Ok,
+                    Err(e) => {
+                        log_warn!(request, "apply_transaction: {}", e);
+                        Response::Error {
+                            message: format!("apply_transaction: {e}"),
+                        }
+                    }
+                }
+            }
+
             Request::Scan { table, columns } => {
                 log_debug!(request, "scan table={}", table);
 
@@ -1347,8 +1510,8 @@ fn reserved_database(name: &str) -> bool {
 /// 每个库拥有独立的目录、字典、句柄和 WAL。
 struct Databases {
     root: Config,
-    system: Server,
-    named: Mutex<HashMap<String, Server>>,
+    system: Arc<Server>,
+    named: Mutex<HashMap<String, Arc<Server>>>,
 }
 
 impl Databases {
@@ -1356,7 +1519,7 @@ impl Databases {
     fn new(cfg: Config) -> std::io::Result<Self> {
         // 系统库按需创建：目录、字典与系统表缺失时自动补齐。
         // 其余库只在 data/databases/<name>/ 下已存在时才装载，启动不会凭空建库。
-        let system = Self::open(&cfg, SYSTEM_DATABASE)?;
+        let system = Arc::new(Self::open(&cfg, SYSTEM_DATABASE)?);
         let root = cfg.data_dir.join("databases");
         std::fs::create_dir_all(&root)?;
         let canonical_root = std::fs::canonicalize(&root)?;
@@ -1379,7 +1542,7 @@ impl Databases {
                 server.rebuild_stats()?;
                 server.migrate_data()?;
                 server.compact_tables()?;
-                named.insert(name, server);
+                named.insert(name, Arc::new(server));
             }
         }
         Ok(Self { root: cfg, system, named: Mutex::new(named) })
@@ -1413,9 +1576,13 @@ impl Databases {
         let mut named = self.named.lock().map_err(|_| Error::other("database lock poisoned"))?;
         match method.as_str() {
             "all_catalogs" => {
+                let servers: Vec<(String, Arc<Server>)> =
+                    std::iter::once((SYSTEM_DATABASE.to_string(), Arc::clone(&self.system)))
+                        .chain(named.iter().map(|(name, server)| (name.clone(), Arc::clone(server))))
+                        .collect();
+                drop(named);
                 let mut schemas = Vec::new();
-                for (database, server) in std::iter::once((SYSTEM_DATABASE, &self.system))
-                    .chain(named.iter().map(|(name, server)| (name.as_str(), server))) {
+                for (database, server) in servers {
                     match server.handle_request(Request::ListCatalog) {
                         Response::Catalog { schemas: entries } => {
                             for mut entry in entries {
@@ -1442,23 +1609,30 @@ impl Databases {
                 std::fs::create_dir(&cfg.data_dir)?;
                 let server = Server::new(cfg)?;
                 server.catalog.lock().map_err(|_| Error::other("catalog lock poisoned"))?.save(server.catalog_path())?;
-                named.insert(name, server);
+                named.insert(name, Arc::new(server));
                 return Ok(Response::Ok);
             }
             "drop_database" => {
                 if name == SYSTEM_DATABASE { return Err(Error::other("cannot drop the system database")); }
-                let server = named.get(&name).ok_or_else(|| Error::other("unknown database"))?;
+                let holder = named.remove(&name).ok_or_else(|| Error::other("unknown database"))?;
+                let server = match Arc::try_unwrap(holder) {
+                    Ok(server) => server,
+                    Err(holder) => {
+                        named.insert(name, holder);
+                        return Err(Error::other("database is in use"));
+                    }
+                };
                 let root = std::fs::canonicalize(self.root.data_dir.join("databases"))?;
                 let target = std::fs::canonicalize(&server.cfg.data_dir)?;
                 if target.parent() != Some(root.as_path()) || !target.join("catalog.json").is_file() {
                     return Err(Error::other("invalid database directory"));
                 }
                 let config = server.cfg.clone();
-                drop(named.remove(&name));
+                drop(server);
                 let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_err(Error::other)?.as_nanos();
                 let tombstone = root.join(format!(".dropped-{name}-{stamp}"));
                 if let Err(error) = std::fs::rename(&target, &tombstone) {
-                    named.insert(name, Server::new(config)?);
+                    named.insert(name, Arc::new(Server::new(config)?));
                     return Err(error);
                 }
                 std::fs::remove_dir_all(&tombstone).map_err(|e| Error::other(format!("database dropped; directory cleanup failed: {e}")))?;
@@ -1482,10 +1656,10 @@ impl Databases {
             | Request::AccountReset { .. } | Request::AccountLogin { .. }
             | Request::AccountDrop { .. } | Request::BootstrapSystem { .. }
             | Request::SystemStatus | Request::Ping);
-        let server = if global { &self.system }
-            else if selected.is_empty() { return Err(Error::other("no database selected")); }
-            else if selected == SYSTEM_DATABASE { &self.system }
-            else { named.get(&selected).ok_or_else(|| Error::other("unknown database"))? };
+        if !global && selected.is_empty() { return Err(Error::other("no database selected")); }
+        let server = if global || selected == SYSTEM_DATABASE { Arc::clone(&self.system) }
+            else { Arc::clone(named.get(&selected).ok_or_else(|| Error::other("unknown database"))?) };
+        drop(named);
         Ok(server.handle_request(request))
     }
 }

chusql-core/storage/src/wal.rs

@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
 
 use crate::protocol::{Account, Row, SchemaColumn, USERS_TABLE};
 
-// 预写日志：先记操作并 fsync，再改数据；重启重放残留。
+// 预写日志：先记操作并 fsync，再改数据；重启按 LSN 重放已提交的组。
 
 const OP_INSERT: u8 = 1;
 const OP_REPLACE_ALL: u8 = 2;
@@ -18,6 +18,7 @@ const OP_ACCOUNTS: u8 = 6;
 const OP_REPLACE_SCHEMA: u8 = 7;
 const OP_HIDE_COLUMN: u8 = 8;
 const OP_COMPACT: u8 = 9;
+const OP_COMMIT: u8 = 10;
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub enum WalOp {
@@ -50,6 +51,8 @@ pub enum WalOp {
         columns: Vec<SchemaColumn>,
         rows: Vec<Row>,
     },
+    /// 一组写操作的提交标记；没有标记的组算未提交
+    Commit,
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -67,6 +70,13 @@ struct ReplaceSchemaPayload {
 pub struct Wal {
     path: PathBuf,
     handle: Mutex<Option<File>>,
+    next_lsn: Mutex<u64>,
+}
+
+/// 一次扫描的结果：记录与完好字节长度
+pub struct WalScan {
+    pub records: Vec<(u64, WalOp)>,
+    pub good_len: usize,
 }
 
 impl Wal {
@@ -75,6 +85,7 @@ impl Wal {
         Wal {
             path: path.as_ref().to_path_buf(),
             handle: Mutex::new(None),
+            next_lsn: Mutex::new(0),
         }
     }
 
@@ -83,6 +94,11 @@ impl Wal {
         &self.path
     }
 
+    /// 检查点文件路径
+    pub fn checkpoint_path(&self) -> PathBuf {
+        self.path.with_extension("checkpoint")
+    }
+
     /// 拿常驻句柄（没有就开一个）
     fn with_handle<R>(&self, f: impl FnOnce(&mut File) -> io::Result<R>) -> io::Result<R> {
         let mut slot = self.handle.lock().unwrap();
@@ -99,6 +115,15 @@ impl Wal {
         f(slot.as_mut().expect("just opened"))
     }
 
+    /// 把一段记录写到文件尾
+    fn write_at_end(&self, bytes: &[u8]) -> io::Result<()> {
+        self.with_handle(|f| {
+            f.seek(SeekFrom::End(0))?;
+            f.write_all(bytes)?;
+            f.sync_data()
+        })
+    }
+
     /// 截断到空并落盘
     pub fn clear(&self) -> io::Result<()> {
         self.with_handle(|f| {
@@ -112,34 +137,136 @@ impl Wal {
         self.with_handle(truncate)
     }
 
-    /// 追加一条并 fsync
-    pub fn append(&self, op: &WalOp) -> io::Result<()> {
-        let bytes = encode(op)?;
+    /// 截到指定长度（去掉撕裂的尾巴）
+    pub fn truncate_to(&self, len: usize) -> io::Result<()> {
         self.with_handle(|f| {
-            f.write_all(&bytes)?;
+            f.set_len(len as u64)?;
+            f.seek(SeekFrom::Start(0))?;
             f.sync_data()
         })
     }
 
-    /// 读当前 WAL；空则 None
+    /// 追加一条并 fsync
+    pub fn append(&self, op: &WalOp) -> io::Result<()> {
+        let bytes = self.encode_next(op)?;
+        self.write_at_end(&bytes)
+    }
+
+    /// 追加一组操作与提交标记，返回提交 LSN
+    pub fn append_group(&self, ops: &[WalOp]) -> io::Result<u64> {
+        let mut frames = Vec::new();
+        for op in ops {
+            frames.extend_from_slice(&self.encode_next(op)?);
+        }
+        let commit_lsn = self.take_lsn()?;
+        frames.extend_from_slice(&encode(&WalOp::Commit, commit_lsn)?);
+        self.write_at_end(&frames)?;
+        Ok(commit_lsn)
+    }
+
+    /// 读当前 WAL 的第一条操作；空则 None
     pub fn read(&self) -> io::Result<Option<WalOp>> {
+        let scan = self.read_all()?;
+        Ok(scan.records.into_iter().find_map(|(_, op)| {
+            if op == WalOp::Commit {
+                None
+            } else {
+                Some(op)
+            }
+        }))
+    }
+
+    /// 读全部记录，撕裂或损坏的尾巴按长度截断
+    pub fn read_all(&self) -> io::Result<WalScan> {
         if !self.path.exists() {
-            return Ok(None);
+            return Ok(WalScan {
+                records: Vec::new(),
+                good_len: 0,
+            });
         }
         let mut f = File::open(&self.path)?;
         let mut buf = Vec::new();
         f.read_to_end(&mut buf)?;
-        if buf.is_empty() {
+        let mut records = Vec::new();
+        let mut offset = 0usize;
+        while offset + 4 <= buf.len() {
+            let body_len =
+                u32::from_le_bytes([buf[offset], buf[offset + 1], buf[offset + 2], buf[offset + 3]])
+                    as usize;
+            let end = offset + 4 + body_len;
+            if end > buf.len() {
+                break;
+            }
+            match decode_body(&buf[offset + 4..end]) {
+                Ok(record) => {
+                    records.push(record);
+                    offset = end;
+                }
+                Err(_) => break,
+            }
+        }
+        Ok(WalScan {
+            records,
+            good_len: offset,
+        })
+    }
+
+    /// 记录检查点：到此为止的 LSN 都已落盘
+    pub fn write_checkpoint(&self, lsn: u64) -> io::Result<()> {
+        let mut f = File::create(self.checkpoint_path())?;
+        f.write_all(&lsn.to_le_bytes())?;
+        f.sync_all()
+    }
+
+    /// 读检查点 LSN
+    pub fn read_checkpoint(&self) -> io::Result<Option<u64>> {
+        let path = self.checkpoint_path();
+        if !path.exists() {
             return Ok(None);
         }
-        match decode(&buf) {
-            Ok(opt) => Ok(opt),
-            Err(e) => Err(io::Error::new(
-                io::ErrorKind::InvalidData,
-                format!("corrupt WAL: {}", e),
-            )),
+        let bytes = std::fs::read(&path)?;
+        if bytes.len() < 8 {
+            return Ok(None);
+        }
+        let mut raw = [0u8; 8];
+        raw.copy_from_slice(&bytes[..8]);
+        Ok(Some(u64::from_le_bytes(raw)))
+    }
+
+    /// 取下一条 LSN
+    fn take_lsn(&self) -> io::Result<u64> {
+        let mut slot = self.next_lsn.lock().unwrap();
+        if *slot == 0 {
+            let last = self.read_all()?.records.last().map(|(lsn, _)| *lsn).unwrap_or(0);
+            *slot = last + 1;
+        }
+        let lsn = *slot;
+        *slot += 1;
+        Ok(lsn)
+    }
+
+    /// 编一条带新 LSN 的记录
+    fn encode_next(&self, op: &WalOp) -> io::Result<Vec<u8>> {
+        let lsn = self.take_lsn()?;
+        encode(op, lsn)
+    }
+}
+
+/// 已提交的记录：提交标记前的整组；无标记的单条按旧格式算已提交
+pub fn committed_ops(records: &[(u64, WalOp)]) -> Vec<(u64, WalOp)> {
+    let mut out = Vec::new();
+    let mut pending: Vec<(u64, WalOp)> = Vec::new();
+    for (lsn, op) in records {
+        if *op == WalOp::Commit {
+            out.append(&mut pending);
+        } else {
+            pending.push((*lsn, op.clone()));
         }
     }
+    if pending.len() == 1 {
+        out.append(&mut pending);
+    }
+    out
 }
 
 /// 清空文件并把读写位置归零
@@ -150,7 +277,7 @@ fn truncate(f: &mut File) -> io::Result<()> {
 }
 
 /// 把操作编成字节
-fn encode(op: &WalOp) -> io::Result<Vec<u8>> {
+fn encode(op: &WalOp, lsn: u64) -> io::Result<Vec<u8>> {
     let (op_type, table, payload) = match op {
         WalOp::HideColumn { table, column } => (OP_HIDE_COLUMN, table.clone(),
             serde_json::to_vec(column).map_err(io::Error::other)?),
@@ -192,11 +319,11 @@ fn encode(op: &WalOp) -> io::Result<Vec<u8>> {
             .map_err(io::Error::other)?;
             (OP_REPLACE_SCHEMA, table.clone(), p)
         }
+        WalOp::Commit => (OP_COMMIT, String::new(), Vec::new()),
     };
 
     let table_bytes = table.as_bytes();
 
-    let lsn: u64 = 1;
     let mut body = Vec::with_capacity(8 + 1 + 2 + table_bytes.len() + payload.len());
     body.extend_from_slice(&lsn.to_le_bytes());
     body.push(op_type);
@@ -210,20 +337,12 @@ fn encode(op: &WalOp) -> io::Result<Vec<u8>> {
     Ok(out)
 }
 
-/// 从字节解回操作
-fn decode(buf: &[u8]) -> Result<Option<WalOp>, String> {
-    if buf.len() < 4 {
-        return Ok(None);
-    }
-    let body_len = u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]) as usize;
-    if buf.len() < 4 + body_len {
-        return Ok(None);
-    }
-    let body = &buf[4..4 + body_len];
+/// 从一个帧体解出 LSN 与操作
+fn decode_body(body: &[u8]) -> Result<(u64, WalOp), String> {
     if body.len() < 8 + 1 + 2 {
         return Err("body too short".into());
     }
-    let _lsn = u64::from_le_bytes([
+    let lsn = u64::from_le_bytes([
         body[0], body[1], body[2], body[3], body[4], body[5], body[6], body[7],
     ]);
     let op_type = body[8];
@@ -236,6 +355,7 @@ fn decode(buf: &[u8]) -> Result<Option<WalOp>, String> {
     let payload = &body[11 + table_len..];
 
     let op = match op_type {
+        OP_COMMIT => WalOp::Commit,
         OP_HIDE_COLUMN => {
             let column = serde_json::from_slice(payload)
                 .map_err(|e| format!("bad hide_column payload: {e}"))?;
@@ -292,5 +412,5 @@ fn decode(buf: &[u8]) -> Result<Option<WalOp>, String> {
         }
         other => return Err(format!("unknown op type: {}", other)),
     };
-    Ok(Some(op))
+    Ok((lsn, op))
 }

chusql-core/storage/tests/runtime_test.rs

@@ -216,6 +216,67 @@ fn concurrent_shard_reads_exclude_writers() -> Result<(), Box<dyn std::error::Er
     Ok(())
 }
 
+/// 跨库并发互不串数据，目录查询不被写饿死
+#[test]
+fn concurrent_requests_across_databases_stay_isolated() -> Result<(), Box<dyn std::error::Error>> {
+    let (_srv, _data) = start_server();
+    let mut c = connect()?;
+    let request = |c: &mut Conn, value: serde_json::Value| -> Result<serde_json::Value, serde_json::Error> {
+        serde_json::from_str(&send(c, &value.to_string()))
+    };
+    let databases = ["alpha", "beta"];
+    for database in databases {
+        assert_eq!(request(&mut c, serde_json::json!({"method":"create_database","database":database}))?["status"], "ok");
+        assert_eq!(
+            request(&mut c, serde_json::json!({"method":"create_table","database":database,"table":"items","columns":[{"name":"id","ty":"int"}]}))?["status"],
+            "ok"
+        );
+    }
+
+    let storage = CURRENT.with(|slot| slot.borrow().as_ref().cloned().expect("storage opened"));
+    let written = 40;
+    std::thread::scope(|scope| {
+        let writers: Vec<_> = databases
+            .iter()
+            .map(|database| {
+                let storage = Arc::clone(&storage);
+                let database = database.to_string();
+                scope.spawn(move || {
+                    for i in 1..=written {
+                        let line = format!(
+                            r#"{{"method":"insert","database":"{database}","table":"items","row":{{"id":{i}}}}}"#
+                        );
+                        let value: serde_json::Value = serde_json::from_str(&storage.request_line(&line)).expect("json");
+                        assert_eq!(value["status"], "ok", "{database}: {value}");
+                    }
+                    database
+                })
+            })
+            .collect();
+        let catalog = {
+            let storage = Arc::clone(&storage);
+            scope.spawn(move || {
+                for _ in 0..40 {
+                    for line in [r#"{"method":"list_databases"}"#, r#"{"method":"all_catalogs"}"#] {
+                        let value: serde_json::Value = serde_json::from_str(&storage.request_line(line)).expect("json");
+                        assert_ne!(value["status"], "error", "{line}: {value}");
+                    }
+                }
+            })
+        };
+        for writer in writers {
+            writer.join().expect("writer thread");
+        }
+        catalog.join().expect("catalog thread");
+    });
+
+    for database in databases {
+        let rows = request(&mut c, serde_json::json!({"method":"scan","database":database,"table":"items"}))?["rows"].clone();
+        assert_eq!(rows.as_array().expect("rows").len(), written, "{database} 行数不对");
+    }
+    Ok(())
+}
+
 /// 删列不改堆字节，重启后不再暴露该列
 #[test]
 fn drop_column_keeps_heap_bytes_and_hides_values_after_restart() -> Result<(), Box<dyn std::error::Error>> {
@@ -744,6 +805,12 @@ fn account_failure_preserves_wal_and_blocks_writes() -> Result<(), Box<dyn std::
     let mut c = connect()?;
     let recovered = send(&mut c, r#"{"method":"accounts_list"}"#);
     assert!(recovered.contains("hash-one"), "{recovered}");
+    let status: serde_json::Value =
+        serde_json::from_str(&send(&mut c, r#"{"method":"system_status"}"#))?;
+    assert!(
+        status["last_lsn"].as_u64().unwrap_or(0) >= 1,
+        "重放后检查点要推进: {status}"
+    );
     Ok(())
 }
 
@@ -1015,6 +1082,125 @@ fn delete_keys_removes_row_and_index_entry() {
     assert!(r.contains(r#""status":"ok""#), "id 2 should be insertable again: {}", r);
 }
 
+/// 事务提交：一批写操作在同一个请求里落地
+#[test]
+fn apply_transaction_lands_upserts_and_deletes_together() {
+    let (_srv, _data) = start_server();
+    let mut c = connect().unwrap();
+
+    for (id, name) in [(1, "Alice"), (2, "Bob")] {
+        let req = format!(
+            r#"{{"method":"insert","table":"txn_t","row":{{"id":{},"name":"{}"}}}}"#,
+            id, name
+        );
+        send(&mut c, &req);
+    }
+
+    let batch = r#"{"method":"apply_transaction","ops":[
+        {"op":"delete","table":"txn_t","ids":[1]},
+        {"op":"upsert","table":"txn_t","rows":[{"id":3,"name":"Carol"}]}
+    ]}"#;
+    let r = send(&mut c, batch);
+    assert!(r.contains(r#""status":"ok""#), "batch: {}", r);
+
+    let r = send(&mut c, r#"{"method":"scan","table":"txn_t"}"#);
+    assert!(!r.contains("Alice"), "deleted row should be gone: {}", r);
+    assert!(r.contains("Bob") && r.contains("Carol"), "kept and added rows: {}", r);
+
+    for line in [
+        r#"{"method":"apply_transaction","ops":[{"op":"upsert","table":"__system_users","rows":[{"id":1}]}]}"#,
+        r#"{"method":"apply_transaction","ops":[{"op":"delete","table":"../evil","ids":[1]}]}"#,
+    ] {
+        let r = send(&mut c, line);
+        assert!(r.contains(r#""status":"error""#), "bad op rejected: {} -> {}", line, r);
+    }
+
+    // 没有整数 id 的表用整表替换提交
+    let r = send(&mut c, r#"{"method":"insert","table":"txn_no_id","row":{"name":"old"}}"#);
+    assert!(r.contains(r#""status":"ok""#), "insert: {}", r);
+    let replace = r#"{"method":"apply_transaction","ops":[{"op":"replace","table":"txn_no_id","rows":[{"name":"only"}]}]}"#;
+    let r = send(&mut c, replace);
+    assert!(r.contains(r#""status":"ok""#), "replace: {}", r);
+    let r = send(&mut c, r#"{"method":"scan","table":"txn_no_id"}"#);
+    assert!(r.contains("only") && !r.contains("old"), "replaced rows: {}", r);
+}
+
+/// 标记已落盘、数据未改的崩溃现场：重启重放并推进检查点
+#[test]
+fn committed_group_replays_after_restart() -> Result<(), Box<dyn std::error::Error>> {
+    use chusql_core_storage::wal::{Wal, WalOp};
+    let (server, data) = start_server();
+    let mut c = connect()?;
+    request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":1,"secret":"old"}}))?;
+    drop(c);
+    drop(server);
+
+    let mut added = std::collections::HashMap::new();
+    added.insert("id".to_string(), serde_json::json!(2));
+    added.insert("secret".to_string(), serde_json::json!("new"));
+    let group = vec![
+        WalOp::InsertBatch {
+            table: "items".into(),
+            rows: vec![added.clone()],
+        },
+        WalOp::DeleteKeys {
+            table: "items".into(),
+            keys: vec![1],
+        },
+    ];
+    let wal = Wal::new(data.path().join("databases/main/wal.log"));
+    let commit_lsn = wal.append_group(&group)?;
+    drop(wal);
+
+    let _server = start_server_in(data.path());
+    let mut c = connect()?;
+    let rows: serde_json::Value =
+        serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"items"}"#))?;
+    assert_eq!(rows["rows"], serde_json::json!([{"id":2,"secret":"new"}]));
+    assert_eq!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len(), 0);
+    let checkpoint = std::fs::read(data.path().join("databases/main/wal.checkpoint"))?;
+    assert_eq!(checkpoint.len(), 8, "检查点要落盘");
+    assert_eq!(
+        u64::from_le_bytes(checkpoint[..8].try_into()?),
+        commit_lsn,
+        "检查点记到提交 LSN"
+    );
+    Ok(())
+}
+
+/// 没有提交标记的记录组在重启时整组丢掉
+#[test]
+fn uncommitted_group_is_discarded_on_restart() -> Result<(), Box<dyn std::error::Error>> {
+    use chusql_core_storage::wal::{Wal, WalOp};
+    let (server, data) = start_server();
+    let mut c = connect()?;
+    request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":1,"secret":"old"}}))?;
+    drop(c);
+    drop(server);
+
+    let mut lost = std::collections::HashMap::new();
+    lost.insert("id".to_string(), serde_json::json!(2));
+    lost.insert("secret".to_string(), serde_json::json!("lost"));
+    let wal = Wal::new(data.path().join("databases/main/wal.log"));
+    wal.append(&WalOp::InsertBatch {
+        table: "items".into(),
+        rows: vec![lost.clone()],
+    })?;
+    wal.append(&WalOp::InsertBatch {
+        table: "items".into(),
+        rows: vec![lost],
+    })?;
+    drop(wal);
+
+    let _server = start_server_in(data.path());
+    let mut c = connect()?;
+    let rows: serde_json::Value =
+        serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"items"}"#))?;
+    assert_eq!(rows["rows"], serde_json::json!([{"id":1,"secret":"old"}]));
+    assert_eq!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len(), 0);
+    Ok(())
+}
+
 /// 给第二列建索引后能按那一列查
 #[test]
 fn create_index_on_secondary_column() {

chusql-core/storage/tests/wal_test.rs

@@ -1,8 +1,8 @@
 use chusql_core_storage::protocol::Row;
-use chusql_core_storage::wal::{Wal, WalOp};
+use chusql_core_storage::wal::{committed_ops, Wal, WalOp};
 use serde_json::json;
 
-// WAL 测试：追加、读取、清空、截断尾巴。
+// WAL 测试：追加、读取、清空、截断尾巴、LSN 与提交标记。
 
 
 /// 用键值对构造一行
@@ -99,3 +99,122 @@ fn truncated_tail_is_ignored() {
     assert!(wal.read().unwrap().is_none());
 }
 
+/// 每条记录带递增的 LSN，重开也从文件里的最大号往下走
+#[test]
+fn every_record_gets_an_increasing_lsn() {
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("wal.log");
+    let wal = Wal::new(&path);
+
+    for id in 1..=3 {
+        wal.append(&WalOp::Insert {
+            table: "t".into(),
+            row: row(&[("id", json!(id))]),
+        })
+        .unwrap();
+    }
+    let scan = wal.read_all().unwrap();
+    let lsns: Vec<u64> = scan.records.iter().map(|(lsn, _)| *lsn).collect();
+    assert_eq!(lsns, vec![1, 2, 3]);
+    assert_eq!(scan.good_len as u64, std::fs::metadata(&path).unwrap().len());
+
+    let reopened = Wal::new(&path);
+    reopened
+        .append(&WalOp::Insert {
+            table: "t".into(),
+            row: row(&[("id", json!(4))]),
+        })
+        .unwrap();
+    assert_eq!(reopened.read_all().unwrap().records.last().unwrap().0, 4);
+}
+
+/// 一组操作写完带提交标记，标记的 LSN 也在记录里
+#[test]
+fn append_group_writes_a_commit_marker() {
+    let dir = tempfile::tempdir().unwrap();
+    let wal = Wal::new(dir.path().join("wal.log"));
+
+    let group = [
+        WalOp::InsertBatch {
+            table: "t".into(),
+            rows: vec![row(&[("id", json!(2))])],
+        },
+        WalOp::DeleteKeys {
+            table: "t".into(),
+            keys: vec![1],
+        },
+    ];
+    let commit_lsn = wal.append_group(&group).unwrap();
+
+    let scan = wal.read_all().unwrap();
+    assert_eq!(scan.records.len(), 3);
+    assert_eq!(scan.records.last().unwrap().1, WalOp::Commit);
+    assert_eq!(scan.records.last().unwrap().0, commit_lsn);
+    assert_eq!(wal.read().unwrap().unwrap(), group[0]);
+    assert_eq!(
+        committed_ops(&scan.records),
+        vec![(1, group[0].clone()), (2, group[1].clone())]
+    );
+}
+
+/// 没有提交标记的记录组算未提交，旧格式的单条仍算已提交
+#[test]
+fn records_without_a_commit_marker_stay_uncommitted() {
+    let dir = tempfile::tempdir().unwrap();
+    let wal = Wal::new(dir.path().join("wal.log"));
+
+    for id in 1..=2 {
+        wal.append(&WalOp::Insert {
+            table: "t".into(),
+            row: row(&[("id", json!(id))]),
+        })
+        .unwrap();
+    }
+    assert!(committed_ops(&wal.read_all().unwrap().records).is_empty());
+
+    wal.clear().unwrap();
+    wal.append(&WalOp::Insert {
+        table: "t".into(),
+        row: row(&[("id", json!(9))]),
+    })
+    .unwrap();
+    assert_eq!(committed_ops(&wal.read_all().unwrap().records).len(), 1);
+}
+
+/// 撕成两半的尾巴能读出完好长度并截掉
+#[test]
+fn trailing_torn_frame_can_be_truncated() {
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("wal.log");
+    let wal = Wal::new(&path);
+
+    for id in 1..=2 {
+        wal.append(&WalOp::Insert {
+            table: "t".into(),
+            row: row(&[("id", json!(id))]),
+        })
+        .unwrap();
+    }
+    let full = std::fs::read(&path).unwrap();
+    std::fs::write(&path, &full[..full.len() - 3]).unwrap();
+
+    let scan = wal.read_all().unwrap();
+    assert_eq!(scan.records.len(), 1);
+    assert!(scan.good_len < full.len() - 3);
+
+    wal.truncate_to(scan.good_len).unwrap();
+    assert_eq!(std::fs::metadata(&path).unwrap().len(), scan.good_len as u64);
+    assert_eq!(wal.read_all().unwrap().records.len(), 1);
+}
+
+/// 检查点写读一致
+#[test]
+fn checkpoint_round_trips() {
+    let dir = tempfile::tempdir().unwrap();
+    let wal = Wal::new(dir.path().join("wal.log"));
+
+    assert_eq!(wal.read_checkpoint().unwrap(), None);
+    wal.write_checkpoint(7).unwrap();
+    assert_eq!(wal.read_checkpoint().unwrap(), Some(7));
+}
+