表结构改动进 WAL
This commit is contained in:
1 parent
045ccc2dc0
commit
4c030e6181
4 files changed
+387
-69
No files matched your search
@@ -280,6 +280,35 @@ impl Server {
|
|||||||
let _ = std::fs::remove_file(self.index_path(table, column));
|
let _ = std::fs::remove_file(self.index_path(table, column));
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
WalOp::CreateTable { table, columns } => {
|
||||||
|
self.with_table(table, |_t| Ok(()))?;
|
||||||
|
let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||||
|
c.create_table(table, columns.clone())?;
|
||||||
|
c.save(self.catalog_path())
|
||||||
|
}
|
||||||
|
WalOp::DropTable { table } => {
|
||||||
|
let dropped_indexes: Vec<String> = {
|
||||||
|
let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||||
|
let idx = c
|
||||||
|
.describe(table)
|
||||||
|
.map(|s| s.indexes.iter().map(|i| i.column.clone()).collect())
|
||||||
|
.unwrap_or_default();
|
||||||
|
if c.describe(table).is_some() {
|
||||||
|
c.drop_table(table)?;
|
||||||
|
c.save(self.catalog_path())?;
|
||||||
|
}
|
||||||
|
idx
|
||||||
|
};
|
||||||
|
self.tables.lock().map_err(|_| std::io::Error::other("table lock poisoned"))?.remove(table);
|
||||||
|
let _ = std::fs::remove_file(self.table_path(table));
|
||||||
|
let _ = std::fs::remove_file(self.index_path(table, "id"));
|
||||||
|
for col in dropped_indexes {
|
||||||
|
let _ = std::fs::remove_file(self.index_path(table, &col));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
WalOp::CreateIndex { table, column } => self.apply_create_index(table, column),
|
||||||
|
WalOp::DropIndex { table, column } => self.apply_drop_index(table, column),
|
||||||
WalOp::ReplaceSchema { table, columns, rows } => {
|
WalOp::ReplaceSchema { table, columns, rows } => {
|
||||||
let old_indexes: Vec<String> = {
|
let old_indexes: Vec<String> = {
|
||||||
let c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
let c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||||
@@ -426,6 +455,8 @@ impl Server {
|
|||||||
let table = match op {
|
let table = match op {
|
||||||
WalOp::HideColumn { .. } => return Ok(()),
|
WalOp::HideColumn { .. } => return Ok(()),
|
||||||
WalOp::Commit => return Ok(()),
|
WalOp::Commit => return Ok(()),
|
||||||
|
// 删表后不需要写回
|
||||||
|
WalOp::DropTable { .. } => return Ok(()),
|
||||||
WalOp::Accounts { .. } => USERS_TABLE,
|
WalOp::Accounts { .. } => USERS_TABLE,
|
||||||
WalOp::Insert { table, .. }
|
WalOp::Insert { table, .. }
|
||||||
| WalOp::InsertBatch { table, .. }
|
| WalOp::InsertBatch { table, .. }
|
||||||
@@ -433,6 +464,9 @@ impl Server {
|
|||||||
| WalOp::ReplaceAll { table, .. }
|
| WalOp::ReplaceAll { table, .. }
|
||||||
| WalOp::DropColumn { table, .. }
|
| WalOp::DropColumn { table, .. }
|
||||||
| WalOp::Compact { table, .. }
|
| WalOp::Compact { table, .. }
|
||||||
|
| WalOp::CreateTable { table, .. }
|
||||||
|
| WalOp::CreateIndex { table, .. }
|
||||||
|
| WalOp::DropIndex { table, .. }
|
||||||
| WalOp::ReplaceSchema { table, .. } => table,
|
| WalOp::ReplaceSchema { table, .. } => table,
|
||||||
};
|
};
|
||||||
self.with_table(table, |t| t.flush())
|
self.with_table(table, |t| t.flush())
|
||||||
@@ -515,10 +549,27 @@ impl Server {
|
|||||||
let c = self.catalog.lock().unwrap();
|
let c = self.catalog.lock().unwrap();
|
||||||
Ok(c.describe(table).is_some_and(|s| !s.columns.iter().any(|col| &col.name == column)))
|
Ok(c.describe(table).is_some_and(|s| !s.columns.iter().any(|col| &col.name == column)))
|
||||||
}
|
}
|
||||||
|
WalOp::CreateTable { table, .. } => self.table_exists(table),
|
||||||
|
WalOp::DropTable { table } => Ok(!self.table_exists(table)?
|
||||||
|
&& !self.table_path(table).exists()
|
||||||
|
&& !self.index_path(table, "id").exists()),
|
||||||
|
WalOp::CreateIndex { table, column } => Ok(self.index_exists(table, column)),
|
||||||
|
WalOp::DropIndex { table, column } => {
|
||||||
|
Ok(!self.index_exists(table, column) && !self.index_path(table, column).exists())
|
||||||
|
}
|
||||||
_ => Ok(false),
|
_ => Ok(false),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 这个列有索引吗
|
||||||
|
fn index_exists(&self, table: &str, column: &str) -> bool {
|
||||||
|
self.catalog
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.describe(table)
|
||||||
|
.is_some_and(|s| s.indexes.iter().any(|i| i.column == column))
|
||||||
|
}
|
||||||
|
|
||||||
/// 这些行是不是都已经在表里
|
/// 这些行是不是都已经在表里
|
||||||
fn applied_rows(&self, table: &str, rows: &[Row]) -> std::io::Result<bool> {
|
fn applied_rows(&self, table: &str, rows: &[Row]) -> std::io::Result<bool> {
|
||||||
if rows.is_empty() {
|
if rows.is_empty() {
|
||||||
@@ -598,15 +649,15 @@ impl Server {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 给一列建索引
|
/// 建索引前的检查:内置列、未知表列、重复索引
|
||||||
fn create_index(&self, table: &str, column: &str) -> std::io::Result<()> { if column == "id" {
|
fn check_create_index(&self, table: &str, column: &str) -> std::io::Result<()> {
|
||||||
|
if column == "id" {
|
||||||
return Err(std::io::Error::new(
|
return Err(std::io::Error::new(
|
||||||
std::io::ErrorKind::AlreadyExists,
|
std::io::ErrorKind::AlreadyExists,
|
||||||
"id is the built-in index and cannot be created twice",
|
"id is the built-in index and cannot be created twice",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
{
|
let c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||||
let c = self.catalog.lock().unwrap();
|
|
||||||
let Some(schema) = c.describe(table) else {
|
let Some(schema) = c.describe(table) else {
|
||||||
return Err(std::io::Error::new(
|
return Err(std::io::Error::new(
|
||||||
std::io::ErrorKind::NotFound,
|
std::io::ErrorKind::NotFound,
|
||||||
@@ -625,8 +676,11 @@ impl Server {
|
|||||||
format!("index on column \"{}\" already exists", column),
|
format!("index on column \"{}\" already exists", column),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 建索引文件并登记到 catalog
|
||||||
|
fn apply_create_index(&self, table: &str, column: &str) -> std::io::Result<()> {
|
||||||
let path = self.index_path(table, column);
|
let path = self.index_path(table, column);
|
||||||
let (page_size, order, pool) = (self.cfg.page_size, self.cfg.btree_order, self.cfg.pool_size);
|
let (page_size, order, pool) = (self.cfg.page_size, self.cfg.btree_order, self.cfg.pool_size);
|
||||||
self.with_table(table, |t| {
|
self.with_table(table, |t| {
|
||||||
@@ -639,33 +693,43 @@ impl Server {
|
|||||||
c.save(self.catalog_path())
|
c.save(self.catalog_path())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 去掉一列的索引(连文件一起删)
|
/// 去索引前的检查:内置列、未知表、索引不存在
|
||||||
fn drop_index(&self, table: &str, column: &str) -> std::io::Result<()> {
|
fn check_drop_index(&self, table: &str, column: &str) -> std::io::Result<()> {
|
||||||
if column == "id" {
|
if column == "id" {
|
||||||
return Err(std::io::Error::new(
|
return Err(std::io::Error::new(
|
||||||
std::io::ErrorKind::InvalidInput,
|
std::io::ErrorKind::InvalidInput,
|
||||||
"id is the built-in index and cannot be dropped",
|
"id is the built-in index and cannot be dropped",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
{
|
let c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||||
let c = self.catalog.lock().unwrap();
|
let Some(schema) = c.describe(table) else {
|
||||||
if c.describe(table).is_none() {
|
|
||||||
return Err(std::io::Error::new(
|
return Err(std::io::Error::new(
|
||||||
std::io::ErrorKind::NotFound,
|
std::io::ErrorKind::NotFound,
|
||||||
format!("unknown table: {}", table),
|
format!("unknown table: {}", table),
|
||||||
));
|
));
|
||||||
|
};
|
||||||
|
if !schema.indexes.iter().any(|i| i.column == column) {
|
||||||
|
return Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::NotFound,
|
||||||
|
format!("no index on column \"{}\"", column),
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 摘掉索引定义并删文件;重放时容忍 catalog 里已没有它
|
||||||
|
fn apply_drop_index(&self, table: &str, column: &str) -> std::io::Result<()> {
|
||||||
|
if self.table_exists(table)? {
|
||||||
self.with_table(table, |t| {
|
self.with_table(table, |t| {
|
||||||
t.detach_index(column);
|
t.detach_index(column);
|
||||||
Ok(())
|
Ok(())
|
||||||
})?;
|
})?;
|
||||||
|
let mut c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||||
let mut c = self.catalog.lock().unwrap();
|
if c.describe(table).is_some_and(|s| s.indexes.iter().any(|i| i.column == column)) {
|
||||||
c.remove_index(table, column)?;
|
c.remove_index(table, column)?;
|
||||||
c.save(self.catalog_path())?;
|
c.save(self.catalog_path())?;
|
||||||
drop(c);
|
}
|
||||||
|
}
|
||||||
let _ = std::fs::remove_file(self.index_path(table, column));
|
let _ = std::fs::remove_file(self.index_path(table, column));
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -892,13 +956,6 @@ impl Server {
|
|||||||
|
|
||||||
/// 持门后分发一条请求
|
/// 持门后分发一条请求
|
||||||
fn dispatch(&self, req: Request) -> Response {
|
fn dispatch(&self, req: Request) -> Response {
|
||||||
let _ddl_guard = if matches!(req, Request::CreateTable { .. } | Request::DropTable { .. }
|
|
||||||
| Request::CreateIndex { .. } | Request::DropIndex { .. }) {
|
|
||||||
match self.write_lock.lock() {
|
|
||||||
Ok(guard) => Some(guard),
|
|
||||||
Err(_) => return Response::Error { message: "write lock poisoned".into() },
|
|
||||||
}
|
|
||||||
} else { None };
|
|
||||||
if self.recovery_required.load(Ordering::Acquire) && !matches!(req, Request::Ping) {
|
if self.recovery_required.load(Ordering::Acquire) && !matches!(req, Request::Ping) {
|
||||||
return Response::Error { message: "storage recovery required".into() };
|
return Response::Error { message: "storage recovery required".into() };
|
||||||
}
|
}
|
||||||
@@ -1168,76 +1225,76 @@ impl Server {
|
|||||||
|
|
||||||
Request::CreateTable { table, columns } => {
|
Request::CreateTable { table, columns } => {
|
||||||
log_debug!(request, "create_table table={} cols={}", table, columns.len());
|
log_debug!(request, "create_table table={} cols={}", table, columns.len());
|
||||||
|
let exists = match self.table_exists(&table) {
|
||||||
{
|
Ok(exists) => exists,
|
||||||
let c = self.catalog.lock().unwrap();
|
Err(e) => {
|
||||||
if c.describe(&table).is_some() {
|
return Response::Error {
|
||||||
|
message: format!("create_table {}: {}", table, e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if exists {
|
||||||
return Response::Error {
|
return Response::Error {
|
||||||
message: format!("create_table {}: table already exists", table),
|
message: format!("create_table {}: table already exists", table),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
let tname = table.clone();
|
||||||
|
let op = WalOp::CreateTable { table, columns };
|
||||||
|
match self.write_via_wal(&op) {
|
||||||
|
Ok(()) => Response::Ok,
|
||||||
|
Err(e) => {
|
||||||
|
log_warn!(request, "create_table {}: {}", tname, e);
|
||||||
|
Response::Error {
|
||||||
|
message: format!("create_table {}: {}", tname, e),
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Err(e) = self.with_table(&table, |_t| Ok(())) {
|
|
||||||
return Response::Error {
|
|
||||||
message: format!("create_table {}: {}", table, e),
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut c = self.catalog.lock().unwrap();
|
|
||||||
if let Err(e) = c.create_table(&table, columns) {
|
|
||||||
return Response::Error {
|
|
||||||
message: format!("create_table {}: {}", table, e),
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
if let Err(e) = c.save(self.catalog_path()) {
|
|
||||||
return Response::Error {
|
|
||||||
message: format!("create_table {}: catalog save: {}", table, e),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
Response::Ok
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Request::DropTable { table } => {
|
Request::DropTable { table } => {
|
||||||
log_debug!(request, "drop_table table={}", table);
|
log_debug!(request, "drop_table table={}", table);
|
||||||
|
let exists = match self.table_exists(&table) {
|
||||||
let dropped_indexes: Vec<String> = {
|
Ok(exists) => exists,
|
||||||
let mut c = self.catalog.lock().unwrap();
|
Err(e) => {
|
||||||
let idx = c
|
|
||||||
.describe(&table)
|
|
||||||
.map(|s| s.indexes.iter().map(|i| i.column.clone()).collect())
|
|
||||||
.unwrap_or_default();
|
|
||||||
if let Err(e) = c.drop_table(&table) {
|
|
||||||
return Response::Error {
|
return Response::Error {
|
||||||
message: format!("drop_table {}: {}", table, e),
|
message: format!("drop_table {}: {}", table, e),
|
||||||
};
|
|
||||||
}
|
}
|
||||||
if let Err(e) = c.save(self.catalog_path()) {
|
}
|
||||||
|
};
|
||||||
|
if !exists {
|
||||||
return Response::Error {
|
return Response::Error {
|
||||||
message: format!("drop_table {}: catalog save: {}", table, e),
|
message: format!("drop_table {}: unknown table: {}", table, table),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
idx
|
let tname = table.clone();
|
||||||
};
|
let op = WalOp::DropTable { table };
|
||||||
|
match self.write_via_wal(&op) {
|
||||||
self.tables.lock().unwrap().remove(&table);
|
Ok(()) => Response::Ok,
|
||||||
let _ = std::fs::remove_file(self.table_path(&table));
|
Err(e) => {
|
||||||
let _ = std::fs::remove_file(self.index_path(&table, "id"));
|
log_warn!(request, "drop_table {}: {}", tname, e);
|
||||||
for col in dropped_indexes {
|
Response::Error {
|
||||||
let _ = std::fs::remove_file(self.index_path(&table, &col));
|
message: format!("drop_table {}: {}", tname, e),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Response::Ok
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Request::CreateIndex { table, column } => {
|
Request::CreateIndex { table, column } => {
|
||||||
log_debug!(request, "create_index table={} column={}", table, column);
|
log_debug!(request, "create_index table={} column={}", table, column);
|
||||||
match self.create_index(&table, &column) {
|
if let Err(e) = self.check_create_index(&table, &column) {
|
||||||
|
log_warn!(request, "create_index {}({}): {}", table, column, e);
|
||||||
|
return Response::Error {
|
||||||
|
message: format!("create_index {}({}): {}", table, column, e),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
let tname = table.clone();
|
||||||
|
let op = WalOp::CreateIndex { table, column };
|
||||||
|
match self.write_via_wal(&op) {
|
||||||
Ok(()) => Response::Ok,
|
Ok(()) => Response::Ok,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log_warn!(request, "create_index {}({}): {}", table, column, e);
|
log_warn!(request, "create_index {}: {}", tname, e);
|
||||||
Response::Error {
|
Response::Error {
|
||||||
message: format!("create_index {}({}): {}", table, column, e),
|
message: format!("create_index {}: {}", tname, e),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1245,12 +1302,20 @@ impl Server {
|
|||||||
|
|
||||||
Request::DropIndex { table, column } => {
|
Request::DropIndex { table, column } => {
|
||||||
log_debug!(request, "drop_index table={} column={}", table, column);
|
log_debug!(request, "drop_index table={} column={}", table, column);
|
||||||
match self.drop_index(&table, &column) {
|
if let Err(e) = self.check_drop_index(&table, &column) {
|
||||||
|
log_warn!(request, "drop_index {}({}): {}", table, column, e);
|
||||||
|
return Response::Error {
|
||||||
|
message: format!("drop_index {}({}): {}", table, column, e),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
let tname = table.clone();
|
||||||
|
let op = WalOp::DropIndex { table, column };
|
||||||
|
match self.write_via_wal(&op) {
|
||||||
Ok(()) => Response::Ok,
|
Ok(()) => Response::Ok,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log_warn!(request, "drop_index {}({}): {}", table, column, e);
|
log_warn!(request, "drop_index {}: {}", tname, e);
|
||||||
Response::Error {
|
Response::Error {
|
||||||
message: format!("drop_index {}({}): {}", table, column, e),
|
message: format!("drop_index {}: {}", tname, e),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,6 +19,10 @@ const OP_REPLACE_SCHEMA: u8 = 7;
|
|||||||
const OP_HIDE_COLUMN: u8 = 8;
|
const OP_HIDE_COLUMN: u8 = 8;
|
||||||
const OP_COMPACT: u8 = 9;
|
const OP_COMPACT: u8 = 9;
|
||||||
const OP_COMMIT: u8 = 10;
|
const OP_COMMIT: u8 = 10;
|
||||||
|
const OP_CREATE_TABLE: u8 = 11;
|
||||||
|
const OP_DROP_TABLE: u8 = 12;
|
||||||
|
const OP_CREATE_INDEX: u8 = 13;
|
||||||
|
const OP_DROP_INDEX: u8 = 14;
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
pub enum WalOp {
|
pub enum WalOp {
|
||||||
@@ -51,6 +55,21 @@ pub enum WalOp {
|
|||||||
columns: Vec<SchemaColumn>,
|
columns: Vec<SchemaColumn>,
|
||||||
rows: Vec<Row>,
|
rows: Vec<Row>,
|
||||||
},
|
},
|
||||||
|
CreateTable {
|
||||||
|
table: String,
|
||||||
|
columns: Vec<SchemaColumn>,
|
||||||
|
},
|
||||||
|
DropTable {
|
||||||
|
table: String,
|
||||||
|
},
|
||||||
|
CreateIndex {
|
||||||
|
table: String,
|
||||||
|
column: String,
|
||||||
|
},
|
||||||
|
DropIndex {
|
||||||
|
table: String,
|
||||||
|
column: String,
|
||||||
|
},
|
||||||
/// 一组写操作的提交标记;没有标记的组算未提交
|
/// 一组写操作的提交标记;没有标记的组算未提交
|
||||||
Commit,
|
Commit,
|
||||||
}
|
}
|
||||||
@@ -67,6 +86,11 @@ struct ReplaceSchemaPayload {
|
|||||||
rows: Vec<Row>,
|
rows: Vec<Row>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
struct CreateTablePayload {
|
||||||
|
columns: Vec<SchemaColumn>,
|
||||||
|
}
|
||||||
|
|
||||||
pub struct Wal {
|
pub struct Wal {
|
||||||
path: PathBuf,
|
path: PathBuf,
|
||||||
handle: Mutex<Option<File>>,
|
handle: Mutex<Option<File>>,
|
||||||
@@ -319,6 +343,23 @@ fn encode(op: &WalOp, lsn: u64) -> io::Result<Vec<u8>> {
|
|||||||
.map_err(io::Error::other)?;
|
.map_err(io::Error::other)?;
|
||||||
(OP_REPLACE_SCHEMA, table.clone(), p)
|
(OP_REPLACE_SCHEMA, table.clone(), p)
|
||||||
}
|
}
|
||||||
|
WalOp::CreateTable { table, columns } => (
|
||||||
|
OP_CREATE_TABLE,
|
||||||
|
table.clone(),
|
||||||
|
serde_json::to_vec(&CreateTablePayload { columns: columns.clone() })
|
||||||
|
.map_err(io::Error::other)?,
|
||||||
|
),
|
||||||
|
WalOp::DropTable { table } => (OP_DROP_TABLE, table.clone(), Vec::new()),
|
||||||
|
WalOp::CreateIndex { table, column } => (
|
||||||
|
OP_CREATE_INDEX,
|
||||||
|
table.clone(),
|
||||||
|
serde_json::to_vec(column).map_err(io::Error::other)?,
|
||||||
|
),
|
||||||
|
WalOp::DropIndex { table, column } => (
|
||||||
|
OP_DROP_INDEX,
|
||||||
|
table.clone(),
|
||||||
|
serde_json::to_vec(column).map_err(io::Error::other)?,
|
||||||
|
),
|
||||||
WalOp::Commit => (OP_COMMIT, String::new(), Vec::new()),
|
WalOp::Commit => (OP_COMMIT, String::new(), Vec::new()),
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -410,6 +451,25 @@ fn decode_body(body: &[u8]) -> Result<(u64, WalOp), String> {
|
|||||||
rows: p.rows,
|
rows: p.rows,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
OP_CREATE_TABLE => {
|
||||||
|
let p: CreateTablePayload = serde_json::from_slice(payload)
|
||||||
|
.map_err(|e| format!("bad create_table payload: {e}"))?;
|
||||||
|
WalOp::CreateTable {
|
||||||
|
table,
|
||||||
|
columns: p.columns,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
OP_DROP_TABLE => WalOp::DropTable { table },
|
||||||
|
OP_CREATE_INDEX => {
|
||||||
|
let column: String = serde_json::from_slice(payload)
|
||||||
|
.map_err(|e| format!("bad create_index payload: {e}"))?;
|
||||||
|
WalOp::CreateIndex { table, column }
|
||||||
|
}
|
||||||
|
OP_DROP_INDEX => {
|
||||||
|
let column: String = serde_json::from_slice(payload)
|
||||||
|
.map_err(|e| format!("bad drop_index payload: {e}"))?;
|
||||||
|
WalOp::DropIndex { table, column }
|
||||||
|
}
|
||||||
other => return Err(format!("unknown op type: {}", other)),
|
other => return Err(format!("unknown op type: {}", other)),
|
||||||
};
|
};
|
||||||
Ok((lsn, op))
|
Ok((lsn, op))
|
||||||
|
|||||||
@@ -1201,6 +1201,199 @@ fn uncommitted_group_is_discarded_on_restart() -> Result<(), Box<dyn std::error:
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 直接删表后文件不留,重建同名表是空表
|
||||||
|
#[test]
|
||||||
|
fn drop_table_removes_files() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let (_srv, data) = start_server();
|
||||||
|
let mut c = connect()?;
|
||||||
|
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"m4_gone","columns":[{"name":"id","ty":"int"},{"name":"note","ty":"text"}]}))?;
|
||||||
|
request_ok(&mut c, serde_json::json!({"method":"create_index","table":"m4_gone","column":"note"}))?;
|
||||||
|
request_ok(&mut c, serde_json::json!({"method":"insert","table":"m4_gone","row":{"id":1,"note":"a"}}))?;
|
||||||
|
assert!(data.path().join("databases/main/m4_gone.db").exists(), "建表要落数据文件");
|
||||||
|
|
||||||
|
request_ok(&mut c, serde_json::json!({"method":"drop_table","table":"m4_gone"}))?;
|
||||||
|
assert!(!data.path().join("databases/main/m4_gone.db").exists(), "数据文件要删掉");
|
||||||
|
assert!(!data.path().join("databases/main/m4_gone.idx").exists(), "id 索引要删掉");
|
||||||
|
assert!(!data.path().join("databases/main/m4_gone.note.idx").exists(), "二级索引要删掉");
|
||||||
|
|
||||||
|
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"m4_gone","columns":[{"name":"id","ty":"int"},{"name":"note","ty":"text"}]}))?;
|
||||||
|
let rows: serde_json::Value =
|
||||||
|
serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"m4_gone"}"#))?;
|
||||||
|
assert_eq!(rows["rows"], serde_json::json!([]), "旧数据不能复活");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 建表操作进了 WAL 后,重启按提交组把表重建出来
|
||||||
|
#[test]
|
||||||
|
fn create_table_replays_after_restart() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
use chusql_core_storage::protocol::SchemaColumn;
|
||||||
|
use chusql_core_storage::wal::{Wal, WalOp};
|
||||||
|
let (server, data) = start_server();
|
||||||
|
drop(server);
|
||||||
|
|
||||||
|
let group = vec![WalOp::CreateTable {
|
||||||
|
table: "m4_create".into(),
|
||||||
|
columns: vec![
|
||||||
|
SchemaColumn {
|
||||||
|
name: "id".into(),
|
||||||
|
ty: "int".into(),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
SchemaColumn {
|
||||||
|
name: "note".into(),
|
||||||
|
ty: "text".into(),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}];
|
||||||
|
let wal = Wal::new(data.path().join("databases/main/wal.log"));
|
||||||
|
let commit_lsn = wal.append_group(&group)?;
|
||||||
|
drop(wal);
|
||||||
|
|
||||||
|
let _server = start_server_in(data.path());
|
||||||
|
let mut c = connect()?;
|
||||||
|
let schema: serde_json::Value =
|
||||||
|
serde_json::from_str(&send(&mut c, r#"{"method":"describe_table","table":"m4_create"}"#))?;
|
||||||
|
assert_eq!(schema["status"], "schema", "{schema}");
|
||||||
|
assert_eq!(schema["columns"][1]["name"], "note", "{schema}");
|
||||||
|
assert!(data.path().join("databases/main/m4_create.db").exists(), "建表要落数据文件");
|
||||||
|
let checkpoint = std::fs::read(data.path().join("databases/main/wal.checkpoint"))?;
|
||||||
|
assert_eq!(u64::from_le_bytes(checkpoint[..8].try_into()?), commit_lsn);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 删表操作进了 WAL 后,重启把表和数据文件一起清掉
|
||||||
|
#[test]
|
||||||
|
fn drop_table_replays_after_restart() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
use chusql_core_storage::wal::{Wal, WalOp};
|
||||||
|
let (server, data) = start_server();
|
||||||
|
let mut c = connect()?;
|
||||||
|
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"m4_drop","columns":[{"name":"id","ty":"int"}]}))?;
|
||||||
|
request_ok(&mut c, serde_json::json!({"method":"insert","table":"m4_drop","row":{"id":1}}))?;
|
||||||
|
drop(c);
|
||||||
|
drop(server);
|
||||||
|
|
||||||
|
let wal = Wal::new(data.path().join("databases/main/wal.log"));
|
||||||
|
wal.append_group(&[WalOp::DropTable {
|
||||||
|
table: "m4_drop".into(),
|
||||||
|
}])?;
|
||||||
|
drop(wal);
|
||||||
|
|
||||||
|
let _server = start_server_in(data.path());
|
||||||
|
let mut c = connect()?;
|
||||||
|
let schema: serde_json::Value =
|
||||||
|
serde_json::from_str(&send(&mut c, r#"{"method":"describe_table","table":"m4_drop"}"#))?;
|
||||||
|
assert_eq!(schema["status"], "error", "{schema}");
|
||||||
|
let left: Vec<String> = std::fs::read_dir(data.path().join("databases/main"))?
|
||||||
|
.map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
|
||||||
|
.collect();
|
||||||
|
assert!(!data.path().join("databases/main/m4_drop.db").exists(), "数据文件要删掉,现存 {left:?}");
|
||||||
|
assert!(!data.path().join("databases/main/m4_drop.idx").exists(), "id 索引要删掉,现存 {left:?}");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 建索引操作进了 WAL 后,重启补建索引文件
|
||||||
|
#[test]
|
||||||
|
fn create_index_replays_after_restart() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
use chusql_core_storage::wal::{Wal, WalOp};
|
||||||
|
let (server, data) = start_server();
|
||||||
|
let mut c = connect()?;
|
||||||
|
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"m4_idx","columns":[{"name":"id","ty":"int"},{"name":"note","ty":"text"}]}))?;
|
||||||
|
request_ok(&mut c, serde_json::json!({"method":"insert","table":"m4_idx","row":{"id":1,"note":"a"}}))?;
|
||||||
|
drop(c);
|
||||||
|
drop(server);
|
||||||
|
|
||||||
|
let wal = Wal::new(data.path().join("databases/main/wal.log"));
|
||||||
|
wal.append_group(&[WalOp::CreateIndex {
|
||||||
|
table: "m4_idx".into(),
|
||||||
|
column: "note".into(),
|
||||||
|
}])?;
|
||||||
|
drop(wal);
|
||||||
|
|
||||||
|
let _server = start_server_in(data.path());
|
||||||
|
let mut c = connect()?;
|
||||||
|
let schema: serde_json::Value =
|
||||||
|
serde_json::from_str(&send(&mut c, r#"{"method":"describe_table","table":"m4_idx"}"#))?;
|
||||||
|
assert_eq!(schema["status"], "schema", "{schema}");
|
||||||
|
let indexed = |schema: &serde_json::Value, column: &str| {
|
||||||
|
schema["indexes"]
|
||||||
|
.as_array()
|
||||||
|
.map(|list| list.iter().any(|i| i["column"] == column))
|
||||||
|
.unwrap_or(false)
|
||||||
|
};
|
||||||
|
assert!(indexed(&schema, "note"), "{schema}");
|
||||||
|
assert!(data.path().join("databases/main/m4_idx.note.idx").exists(), "索引文件要建出来");
|
||||||
|
let found: serde_json::Value = serde_json::from_str(&send(
|
||||||
|
&mut c,
|
||||||
|
r#"{"method":"lookup_by_index","table":"m4_idx","column":"note","key":"a"}"#,
|
||||||
|
))?;
|
||||||
|
assert_eq!(found["rows"], serde_json::json!([{"id":1,"note":"a"}]), "{found}");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 删索引操作进了 WAL 后,重启清掉索引定义和文件
|
||||||
|
#[test]
|
||||||
|
fn drop_index_replays_after_restart() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
use chusql_core_storage::wal::{Wal, WalOp};
|
||||||
|
let (server, data) = start_server();
|
||||||
|
let mut c = connect()?;
|
||||||
|
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"m4_didx","columns":[{"name":"id","ty":"int"},{"name":"note","ty":"text"}]}))?;
|
||||||
|
request_ok(&mut c, serde_json::json!({"method":"create_index","table":"m4_didx","column":"note"}))?;
|
||||||
|
drop(c);
|
||||||
|
drop(server);
|
||||||
|
|
||||||
|
let wal = Wal::new(data.path().join("databases/main/wal.log"));
|
||||||
|
wal.append_group(&[WalOp::DropIndex {
|
||||||
|
table: "m4_didx".into(),
|
||||||
|
column: "note".into(),
|
||||||
|
}])?;
|
||||||
|
drop(wal);
|
||||||
|
|
||||||
|
let _server = start_server_in(data.path());
|
||||||
|
let mut c = connect()?;
|
||||||
|
let schema: serde_json::Value =
|
||||||
|
serde_json::from_str(&send(&mut c, r#"{"method":"describe_table","table":"m4_didx"}"#))?;
|
||||||
|
assert_eq!(schema["status"], "schema", "{schema}");
|
||||||
|
let columns: Vec<&str> = schema["indexes"]
|
||||||
|
.as_array()
|
||||||
|
.map(|list| list.iter().filter_map(|i| i["column"].as_str()).collect())
|
||||||
|
.unwrap_or_default();
|
||||||
|
assert_eq!(columns, vec!["id"], "{schema}");
|
||||||
|
assert!(!data.path().join("databases/main/m4_didx.note.idx").exists(), "索引文件要删掉");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 已生效的建表记录重启时被跳过,不报恢复失败
|
||||||
|
#[test]
|
||||||
|
fn already_applied_create_table_is_skipped_on_restart() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
use chusql_core_storage::protocol::SchemaColumn;
|
||||||
|
use chusql_core_storage::wal::{Wal, WalOp};
|
||||||
|
let (server, data) = start_server();
|
||||||
|
let mut c = connect()?;
|
||||||
|
request_ok(&mut c, serde_json::json!({"method":"create_table","table":"m4_done","columns":[{"name":"id","ty":"int"}]}))?;
|
||||||
|
drop(c);
|
||||||
|
drop(server);
|
||||||
|
|
||||||
|
let wal = Wal::new(data.path().join("databases/main/wal.log"));
|
||||||
|
wal.append_group(&[WalOp::CreateTable {
|
||||||
|
table: "m4_done".into(),
|
||||||
|
columns: vec![SchemaColumn {
|
||||||
|
name: "id".into(),
|
||||||
|
ty: "int".into(),
|
||||||
|
..Default::default()
|
||||||
|
}],
|
||||||
|
}])?;
|
||||||
|
drop(wal);
|
||||||
|
|
||||||
|
let _server = start_server_in(data.path());
|
||||||
|
let mut c = connect()?;
|
||||||
|
let schema: serde_json::Value =
|
||||||
|
serde_json::from_str(&send(&mut c, r#"{"method":"describe_table","table":"m4_done"}"#))?;
|
||||||
|
assert_eq!(schema["status"], "schema", "{schema}");
|
||||||
|
assert_eq!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len(), 0);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// 给第二列建索引后能按那一列查
|
/// 给第二列建索引后能按那一列查
|
||||||
#[test]
|
#[test]
|
||||||
fn create_index_on_secondary_column() {
|
fn create_index_on_secondary_column() {
|
||||||
|
|||||||
@@ -51,7 +51,7 @@ chusql-server
|
|||||||
|
|
||||||
显式事务在会话层实现:`BEGIN` 把当前数据库整库读成内存快照,事务内的语句都在快照上执行;`COMMIT` 计算快照与库的差分,用一条 `apply_transaction` 请求交给存储层应用;`ROLLBACK` 直接丢弃快照。因此隔离级别是快照读加原子提交:事务里看不到别的事务未提交或后来提交的改动,提交对外一次性可见。
|
显式事务在会话层实现:`BEGIN` 把当前数据库整库读成内存快照,事务内的语句都在快照上执行;`COMMIT` 计算快照与库的差分,用一条 `apply_transaction` 请求交给存储层应用;`ROLLBACK` 直接丢弃快照。因此隔离级别是快照读加原子提交:事务里看不到别的事务未提交或后来提交的改动,提交对外一次性可见。
|
||||||
|
|
||||||
WAL 负责崩溃恢复:每条记录带自增 LSN,一组写操作与一条提交标记一次落盘;进程重启时按标记分组重放已提交的记录、丢弃没有标记的整组,重放前逐条做幂等检查,恢复结束把最大 LSN 写进 `wal.checkpoint` 并清空 WAL。`SystemStatus` 应答里的 `last_lsn` 报告该检查点 LSN。
|
WAL 负责崩溃恢复:每条记录带自增 LSN,一组写操作与一条提交标记一次落盘;进程重启时按标记分组重放已提交的记录、丢弃没有标记的整组,重放前逐条做幂等检查,恢复结束把最大 LSN 写进 `wal.checkpoint` 并清空 WAL。`SystemStatus` 应答里的 `last_lsn` 报告该检查点 LSN。建表、删表、建索引、删索引与数据改动走同一条 WAL,恢复时一并重放。
|
||||||
|
|
||||||
系统数据库及部分管理操作仅允许管理员访问。
|
系统数据库及部分管理操作仅允许管理员访问。
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user