ChuNa7a/ChuSQL
1
0 Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

完成rust缓冲池，服务端部分收尾

Browse Source
This commit is contained in:
LIm1tZz7 committed 2026-09-24 23:56:07 +08:00
1 parent 2a4b446252
commit 7f5bcc289e
18 files changed
+482 -316
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

chusql-storage/src/bin/server.rs
+65 -46
View File
@@ -9,7 +9,7 @@ use chusql_storage::catalog::Catalog;
use chusql_storage::config::{self, Config, Loaded};
use chusql_storage::heap::HeapTable;
use chusql_storage::log;
use chusql_storage::protocol::{Request, Response, SchemaColumn};
use chusql_storage::protocol::{Request, Response, SchemaColumn, TableSchemaWire};
use chusql_storage::wal::{Wal, WalOp};
use chusql_storage::{log_debug, log_error, log_info, log_warn};
use interprocess::local_socket::{
@@ -28,20 +28,22 @@ struct Server {
cfg: Config,
tables: Mutex<HashMap<String, Arc<Mutex<HeapTable>>>>,
write_lock: Mutex<()>,
catalog: Mutex<Catalog>,
}
impl Server {
/// 新建服务器状态
fn new(cfg: Config) -> std::io::Result<Self> {
std::fs::create_dir_all(&cfg.data_dir)?;
let catalog = Catalog::load(cfg.data_dir.join("catalog.json"))?;
Ok(Server {
cfg,
tables: Mutex::new(HashMap::new()),
write_lock: Mutex::new(()),
catalog: Mutex::new(catalog),
})
}
// 路径
/// 表名 -> 数据文件
fn table_path(&self, table: &str) -> PathBuf {
@@ -125,7 +127,9 @@ impl Server {
table: &str,
f: impl FnOnce(&mut HeapTable) -> std::io::Result<R>,
) -> std::io::Result<R> {
if !self.table_exists(table) {
// 已经在缓存里的表不用再查磁盘（省一次 metadata 系统调用）
let cached = self.tables.lock().unwrap().contains_key(table);
if !cached && !self.table_exists(table) {
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
format!("unknown table: {}", table),
@@ -143,19 +147,21 @@ impl Server {
None => self.with_table(table, |t| t.insert(row))?,
Some(k) => self.with_indexed_table(table, |t| t.insert_keyed(row, *k))?,
}
let mut c = Catalog::load(self.catalog_path())?;
let row: chusql_storage::protocol::Row = row.clone().into_iter().collect();
c.record_insert(table, &row);
c.save(self.catalog_path())?;
let mut c = self.catalog.lock().unwrap();
if c.record_insert(table, &row) {
c.save(self.catalog_path())?;
}
Ok(())
}
WalOp::ReplaceAll { table, rows } => {
self.with_indexed_table(table, |t| t.replace_all(rows))?;
let mut c = Catalog::load(self.catalog_path())?;
let rows: Vec<chusql_storage::protocol::Row> =
rows.iter().map(|r| r.clone().into_iter().collect()).collect();
c.record_replace_all(table, &rows);
c.save(self.catalog_path())?;
let mut c = self.catalog.lock().unwrap();
if c.record_replace_all(table, &rows) {
c.save(self.catalog_path())?;
}
Ok(())
}
}
@@ -298,6 +304,21 @@ impl Server {
}
}
Request::ListCatalog => {
log_debug!(pipe, "list_catalog");
let c = self.catalog.lock().unwrap();
let schemas: Vec<TableSchemaWire> = c
.all_tables()
.into_iter()
.map(|(name, ts)| TableSchemaWire {
table: name.to_string(),
columns: ts.columns.clone(),
row_count: ts.row_count,
})
.collect();
Response::Catalog { schemas }
}
Request::DescribeTable { table } => {
log_debug!(pipe, "describe_table table={}", table);
match self.describe_table(&table) {
@@ -317,18 +338,13 @@ impl Server {
Request::CreateTable { table, columns } => {
log_debug!(pipe, "create_table table={} cols={}", table, columns.len());
let mut c = match Catalog::load(self.catalog_path()) {
Ok(c) => c,
Err(e) => {
{
let c = self.catalog.lock().unwrap();
if c.describe(&table).is_some() {
return Response::Error {
message: format!("create_table {}: catalog: {}", table, e),
}
message: format!("create_table {}: table already exists", table),
};
}
};
if c.describe(&table).is_some() {
return Response::Error {
message: format!("create_table {}: table already exists", table),
};
}
if let Err(e) = self.with_table(&table, |_t| Ok(())) {
@@ -337,6 +353,7 @@ impl Server {
};
}
let mut c = self.catalog.lock().unwrap();
if let Err(e) = c.create_table(&table, columns) {
return Response::Error {
message: format!("create_table {}: {}", table, e),
@@ -347,31 +364,27 @@ impl Server {
message: format!("create_table {}: catalog save: {}", table, e),
};
}
Response::Ok
}
Request::DropTable { table } => {
log_debug!(pipe, "drop_table table={}", table);
let mut c = match Catalog::load(self.catalog_path()) {
Ok(c) => c,
Err(e) => return Response::Error {
message: format!("drop_table {}: catalog: {}", table, e),
},
};
if let Err(e) = c.drop_table(&table) {
return Response::Error {
message: format!("drop_table {}: {}", table, e),
};
}
if let Err(e) = c.save(self.catalog_path()) {
return Response::Error {
message: format!("drop_table {}: catalog save: {}", table, e),
};
{
let mut c = self.catalog.lock().unwrap();
if let Err(e) = c.drop_table(&table) {
return Response::Error {
message: format!("drop_table {}: {}", table, e),
};
}
if let Err(e) = c.save(self.catalog_path()) {
return Response::Error {
message: format!("drop_table {}: catalog save: {}", table, e),
};
}
}
self.tables.lock().unwrap().remove(&table);
let _ = std::fs::remove_file(self.table_path(&table));
let _ = std::fs::remove_file(self.index_path(&table));
@@ -384,12 +397,12 @@ impl Server {
/// 表名：catalog 与目录取并集
fn list_tables(&self) -> std::io::Result<Vec<String>> {
let mut names = BTreeSet::new();
let c = Catalog::load(self.catalog_path())?;
for t in c.table_names() {
names.insert(t);
{
let c = self.catalog.lock().unwrap();
for t in c.table_names() {
names.insert(t);
}
}
let dir = &self.cfg.data_dir;
if dir.exists() {
for entry in std::fs::read_dir(dir)? {
@@ -400,19 +413,17 @@ impl Server {
}
}
}
Ok(names.into_iter().collect())
}
/// 查一张表的 schema
fn describe_table(&self, table: &str) -> std::io::Result<(Vec<SchemaColumn>, u64)> {
let c = Catalog::load(self.catalog_path())?;
let c = self.catalog.lock().unwrap();
if let Some(schema) = c.describe(table) {
return Ok((schema.columns.clone(), schema.row_count));
}
if self.tables.lock().unwrap().contains_key(table)
|| std::fs::metadata(self.table_path(table)).is_ok()
{
drop(c);
if std::fs::metadata(self.table_path(table)).is_ok() {
return Ok((Vec::new(), 0));
}
Err(std::io::Error::new(
@@ -422,6 +433,14 @@ impl Server {
}
}
impl Drop for Server {
fn drop(&mut self) {
if let Ok(c) = self.catalog.lock() {
let _ = c.save(self.catalog_path());
}
}
}
// 连接
/// 处理一条连接
chusql-storage/src/btree.rs
+28 -41
View File
@@ -250,16 +250,14 @@ impl DiskBTree {
/// 读根页号
fn root(&mut self) -> io::Result<Option<PageId>> {
let meta = self.file.read_page(META_PAGE)?;
let r = read_u64(&meta, OFF_META_ROOT);
let r = self.file.with_page(META_PAGE, |p| read_u64(p, OFF_META_ROOT))?;
Ok(if r == 0 { None } else { Some(r) })
}
/// 写根页号
fn set_root(&mut self, r: Option<PageId>) -> io::Result<()> {
let mut meta = self.file.read_page(META_PAGE)?;
write_u64(&mut meta, OFF_META_ROOT, r.unwrap_or(0));
self.file.write_page(&meta)
self.file
.update_page(META_PAGE, |p| write_u64(p, OFF_META_ROOT, r.unwrap_or(0)))
}
/// 分配新页
@@ -292,8 +290,8 @@ impl DiskBTree {
/// 递归查找
fn get_rec(&mut self, page_id: PageId, key: i64) -> io::Result<Option<u64>> {
let p = self.file.read_page(page_id)?;
match read_node(&p, self.order) {
let order = self.order;
match self.file.with_page(page_id, |p| read_node(p, order))? {
NodeData::Leaf { keys, values, .. } => {
Ok(keys.binary_search(&key).ok().map(|i| values[i]))
}
@@ -345,9 +343,10 @@ impl DiskBTree {
key: i64,
value: u64,
) -> io::Result<(PageId, Option<(i64, PageId)>)> {
let p = self.file.read_page(page_id)?;
let limit = max_keys(self.order);
match read_node(&p, self.order) {
let order = self.order;
let node = self.file.with_page(page_id, |p| read_node(p, order))?;
let limit = max_keys(order);
match node {
NodeData::Leaf { mut keys, mut values, next } => {
match keys.binary_search(&key) {
Ok(i) => values[i] = value,
@@ -357,9 +356,9 @@ impl DiskBTree {
}
}
if keys.len() <= limit {
let mut p2 = self.new_page(page_id);
write_node(&mut p2, &NodeData::Leaf { keys, values, next }, self.order);
self.file.write_page(&p2)?;
self.file.update_page(page_id, |p| {
write_node(p, &NodeData::Leaf { keys, values, next }, order)
})?;
Ok((page_id, None))
} else {
let mid = keys.len() / 2;
@@ -372,17 +371,13 @@ impl DiskBTree {
write_node(
&mut rp,
&NodeData::Leaf { keys: rkeys, values: rvalues, next },
self.order,
order,
);
self.file.write_page(&rp)?;
let mut lp = self.new_page(page_id);
write_node(
&mut lp,
&NodeData::Leaf { keys, values, next: right_id },
self.order,
);
self.file.write_page(&lp)?;
self.file.update_page(page_id, |p| {
write_node(p, &NodeData::Leaf { keys, values, next: right_id }, order)
})?;
Ok((page_id, Some((sep, right_id))))
}
@@ -394,22 +389,18 @@ impl DiskBTree {
children[i] = new_child;
match split {
None => {
let mut p2 = self.new_page(page_id);
write_node(&mut p2, &NodeData::Internal { keys, children }, self.order);
self.file.write_page(&p2)?;
self.file.update_page(page_id, |p| {
write_node(p, &NodeData::Internal { keys, children }, order)
})?;
Ok((page_id, None))
}
Some((sep, right_id)) => {
keys.insert(i, sep);
children.insert(i + 1, right_id);
if keys.len() <= limit {
let mut p2 = self.new_page(page_id);
write_node(
&mut p2,
&NodeData::Internal { keys, children },
self.order,
);
self.file.write_page(&p2)?;
self.file.update_page(page_id, |p| {
write_node(p, &NodeData::Internal { keys, children }, order)
})?;
Ok((page_id, None))
} else {
let mid = keys.len() / 2;
@@ -423,17 +414,13 @@ impl DiskBTree {
write_node(
&mut rp,
&NodeData::Internal { keys: rkeys, children: rchildren },
self.order,
order,
);
self.file.write_page(&rp)?;
let mut lp = self.new_page(page_id);
write_node(
&mut lp,
&NodeData::Internal { keys, children },
self.order,
);
self.file.write_page(&lp)?;
self.file.update_page(page_id, |p| {
write_node(p, &NodeData::Internal { keys, children }, order)
})?;
Ok((page_id, Some((up, right_id))))
}
@@ -454,8 +441,8 @@ impl DiskBTree {
/// 中序遍历
fn walk(&mut self, page_id: PageId, out: &mut Vec<(i64, u64)>) -> io::Result<()> {
let p = self.file.read_page(page_id)?;
match read_node(&p, self.order) {
let order = self.order;
match self.file.with_page(page_id, |p| read_node(p, order))? {
NodeData::Leaf { keys, values, .. } => {
for i in 0..keys.len() {
out.push((keys[i], values[i]));
chusql-storage/src/catalog.rs
+14 -2
View File
@@ -39,6 +39,11 @@ impl Catalog {
serde_json::from_slice(&bytes).map_err(io::Error::other)
}
/// 返回所有表的 (表名, schema)。
pub fn all_tables(&self) -> Vec<(&str, &TableSchema)> {
self.tables.iter().map(|(k, v)| (k.as_str(), v)).collect()
}
/// 写回文件
pub fn save<P: AsRef<Path>>(&self, path: P) -> io::Result<()> {
let bytes = serde_json::to_vec_pretty(self).map_err(io::Error::other)?;
@@ -98,22 +103,27 @@ impl Catalog {
}
}
/// 记一次插入：补列 + 行数 +1
pub fn record_insert(&mut self, table: &str, row: &Row) {
pub fn record_insert(&mut self, table: &str, row: &Row) -> bool {
let entry = self.tables.entry(table.to_string()).or_default();
let mut changed = false;
for (k, v) in row {
if entry.columns.iter().any(|c| &c.name == k) {
continue;
}
if let Some(ty) = infer_type(v) {
entry.columns.push(SchemaColumn { name: k.clone(), ty });
changed = true;
}
}
entry.row_count += 1;
changed
}
/// 记一次整表替换：列照补，行数重置
pub fn record_replace_all(&mut self, table: &str, rows: &[Row]) {
pub fn record_replace_all(&mut self, table: &str, rows: &[Row]) -> bool {
let entry = self.tables.entry(table.to_string()).or_default();
let mut changed = false;
for r in rows {
for (k, v) in r {
if entry.columns.iter().any(|c| &c.name == k) {
@@ -121,10 +131,12 @@ impl Catalog {
}
if let Some(ty) = infer_type(v) {
entry.columns.push(SchemaColumn { name: k.clone(), ty });
changed = true;
}
}
}
entry.row_count = rows.len() as u64;
changed
}
/// 删表
chusql-storage/src/config.rs
+1 -1
View File
@@ -11,7 +11,7 @@ use crate::log::Level;
pub const DEFAULT_CONFIG_PATH: &str = "chusql-storage.toml";
/// 默认池容量（页数）
pub const DEFAULT_POOL_SIZE: usize = 64;
pub const DEFAULT_POOL_SIZE: usize = 1024;
/// 默认页大小
pub const DEFAULT_PAGE_SIZE: usize = 4096;
/// 默认 B+ 树 order
chusql-storage/src/heap.rs
+19 -12
View File
@@ -177,10 +177,12 @@ impl HeapTable {
let n = self.file.num_pages()?;
if n > 0 {
let last_id = n - 1;
let mut p = self.file.read_page(last_id)?;
if let Some(slot) = HeapPage::insert_tuple(&mut p, &tuple) {
self.file.write_page(&p)?;
return Ok((last_id, slot));
let mut slot = None;
self.file.update_page(last_id, |p| {
slot = HeapPage::insert_tuple(p, &tuple);
})?;
if let Some(s) = slot {
return Ok((last_id, s));
}
}
@@ -215,8 +217,8 @@ impl HeapTable {
/// 按位置读一行
pub fn read_at(&mut self, page_id: PageId, slot: u16) -> io::Result<Option<Row>> {
let p = self.file.read_page(page_id)?;
match HeapPage::get_tuple(&p, slot) {
let bytes = self.file.with_page(page_id, |p| HeapPage::get_tuple(p, slot))?;
match bytes {
None => Ok(None),
Some(bytes) => {
let row: Row = serde_json::from_slice(&bytes).map_err(io::Error::other)?;
@@ -230,13 +232,18 @@ impl HeapTable {
let n = self.file.num_pages()?;
let mut rows = Vec::new();
for i in 0..n {
let p = self.file.read_page(i)?;
for slot in HeapPage::iter_slots(&p) {
if let Some(bytes) = HeapPage::get_tuple(&p, slot) {
let row: Row = serde_json::from_slice(&bytes)
.map_err(io::Error::other)?;
rows.push(row);
let tuples = self.file.with_page(i, |p| {
let mut out = Vec::new();
for slot in HeapPage::iter_slots(p) {
if let Some(bytes) = HeapPage::get_tuple(p, slot) {
out.push(bytes);
}
}
out
})?;
for bytes in tuples {
let row: Row = serde_json::from_slice(&bytes).map_err(io::Error::other)?;
rows.push(row);
}
}
Ok(rows)
chusql-storage/src/page.rs
+53 -31
View File
@@ -1,4 +1,4 @@
use std::collections::{HashMap, HashSet, VecDeque};
use std::collections::{HashMap, HashSet};
use std::fs::{File, OpenOptions};
use std::io::{self, Read, Seek, SeekFrom, Write};
use std::path::Path;
@@ -41,7 +41,8 @@ pub struct PageFile {
page_size: usize,
capacity: usize,
cache: HashMap<PageId, Page>,
order: VecDeque<PageId>,
last_used: HashMap<PageId, u64>,
clock: u64,
dirty: HashSet<PageId>,
hits: u64,
misses: u64,
@@ -70,7 +71,8 @@ impl PageFile {
page_size,
capacity,
cache: HashMap::new(),
order: VecDeque::new(),
last_used: HashMap::new(),
clock: 0,
dirty: HashSet::new(),
hits: 0,
misses: 0,
@@ -116,26 +118,28 @@ impl PageFile {
/// 标记为最近使用
fn touch(&mut self, id: PageId) {
if let Some(pos) = self.order.iter().position(|&x| x == id) {
self.order.remove(pos);
}
self.order.push_back(id);
self.clock += 1;
self.last_used.insert(id, self.clock);
}
/// 淘汰一个最冷的页
fn evict_one(&mut self) -> io::Result<bool> {
while let Some(victim) = self.order.pop_front() {
if let Some(p) = self.cache.get(&victim) {
if self.dirty.contains(&victim) {
let p = p.clone();
self.write_raw(&p)?;
self.dirty.remove(&victim);
}
self.cache.remove(&victim);
return Ok(true);
}
let victim = self
.cache
.keys()
.min_by_key(|id| self.last_used.get(id).copied().unwrap_or(0))
.copied();
let Some(victim) = victim else {
return Ok(false);
};
if self.dirty.contains(&victim) {
let p = self.cache[&victim].clone();
self.write_raw(&p)?;
self.dirty.remove(&victim);
}
Ok(false)
self.cache.remove(&victim);
self.last_used.remove(&victim);
Ok(true)
}
/// 直接写文件
@@ -166,20 +170,38 @@ impl PageFile {
/// 读一页（走缓存）
pub fn read_page(&mut self, id: PageId) -> io::Result<Page> {
if let Some(p) = self.cache.get(&id) {
let p = p.clone();
self.note_access(id)?;
Ok(self.cache[&id].clone())
}
/// 只读地用一页（走缓存，不整页拷贝）
pub fn with_page<R>(&mut self, id: PageId, f: impl FnOnce(&Page) -> R) -> io::Result<R> {
self.note_access(id)?;
Ok(f(&self.cache[&id]))
}
/// 就地改一页（走缓存，不整页拷贝）
pub fn update_page<R>(&mut self, id: PageId, f: impl FnOnce(&mut Page) -> R) -> io::Result<R> {
self.note_access(id)?;
let r = f(self.cache.get_mut(&id).expect("cached above"));
self.dirty.insert(id);
Ok(r)
}
/// 记一次访问：命中加 hits，未命中读进来并记 misses
fn note_access(&mut self, id: PageId) -> io::Result<()> {
if self.cache.contains_key(&id) {
self.hits += 1;
self.touch(id);
return Ok(p);
} else {
self.misses += 1;
let p = self.read_raw(id)?;
if self.cache.len() >= self.capacity {
self.evict_one()?;
}
self.cache.insert(id, p);
}
self.misses += 1;
let p = self.read_raw(id)?;
if self.cache.len() >= self.capacity {
self.evict_one()?;
}
self.cache.insert(id, p.clone());
self.order.push_back(id);
Ok(p)
self.touch(id);
Ok(())
}
/// 写一页（走缓存）
@@ -257,7 +279,7 @@ impl PageFile {
/// 清空文件
pub fn truncate(&mut self) -> io::Result<()> {
self.cache.clear();
self.order.clear();
self.last_used.clear();
self.dirty.clear();
self.file.set_len(0)?;
self.file.seek(SeekFrom::Start(0))?;
chusql-storage/src/protocol.rs
+10
View File
@@ -43,6 +43,7 @@ pub enum Request {
columns: Vec<SchemaColumn>,
},
DropTable { table: String },
ListCatalog
}
#[derive(Debug, Serialize, Deserialize)]
@@ -59,4 +60,13 @@ pub enum Response {
},
Ok,
Error { message: String },
Catalog { schemas: Vec<TableSchemaWire> },
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TableSchemaWire {
pub table: String,
pub columns: Vec<SchemaColumn>,
#[serde(default)]
pub row_count: u64,
}
chusql-storage/src/wal.rs
+1 -1
View File
@@ -63,7 +63,7 @@ impl Wal {
.append(true)
.open(&self.path)?;
f.write_all(&bytes)?;
f.sync_all()?;
f.sync_data()?;
Ok(())
}