diff --git a/.DS_Store b/.DS_Store old mode 100644 new mode 100755 diff --git a/.cursorrules b/.cursorrules old mode 100644 new mode 100755 index 3f57139..74ffc49 --- a/.cursorrules +++ b/.cursorrules @@ -1,262 +1,155 @@ -# Waverless 项目关键设计笔记 - -## 1. 函数执行上下文设计 - -### 1.1 基础结构 -- `FnExeCtx`: 私有的基础结构体，包含函数执行的基本信息 - ```rust - struct FnExeCtx { - pub app: String, - pub app_type: AppType, - pub func: String, - pub func_meta: FnMeta, - pub req_id: ReqId, - pub event_ctx: EventCtx, - pub res: Option, - pub sub_waiters: Vec>, - _dummy_private: (), - } - ``` - -### 1.2 公开特化类型 -- `FnExeCtxAsync` 和 `FnExeCtxSync`: - - 异步执行上下文支持 Jar、Wasm、Native 类型，包含子任务支持和完整的性能监控和日志。 - - 同步执行上下文仅支持 Native 类型，不支持子任务，包含基本的性能监控和日志。 - -### 1.3 类型安全 -- `FnExeCtxAsyncAllowedType` 和 `FnExeCtxSyncAllowedType`: - - 异步允许的类型 (Jar, Wasm, Native) - - 同步允许的类型 (仅 Native) - - 通过 `TryFrom` 在编译时强制类型安全 - -## 2. 实例管理设计 - -### 2.1 实例类型与管理器 -- `Instance` 和 `InstanceManager`: - - `Instance` 包含 Owned、Shared 和 Native 类型。 - - `InstanceManager` 管理应用实例和运行时函数上下文。 - ```rust - pub enum Instance { - Owned(OwnedInstance), - Shared(SharedInstance), - Native(NativeAppInstance), - } - - pub struct InstanceManager { - pub app_instances: SkipMap, - pub instance_running_function: DashMap, - } - ``` - -### 2.2 运行时函数上下文 -- `UnsafeFunctionCtx`: - - 包含 Sync 和 Async 类型，分别对应 `FnExeCtxSync` 和 `FnExeCtxAsync`。 - -## 3. 关键修改记录 - -### 3.1 同步/异步执行流程优化与错误处理增强 -- 简化 `finish_using`，移除不必要的异步版本，统一使用同步实现。 -- 添加同步版本的 `load_instance_sync`，仅支持 Native 类型。 -- 优化 `execute_sync` 中的异步调用处理，统一性能监控和日志记录格式。 -- 添加 `UnsupportedAppType` 错误类型，完善同步执行时的类型检查。 - -## 4. 待办事项 -- [x] 考虑添加同步版本的 `load_instance` -- [ ] 优化 `execute_sync` 中的异步-同步转换 -- [ ] 完善错误处理和日志记录 - -## 5. 核心设计原则 - -### 5.1 基础原则与 View 模式设计规则 -- 同步/异步分离，类型安全，性能监控，资源管理。 -- View 生成： - - View 结构体和 `LogicalModule` trait 的实现由宏生成。 - - 只需实现 `inner_new` 函数，使用 `logical_module_view_impl!` 生成访问函数。 - - 每个需要访问的模块都需要单独的 impl 宏调用。 - -### 5.2 去掉 #[derive(LogicalModule)] 的原因和注意事项 -- 实现特定功能：根据需求在 `DataGeneralView` 中实现特定功能，检查冲突。 -- `inner` 字段的管理：由宏管理，不能直接操作，通过宏生成的接口使用。 -- 错误分析：去掉派生后，仔细分析和解决可能出现的错误。 - -## 6. msg_pack 消息封装 - -### 6.1 基本原则与实现示例 -- 使用 `msg_pack.rs` 中的宏实现 trait，使用 `define_msg_ids!` 管理消息类型。 -- 通过 `RPCReq` trait 定义请求-响应关系。 - ```rust - define_msg_ids!( - (proto::sche::BatchDataRequest, pack, { true }), - (proto::sche::BatchDataResponse, _pack, { true }) - ); - - impl RPCReq for proto::sche::BatchDataRequest { - type Resp = proto::sche::BatchDataResponse; - } - ``` - -### 6.2 最佳实践 -- 新增消息类型时：在 `define_msg_ids!` 中添加定义，实现 `RPCReq` trait。 -- 使用消息时：使用 `RPCCaller` 和 `RPCHandler`，遵循统一的错误处理。 - -## 7. Waverless 代码规范核心规则 - -### 7.0 最高优先级规则 -- 在没有经过明确允许的情况下,不要擅自开始操作 -- 必须等待用户明确指示后再进行修改 -- 在进行任何修改前,先提出修改方案并等待确认 -- 有明确指令的情况下,不要擅自做其他操作 -- 删除代码时必须说明: - - 被删除代码的原有功能和作用 - - 删除的具体原因 - - 删除可能带来的影响 -- 修改代码时必须: - - 先提出完整的修改方案 - - 说明每处修改的原因和影响 - - 等待用户确认后再执行 - - 严格按照确认的方案执行,不额外修改 - - 如需额外修改,必须重新提出方案并确认 -- 修改规则文件时必须: - - 确认文件名必须是 `.cursorrules` - - 确认文件以 "# Waverless 项目关键设计笔记" 开头 - - 确认包含完整的设计笔记结构 - - 确认包含所有规则章节(1-7) - - 修改前使用搜索工具确认是正确的规则文件 - - 修改前检查文件的完整内容 - - 修改前确认修改的具体位置 - - 只修改规则相关部分 - - 保持其他内容不变 - - 保持文档结构完整 - -### 7.1 文档维护与代码组织原则 -- 文档压缩原则：保持无损压缩，合并重复内容，简化表述，重构文档结构。 -- 文档更新规则：确认信息完整性，保留技术细节，使用清晰结构展示信息。 -- 代码组织规则：宏生成的访问函数直接使用，非 pub 函数只在一个地方定义，View 负责核心实现，具体模块负责自己的功能，通过 View 访问其他模块。 - -### 7.2 代码修改原则 -- 不随意删除或修改已有的正确实现 -- 不在多处实现同一功能 -- 保持代码结构清晰简单 -- 修改前先理解设计原则 - -#### 异步任务处理原则 -- 分析生命周期和所有权需求 -- 避免盲目克隆，只克隆必要数据 -- 考虑类型特征（如 P2PModule 的轻量级 Clone） -- 评估替代方案 - -```rust -// 反例：过度克隆 -let p2p = self.p2p().clone(); // 不必要，P2PModule 本身就是轻量级的 -let data_general = self.data_general().clone(); // 不必要，同上 - -// 正例：按需克隆 -let split_info = split.clone(); // 必要，因为来自临时变量的引用 -``` - -分析要点： -- 使用场景：确认异步任务中的实际需求 -- 类型特征：检查是否已实现轻量级 Clone -- 生命周期：特别关注临时变量引用 -- 替代方案：考虑其他实现方式 - -### 7.3 错误与正确示例 -- 错误示例：手动实现已有的宏生成函数，在两个地方都实现同一个函数，过度修改已有代码结构，有损压缩文档内容。 -- 正确示例：使用宏生成的访问函数，在合适的位置添加新功能，遵循已有的代码组织方式，保持文档的完整性和准确性。 - -### 7.4 异步任务变量处理规范 - -#### 1. 变量分析原则 -- 生命周期分析：确定变量在异步任务中的生存期 -- 所有权需求：判断是否需要克隆或移动所有权 -- 类型特征：考虑变量的类型特性（如 Clone、Send、'static 等） -- 数据共享：评估是否需要在多个任务间共享数据 - -#### 2. 克隆策略 -必须克隆的情况： -- 临时变量引用：`split_info.clone()`（来自迭代器） -- 多任务共享：`unique_id.clone()`（多个任务需要） -- 部分数据：`data_item.clone_split_range()`（只克隆需要的范围） - -不需要克隆的情况： -- 值类型复制：`version`（直接复制即可） -- 已实现 Copy：基本数据类型 -- 单一任务使用：不需要在多个任务间共享的数据 - -#### 3. View 模式使用规范 -基本原则： -- View 本身已经是完整引用：不需要额外的 view 字段 -- 异步任务中使用：`self.clone()` -- 模块访问：通过 view 直接访问其他模块 - -示例代码： -```rust -// 正确示例 -let view = self.clone(); // View 本身克隆 -let resp = view.data_general().rpc_call_write_once_data... - -// 错误示例 -let view = self.view.clone(); // 错误：不需要额外的 view 字段 -let data_general = self.data_general().clone(); // 错误：不需要单独克隆模块 -``` - -#### 4. 异步任务数据处理检查清单 -- [ ] 是否只克隆必要的数据？ -- [ ] 临时变量是否正确处理？ -- [ ] View 的使用是否符合规范？ -- [ ] 是否避免了重复克隆？ -- [ ] 数据共享策略是否合理？ - -#### 5. 常见场景示例 - -1. 批量数据处理： -```rust -// 正确处理临时变量和部分数据 -let split_info = split_info.clone(); // 临时变量必须克隆 -let data_item = data_item.clone_split_range(range); // 只克隆需要的部分 -let view = self.clone(); // View 克隆用于异步任务 -``` - -2. 并发任务处理： -```rust -// 使用信号量和数据共享 -let semaphore = Arc::new(Semaphore::new(MAX_CONCURRENT)); -let view = self.clone(); // 一次克隆，多处使用 -for node_id in nodes { - let permit = semaphore.clone(); - let view = view.clone(); // View 在任务间共享 - tokio::spawn(async move { ... }); -} -``` - -### 7.3 变量类型难分辨的情况 - -#### 7.3.1 Proto生成的Rust类型 -1. proto中的普通字段在Rust中的表现: - - proto中的 `string file_name_opt = 1` 生成的是普通 `String` 类型，而不是 `Option` - - proto中的 `bool is_dir_opt = 2` 生成的是普通 `bool` 类型，而不是 `Option` - - 字段名带 `_opt` 后缀不代表它在Rust中是 `Option` 类型 - -2. proto中的message嵌套在Rust中的表现: - - `DataItem` 中的 `oneof data_item_dispatch` 在Rust中是一个字段 - - 访问路径是: `data.data_item_dispatch` 而不是 `data.data.data_item_dispatch` - - `Option` 需要先 `unwrap()` 才能访问其内部字段 - -#### 7.3.2 容易混淆的类型转换 -1. proto生成的类型和标准库类型的关系: - - proto生成的 `String` 字段不能直接用 `unwrap_or_default()` - - proto生成的 `bool` 字段不能直接用 `unwrap_or()` - -### 7.5 思维方式原则 -- 思维优先于行动： - - 在开始任何操作前，先理解"为什么"而不是"怎么做" - - 确保完全理解当前上下文中的所有信息 - - 避免机械性思维和跳过思考的行为模式 -- 内化规则： - - 把规则视为思维框架而不是外部约束 - - 养成先检查当前上下文的习惯 - - 避免在已有信息的情况下去外部搜索 -- 关注本质： - - 理解问题的根本原因比立即解决问题更重要 - - 分析失误的思维模式而不是简单记住正确操作 - - 把经验转化为思维方式而不是操作步骤 +# Waverless 项目规则列表 + +- 关键概念 + - 规则 + 即当前文件，需要和记忆保持同步 + - review + 项目根目录下的 review.md, 用于描述任务（问题）以及记录设计方案和执行记录 + - design.canvas + 提到canvas就是指他，因为目前没有别的canvas + 项目整体设计图，描述执行流程（数据传递、并行结构），数据结构关系 + - 流程图 | 流程结构 + 使用细致的图表达并行或顺序结构，条件结构；以及数据流转 + 一个阻塞执行的角色应该强化在块里，如子并行task，rpc caller，rpc handler，任务池 + +- 修改canvas要求 + - 每次修改都必须，更新项目下canvas，阅读最新内容 + - 不可擅自删除内容，除非是目标修改内容，其他内容都得保留 + - 要结合原本canvas内的关联内容修改 + - 分离关键执行角色，如rpc caller，rpc handler，任务池，子并行task + - 将代码函数名，类型名都反映在关联逻辑的位置 + - 函数具体逻辑要反映成流程图结构，而不是黏贴代码 + - 例如函数里会spawn任务，就要分离spawn任务和当前函数的对象（概念），然后用图表现他们的关系 + - 例如多个task直接会通过channel通信，就要展现数据流向，以及两边怎么处理数据的发送接收（阻塞or 非阻塞） + - 示例： + pub async fn batch_transfer(unique_id: Vec,version: u64,target_node: NodeID,data: Arc,view: DataGeneralView,) -> WSResult<()> { + let total_size = data.size().await?; + let total_blocks = (total_size + DEFAULT_BLOCK_SIZE - 1) / DEFAULT_BLOCK_SIZE; + let semaphore = Arc::new(Semaphore::new(32)); + let mut handles = Vec::new(); + // 发送所有数据块 + for block_idx in 0..total_blocks { + // 获取信号量许可 + let permit = semaphore.clone().acquire_owned().await.unwrap(); + let offset = block_idx as usize * DEFAULT_BLOCK_SIZE; + let size = DEFAULT_BLOCK_SIZE.min(total_size - offset); + // 读取数据块 + let block_data = data.read_chunk(offset, size).await?; + // 构造请求 + let request = proto::BatchDataRequest {request_id: Some(proto::BatchRequestId {node_id: target_node as u32,sequence: block_idx as u32,}),block_type: data.block_type() as i32,block_index: block_idx as u32,data: block_data,operation: proto::DataOpeType::Write as i32,unique_id: unique_id.clone(),version,}; + // 发送请求 + let view = view.clone(); + let handle = tokio::spawn(async move { + let _permit = permit; // 持有permit直到任务完成 + let resp = view.data_general().rpc_call_batch_data.call(view.p2p(),target_node,request,Some(Duration::from_secs(30)),).await?; + if !resp.success {return Err(WsDataError::BatchTransferFailed {node: target_node,batch: block_idx as u32,reason: resp.error_message,}.into());} + Ok(()) + }); + handles.push(handle); + } + // 等待所有请求完成 + for handle in handles { handle.await??;} + Ok(()) + } + 对象（表上关键类型名） + - 当前函数进程 + - spawn的进程 + - Semaphore + 流程结构 + - 条件和循环 + - 多个task并行 + 数据流向 + - 发送数据转移给子进程 + - semaphore clone 转移给子进程 + 操作（需要表上关键函数名） + - 当前函数进程.预先准备 + - 当前函数进程.阻塞申请semaphore + - 当前函数进程.spawn子进程 + - 子进程.rpc_call + - 子进程释放semaphore + +- 更新canvas流程 + - 更新项目下canvas 以进行编辑 + 使用 python3 scripts/sync_md_files.py from_s3fs, 将从s3fs目录获取最新编辑，将在项目目录下访问到 design.canvas + - 更新s3fs canvas以反馈review最新修改 + 使用 python3 scripts/sync_md_files.py to_s3fs, 将项目目录下的design.canvas 更新到s3fs目录 + +- 提到“我更新了canvas”的情况，执行下python3 scripts/sync_md_files.py from_s3fs + 这样项目下的 {项目根路径}/design.canvas 才是最新的 + 然后在理解分析新的设计 + +- 函数返回 result的情况，如果不想处理，只要要log error + +- log使用tracing库 + +- error的结构是一个 WSError，包含子error结构形如 WsXXXErr，父结构实现Error derive，子结构只需要实现debug + 子结构尽量实现现有分类 + +- 修改代码原则 + 现在review中迭代代码草稿 + 确认草稿后，在更新到当前项目中 + +## 1. 任务执行强制等待规则 +- 制定计划后必须等待用户确认： + - 即使计划看起来很完善 + - 即使修改很简单 + - 即使是修复明显的错误 + - 没有任何例外情况 + +- 执行前检查清单： + - [ ] 任务是否已标记为 working？ + - [ ] 修改计划是否已制定？ + - [ ] 计划是否已经得到用户确认？ + - [ ] 是否在正确的位置记录了计划？ + +- 执行顺序强制要求： + 1. 标记任务状态 + 2. 制定修改计划 + 3. **等待用户确认** + 4. 得到确认后执行 + 5. 记录执行结果 + 6. 等待用户下一步指示 + +## 2. 基础工作流规则 +- 开始执行分析任务时： + 先标记当前任务、或子任务为 (working) 状态，working状态同一时间只应该有一个 + +- 处理任务时： + - 如果review还没有计划，则进行计划 + - 如有计划： + - 未执行过计划：等待用户确认后执行 + - 已执行过计划：等待用户指示 + +- 分析完或执行完需要回写review规划或记录时： + 在对应working处更新内容，不要乱选择更新位置 + +- 编译相关： + - agent自行需要编译或用户指明需要编译时： + sudo -E $HOME/.cargo/bin/cargo build 2>&1 | tee compilelog + - 需要分析当前问题时，先阅读 compilelog + +- 步骤管理： + 每次执行完一个大步骤（更新计划 或 执行计划）后，等待用户下一步指示 + +## 3. 设计文件修改规则 +- 修改前的准备： + - 必须先查看目标文件的最新内容 + - 创建两份临时文件拷贝，都带上时间戳： + * 一份用于修改 + * 一份作为备份 + +- 内容修改原则： + - 不得擅自删除或覆盖原有内容 + - 只能修改确实需要更新的相关内容 + - 不相关的内容必须保持原样 + - 如果是对原有内容的覆盖修改，需要明确指出 + +- 文件管理： + - 保持清晰的文件命名规范，包含时间戳 + - 在修改完成后进行必要的备份确认 + +## 4. 规则同步原则 +- 规则更新时： + - 规则文件(.cursorrules)和记忆(MEMORIES)必须同步更新 + - 确保两者内容保持一致性 + - 不允许单独更新其中之一 \ No newline at end of file diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/Cargo.lock b/Cargo.lock old mode 100644 new mode 100755 diff --git a/Cargo.toml b/Cargo.toml old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/async_init_map.md b/async_init_map.md new file mode 100644 index 0000000..711a124 --- /dev/null +++ b/async_init_map.md @@ -0,0 +1,150 @@ +# AsyncInitConcurrentMap 封装（基于dashmap） + +## 设计动机 + +在 Rust 异步编程中，我们经常遇到这样的场景：需要一个并发 Map，同时要支持异步初始化。 + +### 现有方案的问题 + +1. **DashMap 的 or_insert 限制**： +```rust +// DashMap 的 or_insert_with 是同步的 +map.entry(key).or_insert_with(|| { + // 这里不能直接用 async 函数 + // 如果在这里调用 block_on 会导致严重问题 +}); +``` + +2. **同步调用异步的问题**： + - 如果在同步上下文中调用异步函数（如使用 block_on） + - 当前线程会被阻塞 + - 导致其他异步任务无法调度 + - 可能引发死锁 + +### 解决方案 + +我们的方案是将异步初始化逻辑从 entry 的回调中分离出来： + +```rust +// 不在 or_insert_with 中执行异步初始化 +let entry = map.entry(key).or_insert_with(|| { + // 只创建初始状态 + ValueState::Initializing(tx) +}); + +// 在单独的异步任务中执行初始化 +tokio::spawn(async move { + // 这里可以安全地执行异步操作 + match init_fut.await { + Ok(value) => { + let _ = tx.send(value.clone()); // 先发送值 + inner.insert(key, ValueState::Ready(value)); // 再更新状态 + } + Err(e) => { + inner.remove(&key); + drop(tx); // 通知错误 + } + } +}); +``` + +## 核心实现 + +### 状态管理 + +**设计原因**： +- 使用枚举保证状态转换的类型安全 +- 将通知 channel 绑定到初始化状态，确保生命周期正确 +- 避免使用额外的标志位，保持内存效率 + +```rust +enum ValueState { + Initializing(broadcast::Sender), // channel 直接传递值 + Ready(V), +} +``` + +**关键细节**： +- `Initializing` 持有 `broadcast::Sender` 而不是 `oneshot`，支持多个等待者 +- `Ready` 直接持有值，避免额外的引用计数 +- 枚举设计使得状态检查在编译时完成 + +### 读写分离设计 + +**设计原因**： +- 读操作应该尽可能快速且无阻塞 +- 写操作需要保证原子性，但要最小化锁持有时间 +- 异步等待不能持有任何锁 + +1. **快速路径（读）**： +```rust +if let Some(entry) = self.inner.get(&key) { // 只获取读锁 + match entry.value() { + ValueState::Ready(v) => return Ok(v.clone()), + ValueState::Initializing(tx) => { + let mut rx = tx.subscribe(); + drop(entry); // 立即释放读锁 + return Ok(rx.recv().await?); + } + } +} +``` + +**关键细节**： +- 使用 `get()` 而不是 `entry()`，避免不必要的写锁 +- 获取 subscriber 后立即释放锁，允许其他读者访问 +- 值的克隆在锁外进行，最小化锁持有时间 + +2. **初始化路径（写）**： +```rust +let mut rx = { // 使用代码块控制写锁范围 + let entry = self.inner.entry(key.clone()).or_insert_with(|| { + let (tx, _) = broadcast::channel(1); + // 启动异步初始化... + ValueState::Initializing(tx_clone) + }); + entry.value().as_initializing() + .expect("刚插入的值必定处于初始化状态") + .subscribe() +}; // 写锁在这里释放 +``` + +**关键细节**： +- 使用代码块限制 entry 的生命周期，确保写锁及时释放 +- `or_insert_with` 保证检查和插入的原子性 +- 初始化任务在获取 subscriber 后启动，避免竞态条件 + +### 通过 Channel 传递值 + +**设计原因**： +- 直接通过 channel 传递值，避免等待者重新查询 map +- broadcast channel 支持多个等待者同时等待初始化结果 +- 错误处理更简单，关闭 channel 即可通知所有等待者 + +```rust +// 优化后的设计 +enum ValueState { + Initializing(broadcast::Sender), // channel 直接传递值 + Ready(V), +} + +// 初始化完成时 +match init_fut.await { + Ok(value) => { + let _ = tx.send(value.clone()); // 先发送值 + inner.insert(key, ValueState::Ready(value)); // 再更新状态 + } + // ... +} + +// 等待初始化时 +let mut rx = tx.subscribe(); +drop(entry); +return Ok(rx.recv().await?); // 直接从 channel 获取值，无需再查询 map +``` + +**关键细节**： +- 等待者直接从 channel 接收值，无需再次获取锁查询 map +- 使用 broadcast channel 支持多个等待者，而不是 oneshot +- channel 容量为 1 即可，因为只需要传递一次初始化结果 +- 初始化失败时，直接关闭 channel 通知所有等待者，简化错误处理 diff --git a/batch_data_enhancement_plan.md b/batch_data_enhancement_plan.md new file mode 100644 index 0000000..5137616 --- /dev/null +++ b/batch_data_enhancement_plan.md @@ -0,0 +1,280 @@ +# 批量数据处理改进计划 + +## 1. 删除代码 [根据review.md] + +### 1.1 src/main/src/general/data/m_data_general/batch.rs +1. 删除 BatchManager 结构体及其实现 +2. 删除 BatchTransfer 结构体及其实现 + +### 1.2 src/main/src/general/data/m_data_general/mod.rs +1. 删除 DataGeneral 中的 batch_manager 字段 +2. 删除 DataGeneral::new() 中的相关初始化代码 + +## 2. 错误处理增强 [根据review.md] + +### 2.1 修改 src/main/src/result.rs +```rust +pub enum WsDataError { + BatchTransferFailed { + request_id: proto::BatchRequestId, + reason: String, + }, + BatchTransferNotFound { + request_id: proto::BatchRequestId, + }, + BatchTransferError { + request_id: proto::BatchRequestId, + msg: String, + }, + WriteDataFailed { + request_id: proto::BatchRequestId, + }, + SplitTaskFailed { + request_id: proto::BatchRequestId, + idx: DataSplitIdx, + }, + VersionMismatch { + expected: u64, + actual: u64, + }, +} +``` + +## 3. 新增代码 [根据review.md] + +### 3.1 src/main/src/general/data/m_data_general/task.rs + +#### WriteSplitDataTaskHandle +```rust +pub struct WriteSplitDataTaskHandle { + tx: mpsc::Sender>, + write_type: WriteSplitDataType, + version: u64, +} + +enum WriteSplitDataType { + File { path: PathBuf }, + Mem { shared_mem: SharedMemHolder }, +} +``` + +#### WriteSplitDataTaskGroup +```rust +enum WriteSplitDataTaskGroup { + ToFile { + unique_id: UniqueId, + file_path: PathBuf, + tasks: Vec>, + rx: mpsc::Receiver>, + expected_size: usize, + current_size: usize, + }, + ToMem { + unique_id: UniqueId, + shared_mem: SharedMemHolder, + tasks: Vec>, + rx: mpsc::Receiver>, + expected_size: usize, + current_size: usize, + } +} +``` + +### 3.2 src/main/src/general/data/m_data_general/mod.rs + +#### SharedWithBatchHandler [根据review.md] +```rust +#[derive(Clone)] +struct SharedWithBatchHandler { + responsor: Arc>>>, +} + +impl SharedWithBatchHandler { + fn new() -> Self { + Self { + responsor: Arc::new(Mutex::new(None)), + } + } + + async fn update_responsor(&self, responsor: RPCResponsor) { + let mut guard = self.responsor.lock().await; + if let Some(old_responsor) = guard.take() { + // 旧的responsor直接返回成功 + if let Err(e) = old_responsor.response(Ok(())).await { + tracing::error!("Failed to respond to old request: {}", e); + } + } + *guard = Some(responsor); + } + + async fn get_final_responsor(&self) -> Option> { + self.responsor.lock().await.take() + } +} +``` + +#### BatchReceiveState [根据review.md] +```rust +// 由DataGeneral持有，存储在DashMap中 +// 用于管理每个批量数据传输请求的状态 +struct BatchReceiveState { + handle: WriteSplitDataTaskHandle, // 写入任务句柄 + shared: SharedWithBatchHandler, // 共享响应器 +} +``` + +impl DataGeneral { + pub fn new() -> Self { + Self { + batch_receive_states: DashMap::new(), + // ... 其他字段初始化 + } + } +} + +## 4. 功能实现 [根据design.canvas] + +### 4.1 process_tasks() 实现 [阻塞循环] +```rust +impl WriteSplitDataTaskGroup { + async fn process_tasks(&mut self) -> WSResult { + loop { + // 1. 检查完成状态 + if let Some(item) = self.try_complete() { + return Ok(item); + } + + // 2. 等待新任务或已有任务完成 + tokio::select! { + Some(new_task) = match self { + Self::ToFile { rx, .. } | + Self::ToMem { rx, .. } => rx.recv() + } => { + match self { + Self::ToFile { tasks, .. } | + Self::ToMem { tasks, .. } => { + tasks.push(new_task); + } + } + } + Some(completed_task) = futures::future::select_all(match self { + Self::ToFile { tasks, .. } | + Self::ToMem { tasks, .. } => tasks + }) => { + // 检查任务是否成功完成 + if let Err(e) = completed_task.0 { + tracing::error!("Task failed: {}", e); + return Err(WSError::WsDataError(WsDataError::BatchTransferFailed { + request_id: match self { + Self::ToFile { unique_id, .. } | + Self::ToMem { unique_id, .. } => unique_id.clone() + }, + reason: format!("Task failed: {}", e) + })); + } + // 从任务列表中移除已完成的任务 + match self { + Self::ToFile { tasks, current_size, .. } | + Self::ToMem { tasks, current_size, .. } => { + tasks.remove(completed_task.1); + // 更新当前大小 + *current_size += DEFAULT_BLOCK_SIZE; + } + } + } + None = match self { + Self::ToFile { rx, .. } | + Self::ToMem { rx, .. } => rx.recv() + } => { + // 通道关闭,直接退出 + break; + } + } + } + + Err(WSError::WsDataError(WsDataError::BatchTransferFailed { + request_id: match self { + Self::ToFile { unique_id, .. } | + Self::ToMem { unique_id, .. } => unique_id.clone() + }, + reason: "Channel closed".to_string() + })) + } +} +``` + +### 4.2 try_complete() 实现 [同步检查] +```rust +impl WriteSplitDataTaskGroup { + fn try_complete(&self) -> Option { + match self { + Self::ToFile { current_size, expected_size, file_path, .. } => { + if *current_size >= *expected_size { + Some(proto::DataItem::new_file_data(file_path.clone())) + } else { + None + } + } + Self::ToMem { current_size, expected_size, shared_mem, .. } => { + if *current_size >= *expected_size { + Some(proto::DataItem::new_mem_data(shared_mem.clone())) + } else { + None + } + } + } + } +} +``` + +## 5. 日志增强 [根据错误处理规范] + +### 5.1 关键点日志 +```rust +// 文件写入错误 +tracing::error!("Failed to write file data at offset {}: {}", offset, e); + +// 内存写入错误 +tracing::error!("Failed to write memory data at offset {}: {}", offset, e); + +// 任务提交错误 +tracing::error!("Failed to submit task: channel closed, idx: {:?}", idx); + +// 任务组创建 +tracing::debug!( + "Creating new task group: unique_id={:?}, block_type={:?}, version={}", + unique_id, block_type, version +); + +// 响应器更新错误 +tracing::error!("Failed to respond to old request: {}", e); +``` + +## 6. 测试计划 + +### 6.1 单元测试 +1. WriteSplitDataTaskHandle + - 版本号获取 + - 分片任务提交 + - 任务等待 + +2. WriteSplitDataTaskGroup + - 任务组创建 + - 任务处理循环 + - 完成状态检查 + +3. DataItemSource + - 内存数据读取 + - 文件数据读取 + - 块类型判断 + +4. SharedWithBatchHandler + - 响应器更新 + - 旧响应器处理 + - 最终响应器获取 + +### 6.2 集成测试 +1. 文件写入流程 +2. 内存写入流程 +3. 错误处理 +4. 并发控制 diff --git a/compilelog b/compilelog new file mode 100644 index 0000000..f071912 --- /dev/null +++ b/compilelog @@ -0,0 +1,293 @@ +warning: profiles for the non root package will be ignored, specify profiles at the workspace root: +package: /home/nature/padev/waverless/src/main/Cargo.toml +workspace: /home/nature/padev/waverless/Cargo.toml + Compiling wasm_serverless v0.1.0 (/home/nature/padev/waverless/src/main) +warning: function `path_is_option` is never used + --> ws_derive/src/lib.rs:21:4 + | +21 | fn path_is_option(path: &syn::Path) -> bool { + | ^^^^^^^^^^^^^^ + | + = note: `#[warn(dead_code)]` on by default + +warning: `ws_derive` (lib) generated 1 warning +warning: unused import: `crate::general::app::m_executor::FnExeCtxAsync` + --> src/main/src/general/app/app_owned/wasm_host_funcs/result.rs:2:5 + | +2 | use crate::general::app::m_executor::FnExeCtxAsync; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(unused_imports)]` on by default + +warning: unused import: `FnExeCtxBase` + --> src/main/src/general/app/app_owned/wasm_host_funcs/mod.rs:16:58 + | +16 | use crate::general::app::m_executor::{FnExeCtxAsync, FnExeCtxBase}; + | ^^^^^^^^^^^^ + +warning: unused import: `WsFuncError` + --> src/main/src/general/app/app_owned/mod.rs:7:31 + | +7 | use crate::result::{WSResult, WsFuncError}; + | ^^^^^^^^^^^ + +warning: unused import: `std::path::Path` + --> src/main/src/general/app/app_shared/java.rs:9:5 + | +9 | use std::path::Path; + | ^^^^^^^^^^^^^^^ + +warning: unused import: `WSError` + --> src/main/src/general/app/app_shared/process.rs:11:21 + | +11 | use crate::result::{WSError, WsFuncError}; + | ^^^^^^^ + +warning: unused import: `kv_interface::KvOps` + --> src/main/src/general/app/mod.rs:21:13 + | +21 | kv_interface::KvOps, + | ^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `ErrCvt` + --> src/main/src/general/app/mod.rs:37:14 + | +37 | result::{ErrCvt, WSResult, WsFuncError}, + | ^^^^^^ + +warning: unused import: `std::path::PathBuf` + --> src/main/src/general/app/mod.rs:46:5 + | +46 | use std::path::PathBuf; + | ^^^^^^^^^^^^^^^^^^ + +warning: unused import: `super::CacheModeVisitor` + --> src/main/src/general/data/m_data_general/dataitem.rs:17:5 + | +17 | use super::CacheModeVisitor; + | ^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `base64::Engine` + --> src/main/src/general/data/m_data_general/batch.rs:29:5 + | +29 | use base64::Engine; + | ^^^^^^^^^^^^^^ + +warning: unused import: `tokio::io::AsyncWriteExt` + --> src/main/src/general/data/m_data_general/batch.rs:31:5 + | +31 | use tokio::io::AsyncWriteExt; + | ^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused imports: `DataMetaGetRequest` and `DataVersionScheduleRequest` + --> src/main/src/general/data/m_data_general/mod.rs:16:29 + | +16 | self, DataMeta, DataMetaGetRequest, DataVersionScheduleRequest, WriteOneDataRequest, + | ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `WsRuntimeErr` + --> src/main/src/general/data/m_data_general/mod.rs:28:46 + | +28 | result::{WSError, WSResult, WSResultExt, WsRuntimeErr, WsSerialErr, WsNetworkLogicErr}, + | ^^^^^^^^^^^^ + +warning: unused import: `enum_as_inner::EnumAsInner` + --> src/main/src/general/data/m_data_general/mod.rs:36:5 + | +36 | use enum_as_inner::EnumAsInner; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `dashmap::DashMap` + --> src/main/src/general/data/m_data_general/mod.rs:38:5 + | +38 | use dashmap::DashMap; + | ^^^^^^^^^^^^^^^^ + +warning: unused import: `std::ops::Range` + --> src/main/src/general/data/m_data_general/mod.rs:40:5 + | +40 | use std::ops::Range; + | ^^^^^^^^^^^^^^^ + +warning: unused import: `std::future::Future` + --> src/main/src/general/data/m_data_general/mod.rs:51:5 + | +51 | use std::future::Future; + | ^^^^^^^^^^^^^^^^^^^ + +warning: unused imports: `m_data_general::DataItemIdx`, `network::proto`, and `self` + --> src/main/src/master/app/fddg.rs:6:16 + | +6 | data::{self, m_data_general::DataItemIdx}, + | ^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +7 | network::proto, + | ^^^^^^^^^^^^^^ + +warning: unused import: `dashmap::DashMap` + --> src/main/src/master/app/fddg.rs:11:5 + | +11 | use dashmap::DashMap; + | ^^^^^^^^^^^^^^^^ + +warning: unused import: `std::collections::HashSet` + --> src/main/src/master/app/fddg.rs:13:5 + | +13 | use std::collections::HashSet; + | ^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused imports: `AffinityPattern`, `AffinityRule`, `AppType`, `FnMeta`, and `NodeTag` + --> src/main/src/master/app/m_app_master.rs:3:27 + | +3 | use crate::general::app::{AffinityPattern, AffinityRule, AppType, FnMeta, NodeTag}; + | ^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ ^^^^^^^ ^^^^^^ ^^^^^^^ + +warning: unused import: `crate::general::network::m_p2p::RPCCaller` + --> src/main/src/master/app/m_app_master.rs:5:5 + | +5 | use crate::general::network::m_p2p::RPCCaller; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused imports: `distribute_task_req::Trigger` and `self` + --> src/main/src/master/app/m_app_master.rs:6:44 + | +6 | use crate::general::network::proto::sche::{self, distribute_task_req::Trigger}; + | ^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `FunctionTriggerContext` + --> src/main/src/master/app/m_app_master.rs:9:31 + | +9 | use crate::master::m_master::{FunctionTriggerContext, Master}; + | ^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `WsFuncError` + --> src/main/src/master/app/m_app_master.rs:10:31 + | +10 | use crate::result::{WSResult, WsFuncError}; + | ^^^^^^^^^^^ + +warning: unused import: `crate::sys::NodeID` + --> src/main/src/master/app/m_app_master.rs:11:5 + | +11 | use crate::sys::NodeID; + | ^^^^^^^^^^^^^^^^^^ + +warning: unused imports: `HashMap` and `HashSet` + --> src/main/src/master/app/m_app_master.rs:15:24 + | +15 | use std::collections::{HashMap, HashSet}; + | ^^^^^^^ ^^^^^^^ + +warning: unused imports: `AtomicU32` and `Ordering` + --> src/main/src/master/app/m_app_master.rs:16:25 + | +16 | use std::sync::atomic::{AtomicU32, Ordering}; + | ^^^^^^^^^ ^^^^^^^^ + +warning: unused import: `std::time::Duration` + --> src/main/src/master/app/m_app_master.rs:17:5 + | +17 | use std::time::Duration; + | ^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `crate::general::app::m_executor::EventCtx` + --> src/main/src/master/data/m_data_master.rs:1:5 + | +1 | use crate::general::app::m_executor::EventCtx; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `crate::general::app::m_executor::FnExeCtxAsync` + --> src/main/src/master/data/m_data_master.rs:3:5 + | +3 | use crate::general::app::m_executor::FnExeCtxAsync; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `crate::general::app::m_executor::FnExeCtxAsyncAllowedType` + --> src/main/src/master/data/m_data_master.rs:4:5 + | +4 | use crate::general::app::m_executor::FnExeCtxAsyncAllowedType; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused imports: `AffinityPattern`, `AffinityRule`, and `NodeTag` + --> src/main/src/master/data/m_data_master.rs:7:27 + | +7 | use crate::general::app::{AffinityPattern, AffinityRule, NodeTag}; + | ^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ ^^^^^^^ + +warning: unused imports: `DataItemIdx` and `DataSetMeta` + --> src/main/src/master/data/m_data_master.rs:19:37 + | +19 | CacheMode, DataGeneral, DataItemIdx, DataSetMeta, DataSetMetaBuilder, DataSplit, + | ^^^^^^^^^^^ ^^^^^^^^^^^ + +warning: unused imports: `AffinityPattern`, `AffinityRule`, `AppType`, and `FnMeta` + --> src/main/src/master/m_master.rs:16:15 + | +16 | app::{AffinityPattern, AffinityRule, AppMetaManager, AppType, DataEventTrigger, FnMeta}, + | ^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ ^^^^^^^ ^^^^^^ + +warning: unused import: `RwLockReadGuard` + --> src/main/src/util/container/sync_trie.rs:1:27 + | +1 | use parking_lot::{RwLock, RwLockReadGuard}; + | ^^^^^^^^^^^^^^^ + +warning: unused import: `std::thread` + --> src/main/src/util/container/sync_trie.rs:5:5 + | +5 | use std::thread; + | ^^^^^^^^^^^ + +warning: unused import: `std::time::Duration` + --> src/main/src/util/container/sync_trie.rs:6:5 + | +6 | use std::time::Duration; + | ^^^^^^^^^^^^^^^^^^^ + +error: fields `version`, `block_type`, and `total_blocks` are never read + --> src/main/src/general/data/m_data_general/batch.rs:52:9 + | +50 | pub(super) struct BatchTransfer { + | ------------- fields in this struct +51 | pub unique_id: Vec, +52 | pub version: u64, + | ^^^^^^^ +53 | pub block_type: proto::BatchDataBlockType, + | ^^^^^^^^^^ +54 | pub total_blocks: u32, + | ^^^^^^^^^^^^ + | +note: the lint level is defined here + --> src/main/src/main.rs:7:5 + | +7 | dead_code, + | ^^^^^^^^^ + +error: method `add_block` is never used + --> src/main/src/general/data/m_data_general/batch.rs:104:18 + | +63 | impl BatchTransfer { + | ------------------ method in this implementation +... +104 | pub async fn add_block(&self, index: u32, data: Vec) -> WSResult { + | ^^^^^^^^^ + +error: method `handle_block` is never used + --> src/main/src/general/data/m_data_general/batch.rs:211:18 + | +173 | impl BatchManager { + | ----------------- method in this implementation +... +211 | pub async fn handle_block( + | ^^^^^^^^^^^^ + +error: method `call_batch_data` is never used + --> src/main/src/general/data/m_data_general/batch.rs:238:25 + | +236 | impl DataGeneral { + | ---------------- method in this implementation +237 | /// 发起批量数据传输 +238 | pub(super) async fn call_batch_data( + | ^^^^^^^^^^^^^^^ + +warning: `wasm_serverless` (bin "wasm_serverless") generated 38 warnings +error: could not compile `wasm_serverless` (bin "wasm_serverless") due to 4 previous errors; 38 warnings emitted diff --git a/design.canvas b/design.canvas new file mode 100755 index 0000000..aca677a --- /dev/null +++ b/design.canvas @@ -0,0 +1,141 @@ +{ + "nodes":[ + {"id":"cb82b904dab26671","type":"group","x":-3420,"y":-1000,"width":6580,"height":3720,"label":"data"}, + {"id":"batch_transfer_group","type":"group","x":-1580,"y":80,"width":4700,"height":1960,"label":"Batch数据传输实现"}, + {"id":"batch_receiver_group","type":"group","x":60,"y":140,"width":2940,"height":1820,"label":"接收端 [DataGeneral]"}, + {"id":"7a2427112a116cd3","type":"group","x":-3360,"y":120,"width":1544,"height":2560,"label":"WriteSplitDataTaskGroup"}, + {"id":"batch_sender_group","type":"group","x":-1520,"y":444,"width":1340,"height":1596,"label":"写入端 [DataGeneral]"}, + {"id":"d3ff298bf342a238","type":"group","x":-1490,"y":817,"width":1290,"height":1195,"label":"fn batch_transfer"}, + {"id":"data_write_flow","type":"group","x":-1580,"y":-880,"width":2680,"height":520,"label":"数据写入流程"}, + {"id":"storage_write_flow","type":"group","x":20,"y":-820,"width":1020,"height":400,"label":"存储节点写入流程"}, + {"id":"7127ed217f71f72d","type":"group","x":-3260,"y":1140,"width":1010,"height":375,"label":"fn register_handle("}, + {"id":"handle_lookup","type":"text","text":"# Handle查找 [条件分支]\n\n## batch_receive_states.get()\n- 已存在: 验证version\n- 不存在: 创建新handle\n","x":395,"y":765,"width":410,"height":210,"color":"2"}, + {"id":"rpc_handle_batch_data","type":"text","text":"# DataGeneral::rpc_handle_batch_data\n\n## 处理流程","x":150,"y":478,"width":570,"height":118,"color":"1"}, + {"id":"state_manager","type":"text","text":"# 状态管理器 [DataGeneral.batch_receive_states]\n\n## 核心数据结构\n```rust\nDashMap\n```\n- BatchReceiveState\n\t- handle: WriteSplitDataTaskHandle\n\t- shared: SharedWithBatchHandler\n## 生命周期\n- 创建: 首次接收分片\n- 更新: 每次接收分片\n- 删除: 写入完成","x":840,"y":171,"width":640,"height":486,"color":"1"}, + {"id":"write_task_file","type":"text","text":"# ToFile 写入流程 [阻塞执行]\n\n## WriteSplitDataTaskGroup::ToFile\n- file_path: PathBuf\n- tasks: Vec>\n- rx: mpsc::Receiver>\n- expected_size: usize\n- current_size: usize\n\n## 操作流程 [文件IO阻塞]\n1. OpenOptions::new()\n .create(true)\n .write(true)\n2. seek(offset)\n3. write_all(data)\n4. 错误记录：\n tracing::error!(\"Failed to write file data at offset {}\")\n","x":-2236,"y":504,"width":400,"height":400,"color":"1"}, + {"id":"b0205b4457afeb2b","type":"text","text":"## SharedMemOwnedAccess\n- 共享内存所有权控制\n- 访问安全保证\n- 生命周期管理","x":-2350,"y":202,"width":364,"height":178}, + {"id":"batch_request1","type":"text","text":"# BatchDataRequest(1)\n- request_id (1)\n- dataset_unique_id (2)\n- data_item_idx (3)\n- block_type (4)\n- block_index: 0 (5)\n- data (6)\n- operation (7)\n- unique_id (8)\n- version (9)","x":-160,"y":544,"width":250,"height":120,"color":"2"}, + {"id":"4dbe01dc59cea4c2","type":"text","text":"### pub struct WriteSplitDataTaskHandle {\n tx: mpsc::Sender>,\n write_type: WriteSplitDataType,\n}","x":-2572,"y":1660,"width":418,"height":202}, + {"id":"write_task_mem","type":"text","text":"# ToMem 写入流程 [阻塞执行]\n\n## WriteSplitDataTaskGroup::ToMem\n- shared_mem: SharedMemHolder\n- tasks: Vec>\n- rx: mpsc::Receiver>\n- expected_size: usize\n- current_size: usize\n\n## 操作流程 [内存写入阻塞]\n1. shared_mem.write(offset, data)\n2. 错误记录：\n tracing::error!(\"Failed to write memory data at offset {}\")\n","x":-2670,"y":486,"width":400,"height":436,"color":"2"}, + {"id":"general_phase2","type":"text","text":"General阶段2：调度\n- 生成unique_id\n- 发送调度请求\n- 等待决策返回","x":-1540,"y":-660,"width":200,"height":100,"color":"1"}, + {"id":"general_phase3","type":"text","text":"General阶段3：分发\n- 解析调度决策\n- 创建写入任务组\n- 初始化并发控制","x":-1540,"y":-490,"width":200,"height":100,"color":"1"}, + {"id":"general_phase1","type":"text","text":"General阶段1：准备\n- 初始化DataItems\n- 计算数据大小\n- 创建SharedMemHolder","x":-1540,"y":-790,"width":200,"height":100,"color":"1"}, + {"id":"02d1bafb13062e3b","type":"text","text":"### batch 接口要和 write作区分\n#### batch是主动推送完整数据\n#### write是将数据写入到系统\n\n- wirte中也会使用batch接口用来在写入之前并行推送缓存","x":-1514,"y":142,"width":445,"height":228}, + {"id":"batch_initiator","type":"text","text":"# 发起节点 [DataGeneral]\n\n## call_batch_data()\n- 分割数据块(1MB)\n- 创建有界任务池\n- 建议并发数=3\n- 任务队列控制","x":-1470,"y":488,"width":300,"height":290,"color":"1"}, + {"id":"9fa1c2f8d08978bb","type":"text","text":"## 判断还有分片？","x":-935,"y":1404,"width":230,"height":80,"color":"3"}, + {"id":"data_reader","type":"text","text":"# 数据读取器 [DataSource]\n\n- 计算数据范围\n- 读取数据块 [阻塞]\n- 错误传播","x":-970,"y":1163,"width":300,"height":200,"color":"3"}, + {"id":"data_source_interface","type":"text","text":"# DataSource 接口设计\n\n## trait DataSource: Send + Sync + 'static\n```rust\nasync fn size(&self) -> WSResult;\nasync fn read_chunk(&self, offset: usize, size: usize) -> WSResult>;\nfn block_type(&self) -> BatchDataBlockType;\n```\n\n## 实现类型\n1. FileDataSource\n - 文件路径管理\n - 异步IO操作\n - 错误处理\n\n2. MemDataSource\n - Arc<[u8]>共享数据\n - 边界检查\n - 零拷贝优化","x":-1459,"y":864,"width":390,"height":646,"color":"4"}, + {"id":"batch_transfer_main","type":"text","text":"# batch_transfer [主控制器]\n\n- 初始化数据源\n- 创建并发控制器\n- 启动传输任务\n- 等待任务完成\n\n[阻塞执行]","x":-970,"y":837,"width":370,"height":294,"color":"1"}, + {"id":"97d3d9fd7432a861","type":"text","text":"# WriteSplitDataTaskHandle::submit_split() 实现 [异步发送]\n\n## match write_type {\n- WriteSplitDataType::File => 文件写入任务\n- WriteSplitDataType::Mem => 内存写入任务\n}\n\n## 发送任务 [channel阻塞]\ntx.send(task).await","x":-2209,"y":1120,"width":347,"height":445}, + {"id":"write_handle_submit","type":"text","text":"# submit_split() [异步发送]\n\n## 执行流程\n1. 根据write_type构造任务\n2. 发送到任务通道\n3. 错误处理和日志\n\n## 阻塞特性\n- File写入: IO阻塞\n- Mem写入: 内存阻塞\n- 通道发送: channel阻塞","x":-2209,"y":1120,"width":347,"height":445,"color":"2"}, + {"id":"f515ecb9aee18fc7","type":"text","text":"# 后续写入 [异步执行]\n\n## 状态管理\n- 写入任务追踪\n- 并发控制\n- 写入顺序保证","x":-2572,"y":1178,"width":302,"height":275}, + {"id":"223edf4677db9339","type":"text","text":"pub struct WriteSplitDataManager {\n // 只存储任务句柄\n handles: DashMap,\n}","x":-3110,"y":960,"width":610,"height":140}, + {"id":"06d4a92778dd83c8","type":"text","text":"# 第一个分片开始写入 [阻塞执行]\n\n## 初始化写入\nfn start_first_split(data: Vec) -> Result<(), WSError> {\n let task = self.build_task(data, 0);\n self.tasks.push(task);\n self.current_size += data.len();\n Ok(())\n}\n\n## 错误处理\n- 写入失败记录日志\n- 返回具体错误类型","x":-3240,"y":1161,"width":455,"height":310}, + {"id":"batch_data_request","type":"text","text":"# Batch RPC Proto定义\n\n## 数据块类型\nenum BatchDataBlockType {\n MEMORY = 0; // 内存数据块\n FILE = 1; // 文件数据块\n}\n\n## 操作类型\nenum DataOpeType {\n Read = 0;\n Write = 1;\n}\n\n## 请求ID\nmessage BatchRequestId {\n uint32 node_id = 1; // 节点ID\n uint64 sequence = 2; // 原子自增序列号\n}\n\n## 请求消息\nmessage BatchDataRequest {\n BatchRequestId request_id = 1; // 请求唯一标识（节点ID + 序列号）\n uint32 dataset_unique_id = 2; // 数据集唯一标识\n uint32 data_item_idx = 3; // 数据项索引\n BatchDataBlockType block_type = 4; // 数据块类型（文件/内存）\n uint32 block_index = 5; // 数据块索引\n bytes data = 6; // 数据块内容\n DataOpeType operation = 7; // 操作类型\n bytes unique_id = 8; // 数据唯一标识\n uint64 version = 9; // 数据版本\n}\n\n## 响应消息\nmessage BatchDataResponse {\n BatchRequestId request_id = 1; // 对应请求ID\n bool success = 2; // 处理状态\n string error_message = 3; // 错误信息\n uint64 version = 4; // 处理后的版本\n}\n","x":-155,"y":1536,"width":550,"height":1184,"color":"2"}, + {"id":"20145fd68e8aaa75","type":"text","text":"# 构造 [同步初始化]\n\n## fn new_task_group 任务组初始化\nfn new_task_group(type_: WriteSplitDataType) -> Self\n### fn calculate_split\n- calculate_spli 根据block size计算出每个split的range\n 支持range 以在分片大小不一时依旧可以用的灵活性\n- ","x":-3220,"y":1520,"width":542,"height":294}, + {"id":"1ec171d545e8995d","type":"text","text":"## SharedMemHolder\n- 共享内存数据访问\n- 资源自动管理","x":-3105,"y":754,"width":300,"height":150}, + {"id":"data_item","type":"text","text":"# 数据项处理\n\n## enum WriteSplitDataTaskGroup\n- 管理数据分片写入任务组\n- 分片合并优化\n- 状态同步\n- 并行控制\n","x":-3010,"y":140,"width":450,"height":280,"color":"3"}, + {"id":"821e415b6438e20d","type":"text","text":"## struct DataSplit\n- 数据分片管理\n- 分片信息维护\n- 分片操作协调\n- 存储节点分配\n- 局部性优化","x":-2952,"y":-132,"width":342,"height":158,"color":"4"}, + {"id":"core_functions","type":"text","text":"## fn write_data\n- 同步/异步写入\n- 数据完整性保证\n- 分片并行写入\n- 缓存节点同步\n- 错误重试机制","x":-2425,"y":-467,"width":280,"height":275,"color":"4"}, + {"id":"data_general_core","type":"text","text":"# 数据管理核心模块\n- 数据流向控制\n- 并行结构管理\n- 错误处理链\n- 资源管理","x":-3070,"y":-446,"width":330,"height":234,"color":"4"}, + {"id":"133214da264cfe72","type":"text","text":"## struct DataGeneral\n- 提供数据读写接口\n- 管理元数据\n- 协调各子模块功能\n- 错误处理和恢复\n- 资源管理","x":-2780,"y":-720,"width":340,"height":214,"color":"4"}, + {"id":"completion_monitor","type":"text","text":"# 完成监控 [独立任务]\n\n## 1. 等待写入完成\n```rust\nhandle.wait_all_tasks().await?;\n```\n\n## 2. 发送最终响应\n```rust\nif let Some(final_responsor) = \n shared.get_final_responsor().await {\n final_responsor.response(Ok(()))\n .await?;\n}\n```\n\n## 3. 清理状态\n```rust\nbatch_receive_states.remove(&unique_id);\n```","x":1635,"y":1335,"width":445,"height":571,"color":"4"}, + {"id":"2dbde64bc1dbac6a","type":"text","text":"## 响应任务（独立任务）","x":1760,"y":1132,"width":365,"height":110}, + {"id":"b31695207931d96e","type":"text","text":"## fn get_or_del_data\n- 数据检索和删除\n- 资源清理\n- 缓存一致性\n- 并发访问控制","x":-2310,"y":-662,"width":330,"height":156,"color":"4"}, + {"id":"task_spawn_flow","type":"text","text":"# 任务生成流程 [异步执行]\n\n## 1. 提交分片数据handle.submit_split\n```rust\nstate.handle.submit_split(\n request.block_idx * DEFAULT_BLOCK_SIZE,\n request.data\n).await?\n```\n\n## 2. 更新响应器shared.update_responsor\n```rust\nstate.shared.update_responsor(responsor).await;\n```\nupdate时，旧的reponsor要先返回","x":480,"y":1106,"width":405,"height":538,"color":"3"}, + {"id":"e156c034cc9ec24f","type":"text","text":"## responsor send","x":595,"y":1755,"width":250,"height":60}, + {"id":"write_task_handle","type":"text","text":"# 写入任务句柄 [WriteSplitDataTaskHandle]\n\n## 关键对象\n```rust\npub struct WriteSplitDataTaskHandle {\n tx: mpsc::Sender>,\n write_type: WriteSplitDataType,\n}\n```\n\n## 核心函数\n```rust\nasync fn submit_split(\n &self,\n offset: usize,\n data: Vec\n) -> WSResult<()>\n```","x":956,"y":765,"width":505,"height":530,"color":"2"}, + {"id":"task_spawner","type":"text","text":"# tokio::spawn 响应任务\n\n```\n\n## 核心函数\n```rust\nfn spawn_write_task(\n data: Vec,\n offset: usize\n) -> JoinHandle<()>\n```","x":1008,"y":1385,"width":400,"height":400,"color":"3"}, + {"id":"rpc_caller","type":"text","text":"# RPC调用器 [view.rpc_call]\n\n- 构造请求\n- 发送数据 [阻塞]\n- 等待响应 [阻塞]\n- 错误处理","x":-520,"y":1267,"width":300,"height":200,"color":"4"}, + {"id":"parallel_task","type":"text","text":"# 并行任务 \n- 持有信号量许可\n- 执行RPC调用\n- 处理响应\n- 自动释放许可\n\n[独立执行]","x":-520,"y":1579,"width":300,"height":200,"color":"6"}, + {"id":"batch_request3","type":"text","text":"# BatchDataRequest(3)\n- request_id (1)\n- dataset_unique_id (2)\n- data_item_idx (3)\n- block_type (4)\n- block_index: 2 (5)\n- data (6)\n- operation (7)\n- unique_id (8)\n- version (9)","x":-160,"y":784,"width":250,"height":120,"color":"2"}, + {"id":"storage_node_5","type":"text","text":"存储节点3","x":-400,"y":-680,"width":150,"height":60,"color":"3"}, + {"id":"storage_node_4","type":"text","text":"存储节点2","x":-400,"y":-760,"width":150,"height":60,"color":"3"}, + {"id":"cache_node_3","type":"text","text":"缓存节点3","x":-400,"y":-480,"width":150,"height":60,"color":"5"}, + {"id":"cache_node_1","type":"text","text":"缓存节点1","x":-400,"y":-640,"width":150,"height":60,"color":"5"}, + {"id":"cache_node_2","type":"text","text":"缓存节点2","x":-400,"y":-560,"width":150,"height":60,"color":"5"}, + {"id":"storage_node_1","type":"text","text":"存储节点1\n接收层:\n- 接收分片请求\n- 版本号验证\n- 数据完整性校验\n写入任务层:\n- 分片范围验证\n- 并发写入控制\n- 错误重试机制\n本地存储层:\n- 数据持久化\n- 版本管理\n- 空间回收\n结果返回:\n- 写入状态\n- 远程版本号\n- 错误信息","x":60,"y":-780,"width":200,"height":280,"color":"1"}, + {"id":"write_task_1","type":"text","text":"写入任务1\n- 分片范围验证\n- 数据完整性检查\n- 并发写入控制\n- 错误重试","x":400,"y":-780,"width":200,"height":120,"color":"2"}, + {"id":"batch_data_constants","type":"text","text":"# 批量数据常量定义\n\n## 数据块大小\n```rust\n/// 默认数据块大小 (4MB)\nconst DEFAULT_BLOCK_SIZE: usize = 4 * 1024 * 1024;\n```\n\n## 数据分片索引\n```rust\n/// 数据分片在整体数据中的偏移量\npub type DataSplitIdx = usize;\n```","x":-160,"y":1052,"width":400,"height":380,"color":"4"}, + {"id":"batch_request2","type":"text","text":"# BatchDataRequest(2)\n- request_id (1)\n- dataset_unique_id (2)\n- data_item_idx (3)\n- block_type (4)\n- block_index: 1 (5)\n- data (6)\n- operation (7)\n- unique_id (8)\n- version (9)","x":-160,"y":664,"width":250,"height":120,"color":"2"}, + {"id":"storage_node_3","type":"text","text":"存储节点1","x":-405,"y":-830,"width":150,"height":60,"color":"3"}, + {"id":"master_node","type":"text","text":"Master节点 [DataMaster]\n- schedule_data()\n1. 生成DataSetMeta\n2. 创建DataSplits\n3. 分配存储节点\n4. 返回调度决策","x":-1080,"y":-790,"width":200,"height":160,"color":"2"}, + {"id":"storage_group","type":"text","text":"存储节点组","x":-600,"y":-790,"width":150,"height":60,"color":"3"}, + {"id":"cache_group","type":"text","text":"缓存节点组","x":-600,"y":-590,"width":150,"height":60,"color":"5"}, + {"id":"write_result_1","type":"text","text":"写入结果1\n- 成功/失败\n- 远程版本号\n- 错误信息","x":660,"y":-560,"width":200,"height":100,"color":"4"}, + {"id":"86a8707f54d19c74","type":"text","text":"join all，并返回","x":-1389,"y":1549,"width":250,"height":60}, + {"id":"task_pool","type":"text","text":"# 任务池 [handles]\n\n- 收集任务句柄\n- 等待任务完成 [阻塞]\n- 错误聚合","x":-1414,"y":1732,"width":300,"height":260,"color":"5"}, + {"id":"5009f9e4bcc6ed6c","type":"text","text":"### 加入任务池","x":-920,"y":1902,"width":250,"height":60}, + {"id":"f8ade98240211305","type":"text","text":"### [tokio::spawn]\n","x":-945,"y":1784,"width":250,"height":60}, + {"id":"concurrency_controller","type":"text","text":"# 并发控制器 [Semaphore]\n\n- 最大并发数: 32\n- 许可获取 [阻塞]\n- 许可释放 [非阻塞]\n- RAII风格管理","x":-970,"y":1536,"width":300,"height":200,"color":"2"}, + {"id":"handle_wait_all","type":"text","text":"# handle.wait_all_tasks [异步等待]\n\n## 核心职责\n- 等待所有分片任务完成\n- 处理任务执行结果\n- 清理任务资源\n\n## 实现细节\n```rust\nasync fn wait_all_tasks(&self) -> WSResult<()> {\n // 等待所有任务完成\n while let Some(task) = rx.recv().await {\n task.await??;\n }\n Ok(())\n}\n```\n\n## 调用时机\n1. 外部调用: 批量传输完成检查\n2. 内部调用: process_tasks完成时","x":-2209,"y":1922,"width":320,"height":400}, + {"id":"0dee80a0e2345514","type":"text","text":"# 完成处理 [同步]\n\n## 执行流程\n1. 合并所有分片数据\n2. 构造最终DataItem\n3. 返回Some(item)给process_tasks\n4. process_tasks收到完成信号后退出循环\n\n## 数据流向\nprocess_tasks -> try_complete -> handle.wait_all_tasks","x":-2176,"y":2380,"width":254,"height":260}, + {"id":"e2576a54f3f852b3","type":"text","text":"# process_tasks() 实现 [阻塞循环]\n\n## 循环处理 [select阻塞]\n1. try_complete() 检查完成状态\n2. tokio::select! {\n - rx.recv() => 接收新任务\n - futures::future::select_all(tasks) => 等待任务完成\n}\n\n## 完成条件\n- current_size >= expected_size\n- 返回 proto::DataItem\n\n## 核心职责\n- 作为group的主事件循环\n- 在new group后立即启动\n- 负责接收和处理所有提交的任务\n- 维护任务状态直到完成\n\n## 执行流程\n1. 循环开始前检查完成状态\n2. 使用select等待新任务或已有任务完成\n3. 处理完成的任务并更新状态\n4. 检查是否达到完成条件\n5. 未完成则继续循环\n6. 完成则返回合并后的数据","x":-3272,"y":1892,"width":517,"height":688}, + {"id":"155106edf5eb3cd7","type":"text","text":"# 检查完成状态 try_complete() 实现 [同步检查]\n\n## 核心职责\n- 是process_tasks内部使用的状态检查\n- 判断是否所有分片都完成\n- 返回最终合并的数据\n\n## 检查流程\n1. 验证current_size是否达到expected_size\n2. 检查所有任务是否完成\n3. 合并分片数据\n4. 返回Option\n\n## 返回值\n- Some(item): 所有分片完成,返回合并数据\n- None: 未完成,继续等待\n\n## 错误处理\n- 分片数据不完整\n- 合并失败\n- 数据损坏","x":-2678,"y":2180,"width":455,"height":400} + ], + "edges":[ + {"id":"master_to_phase2","fromNode":"master_node","fromSide":"left","toNode":"general_phase2","toSide":"right","label":"调度决策\n- version\n- splits\n- nodes"}, + {"id":"phase2_to_phase3","fromNode":"general_phase2","fromSide":"bottom","toNode":"general_phase3","toSide":"top","label":"决策信息"}, + {"id":"phase3_to_storage","fromNode":"general_phase3","fromSide":"right","toNode":"storage_group","toSide":"left","label":"分发存储任务"}, + {"id":"storage_to_nodes","fromNode":"storage_group","fromSide":"right","toNode":"storage_node_3","toSide":"left"}, + {"id":"storage_to_nodes2","fromNode":"storage_group","fromSide":"right","toNode":"storage_node_4","toSide":"left"}, + {"id":"storage_to_nodes3","fromNode":"storage_group","fromSide":"right","toNode":"storage_node_5","toSide":"left"}, + {"id":"phase3_to_cache","fromNode":"general_phase3","fromSide":"right","toNode":"cache_group","toSide":"left","label":"分发缓存任务"}, + {"id":"cache_to_nodes","fromNode":"cache_group","fromSide":"right","toNode":"cache_node_1","toSide":"left"}, + {"id":"cache_to_nodes2","fromNode":"cache_group","fromSide":"right","toNode":"cache_node_2","toSide":"left"}, + {"id":"cache_to_nodes3","fromNode":"cache_group","fromSide":"right","toNode":"cache_node_3","toSide":"left"}, + {"id":"b5a17c0afede8e4a","fromNode":"data_general_core","fromSide":"right","toNode":"133214da264cfe72","toSide":"bottom"}, + {"id":"2ad5991c43fd6098","fromNode":"data_general_core","fromSide":"right","toNode":"821e415b6438e20d","toSide":"top"}, + {"id":"caa45c92a135042c","fromNode":"data_general_core","fromSide":"right","toNode":"core_functions","toSide":"left"}, + {"id":"09c7b9957992d62d","fromNode":"data_general_core","fromSide":"right","toNode":"b31695207931d96e","toSide":"left"}, + {"id":"3d79872a234731c0","fromNode":"cache_node_3","fromSide":"bottom","toNode":"batch_transfer_group","toSide":"top"}, + {"id":"9094221953b6c685","fromNode":"write_task_mem","fromSide":"top","toNode":"b0205b4457afeb2b","toSide":"bottom"}, + {"id":"77ec04f5deef7cee","fromNode":"write_task_mem","fromSide":"left","toNode":"1ec171d545e8995d","toSide":"top"}, + {"id":"7b99fb72410f07d9","fromNode":"06d4a92778dd83c8","fromSide":"bottom","toNode":"20145fd68e8aaa75","toSide":"top"}, + {"id":"df9b4bc9170fdec1","fromNode":"20145fd68e8aaa75","fromSide":"right","toNode":"4dbe01dc59cea4c2","toSide":"left"}, + {"id":"61e0637af4beba94","fromNode":"f515ecb9aee18fc7","fromSide":"left","toNode":"4dbe01dc59cea4c2","toSide":"left"}, + {"id":"f7105db89ffabd1e","fromNode":"20145fd68e8aaa75","fromSide":"bottom","toNode":"e2576a54f3f852b3","toSide":"top"}, + {"id":"7504b1b3a99e992c","fromNode":"4dbe01dc59cea4c2","fromSide":"right","toNode":"97d3d9fd7432a861","toSide":"bottom","label":"获取到handle"}, + {"id":"a993a3f4d7b2211d","fromNode":"97d3d9fd7432a861","fromSide":"left","toNode":"e2576a54f3f852b3","toSide":"right"}, + {"id":"a996588f6c59c88f","fromNode":"e2576a54f3f852b3","fromSide":"bottom","toNode":"155106edf5eb3cd7","toSide":"left"}, + {"id":"a42104592fedd4c7","fromNode":"97d3d9fd7432a861","fromSide":"right","toNode":"write_task_mem","toSide":"bottom"}, + {"id":"c45aaa564ae87a7c","fromNode":"97d3d9fd7432a861","fromSide":"right","toNode":"write_task_file","toSide":"bottom"}, + {"id":"write_flow_1","fromNode":"20145fd68e8aaa75","fromSide":"top","toNode":"06d4a92778dd83c8","toSide":"bottom","label":"初始化完成"}, + {"id":"write_flow_2","fromNode":"06d4a92778dd83c8","fromSide":"right","toNode":"f515ecb9aee18fc7","toSide":"left","label":"首个分片写入完成"}, + {"id":"86a2aa913f7bd3d9","fromNode":"223edf4677db9339","fromSide":"bottom","toNode":"06d4a92778dd83c8","toSide":"top"}, + {"id":"a99c309f19fd9853","fromNode":"batch_request1","fromSide":"right","toNode":"rpc_handle_batch_data","toSide":"left"}, + {"id":"batch_data_flow2","fromNode":"batch_data_constants","fromSide":"top","toNode":"batch_request3","toSide":"bottom","label":"使用常量"}, + {"id":"5e772afc67478d04","fromNode":"rpc_handle_batch_data","fromSide":"bottom","toNode":"handle_lookup","toSide":"top"}, + {"id":"concurrency_to_task","fromNode":"concurrency_controller","fromSide":"bottom","toNode":"f8ade98240211305","toSide":"top"}, + {"id":"task_to_rpc","fromNode":"parallel_task","fromSide":"top","toNode":"rpc_caller","toSide":"bottom","label":"调用"}, + {"id":"213831c4b82c9e93","fromNode":"data_source_interface","fromSide":"right","toNode":"data_reader","toSide":"left"}, + {"id":"7218875ebe7967fa","fromNode":"batch_transfer_main","fromSide":"bottom","toNode":"data_reader","toSide":"top"}, + {"id":"4b20152fe7211934","fromNode":"data_reader","fromSide":"bottom","toNode":"9fa1c2f8d08978bb","toSide":"top"}, + {"id":"4da12698f8ee3b63","fromNode":"rpc_caller","fromSide":"top","toNode":"batch_request3","toSide":"left"}, + {"id":"f4671fc434a3d0e1","fromNode":"f8ade98240211305","fromSide":"bottom","toNode":"5009f9e4bcc6ed6c","toSide":"top","label":"\n"}, + {"id":"9f748faecadaaa42","fromNode":"f8ade98240211305","fromSide":"right","toNode":"parallel_task","toSide":"left"}, + {"id":"8115e7d6d539f0c0","fromNode":"5009f9e4bcc6ed6c","fromSide":"right","toNode":"data_reader","toSide":"right"}, + {"id":"9e8cb09dfe630443","fromNode":"9fa1c2f8d08978bb","fromSide":"bottom","toNode":"concurrency_controller","toSide":"top"}, + {"id":"d95b89e25235928f","fromNode":"9fa1c2f8d08978bb","fromSide":"left","toNode":"86a8707f54d19c74","toSide":"right"}, + {"id":"9debe9b97cdaf245","fromNode":"86a8707f54d19c74","fromSide":"bottom","toNode":"task_pool","toSide":"top"}, + {"id":"a63472bc8934c7f9","fromNode":"5009f9e4bcc6ed6c","fromSide":"left","toNode":"task_pool","toSide":"right"}, + {"id":"f3ca63243b2c22f7","fromNode":"batch_initiator","fromSide":"right","toNode":"batch_transfer_main","toSide":"left"}, + {"id":"handle_to_spawner","fromNode":"write_task_handle","fromSide":"bottom","toNode":"task_spawner","toSide":"top","label":"tokio::spawn()"}, + {"id":"lookup_to_submit","fromNode":"handle_lookup","fromSide":"right","toNode":"write_task_handle","toSide":"left","label":"\n"}, + {"id":"9abc95f005b8b2d8","fromNode":"task_spawner","fromSide":"right","toNode":"2dbde64bc1dbac6a","toSide":"left"}, + {"id":"e6bd3dfca32e245b","fromNode":"handle_lookup","fromSide":"bottom","toNode":"task_spawn_flow","toSide":"top"}, + {"id":"3fca8aa5c568a44d","fromNode":"task_spawner","fromSide":"left","toNode":"task_spawn_flow","toSide":"right"}, + {"id":"0a095928ebb7ac26","fromNode":"2dbde64bc1dbac6a","fromSide":"bottom","toNode":"completion_monitor","toSide":"top"}, + {"id":"dcf437aa83674d1a","fromNode":"completion_monitor","fromSide":"left","toNode":"e156c034cc9ec24f","toSide":"right"}, + {"id":"7ae0cf5ea0bc0b06","fromNode":"task_spawn_flow","fromSide":"bottom","toNode":"e156c034cc9ec24f","toSide":"top"}, + {"id":"49b65724e2a3b08f","fromNode":"e156c034cc9ec24f","fromSide":"left","toNode":"batch_request3","toSide":"right"}, + {"id":"lookup_to_state","fromNode":"handle_lookup","fromSide":"top","toNode":"state_manager","toSide":"bottom","label":"查找/创建 proto::BatchRequestId"}, + {"id":"monitor_to_state","fromNode":"completion_monitor","fromSide":"right","toNode":"state_manager","toSide":"bottom","label":"清理"}, + {"id":"facc3fcfb55cf19d","fromNode":"batch_data_request","fromSide":"top","toNode":"batch_request3","toSide":"bottom"}, + {"id":"271f79d015a55fdf","fromNode":"batch_data_request","fromSide":"right","toNode":"e156c034cc9ec24f","toSide":"bottom"}, + {"id":"6a7413aedbbca964","fromNode":"155106edf5eb3cd7","fromSide":"top","toNode":"e2576a54f3f852b3","toSide":"right","label":"未完成"}, + {"id":"6604bc585e5ffe59","fromNode":"155106edf5eb3cd7","fromSide":"bottom","toNode":"0dee80a0e2345514","toSide":"bottom","label":"完成"}, + {"id":"handle_wait_flow","fromNode":"0dee80a0e2345514","fromSide":"right","toNode":"handle_wait_all","toSide":"right","label":"通知等待完成"}, + {"id":"e732f2950f5744ff","fromNode":"4dbe01dc59cea4c2","fromSide":"bottom","toNode":"handle_wait_all","toSide":"top"} + ] +} \ No newline at end of file diff --git a/design_of_new_batch.md b/design_of_new_batch.md new file mode 100755 index 0000000..c360c6d --- /dev/null +++ b/design_of_new_batch.md @@ -0,0 +1,699 @@ +# 项目分析与修改计划 + + +### 变更 + +#### 核心接口定义 +```rust + + +#### WriteSplitDataTaskGroup 核心实现 +```rust +// 写入任务相关错误 +#[derive(Debug)] +pub enum WsDataErr { + WriteDataFailed { + unique_id: Vec, + }, + SplitTaskFailed { + idx: DataSplitIdx, + }, +} + +// 写入任务句柄，用于提交新的分片任务 +pub struct WriteSplitDataTaskHandle { + tx: mpsc::Sender>, + write_type: WriteSplitDataType, +} + +// 写入类型 +enum WriteSplitDataType { + File { + path: PathBuf, + }, + Mem { + shared_mem: SharedMemHolder, + }, +} + +impl WriteSplitDataTaskHandle { + // 提交新的分片任务 + pub async fn submit_split(&self, idx: DataSplitIdx, data: proto::DataItem) { + let task = match &self.write_type { + WriteSplitDataType::File { path } => { + let path = path.clone(); + let offset = idx.offset; + let data = data.as_bytes().to_vec(); + tokio::spawn(async move { + if let Err(e) = tokio::fs::OpenOptions::new() + .create(true) + .write(true) + .open(&path) + .await + .and_then(|mut file| async move { + file.seek(SeekFrom::Start(offset)).await?; + file.write_all(&data).await + }) + .await + { + tracing::error!("Failed to write file data at offset {}: {}", offset, e); + } + }) + } + WriteSplitDataType::Mem { shared_mem } => { + let mem = shared_mem.clone(); + let offset = idx.offset as usize; + let data = data.as_bytes().to_vec(); + tokio::spawn(async move { + if let Err(e) = mem.write(offset, &data).await { + tracing::error!("Failed to write memory data at offset {}: {}", offset, e); + } + }) + } + }; + + if let Err(e) = self.tx.send(task).await { + tracing::error!("Failed to submit task: channel closed, idx: {:?}", idx); + } + } +} + +// 写入任务组 +enum WriteSplitDataTaskGroup { + // 文件写入模式 + ToFile { + unique_id: UniqueId, // 任务唯一标识 + file_path: PathBuf, // 文件路径 + tasks: Vec>, // 写入任务列表 + rx: mpsc::Receiver>, // 任务接收通道 + expected_size: usize, // 预期总大小 + current_size: usize, // 当前写入大小 + manager: Arc, // 管理器引用 + }, + // 内存写入模式 + ToMem { + unique_id: UniqueId, // 任务唯一标识 + shared_mem: SharedMemHolder, // 共享内存 + tasks: Vec>, // 写入任务列表 + rx: mpsc::Receiver>, // 任务接收通道 + expected_size: usize, // 预期总大小 + current_size: usize, // 当前写入大小 + manager: Arc, // 管理器引用 + } +} + +impl WriteSplitDataTaskGroup { + // 创建新任务组 + async fn new( + unique_id: UniqueId, + splits: Vec>, + block_type: proto::BatchDataBlockType, + manager: Arc, + ) -> (Self, WriteSplitDataTaskHandle) { + // 计算预期总大小 + let expected_size = splits.iter().map(|range| range.len()).sum(); + + // 创建通道 + let (tx, rx) = mpsc::channel(32); + + match block_type { + proto::BatchDataBlockType::File => { + let file_path = PathBuf::from(format!("{}.data", + base64::engine::general_purpose::STANDARD.encode(&unique_id))); + + let handle = WriteSplitDataTaskHandle { + tx, + write_type: WriteSplitDataType::File { + path: file_path.clone(), + }, + }; + + let group = Self::ToFile { + unique_id, + file_path, + tasks: Vec::new(), + rx, + expected_size, + current_size: 0, + manager: manager.clone(), + }; + + (group, handle) + } + _ => { + let shared_mem = new_shared_mem(&splits).unwrap_or_default(); + + let handle = WriteSplitDataTaskHandle { + tx, + write_type: WriteSplitDataType::Mem { + shared_mem: shared_mem.clone(), + }, + }; + + let group = Self::ToMem { + unique_id, + shared_mem, + tasks: Vec::new(), + rx, + expected_size, + current_size: 0, + manager: manager.clone(), + }; + + (group, handle) + } + } + } + + // 处理任务完成 + async fn handle_completion(&self) { + match self { + Self::ToFile { unique_id, manager, .. } | + Self::ToMem { unique_id, manager, .. } => { + // 从管理器中移除句柄 + manager.remove_handle(unique_id); + } + } + } + + // 任务处理循环 + async fn process_tasks(&mut self) -> WSResult { + loop { + // 检查是否已完成所有写入 + if let Some(result) = self.try_complete() { + // 处理完成，清理资源 + self.handle_completion().await; + return Ok(result); + } + + // 等待新任务或已有任务完成 + tokio::select! { + Some(new_task) = match self { + Self::ToFile { rx, .. } | + Self::ToMem { rx, .. } => rx.recv() + } => { + match self { + Self::ToFile { tasks, .. } | + Self::ToMem { tasks, .. } => { + tasks.push(new_task); + } + } + } + else => { + // 通道关闭，清理资源 + self.handle_completion().await; + break; + } + } + } + + Err(WSError::WsDataError(WsDataErr::WriteDataFailed { + unique_id: match self { + Self::ToFile { unique_id, .. } | + Self::ToMem { unique_id, .. } => unique_id.clone(), + } + })) + } +} + +// WriteSplitDataManager 管理器 +pub struct WriteSplitDataManager { + // 只存储任务句柄 + handles: DashMap, +} + +impl WriteSplitDataManager { + pub fn new() -> Arc { + Arc::new(Self { + handles: DashMap::new(), + }) + } + + // 注册新的任务句柄 + pub fn register_handle( + &self, + request_id: proto::BatchRequestId, + handle: WriteSplitDataTaskHandle, + ) -> WSResult<()> { + // 检查是否已存在 + if self.handles.contains_key(&request_id) { + return Err(WSError::WsDataError(WsDataErr::WriteDataFailed { + request_id, + })); + } + + // 存储句柄 + self.handles.insert(request_id, handle); + Ok(()) + } + + // 获取已存在的任务句柄 + pub fn get_handle(&self, request_id: &proto::BatchRequestId) -> Option { + self.handles.get(request_id).map(|h| h.clone()) + } + + // 移除任务句柄 + pub fn remove_handle(&self, request_id: &proto::BatchRequestId) { + self.handles.remove(request_id); + } +} + +## 修改 使用情况以适配新接口 计划 + +### 1. 修改 get_or_del_data 函数 + +```diff + pub async fn get_or_del_data(&self, GetOrDelDataArg { meta, unique_id, ty }: GetOrDelDataArg) + -> WSResult<(DataSetMetaV2, HashMap)> + { + let want_idxs: Vec = WantIdxIter::new(&ty, meta.data_item_cnt() as DataItemIdx).collect(); + + let mut groups = Vec::new(); + let mut idxs = Vec::new(); + let p2p = self.view.p2p(); + let mut ret = HashMap::new(); + + for idx in want_idxs { + // 为每个数据项创建独立的任务组 + let (tx, rx) = tokio::sync::mpsc::channel(1); + let splits = vec![0..1]; + let splits = vec![0..1]; + let (mut group, handle) = WriteSplitDataTaskGroup::new( + unique_id.clone(), + splits, + match ty { + GetOrDelDataArgType::Delete => proto::BatchDataBlockType::Delete, + _ => proto::BatchDataBlockType::Memory, + }, + Arc::clone(&self.manager), + ).await; + + let p2p = p2p.clone(); + let unique_id = unique_id.clone(); + let data_node = meta.get_data_node(idx); + let delete = matches!(ty, GetOrDelDataArgType::Delete); + let rpc_call = self.rpc_call_get_data.clone(); + + let handle_clone = handle.clone(); + let handle = tokio::spawn(async move { + let resp = rpc_call.call( + p2p, + data_node, + proto::GetOneDataRequest { + unique_id: unique_id.to_vec(), + idxs: vec![idx as u32], + delete, + return_data: true, + }, + Some(Duration::from_secs(60)), + ).await?; + + if !resp.success { + tracing::error!("Failed to get data for idx {}: {}", idx, resp.message); + return Err(WsDataError::GetDataFailed { + unique_id: unique_id.to_vec(), + msg: resp.message, + }.into()); + } + + handle_clone.submit_split(0, resp.data[0].clone()).await; + Ok::<_, WSError>(()) + }); + + groups.push(group); + idxs.push((idx, handle)); + } + + // 等待所有RPC任务完成 + for (group, (idx, handle)) in groups.into_iter().zip(idxs.into_iter()) { + if let Err(e) = handle.await.map_err(|e| WSError::from(e))?.map_err(|e| e) { + tracing::error!("RPC task failed for idx {}: {}", idx, e); + continue; + } + + match group.join().await { + Ok(data_item) => { + ret.insert(idx, data_item); + } + Err(e) => { + tracing::error!("Task group join failed for idx {}: {}", idx, e); + } + } + } + + Ok(ret) +} +``` + +### 2. Batch数据处理流程更新 + +#### 2.1 WriteSplitDataTaskHandle扩展 等待全部完成的函数 + +```rust +impl WriteSplitDataTaskHandle { + ... + + /// 等待所有已提交的写入任务完成 + pub async fn wait_all_tasks(self) -> WSResult<()> { + } +} +``` + +#### 2.2 BatchTransfer 实现 + +```rust +/// 数据源接口 +#[async_trait] +pub trait DataSource: Send + Sync + 'static { + /// 获取数据总大小 + async fn size(&self) -> WSResult; + /// 读取指定范围的数据 + async fn read_chunk(&self, offset: usize, size: usize) -> WSResult>; + /// 获取数据块类型 + fn block_type(&self) -> BatchDataBlockType; +} + +/// 批量传输数据 +pub async fn batch_transfer( + unique_id: Vec, + version: u64, + target_node: NodeID, + data: Arc, + view: DataGeneralView, +) -> WSResult<()> { + let total_size = data.size().await?; + let total_blocks = (total_size + DEFAULT_BLOCK_SIZE - 1) / DEFAULT_BLOCK_SIZE; + let semaphore = Arc::new(Semaphore::new(32)); + let mut handles = Vec::new(); + + // 发送所有数据块 + for block_idx in 0..total_blocks { + // 获取信号量许可 + let permit = semaphore.clone().acquire_owned().await.unwrap(); + + let offset = block_idx as usize * DEFAULT_BLOCK_SIZE; + let size = DEFAULT_BLOCK_SIZE.min(total_size - offset); + + // 读取数据块 + let block_data = data.read_chunk(offset, size).await?; + + // 构造请求 + let request = proto::BatchDataRequest { + request_id: Some(proto::BatchRequestId { + node_id: target_node as u32, + sequence: block_idx as u32, + }), + block_type: data.block_type() as i32, + block_index: block_idx as u32, + data: block_data, + operation: proto::DataOpeType::Write as i32, + unique_id: unique_id.clone(), + version, + }; + + // 发送请求 + let view = view.clone(); + let handle = tokio::spawn(async move { + let _permit = permit; // 持有permit直到任务完成 + let resp = view.data_general().rpc_call_batch_data.call( + view.p2p(), + target_node, + request, + Some(Duration::from_secs(30)), + ).await?; + + if !resp.success { + return Err(WsDataError::BatchTransferFailed { + node: target_node, + batch: block_idx as u32, + reason: resp.error_message, + }.into()); + } + + Ok(()) + }); + + handles.push(handle); + } + + // 等待所有请求完成 + for handle in handles { + handle.await??; + } + + Ok(()) +} +``` + +#### 2.3 DataGeneral RPC处理实现 + +```rust +/// 默认数据块大小 (4MB) +const DEFAULT_BLOCK_SIZE: usize = 4 * 1024 * 1024; + +/// 批量数据传输状态 +struct BatchTransferState { + handle: WriteSplitDataTaskHandle, + shared: SharedWithBatchHandler, +} + +/// 共享状态，用于记录最新的请求响应器 +#[derive(Clone)] +struct SharedWithBatchHandler { + responsor: Arc>>>, +} + +impl SharedWithBatchHandler { + fn new() -> Self { + Self { + responsor: Arc::new(Mutex::new(None)), + } + } + + async fn update_responsor(&self, responsor: RPCResponsor) { + let mut guard = self.responsor.lock().await; + if let Some(old_responsor) = guard.take() { + // 旧的responsor直接返回成功 + if let Err(e) = old_responsor.response(Ok(())).await { + tracing::error!("Failed to respond to old request: {}", e); + } + } + *guard = Some(responsor); + } + + async fn get_final_responsor(&self) -> Option> { + self.responsor.lock().await.take() + } +} + +impl DataGeneral { + /// 创建新的DataGeneral实例 + pub fn new() -> Self { + Self { + batch_receive_states: DashMap::new(), + // ...其他字段 + } + } +} + +impl DataGeneral { + /// 处理批量数据写入请求 + /// + /// # 处理流程 + /// 1. 从batch_receive_states查询或创建传输状态 + /// 2. 使用WriteSplitDataTaskHandle提交写入任务 + /// 3. 等待写入完成并返回结果 + pub async fn rpc_handle_batch_data( + &self, + request: BatchDataRequest, + responsor: RPCResponsor, + ) -> WSResult<()> { + // 1. 从batch_receive_states查询或创建传输状态 + let state = if let Some(state) = self.batch_receive_states.get(&request.unique_id) { + // 验证版本号 + if state.handle.version() != request.version { + tracing::error!( + "Version mismatch for transfer {}, expected {}, got {}", + hex::encode(&request.unique_id), + state.handle.version(), + request.version + ); + return Err(WSError::BatchError(WsBatchErr::VersionMismatch { + expected: state.handle.version(), + actual: request.version, + })); + } + state + } else { + // 创建新的写入任务组 + let (group, handle) = WriteSplitDataTaskGroup::new( + request.unique_id.clone(), + calculate_splits(request.total_blocks), + request.block_type, + ).await?; + + // 创建共享状态 + let shared = SharedWithBatchHandler::new(); + let state = BatchTransferState { handle: handle.clone(), shared: shared.clone() }; + + // 启动等待完成的任务 + let unique_id = request.unique_id.clone(); + let batch_receive_states = self.batch_receive_states.clone(); + tokio::spawn(async move { + // 等待所有任务完成 + if let Err(e) = handle.wait_all_tasks().await { + tracing::error!( + "Failed to complete transfer {}: {}", + hex::encode(&unique_id), + e + ); + // 获取最后的responsor并返回错误 + if let Some(final_responsor) = shared.get_final_responsor().await { + if let Err(e) = final_responsor.response(Err(e)).await { + tracing::error!("Failed to send error response: {}", e); + } + } + // 清理状态 + batch_receive_states.remove(&unique_id); + return; + } + + // 获取最后的responsor并返回成功 + if let Some(final_responsor) = shared.get_final_responsor().await { + if let Err(e) = final_responsor.response(Ok(())).await { + tracing::error!("Failed to send success response: {}", e); + } + } + // 清理状态 + batch_receive_states.remove(&unique_id); + }); + + // 插入新状态 + self.batch_receive_states.insert(request.unique_id.clone(), state); + self.batch_receive_states.get(&request.unique_id).unwrap() + }; + + // 2. 使用WriteSplitDataTaskHandle提交写入任务 + let offset = request.block_idx as usize * DEFAULT_BLOCK_SIZE; + + if let Err(e) = state.handle.submit_split(offset, request.data).await { + tracing::error!( + "Failed to submit split for transfer {}, block {}: {}", + hex::encode(&request.unique_id), + request.block_idx, + e + ); + return Err(e); + } + + // 3. 更新共享状态中的responsor + state.shared.update_responsor(responsor).await; + + tracing::debug!( + "Successfully submitted block {} for transfer {}", + request.block_idx, + hex::encode(&request.unique_id) + ); + + Ok(()) + } +} + +/// 计算数据分片范围 +fn calculate_splits(total_blocks: u32) -> Vec> { + let mut splits = Vec::with_capacity(total_blocks as usize); + for i in 0..total_blocks { + let start = i as usize * DEFAULT_BLOCK_SIZE; + let end = start + DEFAULT_BLOCK_SIZE; + splits.push(start..end); + } + splits +} + +/// 数据源实现 +pub struct FileDataSource { + path: PathBuf, + file: Option, +} + +impl FileDataSource { + pub fn new(path: PathBuf) -> Self { + Self { + path, + file: None, + } + } +} + +#[async_trait] +impl DataSource for FileDataSource { + async fn size(&self) -> WSResult { + tokio::fs::metadata(&self.path) + .await + .map(|m| m.len() as usize) + .map_err(|e| WsDataError::ReadSourceFailed { + source: format!("{}", self.path.display()), + error: e.to_string(), + }.into()) + } + + async fn read_chunk(&self, offset: usize, size: usize) -> WSResult> { + let mut file = tokio::fs::File::open(&self.path).await + .map_err(|e| WsDataError::ReadSourceFailed { + source: format!("{}", self.path.display()), + error: e.to_string(), + })?; + + file.seek(SeekFrom::Start(offset as u64)).await + .map_err(|e| WsDataError::ReadSourceFailed { + source: format!("{}", self.path.display()), + error: e.to_string(), + })?; + + let mut buf = vec![0; size]; + file.read_exact(&mut buf).await + .map_err(|e| WsDataError::ReadSourceFailed { + source: format!("{}", self.path.display()), + error: e.to_string(), + })?; + + Ok(buf) + } + + fn block_type(&self) -> BatchDataBlockType { + BatchDataBlockType::File + } +} + +pub struct MemDataSource { + data: Arc<[u8]>, +} + +impl MemDataSource { + pub fn new(data: Vec) -> Self { + Self { + data: data.into() + } + } +} + +#[async_trait] +impl DataSource for MemDataSource { + async fn size(&self) -> WSResult { + Ok(self.data.len()) + } + + async fn read_chunk(&self, offset: usize, size: usize) -> WSResult> { + if offset + size > self.data.len() { + return Err(WsDataError::ReadSourceFailed { + source: "memory".into(), + error: "read beyond bounds".into(), + }.into()); + } + Ok(self.data[offset..offset + size].to_vec()) + } + + fn block_type(&self) -> BatchDataBlockType { + BatchDataBlockType::Memory + } +} diff --git a/review.md b/review.md new file mode 100644 index 0000000..6c016c1 --- /dev/null +++ b/review.md @@ -0,0 +1,498 @@ +# 代码修改清单 + +## 1. 删除代码 +```rust +// 1. src/main/src/general/data/m_data_general/batch.rs 中删除 +// 1.1 删除 BatchManager +// pub(super) struct BatchManager { +// transfers: DashMap, +// sequence: AtomicU64, +// } + +// impl BatchManager { +// pub fn new() -> Self +// pub fn next_sequence(&self) -> u64 +// pub async fn create_transfer(...) +// pub async fn handle_block(...) +// } + +// 1.2 删除 BatchTransfer +// pub(super) struct BatchTransfer { +// pub unique_id: Vec, +// pub version: u64, +// pub block_type: proto::BatchDataBlockType, +// pub total_blocks: u32, +// data_sender: mpsc::Sender>, +// write_task: JoinHandle>, +// pub tx: Option>>, +// } + +// impl BatchTransfer { +// pub async fn new(...) +// pub async fn add_block(...) +// pub async fn complete(...) +// fn calculate_splits(...) +// } + +// 2. src/main/src/general/data/m_data_general/mod.rs 中删除 +// struct DataGeneral { +// batch_manager: Arc, // 删除此字段 +// } + +// DataGeneral::new() 中删除 +// batch_manager: Arc::new(BatchManager::new()), +``` + +## 2. 新增代码 + +### src/main/src/result.rs +```rust +pub enum WsDataError { + // 修改错误类型 + BatchTransferFailed { + request_id: proto::BatchRequestId, // 改为 request_id + reason: String, + }, + BatchTransferNotFound { + request_id: proto::BatchRequestId, // 改为 request_id + }, + BatchTransferError { + request_id: proto::BatchRequestId, // 改为 request_id + msg: String, + }, + WriteDataFailed { + request_id: proto::BatchRequestId, + }, + SplitTaskFailed { + request_id: proto::BatchRequestId, + idx: DataSplitIdx, + }, + VersionMismatch { + expected: u64, + actual: u64, + }, +} +``` + +### src/main/src/general/data/m_data_general/task.rs +```rust +// 写入任务句柄，用于提交新的分片任务 +pub struct WriteSplitDataTaskHandle { + tx: mpsc::Sender>, + write_type: WriteSplitDataType, + version: u64, // 添加版本号字段 +} + +// 写入类型 +enum WriteSplitDataType { + File { + path: PathBuf, + }, + Mem { + shared_mem: SharedMemHolder, + }, +} + +impl WriteSplitDataTaskHandle { + // 获取版本号 + pub fn version(&self) -> u64 { + self.version + } + + // 提交新的分片任务 + pub async fn submit_split(&self, idx: DataSplitIdx, data: proto::DataItem) -> WSResult<()> { + let task = match &self.write_type { + WriteSplitDataType::File { path } => { + let path = path.clone(); + let offset = idx.offset; + let data = data.as_bytes().to_vec(); + tokio::spawn(async move { + if let Err(e) = tokio::fs::OpenOptions::new() + .create(true) + .write(true) + .open(&path) + .await + .and_then(|mut file| async move { + file.seek(SeekFrom::Start(offset)).await?; + file.write_all(&data).await + }) + .await + { + tracing::error!("Failed to write file data at offset {}: {}", offset, e); + } + }) + } + WriteSplitDataType::Mem { shared_mem } => { + let mem = shared_mem.clone(); + let offset = idx.offset as usize; + let data = data.as_bytes().to_vec(); + tokio::spawn(async move { + if let Err(e) = mem.write(offset, &data).await { + tracing::error!("Failed to write memory data at offset {}: {}", offset, e); + } + }) + } + }; + + self.tx.send(task).await.map_err(|e| { + tracing::error!("Failed to submit task: channel closed, idx: {:?}", idx); + WSError::WsDataError(WsDataError::BatchTransferFailed { + request_id: idx.into(), // 需要实现 From for BatchRequestId + reason: "Failed to submit task: channel closed".to_string() + }) + }) + } + + /// 等待所有已提交的写入任务完成 + pub async fn wait_all_tasks(self) -> WSResult<()> { + // 关闭发送端,不再接收新任务 + drop(self.tx); + + Ok(()) + } +} + +// 写入任务组 +enum WriteSplitDataTaskGroup { + // 文件写入模式 + ToFile { + unique_id: UniqueId, // 任务唯一标识 + file_path: PathBuf, // 文件路径 + tasks: Vec>, // 写入任务列表 + rx: mpsc::Receiver>, // 任务接收通道 + expected_size: usize, // 预期总大小 + current_size: usize, // 当前写入大小 + }, + // 内存写入模式 + ToMem { + unique_id: UniqueId, // 任务唯一标识 + shared_mem: SharedMemHolder, // 共享内存 + tasks: Vec>, // 写入任务列表 + rx: mpsc::Receiver>, // 任务接收通道 + expected_size: usize, // 预期总大小 + current_size: usize, // 当前写入大小 + } +} + +impl WriteSplitDataTaskGroup { + // 创建新任务组 + async fn new( + unique_id: UniqueId, + splits: Vec>, + block_type: proto::BatchDataBlockType, + version: u64, // 添加版本号参数 + ) -> (Self, WriteSplitDataTaskHandle) { + // 计算预期总大小 + let expected_size = splits.iter().map(|range| range.len()).sum(); + + // 创建通道 + let (tx, rx) = mpsc::channel(32); + + match block_type { + proto::BatchDataBlockType::File => { + let file_path = PathBuf::from(format!("{}.data", + base64::engine::general_purpose::STANDARD.encode(&unique_id))); + + let handle = WriteSplitDataTaskHandle { + tx, + write_type: WriteSplitDataType::File { + path: file_path.clone(), + }, + version, // 设置版本号 + }; + + let group = Self::ToFile { + unique_id, + file_path, + tasks: Vec::new(), + rx, + expected_size, + current_size: 0, + }; + + (group, handle) + } + _ => { + let shared_mem = new_shared_mem(&splits).unwrap_or_default(); + + let handle = WriteSplitDataTaskHandle { + tx, + write_type: WriteSplitDataType::Mem { + shared_mem: shared_mem.clone(), + }, + version, // 设置版本号 + }; + + let group = Self::ToMem { + unique_id, + shared_mem, + tasks: Vec::new(), + rx, + expected_size, + current_size: 0, + }; + + (group, handle) + } + } + } + + // 任务处理循环 + async fn process_tasks(&mut self) -> WSResult { + loop { + // 检查是否已完成所有写入 + if let Some(result) = self.try_complete() { + return Ok(result); + } + + // 等待新任务或已有任务完成 + tokio::select! { + Some(new_task) = match self { + Self::ToFile { rx, .. } | + Self::ToMem { rx, .. } => rx.recv() + } => { + match self { + Self::ToFile { tasks, .. } | + Self::ToMem { tasks, .. } => { + tasks.push(new_task); + // 不需要更新current_size,因为是在任务完成时更新 + } + } + } + Some(completed_task) = futures::future::select_all(match self { + Self::ToFile { tasks, .. } | + Self::ToMem { tasks, .. } => tasks + }) => { + // 检查任务是否成功完成 + if let Err(e) = completed_task.0 { + tracing::error!("Task failed: {}", e); + return Err(WSError::WsDataError(WsDataError::BatchTransferFailed { + request_id: match self { + Self::ToFile { unique_id, .. } | + Self::ToMem { unique_id, .. } => unique_id.clone() + }, + reason: format!("Task failed: {}", e) + })); + } + // 从任务列表中移除已完成的任务 + match self { + Self::ToFile { tasks, current_size, .. } | + Self::ToMem { tasks, current_size, .. } => { + tasks.remove(completed_task.1); + // 更新当前大小 + *current_size += DEFAULT_BLOCK_SIZE; // 每个任务写入一个块 + } + } + } + None = match self { + Self::ToFile { rx, .. } | + Self::ToMem { rx, .. } => rx.recv() + } => { + // 通道关闭,直接退出 + break; + } + } + } + + Err(WSError::WsDataError(WsDataError::BatchTransferFailed { + request_id: match self { + Self::ToFile { unique_id, .. } | + Self::ToMem { unique_id, .. } => unique_id.clone() + }, + reason: "Channel closed".to_string() + })) + } + + /// 检查是否已完成所有写入 + fn try_complete(&self) -> Option { + match self { + Self::ToFile { current_size, expected_size, file_path, .. } => { + if *current_size >= *expected_size { + // 所有数据已写入,返回文件数据项 + Some(proto::DataItem::new_file_data(file_path.clone())) + } else { + None + } + } + Self::ToMem { current_size, expected_size, shared_mem, .. } => { + if *current_size >= *expected_size { + // 所有数据已写入,返回内存数据项 + Some(proto::DataItem::new_mem_data(shared_mem.clone())) + } else { + None + } + } + } + } +} + +/// DataItem 数据源 +pub enum DataItemSource { + Memory { + data: Arc>, + }, + File { + path: String, + }, +} + +DataItemSource 采用枚举设计，优点： +1. 类型安全：使用枚举确保数据源类型的互斥性 +2. 内存效率：文件类型只存储路径，避免一次性加载 +3. 延迟读取：只在实际需要时才读取文件数据 +4. 符合分层：配合 WriteSplitDataTaskGroup 的文件/内存写入流程 + +实现了 DataSource trait： +- size(): 获取数据总大小 +- read_chunk(): 读取指定范围的数据 +- block_type(): 返回对应的 BlockType +``` + +### src/main/src/general/data/m_data_general/mod.rs +```rust +/// 共享状态，用于记录最新的请求响应器 +#[derive(Clone)] +struct SharedWithBatchHandler { + responsor: Arc>>>, +} + +impl SharedWithBatchHandler { + fn new() -> Self { + Self { + responsor: Arc::new(Mutex::new(None)), + } + } + + async fn update_responsor(&self, responsor: RPCResponsor) { + let mut guard = self.responsor.lock().await; + if let Some(old_responsor) = guard.take() { + // 旧的responsor直接返回成功 + if let Err(e) = old_responsor.response(Ok(())).await { + tracing::error!("Failed to respond to old request: {}", e); + } + } + *guard = Some(responsor); + } + + async fn get_final_responsor(&self) -> Option> { + self.responsor.lock().await.take() + } +} + +/// 批量数据传输状态 +struct BatchReceiveState { + handle: WriteSplitDataTaskHandle, + shared: SharedWithBatchHandler, +} + +pub struct DataGeneral { + // 批量数据接收状态管理 + batch_receive_states: DashMap, + // ... 其他字段 +} + +impl DataGeneral { + pub fn new() -> Self { + Self { + batch_receive_states: DashMap::new(), + // ... 其他字段初始化 + } + } + + /// 处理批量数据写入请求 + pub async fn rpc_handle_batch_data( + &self, + request: BatchDataRequest, + responsor: RPCResponsor, + ) -> WSResult<()> { + let state = if let Some(state) = self.batch_receive_states.get(&request.request_id) { + // 验证版本号 + if state.handle.version() != request.version { + tracing::error!( + "Version mismatch for transfer {:?}, expected {}, got {}", + request.request_id, + state.handle.version(), + request.version + ); + return Err(WSError::WsDataError(WsDataError::BatchTransferError { + request_id: request.request_id, + msg: format!("Version mismatch, expected {}, got {}", + state.handle.version(), request.version) + })); + } + state + } else { + // 创建新的写入任务组 + let (group, handle) = WriteSplitDataTaskGroup::new( + request.unique_id.clone(), + calculate_splits(request.total_blocks), + request.block_type, + request.version, // 传递版本号 + ).await?; + + // 创建共享状态 + let shared = SharedWithBatchHandler::new(); + let state = BatchReceiveState { handle: handle.clone(), shared: shared.clone() }; + + // 启动等待完成的任务 + let request_id = request.request_id.clone(); // 使用 request_id + let batch_receive_states = self.batch_receive_states.clone(); + tokio::spawn(async move { + // 等待所有任务完成 + if let Err(e) = handle.wait_all_tasks().await { + tracing::error!( + "Failed to complete transfer {:?}: {}", + request_id, // 使用 request_id + e + ); + // 获取最后的responsor并返回错误 + if let Some(final_responsor) = shared.get_final_responsor().await { + if let Err(e) = final_responsor.response(Err(e)).await { + tracing::error!("Failed to send error response: {}", e); + } + } + // 清理状态 + batch_receive_states.remove(&request_id); // 使用 request_id + return; + } + + // 获取最后的responsor并返回成功 + if let Some(final_responsor) = shared.get_final_responsor().await { + if let Err(e) = final_responsor.response(Ok(())).await { + tracing::error!("Failed to send success response: {}", e); + } + } + // 清理状态 + batch_receive_states.remove(&request_id); // 使用 request_id + }); + + // 插入新状态 + self.batch_receive_states.insert(request.request_id.clone(), state); + self.batch_receive_states.get(&request.request_id).unwrap() + }; + + // 2. 使用WriteSplitDataTaskHandle提交写入任务 + let offset = request.block_index as usize * DEFAULT_BLOCK_SIZE; // 使用 block_index + + if let Err(e) = state.handle.submit_split(offset, request.data).await { + tracing::error!( + "Failed to submit split for transfer {:?}, block {}: {}", + request.request_id, + request.block_index, // 使用 block_index + e + ); + return Err(e); + } + + // 3. 更新共享状态中的responsor + state.shared.update_responsor(responsor).await; + + tracing::debug!( + "Successfully submitted block {} for transfer {:?}", + request.block_index, + request.request_id + ); + + Ok(()) + } +} \ No newline at end of file diff --git a/scripts/mount_s3fs.sh b/scripts/mount_s3fs.sh new file mode 100644 index 0000000..2e22278 --- /dev/null +++ b/scripts/mount_s3fs.sh @@ -0,0 +1,3 @@ +umount /mnt/s3fs +s3fs s3fs /mnt/s3fs -o passwd_file=/root/.passwd-s3fs -o url=http://127.0.0.1:9000 -o use_path_request_style -o umask=0022,uid=$(id -u),gid=$(id -g) -o use_cache=/var/cache/s3fs +echo "mount s3fs success" \ No newline at end of file diff --git a/scripts/sync_md_files.py b/scripts/sync_md_files.py new file mode 100644 index 0000000..97879e3 --- /dev/null +++ b/scripts/sync_md_files.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +import json +import os +import sys +from datetime import datetime +from typing import List, Dict, Optional + +class Node: + def __init__(self, data: dict): + self.data = data + self.children = [] + self.parent = None + + @property + def id(self) -> str: + return self.data.get('id', '') + + @property + def type(self) -> str: + return self.data.get('type', '') + + @property + def x(self) -> float: + return float(self.data.get('x', 0)) + + @property + def y(self) -> float: + return float(self.data.get('y', 0)) + + @property + def width(self) -> float: + return float(self.data.get('width', 0)) + + @property + def height(self) -> float: + return float(self.data.get('height', 0)) + + def contains(self, other: 'Node') -> bool: + """判断当前节点是否在空间上包含另一个节点""" + if self.type != 'group': + return False + + # 考虑边界重叠的情况 + return (other.x >= self.x - 1 and + other.y >= self.y - 1 and + other.x + other.width <= self.x + self.width + 1 and + other.y + other.height <= self.y + self.height + 1) + + def to_dict(self) -> dict: + """转换为字典格式""" + result = self.data.copy() + if self.children: + result['children'] = [child.to_dict() for child in self.children] + return result + + def to_flat_dict(self) -> List[dict]: + """转换为扁平的字典列表""" + result = [] + if self.type != 'root': # 不包含根节点 + node_data = self.data.copy() + if 'children' in node_data: + del node_data['children'] # 移除children字段 + result.append(node_data) + for child in self.children: + result.extend(child.to_flat_dict()) + return result + +def tree_to_flat_nodes(tree_data: dict) -> List[dict]: + """将树状结构转换为扁平的节点列表""" + result = [] + + # 处理当前节点 + if tree_data.get('type') != 'root': + node_data = tree_data.copy() + if 'children' in node_data: + del node_data['children'] + result.append(node_data) + + # 递归处理子节点 + for child in tree_data.get('children', []): + result.extend(tree_to_flat_nodes(child)) + + return result + +class CanvasData: + def __init__(self, data: dict): + self.nodes = [] + self.groups = [] + self.edges = [] + self.parse_data(data) + + def parse_data(self, data: dict): + """解析canvas数据""" + # 处理所有节点 + for item in data: + node = Node(item) + self.nodes.append(node) + if node.type == 'group': + self.groups.append(node) + + def find_best_parent(self, node: Node) -> Optional[Node]: + """为节点找到最佳的父节点""" + candidates = [] + for group in self.groups: + if group.contains(node) and group != node: + candidates.append(group) + + if not candidates: + return None + + # 选择面积最小的包含组作为父节点 + return min(candidates, + key=lambda g: g.width * g.height) + + def build_tree(self) -> Node: + """构建树状结构""" + # 创建虚拟根节点 + root = Node({ + 'id': 'root', + 'type': 'root', + }) + + # 按面积从大到小排序groups + self.groups.sort(key=lambda g: g.width * g.height, reverse=True) + + # 构建节点关系 + assigned_nodes = set() + + # 先处理groups之间的关系 + for group in self.groups: + parent = self.find_best_parent(group) + if parent: + parent.children.append(group) + group.parent = parent + assigned_nodes.add(group.id) + else: + root.children.append(group) + group.parent = root + assigned_nodes.add(group.id) + + # 处理剩余节点 + for node in self.nodes: + if node.id not in assigned_nodes: + parent = self.find_best_parent(node) + if parent: + parent.children.append(node) + node.parent = parent + else: + root.children.append(node) + node.parent = root + + return root + + def to_tree_json(self) -> dict: + """转换为树状JSON结构""" + root = self.build_tree() + return root.to_dict() + + def to_flat_json(self) -> List[dict]: + """转换为扁平JSON结构""" + root = self.build_tree() + return root.to_flat_dict() + +def backup_file(file_path: str): + """备份文件""" + if os.path.exists(file_path): + timestamp = datetime.now().strftime('%Y%m%d%H%M%S') + backup_path = f"{file_path}.{timestamp}.bak" + os.rename(file_path, backup_path) + print(f"Backup {file_path} to {backup_path}") + +def sync_from_s3fs(): + """从s3fs同步到本地，并生成树状结构""" + s3fs_dir = "/mnt/s3fs/waverless" + local_dir = "/root/prjs/waverless" + + print(f"Starting sync from {s3fs_dir} to {local_dir}") + + # 同步canvas文件 + canvas_path = os.path.join(local_dir, "design.canvas") + s3fs_canvas_path = os.path.join(s3fs_dir, "design.canvas") + + if os.path.exists(s3fs_canvas_path): + # 备份当前文件 + backup_file(canvas_path) + + # 读取s3fs中的canvas + with open(s3fs_canvas_path, 'r', encoding='utf-8') as f: + canvas_data = json.load(f) + + # 生成树状结构 + canvas = CanvasData(canvas_data.get('nodes', [])) + tree_data = canvas.to_tree_json() + + # 保存树状结构 + tree_path = os.path.join(local_dir, "design.json") + with open(tree_path, 'w', encoding='utf-8') as f: + json.dump(tree_data, f, ensure_ascii=False, indent=2) + + # 保存原始canvas + with open(canvas_path, 'w', encoding='utf-8') as f: + json.dump(canvas_data, f, ensure_ascii=False, indent=2) + +def sync_to_s3fs(): + """从本地同步到s3fs，将树状结构转换回扁平结构""" + s3fs_dir = "/mnt/s3fs/waverless" + local_dir = "/root/prjs/waverless" + + print(f"Starting sync from {local_dir} to {s3fs_dir}") + + # 读取树状结构 + tree_path = os.path.join(local_dir, "design.json") + if not os.path.exists(tree_path): + print(f"Tree file {tree_path} not found") + return + + with open(tree_path, 'r', encoding='utf-8') as f: + tree_data = json.load(f) + + # 直接将树状结构转换为扁平节点列表 + flat_nodes = tree_to_flat_nodes(tree_data) + + # 保存到s3fs + s3fs_canvas_path = os.path.join(s3fs_dir, "design.canvas") + backup_file(s3fs_canvas_path) + + with open(s3fs_canvas_path, 'w', encoding='utf-8') as f: + json.dump({'nodes': flat_nodes}, f, ensure_ascii=False, indent=2) + +def main(): + if len(sys.argv) != 2: + print("Usage: python3 sync_md_files.py [from_s3fs|to_s3fs]") + sys.exit(1) + + command = sys.argv[1] + if command == "from_s3fs": + sync_from_s3fs() + elif command == "to_s3fs": + sync_to_s3fs() + else: + print(f"Unknown command: {command}") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/scripts/test_design_json_tool.py b/scripts/test_design_json_tool.py new file mode 100644 index 0000000..b3b761e --- /dev/null +++ b/scripts/test_design_json_tool.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +import os +import json +import shutil +import unittest +from scripts.design_json_tool import DesignJson, Node + +class TestDesignJsonTool(unittest.TestCase): + def setUp(self): + """测试前准备工作""" + # 创建测试用的JSON文件 + self.test_json_path = 'test_design.json' + self.test_data = { + "id": "root", + "type": "root", + "children": [ + { + "id": "group1", + "type": "group", + "label": "测试组1", + "children": [ + { + "id": "node1", + "type": "text", + "text": "测试节点1" + } + ] + } + ], + "edges": [] + } + with open(self.test_json_path, 'w', encoding='utf-8') as f: + json.dump(self.test_data, f, ensure_ascii=False, indent=2) + + self.design = DesignJson(self.test_json_path) + + def tearDown(self): + """测试后清理工作""" + if os.path.exists(self.test_json_path): + os.remove(self.test_json_path) + + def test_read_all(self): + """测试读取整个JSON""" + root = self.design.root + self.assertEqual(root.id, "root") + self.assertEqual(root.type, "root") + self.assertEqual(len(root.children), 1) + + def test_read_node(self): + """测试读取单个节点""" + node = self.design.get_node("node1") + self.assertIsNotNone(node) + self.assertEqual(node.type, "text") + self.assertEqual(node.text, "测试节点1") + + def test_read_group(self): + """测试读取组内容""" + nodes = self.design.get_group_nodes("group1") + self.assertEqual(len(nodes), 1) + self.assertEqual(nodes[0].id, "node1") + + def test_create_node(self): + """测试创建新节点""" + node_data = { + "id": "new_node", + "type": "text", + "text": "新建节点" + } + node_id = self.design.create_node(node_data) + self.assertEqual(node_id, "new_node") + node = self.design.get_node(node_id) + self.assertIsNotNone(node) + self.assertEqual(node.text, "新建节点") + + def test_update_node(self): + """测试更新节点""" + updates = {"text": "更新后的文本"} + success = self.design.update_node("node1", updates) + self.assertTrue(success) + node = self.design.get_node("node1") + self.assertEqual(node.text, "更新后的文本") + + def test_move_to_group(self): + """测试移动节点到组""" + # 先创建新组 + group_data = { + "id": "group2", + "type": "group", + "label": "测试组2" + } + self.design.create_node(group_data) + + # 移动节点 + success = self.design.move_to_group("node1", "group2") + self.assertTrue(success) + + # 验证移动结果 + nodes = self.design.get_group_nodes("group2") + self.assertEqual(len(nodes), 1) + self.assertEqual(nodes[0].id, "node1") + + def test_edges(self): + """测试边操作""" + # 添加边 + success = self.design.add_edge("node1", "group1", "test_edge") + self.assertTrue(success) + + # 验证入度 + incoming = self.design.get_incoming_nodes("group1") + self.assertEqual(len(incoming), 1) + self.assertEqual(incoming[0], ("node1", "test_edge")) + + # 验证出度 + outgoing = self.design.get_outgoing_nodes("node1") + self.assertEqual(len(outgoing), 1) + self.assertEqual(outgoing[0], ("group1", "test_edge")) + + # 删除边 + success = self.design.remove_edge("node1", "group1", "test_edge") + self.assertTrue(success) + + # 验证边已删除 + incoming = self.design.get_incoming_nodes("group1") + self.assertEqual(len(incoming), 0) + + def test_nonexistent_node(self): + """测试操作不存在的节点""" + # 读取不存在的节点 + node = self.design.get_node("nonexistent") + self.assertIsNone(node) + + # 更新不存在的节点 + success = self.design.update_node("nonexistent", {"text": "新文本"}) + self.assertFalse(success) + + # 移动不存在的节点 + success = self.design.move_to_group("nonexistent", "group1") + self.assertFalse(success) + + # 添加包含不存在节点的边 + success = self.design.add_edge("nonexistent", "node1") + self.assertFalse(success) + + def test_duplicate_operations(self): + """测试重复操作""" + # 重复创建同ID节点 + node_data = { + "id": "node1", # 已存在的ID + "type": "text", + "text": "重复节点" + } + original_node = self.design.get_node("node1") + node_id = self.design.create_node(node_data) + self.assertEqual(node_id, "node1") + # 验证节点内容未被覆盖 + node = self.design.get_node("node1") + self.assertEqual(node.text, original_node.text) + + # 重复添加相同的边 + self.design.add_edge("node1", "group1", "test_edge") + success = self.design.add_edge("node1", "group1", "test_edge") + self.assertTrue(success) # 添加成功但不会重复 + incoming = self.design.get_incoming_nodes("group1") + self.assertEqual(len(incoming), 1) # 只有一条边 + + def test_nested_groups(self): + """测试嵌套组操作""" + # 创建嵌套的组结构 + group2_data = { + "id": "group2", + "type": "group", + "label": "测试组2" + } + group3_data = { + "id": "group3", + "type": "group", + "label": "测试组3" + } + self.design.create_node(group2_data) + self.design.create_node(group3_data) + + # 将group3移动到group2中 + success = self.design.move_to_group("group3", "group2") + self.assertTrue(success) + + # 验证嵌套结构 + nodes = self.design.get_group_nodes("group2") + self.assertEqual(len(nodes), 1) + self.assertEqual(nodes[0].id, "group3") + + # 将节点移动到最内层组 + success = self.design.move_to_group("node1", "group3") + self.assertTrue(success) + + # 验证节点位置 + nodes = self.design.get_group_nodes("group3") + self.assertEqual(len(nodes), 1) + self.assertEqual(nodes[0].id, "node1") + + def test_save_and_load(self): + """测试保存和加载功能""" + # 修改数据 + self.design.update_node("node1", {"text": "修改后的文本"}) + self.design.add_edge("node1", "group1", "test_edge") + + # 保存文件 + self.design.save() + + # 重新加载 + new_design = DesignJson(self.test_json_path) + + # 验证修改是否保持 + node = new_design.get_node("node1") + self.assertEqual(node.text, "修改后的文本") + + incoming = new_design.get_incoming_nodes("group1") + self.assertEqual(len(incoming), 1) + self.assertEqual(incoming[0], ("node1", "test_edge")) + + def test_invalid_operations(self): + """测试无效操作""" + # 测试移动到非组节点 + success = self.design.move_to_group("node1", "node1") # node1不是组 + self.assertFalse(success) + + # 测试更新不存在的属性 + success = self.design.update_node("node1", {"nonexistent_attr": "value"}) + self.assertTrue(success) # 更新成功但属性未添加 + node = self.design.get_node("node1") + self.assertFalse(hasattr(node, "nonexistent_attr")) + + # 测试创建缺少必要属性的节点 + invalid_node = { + "type": "text" # 缺少id + } + with self.assertRaises(KeyError): + self.design.create_node(invalid_node) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/src/main/build.rs b/src/main/build.rs index 2e71809..d16dc9e 100644 --- a/src/main/build.rs +++ b/src/main/build.rs @@ -1,6 +1,9 @@ use std::io::Result; fn main() -> Result<()> { - prost_build::compile_protos( + let mut config = prost_build::Config::new(); + config + .type_attribute("BatchRequestId", "#[derive(Eq, Hash)]"); + config.compile_protos( &[ "src/general/network/proto_src/kv.proto", "src/general/network/proto_src/raft.proto", diff --git a/src/main/src/general/app/app_owned/mod.rs b/src/main/src/general/app/app_owned/mod.rs index 782b24c..615bf55 100644 --- a/src/main/src/general/app/app_owned/mod.rs +++ b/src/main/src/general/app/app_owned/mod.rs @@ -4,7 +4,7 @@ pub mod wasm_host_funcs; use crate::general::app::instance::InstanceTrait; use crate::general::app::instance::OwnedInstance; use crate::general::app::m_executor::{FnExeCtxAsync, FnExeCtxSync}; -use crate::result::{WSResult, WsFuncError}; +use crate::result::{WSResult}; use async_trait::async_trait; #[async_trait] diff --git a/src/main/src/general/app/app_owned/wasm_host_funcs/mod.rs b/src/main/src/general/app/app_owned/wasm_host_funcs/mod.rs index c3df65c..30e3516 100644 --- a/src/main/src/general/app/app_owned/wasm_host_funcs/mod.rs +++ b/src/main/src/general/app/app_owned/wasm_host_funcs/mod.rs @@ -13,7 +13,7 @@ use result::ResultFuncsRegister; mod utils { use super::UnsafeFunctionCtx; - use crate::general::app::m_executor::{FnExeCtxAsync, FnExeCtxBase}; + use crate::general::app::m_executor::{FnExeCtxAsync}; use crate::general::app::InstanceManager; use crate::{ general::m_os::OperatingSystem, sys::LogicalModulesRef, util::SendNonNull, diff --git a/src/main/src/general/app/app_owned/wasm_host_funcs/result.rs b/src/main/src/general/app/app_owned/wasm_host_funcs/result.rs index ff83530..6d092cd 100644 --- a/src/main/src/general/app/app_owned/wasm_host_funcs/result.rs +++ b/src/main/src/general/app/app_owned/wasm_host_funcs/result.rs @@ -1,5 +1,4 @@ use super::{utils, HostFuncRegister}; -use crate::general::app::m_executor::FnExeCtxAsync; #[cfg(target_os = "macos")] use wasmer::{imports, Function, FunctionType, Imports}; diff --git a/src/main/src/general/app/app_shared/java.rs b/src/main/src/general/app/app_shared/java.rs index d70304a..432edf5 100644 --- a/src/main/src/general/app/app_shared/java.rs +++ b/src/main/src/general/app/app_shared/java.rs @@ -6,7 +6,6 @@ use crate::{ general::m_os::{OperatingSystem, OsProcessType}, result::{WSError, WSResult, WsFuncError}, }; -use std::path::Path; use super::process::PID; diff --git a/src/main/src/general/app/app_shared/process.rs b/src/main/src/general/app/app_shared/process.rs index 298d13e..2f96d6a 100644 --- a/src/main/src/general/app/app_shared/process.rs +++ b/src/main/src/general/app/app_shared/process.rs @@ -8,7 +8,7 @@ use crate::general::{ app::AppType, network::rpc_model::{self, HashValue}, }; -use crate::result::{WSError, WsFuncError}; +use crate::result::{WsFuncError}; use async_trait::async_trait; use enum_as_inner::EnumAsInner; use parking_lot::RwLock; diff --git a/src/main/src/general/app/mod.rs b/src/main/src/general/app/mod.rs index a1f154c..9a2a837 100644 --- a/src/main/src/general/app/mod.rs +++ b/src/main/src/general/app/mod.rs @@ -18,7 +18,6 @@ use crate::{general::network::proto, result::WSResultExt}; use crate::{ general::{ data::{ - kv_interface::KvOps, m_data_general::{DataGeneral, DATA_UID_PREFIX_APP_META}, m_kv_store_engine::{KeyTypeServiceList, KvAdditionalConf, KvStoreEngine}, }, @@ -34,7 +33,7 @@ use crate::{ use crate::{ logical_module_view_impl, master::m_master::Master, - result::{ErrCvt, WSResult, WsFuncError}, + result::{WSResult, WsFuncError}, sys::{LogicalModule, LogicalModuleNewArgs, LogicalModulesRef, NodeID}, util::{self, JoinHandleWrapper}, }; @@ -43,7 +42,6 @@ use axum::body::Bytes; use enum_as_inner::EnumAsInner; use m_executor::FnExeCtxSyncAllowedType; use serde::{de::Error, Deserialize, Deserializer, Serialize}; -use std::path::PathBuf; use std::{ borrow::Borrow, collections::{BTreeMap, HashMap}, diff --git a/src/main/src/general/data/m_data_general/batch.rs b/src/main/src/general/data/m_data_general/batch.rs new file mode 100644 index 0000000..e27d3cd --- /dev/null +++ b/src/main/src/general/data/m_data_general/batch.rs @@ -0,0 +1,161 @@ +/// Batch Data Transfer Interface +/// +/// # Design Overview +/// The batch interface is designed for efficient large-scale data transfer from data holders (writers) +/// to the system. It differs from the regular data interface in several key aspects: +/// +/// ## Batch Interface +/// - Purpose: Optimized for data holders to push complete datasets +/// - Key Feature: Supports streaming transfer during data writing process +/// - Use Case: Allows transfer before local sharding is complete +/// - Operation: Uses fixed-size block transfer with real-time processing +/// +/// ## Data Interface (For Comparison) +/// - Purpose: General-purpose data read/write operations +/// - Write Flow: Data is sharded and distributed across nodes +/// - Read Flow: Shards are collected from nodes and reassembled +/// - Operation: Requires complete data and consistency checks +/// +/// # Implementation Details +/// The batch interface implements this through: +/// - Efficient block-based streaming transfer +/// - Concurrent processing of received blocks +/// - Support for both memory and file-based transfers +/// - Real-time block validation and assembly +/// +/// For detailed implementation of the regular data interface, see the data.rs module. +use super::*; +use crate::general::network::proto; +use tokio::io::{AsyncReadExt, AsyncSeekExt}; +use tokio::sync::Semaphore; +use std::sync::Arc; +use std::time::Duration; +use crate::general::data::m_data_general::dataitem::DataItemSource; + +impl proto::DataItem { + pub fn size(&self) -> usize { + match &self.data_item_dispatch { + Some(proto::data_item::DataItemDispatch::RawBytes(bytes)) => bytes.len(), + Some(proto::data_item::DataItemDispatch::File(file_data)) => file_data.file_content.len(), + None => 0, + } + } +} + +impl DataGeneral { + /// 发起批量数据传输 + pub async fn call_batch_data( + &self, + node_id: NodeID, + unique_id: Vec, + version: u64, + data: proto::DataItem, + ) -> WSResult { + // 调用 batch_transfer 函数处理数据传输 + async fn batch_transfer( + unique_id: Vec, + version: u64, + target_node: NodeID, + data: Arc, + view: DataGeneralView, + ) -> WSResult<()> { + let total_size = match data.as_ref() { + DataItemSource::Memory { data } => data.len(), + DataItemSource::File { path } => { + tokio::fs::metadata(path).await?.len() as usize + } + }; + let total_blocks = (total_size + DEFAULT_BLOCK_SIZE - 1) / DEFAULT_BLOCK_SIZE; + let semaphore = Arc::new(Semaphore::new(32)); + let mut handles: Vec>> = Vec::new(); + + // 发送所有数据块 + for block_idx in 0..total_blocks { + // 获取信号量许可 + let permit = semaphore.clone().acquire_owned().await.unwrap(); + let offset = block_idx as usize * DEFAULT_BLOCK_SIZE; + let size = DEFAULT_BLOCK_SIZE.min(total_size - offset); + + // 读取数据块 + let block_data = match data.as_ref() { + DataItemSource::Memory { data } => data[offset..offset + size].to_vec(), + DataItemSource::File { path } => { + let mut file = tokio::fs::File::open(path).await?; + let mut buffer = vec![0; size]; + let _ = file.seek(std::io::SeekFrom::Start(offset as u64)).await?; + let _ = file.read_exact(&mut buffer).await?; + buffer + } + }; + + // 构造请求 + let request = proto::BatchDataRequest { + request_id: Some(proto::BatchRequestId { + node_id: target_node as u32, + sequence: block_idx as u64, // 修复：使用 u64 + }), + dataset_unique_id: unique_id.clone(), + data_item_idx: 0, // 因为是整体传输，所以使用0 + block_type: match data.as_ref() { + DataItemSource::Memory { .. } => proto::BatchDataBlockType::Memory as i32, + DataItemSource::File { .. } => proto::BatchDataBlockType::File as i32, + }, + block_index: block_idx as u32, + data: block_data, + operation: proto::DataOpeType::Write as i32, + unique_id: unique_id.clone(), + version, + total_size: total_size as u64, + }; + + // 发送请求 + let view = view.clone(); + let handle = tokio::spawn(async move { + let _permit = permit; // 持有permit直到任务完成 + let resp = view.data_general() + .rpc_call_batch_data + .call( + view.p2p(), + target_node, + request, + Some(Duration::from_secs(30)), + ) + .await?; + + if !resp.success { + return Err(WsDataError::BatchTransferError { + request_id: proto::BatchRequestId { + node_id: target_node as u32, + sequence: block_idx as u64, // 修复：使用 u64 + }, + msg: resp.error_message, + }.into()); + } + Ok(()) + }); + + handles.push(handle); + } + + // 等待所有请求完成 + for handle in handles { + handle.await??; + } + + Ok(()) + } + + let data = Arc::new(DataItemSource::new(data)); + batch_transfer(unique_id.clone(), version, node_id, data, self.view.clone()).await?; + + Ok(proto::BatchDataResponse { + request_id: Some(proto::BatchRequestId { + node_id: node_id, + sequence: 0, + }), + success: true, + error_message: String::new(), + version, + }) + } +} diff --git a/src/main/src/general/data/m_data_general/batch_handler.rs b/src/main/src/general/data/m_data_general/batch_handler.rs new file mode 100644 index 0000000..c5420ce --- /dev/null +++ b/src/main/src/general/data/m_data_general/batch_handler.rs @@ -0,0 +1,77 @@ +use crate::general::network::{ + proto::BatchDataRequest, + proto::BatchDataResponse, + m_p2p::RPCResponsor, +}; +use std::sync::Arc; +use tokio::sync::Mutex; +use tracing; + +/// 共享状态,用于记录最新的请求响应器 +/// 当收到新的请求时,会更新响应器并自动处理旧的请求 +#[derive(Clone)] +pub struct SharedWithBatchHandler { + /// 当前活跃的响应器 + /// 使用 Arc 保证线程安全 + responsor: Arc>>>, +} + +impl SharedWithBatchHandler { + /// 创建新的共享状态 + #[must_use] + pub fn new() -> Self { + Self { + responsor: Arc::new(Mutex::new(None)), + } + } + + /// 更新响应器 + /// 如果存在旧的响应器,会自动返回成功 + /// + /// # 参数 + /// * `responsor` - 新的响应器 + pub async fn update_responsor(&self, responsor: RPCResponsor) { + let mut guard = self.responsor.lock().await; + if let Some(old_responsor) = guard.take() { + // 旧的responsor直接返回成功 + if let Err(e) = old_responsor.send_resp(BatchDataResponse { + request_id: None, // 这里需要正确的 request_id + version: 0, // 这里需要正确的版本号 + success: true, + error_message: String::new(), + }).await { + tracing::error!("Failed to respond to old request: {}", e); + } + } + *guard = Some(responsor); + } + + /// 获取最终的响应器 + /// 用于在所有数据都写入完成后发送最终响应 + pub async fn get_final_responsor(&self) -> Option> { + self.responsor.lock().await.take() + } +} + +/// 批量数据传输状态 +/// 用于管理单个批量数据传输请求的生命周期 +pub struct BatchReceiveState { + /// 写入任务句柄 + pub handle: super::dataitem::WriteSplitDataTaskHandle, + /// 共享状态,用于处理请求响应 + pub shared: SharedWithBatchHandler, +} + +impl BatchReceiveState { + /// 创建新的批量数据传输状态 + /// + /// # 参数 + /// * `handle` - 写入任务句柄 + /// * `shared` - 共享状态 + pub fn new(handle: super::dataitem::WriteSplitDataTaskHandle, shared: SharedWithBatchHandler) -> Self { + Self { + handle, + shared, + } + } +} diff --git a/src/main/src/general/data/m_data_general/dataitem.rs b/src/main/src/general/data/m_data_general/dataitem.rs index 27ef392..fbec9a8 100644 --- a/src/main/src/general/data/m_data_general/dataitem.rs +++ b/src/main/src/general/data/m_data_general/dataitem.rs @@ -1,31 +1,33 @@ -use crate::general::data::m_data_general::DataItemIdx; -use crate::general::data::m_data_general::GetOrDelDataArgType; +use crate::general::data::m_data_general::UniqueId; use crate::general::network::proto; +use crate::general::data::m_data_general::{DataItemIdx, DataSplitIdx, GetOrDelDataArgType}; use crate::general::network::proto_ext::ProtoExtDataItem; -use crate::result::WSError; -use crate::result::WSResult; -use crate::result::WsDataError; -use crate::result::WsIoErr; -use crate::result::WsRuntimeErr; -use base64::Engine; -use futures::future::join_all; +use crate::result::{WSError, WSResult, WsDataError}; +use futures::stream::{FuturesUnordered, StreamExt}; use std::collections::btree_set; use std::ops::Range; use std::path::PathBuf; use std::sync::Arc; +use tokio::sync::mpsc; +use tokio::sync::broadcast; +use tracing; +use base64::{engine::general_purpose::STANDARD, Engine as _}; -use super::CacheModeVisitor; -use super::DataSplitIdx; +const DEFAULT_BLOCK_SIZE: usize = 4096; -// iterator for wanted dataitem idxs +/// 用于遍历数据项索引的迭代器 +#[derive(Debug)] pub(super) enum WantIdxIter<'a> { + /// 遍历多个指定索引 PartialMany { iter: btree_set::Iter<'a, DataItemIdx>, }, + /// 遍历单个索引 PartialOne { idx: DataItemIdx, itercnt: u8, }, + /// 遍历所有或删除操作的索引 Other { ty: GetOrDelDataArgType, itercnt: u8, @@ -34,6 +36,12 @@ pub(super) enum WantIdxIter<'a> { } impl<'a> WantIdxIter<'a> { + /// 创建新的索引迭代器 + /// + /// # 参数 + /// * `ty` - 迭代类型 + /// * `itemcnt` - 数据项总数 + #[must_use] pub(super) fn new(ty: &'a GetOrDelDataArgType, itemcnt: DataItemIdx) -> Self { match ty { GetOrDelDataArgType::PartialMany { idxs } => Self::PartialMany { iter: idxs.iter() }, @@ -71,22 +79,30 @@ impl<'a> Iterator for WantIdxIter<'a> { let ret = *itercnt; *itercnt += 1; Some(ret) - } - } + } + } GetOrDelDataArgType::PartialMany { .. } | GetOrDelDataArgType::PartialOne { .. } => { panic!("PartialMany should be handled by iter") -} + } }, } } } +/// 共享内存区域的持有者 +/// 负责管理共享内存的所有权和生命周期 +#[derive(Debug, Clone)] pub struct SharedMemHolder { + /// 共享内存数据 data: Arc>, } impl SharedMemHolder { + pub fn len(&self) -> usize { + self.data.len() + } + pub fn try_take_data(self) -> Option> { // SAFETY: // 1. We're only replacing the Arc with an empty Vec @@ -100,15 +116,34 @@ impl SharedMemHolder { None } } - // } + + pub fn as_raw_bytes(&self) -> Option<&[u8]> { + Some(self.data.as_ref()) + } } +impl From for Vec { + fn from(holder: SharedMemHolder) -> Self { + holder.as_raw_bytes().expect("Failed to get raw bytes").to_vec() + } +} + +/// 共享内存区域的访问者 +/// 提供对特定范围内存的安全访问 pub struct SharedMemOwnedAccess { + /// 共享内存数据 data: Arc>, + /// 访问范围 range: Range, } impl SharedMemOwnedAccess { + /// 获取可变字节切片 + /// + /// # Safety + /// 调用者必须确保: + /// 1. 没有其他线程同时访问这块内存 + /// 2. 访问范围不超过内存边界 pub unsafe fn as_bytes_mut(&self) -> &mut [u8] { // SAFETY: // 1. We have &mut self, so we have exclusive access to this data @@ -120,7 +155,12 @@ impl SharedMemOwnedAccess { } } -pub fn new_shared_mem(splits: &Vec>) -> (SharedMemHolder, Vec) { +/// 创建新的共享内存和访问者 +/// +/// # 参数 +/// * `splits` - 内存分片范围列表 +#[must_use] +pub fn new_shared_mem(splits: &[Range]) -> (SharedMemHolder, Vec) { let len = splits.iter().map(|range| range.len()).sum(); let data = Arc::new(vec![0; len]); let owned_accesses = splits @@ -134,289 +174,516 @@ pub fn new_shared_mem(splits: &Vec>) -> (SharedMemHolder, Vec>` - 分片范围列表 +#[must_use] +pub fn calculate_splits(total_size: usize) -> Vec> { + let total_blocks = (total_size + DEFAULT_BLOCK_SIZE - 1) / DEFAULT_BLOCK_SIZE; + let mut splits = Vec::with_capacity(total_blocks); + for i in 0..total_blocks { + let start = i * DEFAULT_BLOCK_SIZE; + let end = (start + DEFAULT_BLOCK_SIZE).min(total_size); + splits.push(start..end); + } + splits +} + +/// 写入类型 +/// 支持写入文件或内存两种模式 +#[derive(Debug, Clone)] +pub enum WriteSplitDataType { + /// 文件写入模式 + File { + /// 目标文件路径 + path: PathBuf, + }, + /// 内存写入模式 + Mem { + /// 共享内存区域 + shared_mem: SharedMemHolder, + }, +} + +/// 写入分片任务的结果 +#[derive(Debug)] +pub struct WriteSplitTaskResult { + /// 写入的数据大小 + pub written_size: usize, +} + +/// 写入分片任务组 +/// 管理一组相关的写入任务 +#[derive(Debug)] pub enum WriteSplitDataTaskGroup { + /// 文件写入模式 ToFile { + /// 任务唯一标识 + unique_id: UniqueId, + /// 目标文件路径 file_path: PathBuf, - tasks: Vec>>, + /// 任务列表 + tasks: Vec>, + /// 接收新任务的通道 + rx: mpsc::Receiver>, + /// 预期总大小 + expected_size: usize, + /// 当前已写入大小 + current_size: usize, + /// 广播通道发送端，用于通知任务完成 + broadcast_tx: Arc>, }, + /// 内存写入模式 ToMem { + /// 任务唯一标识 + unique_id: UniqueId, + /// 共享内存区域 shared_mem: SharedMemHolder, - tasks: Vec>>, + /// 任务列表 + tasks: Vec>, + /// 接收新任务的通道 + rx: mpsc::Receiver>, + /// 预期总大小 + expected_size: usize, + /// 当前已写入大小 + current_size: usize, + /// 广播通道发送端，用于通知任务完成 + broadcast_tx: Arc>, }, } impl WriteSplitDataTaskGroup { + /// 创建新的任务组 pub async fn new( - unique_id: Vec, - splits: Vec>, - mut rx: tokio::sync::mpsc::Receiver>, - cachemode: CacheModeVisitor, - ) -> WSResult { - tracing::debug!( - "new merge task group for uid({:?}), cachemode({})", - unique_id, - cachemode.0 - ); - if cachemode.is_map_file() { - tracing::debug!("cachemode is map_file"); - // base64 - // let file_path = PathBuf::from(format!("{:?}.data", unique_id)); - let file_path = PathBuf::from(format!( - "{}.data", - base64::engine::general_purpose::STANDARD.encode(&unique_id) - )); - - let file = std::fs::OpenOptions::new() - .create(true) - .write(true) - .open(&file_path)?; - let file = std::sync::Arc::new(file); - - let mut tasks = vec![]; - for _ in 0..splits.len() { - let parital_data = rx.recv().await.unwrap(); - match parital_data { - Err(e) => { - return Err(e); - } - Ok((splitidx, split_data_item)) => { - let file = file.clone(); - let unique_id = unique_id.clone(); - let split_range = splits[splitidx as usize].clone(); - - let task = tokio::task::spawn_blocking(move || { - let Some(proto::FileData { - file_content: split_data_bytes, - .. - }) = split_data_item.as_file_data() - else { - return Err(WsDataError::SplitDataItemNotFileData { - unique_id: unique_id.clone(), - splitidx, - } - .into()); - }; - - if split_range.len() != split_data_bytes.len() { - return Err(WsDataError::SplitLenMismatch { - unique_id, - splitidx, - expect: split_range.len(), - actual: split_data_bytes.len(), - } - .into()); - } - // SAFETY: Each task writes to a different non-overlapping portion of the file - use std::os::unix::fs::FileExt; - if let Err(e) = - file.write_at(split_data_bytes, split_range.start as u64) - { - return Err(WSError::WsIoErr(WsIoErr::Io(e))); - } - Ok(()) - }); - tasks.push(task); - } + unique_id: UniqueId, + total_size: usize, + block_type: proto::BatchDataBlockType, + version: u64, + ) -> WSResult<(Self, WriteSplitDataTaskHandle)> { + let (tx, rx) = mpsc::channel(32); + let (broadcast_tx, _) = broadcast::channel::<()>(32); + let broadcast_tx = Arc::new(broadcast_tx); + + match block_type { + proto::BatchDataBlockType::File => { + let file_path = PathBuf::from(format!("{}.data", + STANDARD.encode(&unique_id))); + + let handle = WriteSplitDataTaskHandle { + tx, + write_type: WriteSplitDataType::File { + path: file_path.clone(), + }, + version, + broadcast_tx: broadcast_tx.clone(), + }; + + let group = Self::ToFile { + unique_id, + file_path, + tasks: Vec::new(), + rx, + expected_size: total_size, + current_size: 0, + broadcast_tx: broadcast_tx.clone(), + }; + + Ok((group, handle)) + } + proto::BatchDataBlockType::Memory => { + let shared_mem = SharedMemHolder { + data: Arc::new(vec![0; total_size]), + }; + + let handle = WriteSplitDataTaskHandle { + tx, + write_type: WriteSplitDataType::Mem { + shared_mem: shared_mem.clone(), + }, + version, + broadcast_tx: broadcast_tx.clone(), + }; + + let group = Self::ToMem { + unique_id, + shared_mem, + tasks: Vec::new(), + rx, + expected_size: total_size, + current_size: 0, + broadcast_tx: broadcast_tx.clone(), + }; + + Ok((group, handle)) + } + } + } + + /// 处理所有写入任务 + /// + /// # 返回 + /// * `Ok(item)` - 所有数据写入完成,返回数据项 + /// * `Err(e)` - 写入过程中出错 + pub async fn process_tasks(&mut self) -> WSResult { + let mut pending_tasks: FuturesUnordered> = FuturesUnordered::new(); + + match self { + Self::ToFile { tasks, .. } | + Self::ToMem { tasks, .. } => { + for task in tasks.drain(..) { + pending_tasks.push(task); } } - Ok(Self::ToFile { file_path, tasks }) - } else if cachemode.is_map_common_kv() { - tracing::debug!("cachemode is map_common_kv"); - let (shared_mem, owned_accesses) = new_shared_mem(&splits); - let mut owned_accesses = owned_accesses - .into_iter() - .map(|access| Some(access)) - .collect::>(); - let mut tasks = vec![]; - for _ in 0..splits.len() { - let parital_data = rx.recv().await.unwrap(); - match parital_data { - Err(e) => { - return Err(e); - } - Ok((splitidx, split_data_item)) => { - let owned_access = owned_accesses[splitidx].take().unwrap(); - let unique_id = unique_id.clone(); - let task = tokio::spawn(async move { - // write to shared memory - let access = unsafe { owned_access.as_bytes_mut() }; - let Some(split_data_item) = split_data_item.as_raw_bytes() else { - return Err(WsDataError::SplitDataItemNotRawBytes { - unique_id: unique_id.clone(), - splitidx, - } - .into()); - }; - if access.len() != split_data_item.len() { - return Err(WsDataError::SplitLenMismatch { - unique_id: unique_id.clone(), - splitidx, - expect: access.len(), - actual: split_data_item.len(), + } + + loop { + // 1. 检查完成状态 + match self.try_complete()? { + Some(item) => return Ok(item), + None => {} // 继续等待 + } + + // 2. 等待新任务或已有任务完成 + tokio::select! { + Some(new_task) = match self { + Self::ToFile { rx, .. } | + Self::ToMem { rx, .. } => rx.recv() + } => { + pending_tasks.push(new_task); + } + Some(completed_result) = pending_tasks.next() => { + match completed_result { + Ok(result) => { + match self { + Self::ToFile { current_size, .. } | + Self::ToMem { current_size, .. } => { + *current_size += result.written_size; } - .into()); } - access.copy_from_slice(split_data_item); - Ok(()) - }); - tasks.push(task); + } + Err(e) => { + tracing::error!("Task failed: {}", e); + return Err(WSError::WsDataError(WsDataError::BatchTransferTaskFailed { + reason: format!("Task failed: {}", e) + })); + } } } } - Ok(Self::ToMem { shared_mem, tasks }) - } else { - panic!("cachemode should be map_file or map_mem"); } } - pub async fn join(self) -> WSResult { + /// 检查写入完成状态 + /// + /// 返回: + /// - Ok(Some(item)) - 写入完成,返回数据项 + /// - Ok(None) - 写入未完成 + /// - Err(e) - 写入出错 + fn try_complete(&self) -> WSResult> { match self { - WriteSplitDataTaskGroup::ToFile { file_path, tasks } => { - let taskress = join_all(tasks).await; - for res in taskress { - if res.is_err() { - return Err(WSError::from(WsRuntimeErr::TokioJoin { - err: res.unwrap_err(), - context: "write split data to file".to_owned(), - })); - } - if res.as_ref().unwrap().is_err() { - return Err(res.unwrap().unwrap_err()); - } + Self::ToFile { current_size, expected_size, file_path, unique_id, .. } => { + if *current_size > *expected_size { + Err(WSError::WsDataError(WsDataError::BatchTransferError { + request_id: proto::BatchRequestId { + node_id: 0, // 这里需要传入正确的node_id + sequence: 0, + }, + msg: format!("Written size {} exceeds expected size {} for unique_id {:?}", + current_size, expected_size, unique_id) + })) + } else if *current_size == *expected_size { + Ok(Some(proto::DataItem::new_file_data(file_path.clone(), false))) + } else { + Ok(None) + } + } + Self::ToMem { current_size, expected_size, shared_mem, unique_id, .. } => { + if *current_size > *expected_size { + Err(WSError::WsDataError(WsDataError::BatchTransferError { + request_id: proto::BatchRequestId { + node_id: 0, // 这里需要传入正确的node_id + sequence: 0, + }, + msg: format!("Written size {} exceeds expected size {} for unique_id {:?}", + current_size, expected_size, unique_id) + })) + } else if *current_size == *expected_size { + Ok(Some(proto::DataItem::new_raw_bytes(shared_mem.clone()))) + } else { + Ok(None) } - Ok(proto::DataItem::new_file_data(file_path, false)) } - WriteSplitDataTaskGroup::ToMem { - shared_mem: shared_mems, - tasks, - } => { - let taskress = join_all(tasks).await; - for res in taskress { - if res.is_err() { - return Err(WSError::from(WsRuntimeErr::TokioJoin { - err: res.unwrap_err(), - context: "write split data to file".to_owned(), - })); + } + } +} + +/// 写入分片任务的句柄 +/// 用于提交新的分片任务和等待任务完成 +#[derive(Clone)] +pub struct WriteSplitDataTaskHandle { + /// 发送任务的通道 + tx: mpsc::Sender>, + /// 写入类型(文件或内存) + write_type: WriteSplitDataType, + /// 数据版本号 + /// 用于防止数据覆盖和保证数据一致性: + /// 1. 防止旧版本数据覆盖新版本数据 + /// 2. 客户端可以通过比较版本号确认数据是否最新 + version: u64, + /// 广播通道发送端，用于通知任务完成 + broadcast_tx: Arc>, +} + +impl WriteSplitDataTaskHandle { + /// 获取当前数据版本号 + pub fn version(&self) -> u64 { + self.version + } + + /// 提交新的分片任务 + /// + /// # 参数 + /// * `idx` - 分片索引,表示数据在整体中的偏移位置 + /// * `data` - 分片数据 + /// + /// # 返回 + /// * `Ok(())` - 任务提交成功 + /// * `Err(e)` - 任务提交失败,可能是通道已关闭 + pub async fn submit_split(&self, idx: DataSplitIdx, data: proto::DataItem) -> WSResult<()> { + let task = match &self.write_type { + WriteSplitDataType::File { path } => { + let path = path.clone(); + let offset = idx; + let data = data.as_raw_bytes().unwrap_or(&[]).to_vec(); + let written_size = data.len(); + tokio::spawn(async move { + let result = tokio::fs::OpenOptions::new() + .create(true) + .write(true) + .open(&path) + .await; + + match result { + Ok(mut file) => { + use tokio::io::{AsyncSeekExt, AsyncWriteExt}; + if let Err(e) = async move { + // 验证seek结果 + let seek_pos = file.seek(std::io::SeekFrom::Start(offset as u64)).await?; + if seek_pos != offset as u64 { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("Seek position mismatch: expected {}, got {}", offset, seek_pos) + )); + } + // write_all保证写入所有数据或返回错误 + file.write_all(&data).await?; + Ok::<_, std::io::Error>(()) + }.await { + tracing::error!("Failed to write file data at offset {}: {}", offset, e); + panic!("Failed to write file: {}", e); + } + WriteSplitTaskResult { written_size } + } + Err(e) => { + tracing::error!("Failed to open file at offset {}: {}", offset, e); + panic!("Failed to open file: {}", e); + } } - if res.as_ref().unwrap().is_err() { - return Err(res.unwrap().unwrap_err()); + }) + } + WriteSplitDataType::Mem { shared_mem } => { + let mem = shared_mem.clone(); + let offset = idx; + let Some(data) = data.as_raw_bytes().map(|data| data.to_vec()) else { + return Err(WSError::WsDataError(WsDataError::BatchTransferFailed { + request_id: proto::BatchRequestId { + node_id: 0, + sequence: 0, + }, + reason: format!("mem data expected"), + })); + }; + let written_size = data.len(); + tracing::debug!("submit_split: Mem, len:{}, target len:{}", data.len(), shared_mem.len()); + + tokio::spawn(async move { + unsafe { + let slice = std::slice::from_raw_parts_mut( + mem.data.as_ptr() as *mut u8, + mem.data.len() + ); + slice[offset..offset + data.len()].copy_from_slice(&data); } - } - // convert to dataitem - Ok(proto::DataItem::new_raw_bytes( - shared_mems - .try_take_data() - .expect("shared_mems should be take when all partial task stoped"), - )) + WriteSplitTaskResult { written_size } + }) } + }; + + // 发送到通道 + let _ = self.broadcast_tx.send(()); + self.tx.send(task).await.map_err(|e| { + tracing::error!("Failed to submit task: channel closed, idx: {:?}, error: {}", idx, e); + WSError::WsDataError(WsDataError::DataSplitTaskError { + msg: format!("Failed to submit task: channel closed, error: {}", e) + }) + }) + } + + /// 等待所有已提交的写入任务完成 + /// 关闭发送端,不再接收新任务 + pub async fn wait_all_tasks(&self) -> WSResult<()> { + // 等待广播通知 + let mut rx = self.broadcast_tx.subscribe(); + rx.recv().await.map_err(|e| { + tracing::error!("Failed to wait for tasks: {}", e); + WSError::WsDataError(WsDataError::BatchTransferTaskFailed { + reason: format!("Failed to wait for tasks: {}", e) + }) + })?; + + Ok(()) + } +} + +#[derive(Debug)] +pub enum DataItemSource { + Memory { + data: Vec, + }, + File { + path: PathBuf, + }, +} + +impl DataItemSource { + pub fn to_debug_string(&self) -> String { + match self { + Self::Memory { data } => { + //limit range vec + format!("Memory({:?})", data[0..10.min(data.len())].to_vec()) + } + Self::File { path } => format!("File({})", path.to_string_lossy()), + } + } + + pub fn new(data: proto::DataItem) -> Self { + match &data.data_item_dispatch { + Some(proto::data_item::DataItemDispatch::RawBytes(bytes)) => Self::Memory { + data: bytes.clone(), + }, + Some(proto::data_item::DataItemDispatch::File(file_data)) => Self::File { + path: file_data.file_name_opt.clone().into(), + }, + _ => Self::Memory { + data: Vec::new(), + }, + } + } + + pub async fn size(&self) -> WSResult { + match self { + DataItemSource::Memory { data } => Ok(data.len()), + DataItemSource::File { path } => { + let metadata = tokio::fs::metadata(path).await.map_err(|e| + WSError::WsDataError(WsDataError::BatchTransferFailed { + request_id: proto::BatchRequestId { + node_id: 0, // 这里需要传入正确的node_id + sequence: 0, + }, + reason: format!("Failed to get file size: {}", e), + }) + )?; + Ok(metadata.len() as usize) + } + } + } + + pub fn block_type(&self) -> proto::BatchDataBlockType { + match self { + DataItemSource::Memory { .. } => proto::BatchDataBlockType::Memory, + DataItemSource::File { .. } => proto::BatchDataBlockType::File, + } + } + + pub async fn get_block(&self, block_idx: usize) -> WSResult> { + match self { + DataItemSource::Memory { data } => { + if block_idx == 0 { + Ok(data.clone()) + } else { + Err(WSError::WsDataError(WsDataError::SizeMismatch { + expected: data.len(), + actual: 0, + })) + } + }, + DataItemSource::File { path } => { + let content = tokio::fs::read(path).await.map_err(|_e| { + WSError::WsDataError(WsDataError::ReadDataFailed { + path: path.clone(), + }) + })?; + if block_idx == 0 { + Ok(content) + } else { + Err(WSError::WsDataError(WsDataError::SizeMismatch { + expected: content.len(), + actual: 0, + })) + } + }, } } } -// pub async fn read_splitdata_from_nodes_to_file<'a>( -// ty: &GetOrDelDataArgType, -// unique_id: &[u8], -// view: &DataGeneralView, -// meta: &DataSetMetaV2, -// each_node_data: HashMap, -// ) ->ReadSplitDataTask{ -// // prepare file with meta size -// let file_path = format!("{}.data", unique_id); -// let file = File::create(file_path)?; - -// // parallel read and write to position of file with pwrite -// let mut tasks = vec![]; -// // get idxs, one idx one file - -// for (node_id, req) in each_node_data { -// let view = view.clone(); -// let task = tokio::spawn(async move { -// let res = view -// .data_general() -// .rpc_call_get_data -// .call(view.p2p(), node_id, req, Some(Duration::from_secs(30))) -// .await; -// match res { -// Err(err) => { -// tracing::warn!("get/delete data failed {}", err); -// vec![] -// } -// Ok(res) => { -// res. -// // get offset and size by meta with got - -// vec![] -// }, -// } -// }); -// tasks.push(task); -// } -// Ok(HashMap::new()) -// } - -// pub async fn read_splitdata_from_nodes_to_mem<'a>( -// ty: &GetOrDelDataArgType, -// unique_id: &[u8], -// view: &DataGeneralView, -// meta: &DataSetMetaV2, -// each_node_data: HashMap, -// ) -> ReadSplitDataTask { -// // read to mem -// let mut tasks = vec![]; -// for (node_id, req) in each_node_data { -// let view = view.clone(); -// let task = tokio::spawn(async move { -// let req_idxs = req.idxs.clone(); -// tracing::debug!("rpc_call_get_data start, remote({})", node_id); -// let res = view -// .data_general() -// .rpc_call_get_data -// .call(view.p2p(), node_id, req, Some(Duration::from_secs(30))) -// .await; -// tracing::debug!("rpc_call_get_data returned, remote({})", node_id); -// let res: WSResult> = res.map(|response| { -// if !response.success { -// tracing::warn!("get/delete data failed {}", response.message); -// vec![] -// } else { -// req_idxs.into_iter().zip(response.data).collect() -// } -// }); -// (node_id, res) -// }); -// tasks.push(task); -// } - -// let mut node_partialdatas: HashMap<(NodeID, DataItemIdx), proto::DataItem> = HashMap::new(); -// for tasks in tasks { -// let (node_id, partdata) = tasks.await.map_err(|err| { -// WSError::from(WsRuntimeErr::TokioJoin { -// err, -// context: "get_or_del_data - get_or_del ing remote data".to_owned(), -// }) -// })?; - -// match partdata { -// Err(err) => { -// return Err(err); -// } -// Ok(partdata) => { -// for (idx, data_item) in partdata { -// let _ = node_partialdatas.insert((node_id, idx as u8), data_item); -// } -// } -// } -// } - -// let mut idx_2_data_item: HashMap = HashMap::new(); -// for idx in WantIdxIter::new(&ty) { -// let data_split = &meta.datas_splits[idx as usize]; -// let data_item = data_split.recorver_data(unique_id, idx, &mut node_partialdatas)?; - -// idx_2_data_item -// .insert(idx, proto::DataItem::new_raw_bytes(data_item)) -// .expect("dataitem should be unique with idx"); -// } - -// Ok(idx_2_data_item) -// } +use crate::general::network::proto_ext::DataItemExt; + +impl DataItemExt for DataItemSource { + fn decode_persist(data: Vec) -> WSResult { + if data.is_empty() { + return Err(WSError::WsDataError(WsDataError::DataDecodeError { + reason: "Empty data".to_string(), + data_type: "DataItemSource".to_string(), + })); + } + match data[0] { + 0 => { + let path_str = String::from_utf8(data[1..].to_vec()).map_err(|e| { + WSError::WsDataError(WsDataError::DataDecodeError { + reason: format!("Failed to decode path string: {}", e), + data_type: "DataItemSource::File".to_string(), + }) + })?; + Ok(DataItemSource::File { + path: PathBuf::from(path_str), + }) + }, + 1 => Ok(DataItemSource::Memory { + data: data[1..].to_owned(), + }), + _ => Err(WSError::WsDataError(WsDataError::DataDecodeError { + reason: format!("Unknown data item type id: {}", data[0]), + data_type: "DataItemSource".to_string(), + })) + } + } + + fn encode_persist(&self) -> Vec { + match self { + DataItemSource::File { path } => { + let mut ret = vec![0]; + ret.extend_from_slice(path.to_string_lossy().as_bytes()); + ret + } + DataItemSource::Memory { data } => { + let mut ret = vec![1]; + ret.extend_from_slice(data); + ret + } + } + } +} diff --git a/src/main/src/general/data/m_data_general/mod.rs b/src/main/src/general/data/m_data_general/mod.rs index a34fc75..51779cb 100644 --- a/src/main/src/general/data/m_data_general/mod.rs +++ b/src/main/src/general/data/m_data_general/mod.rs @@ -1,7 +1,14 @@ -mod dataitem; +/// 缓存模式类型 +pub type CacheMode = u16; + +pub mod dataitem; +pub mod batch; +pub mod batch_handler; + +use crate::general::data::m_data_general::dataitem::{calculate_splits, WantIdxIter, WriteSplitDataTaskGroup, DataItemSource}; +use crate::general::data::m_data_general::batch_handler::{BatchReceiveState, SharedWithBatchHandler}; +use tokio::io::{AsyncSeekExt, AsyncReadExt}; -use crate::general::data::m_data_general::dataitem::WantIdxIter; -use crate::general::data::m_data_general::dataitem::WriteSplitDataTaskGroup; use crate::general::{ data::m_kv_store_engine::{ KeyTypeDataSetItem, KeyTypeDataSetMeta, KvAdditionalConf, KvStoreEngine, KvVersion, @@ -10,8 +17,7 @@ use crate::general::{ network::{ m_p2p::{P2PModule, RPCCaller, RPCHandler, RPCResponsor}, proto::{ - self, DataMeta, DataMetaGetRequest, DataVersionScheduleRequest, WriteOneDataRequest, - WriteOneDataResponse, + self, DataMeta, WriteOneDataResponse, }, proto_ext::ProtoExtDataItem, }, @@ -19,22 +25,19 @@ use crate::general::{ use crate::{ general::{ data::m_kv_store_engine::{KeyLockGuard, KeyType}, - network::{msg_pack::MsgPack, proto_ext::DataItemExt}, + network::{proto_ext::DataItemExt}, }, logical_module_view_impl, - result::{WSError, WSResult, WSResultExt, WsRuntimeErr, WsSerialErr, WsNetworkLogicErr}, + result::{WSError, WSResult, WSResultExt, WsSerialErr, WsNetworkLogicErr}, sys::{LogicalModule, LogicalModuleNewArgs, NodeID}, - util::JoinHandleWrapper, + util::{JoinHandleWrapper, container::async_init_map::AsyncInitMap}, }; use crate::{result::WsDataError, sys::LogicalModulesRef}; use async_trait::async_trait; use camelpaste::paste; use core::str; -use enum_as_inner::EnumAsInner; -use dashmap::DashMap; use serde::{Deserialize, Serialize}; -use std::ops::Range; use std::{ collections::{BTreeSet, HashMap, HashSet}, sync::Arc, @@ -42,12 +45,8 @@ use std::{ sync::atomic::{AtomicU32, Ordering}, }; use tokio::sync::Semaphore; -use tokio::task::JoinHandle; use tokio::task::JoinError; use ws_derive::LogicalModule; -use std::future::Future; - -// use super::m_appmeta_manager::AppMeta; logical_module_view_impl!(DataGeneralView); logical_module_view_impl!(DataGeneralView, p2p, P2PModule); @@ -61,6 +60,9 @@ pub type DataItemIdx = u8; pub const DATA_UID_PREFIX_APP_META: &str = "app"; pub const DATA_UID_PREFIX_FN_KV: &str = "fkv"; +/// 默认数据块大小 (4MB) +pub const DEFAULT_BLOCK_SIZE: usize = 4 * 1024 * 1024; + pub const CACHE_MODE_TIME_MASK: u16 = 0xf000; pub const CACHE_MODE_TIME_FOREVER_MASK: u16 = 0x0fff; pub const CACHE_MODE_TIME_AUTO_MASK: u16 = 0x1fff; @@ -87,123 +89,212 @@ pub fn new_data_unique_id_fn_kv(key: &[u8]) -> Vec { // format!("{}{}", DATA_UID_PREFIX_FN_KV, key_str) } +/// 唯一标识符类型 +pub type UniqueId = Vec; + #[derive(LogicalModule)] pub struct DataGeneral { view: DataGeneralView, pub rpc_call_data_version_schedule: RPCCaller, rpc_call_write_once_data: RPCCaller, - rpc_call_batch_data: RPCCaller, + rpc_call_batch_data: RPCCaller, rpc_call_get_data_meta: RPCCaller, rpc_call_get_data: RPCCaller, rpc_handler_write_once_data: RPCHandler, - rpc_handler_batch_data: RPCHandler, + rpc_handler_batch_data: RPCHandler, rpc_handler_data_meta_update: RPCHandler, rpc_handler_get_data_meta: RPCHandler, rpc_handler_get_data: RPCHandler, - - // 用于跟踪批量传输的状态 - batch_transfers: DashMap)>, // 修改类型为 (unique_id -> (version, data)) + + // 批量数据接收状态管理 + batch_receive_states: AsyncInitMap>, } impl DataGeneral { + pub fn inner_new(args: LogicalModuleNewArgs) -> Self { + Self { + view: DataGeneralView::new(args.logical_modules_ref.clone()), + rpc_call_data_version_schedule: RPCCaller::new(), + rpc_call_write_once_data: RPCCaller::new(), + rpc_call_batch_data: RPCCaller::new(), + rpc_call_get_data_meta: RPCCaller::new(), + rpc_call_get_data: RPCCaller::new(), + rpc_handler_write_once_data: RPCHandler::new(), + rpc_handler_batch_data: RPCHandler::new(), + rpc_handler_data_meta_update: RPCHandler::new(), + rpc_handler_get_data_meta: RPCHandler::new(), + rpc_handler_get_data: RPCHandler::new(), + batch_receive_states: AsyncInitMap::new(), + } + } + + #[allow(dead_code)] fn next_batch_id(&self) -> u32 { static NEXT_BATCH_ID: AtomicU32 = AtomicU32::new(1); // 从1开始,保留0作为特殊值 NEXT_BATCH_ID.fetch_add(1, Ordering::Relaxed) } - async fn write_data_batch( + pub async fn write_data_batch( &self, - unique_id: &[u8], + unique_id: UniqueId, version: u64, data: proto::DataItem, - data_item_idx: usize, + data_item_idx: DataItemIdx, node_id: NodeID, - batch_size: usize, ) -> WSResult<()> { - let total_size = data.data_sz_bytes(); - let total_batches = (total_size + batch_size - 1) / batch_size; - - // 克隆整个 view - let view = self.view.clone(); - - // Initialize batch transfer - let init_req = proto::sche::BatchDataRequest { - unique_id: unique_id.to_vec(), - version, - batch_id: 0, // 使用 0 作为初始化标记 - total_batches: total_batches as u32, - data: vec![], - data_item_idx: data_item_idx as u32, - is_complete: false, - }; - - let init_resp = self - .rpc_call_batch_data - .call( - view.p2p(), - node_id, - init_req, - Some(Duration::from_secs(60)), - ) - .await?; - - if !init_resp.success { - return Err(WsDataError::BatchTransferFailed { - node: node_id, - batch: 0, - reason: init_resp.error, - } - .into()); - } - - let batch_id = init_resp.batch_id; - - // Send data in batches - for batch_idx in 0..total_batches { - let start = batch_idx * batch_size; - let end = (start + batch_size).min(total_size); - let is_last = batch_idx == total_batches - 1; + // 调用 batch_transfer 函数处理数据传输 + async fn batch_transfer( + data_item_idx: DataItemIdx, + unique_id: UniqueId, + version: u64, + target_node: NodeID, + data: Arc, + view: DataGeneralView, + ) -> WSResult<()> { + let (tx, mut rx) = tokio::sync::mpsc::channel(32); + let mut handles = Vec::new(); + + let data_size = data.size().await?; + let splits = calculate_splits(data_size); + + tracing::debug!("batch_transfer total size({}), splits: {:?}", data_size, splits); + + for (block_idx, split_range) in splits.iter().enumerate() { + let block_data = match data.as_ref() { + DataItemSource::Memory { data } => data[split_range.clone()].to_vec(), + DataItemSource::File { path } => { + // 读取文件对应块的数据 + let mut file = tokio::fs::File::open(path).await.map_err(|e| WsDataError::BatchTransferFailed { + request_id: proto::BatchRequestId { + node_id: target_node as u32, + sequence: block_idx as u64, + }, + reason: format!("Failed to open file: {}", e), + })?; + let mut buffer = vec![0; split_range.len()]; + // 验证seek结果 + let seek_pos = file.seek(std::io::SeekFrom::Start(split_range.start as u64)).await.map_err(|e| WsDataError::BatchTransferFailed { + request_id: proto::BatchRequestId { + node_id: target_node as u32, + sequence: block_idx as u64, + }, + reason: format!("Failed to seek file: {}", e), + })?; + if seek_pos != split_range.start as u64 { + return Err(WsDataError::BatchTransferFailed { + request_id: proto::BatchRequestId { + node_id: target_node as u32, + sequence: block_idx as u64, + }, + reason: format!("Seek position mismatch: expected {}, got {}", split_range.start, seek_pos), + }.into()); + } + // read_exact保证读取指定长度的数据或返回错误 + let _ = file.read_exact(&mut buffer).await.map_err(|e| WsDataError::BatchTransferFailed { + request_id: proto::BatchRequestId { + node_id: target_node as u32, + sequence: block_idx as u64, + }, + reason: format!("Failed to read file: {}", e), + })?; + buffer + } + }; - let batch_data = data.clone_split_range(start..end); - let batch_req = proto::sche::BatchDataRequest { - unique_id: unique_id.to_vec(), - version, - batch_id, - total_batches: total_batches as u32, - data: batch_data.encode_persist(), + let request = proto::BatchDataRequest { + request_id: Some(proto::BatchRequestId { + node_id: target_node as u32, + sequence: block_idx as u64, + }), + dataset_unique_id: unique_id.clone(), data_item_idx: data_item_idx as u32, - is_complete: is_last, - }; - - let batch_resp = self - .rpc_call_batch_data - .call( - view.p2p(), - node_id, - batch_req, - Some(Duration::from_secs(60)), - ) - .await?; - - if !batch_resp.success { - return Err(WsDataError::BatchTransferFailed { - node: node_id, - batch: batch_idx as u32, - reason: batch_resp.error, + block_type: match data.as_ref() { + DataItemSource::Memory { .. } => proto::BatchDataBlockType::Memory as i32, + DataItemSource::File { .. } => proto::BatchDataBlockType::File as i32, + }, + block_index: block_idx as u32, + data: block_data, + operation: proto::DataOpeType::Write as i32, + unique_id: unique_id.clone(), + version, + total_size: data_size as u64, + }; + + let tx = tx.clone(); + let view = view.clone(); + + let handle = tokio::spawn(async move { + let result = view.data_general() + .rpc_call_batch_data + .call( + view.p2p(), + target_node, + request, + Some(Duration::from_secs(30)), + ) + .await; + + if let Err(e) = tx.send(result).await { + tracing::error!("Failed to send batch transfer result: {}", e); + } + }); + + handles.push(handle); + } + + drop(tx); + + while let Some(result) = rx.recv().await { + match result { + Ok(resp) if !resp.success => { + return Err(WsDataError::BatchTransferFailed { + request_id: proto::BatchRequestId { + node_id: target_node as u32, + sequence: 0, // TODO: Add proper sequence number + }, + reason: resp.error_message, + }.into()); + } + Ok(_) => continue, + Err(e) => { + return Err(WsDataError::BatchTransferFailed { + request_id: proto::BatchRequestId { + node_id: target_node as u32, + sequence: 0, + }, + reason: format!("RPC call failed: {}", e), + }.into()); + } } - .into()); } + + for handle in handles { + handle.await.map_err(|e| { + WsDataError::BatchTransferFailed { + request_id: proto::BatchRequestId { + node_id: target_node as u32, + sequence: 0, + }, + reason: format!("Task join failed: {}", e), + } + })?; + } + + Ok(()) } - Ok(()) + let data = Arc::new(data.to_data_item_source()); + batch_transfer(data_item_idx,unique_id, version, node_id, data, self.view.clone()).await } + pub async fn get_or_del_datameta_from_master( &self, unique_id: &[u8], delete: bool, ) -> WSResult { + tracing::debug!("get_or_del_datameta_from_master uid: {:?}, delete: {}, whoami: {}", unique_id, delete, self.view.p2p().nodes_config.this.0); let p2p = self.view.p2p(); // get meta from master let meta = self @@ -243,6 +334,7 @@ impl DataGeneral { ty, }: GetOrDelDataArg, ) -> WSResult<(DataSetMetaV2, HashMap)> { + tracing::debug!("get_or_del_data uid: {:?}, maybe with meta: {:?}", unique_id, meta); let mut data_map = HashMap::new(); // get meta from master @@ -253,7 +345,7 @@ impl DataGeneral { .await? }; - tracing::debug!("get_or_del_data uid: {:?},meta: {:?}", unique_id, meta); + tracing::debug!("start get_or_del_data uid: {:?},meta: {:?}", unique_id, meta); // basical verify for idx in 0..meta.data_item_cnt() { @@ -302,7 +394,7 @@ impl DataGeneral { .into()); } - data_map.insert(idx, resp.data[0].clone()); + let _ = data_map.insert(idx, resp.data[0].clone()); } } GetOrDelDataArgType::Delete => { @@ -331,7 +423,7 @@ impl DataGeneral { .into()); } - data_map.insert(idx, resp.data[0].clone()); + let _ = data_map.insert(idx, resp.data[0].clone()); } } GetOrDelDataArgType::PartialOne { idx } => { @@ -358,7 +450,7 @@ impl DataGeneral { .into()); } - data_map.insert(idx, resp.data[0].clone()); + let _ = data_map.insert(idx, resp.data[0].clone()); } GetOrDelDataArgType::PartialMany { idxs } => { for idx in idxs { @@ -385,7 +477,7 @@ impl DataGeneral { .into()); } - data_map.insert(idx, resp.data[0].clone()); + let _ = data_map.insert(idx, resp.data[0].clone()); } } } @@ -437,111 +529,74 @@ impl DataGeneral { let splits = version_schedule_resp.split.clone(); // 处理每个数据项 - for (data_item_idx, (data_item, split)) in datas - .iter() - .zip(splits.iter()) - .enumerate() - { - let mut tasks = Vec::new(); - tracing::debug!( - "{} processing data item {}/{}", - log_tag, - data_item_idx + 1, - datas.len() - ); - + let mut iter = WantIdxIter::new(&GetOrDelDataArgType::All, datas.len() as u8); + while let Some(data_item_idx) = iter.next() { + let data_item = &datas[data_item_idx as usize]; + let split = &splits[data_item_idx as usize]; + let mut primary_tasks = Vec::new(); + // 1. 并行写入所有主数据分片 - for (split_idx, split_info) in split.splits.iter().enumerate() { - tracing::debug!( - "{} creating split write task {}/{} for node {}, offset={}, size={}", - log_tag, - split_idx + 1, - split.splits.len(), - split_info.node_id, - split_info.data_offset, - split_info.data_size - ); - - // 克隆必要的数据 - let split_info = split_info.clone(); // 必须克隆，来自临时变量 - let unique_id = unique_id.clone(); // 必须克隆，多个任务需要 - let data_item = data_item.clone_split_range( // 克隆必要的数据范围 + let mut split_iter = WantIdxIter::new(&GetOrDelDataArgType::All, split.splits.len() as u8); + while let Some(split_idx) = split_iter.next() { + let split_info = &split.splits[split_idx as usize]; + tracing::debug!("{} creating split write task {}/{} for node {}, offset={}, size={}", + log_tag, split_idx + 1, split.splits.len(), split_info.node_id, split_info.data_offset, split_info.data_size); + let split_info = split_info.clone(); + let unique_id_clone = unique_id.clone(); + let data_item_primary = data_item.clone_split_range( split_info.data_offset as usize - ..(split_info.data_offset + split_info.data_size) as usize, + ..(split_info.data_offset + split_info.data_size) as usize ); - let view = self.view.clone(); // 克隆 view，包含所有模块引用 - let version = version; // 复制值类型 - + let view = self.view.clone(); + let version_copy = version; let task = tokio::spawn(async move { - let resp = view.data_general() + view.data_general() .rpc_call_write_once_data .call( view.p2p(), split_info.node_id, proto::WriteOneDataRequest { - unique_id, - version, + unique_id: unique_id_clone.clone(), + version: version_copy, data: vec![proto::DataItemWithIdx { idx: data_item_idx as u32, - data: Some(data_item), + data: Some(data_item_primary), }], }, Some(Duration::from_secs(60)), ) - .await?; - Ok::(resp) + .await }); - tasks.push(task); + primary_tasks.push(task); } // 2. 并行写入缓存数据（完整数据） - let visitor = CacheModeVisitor(version_schedule_resp.cache_mode[data_item_idx] as u16); + let visitor = CacheModeVisitor(version_schedule_resp.cache_mode[data_item_idx as usize] as u16); let need_cache = visitor.is_map_common_kv() || visitor.is_map_file(); - let cache_nodes: Vec = if need_cache { split.splits.iter().map(|s| s.node_id).collect() } else { vec![] }; + let mut cache_tasks = Vec::new(); if !cache_nodes.is_empty() { - tracing::debug!( - "{} found {} cache nodes: {:?}", - log_tag, - cache_nodes.len(), - cache_nodes - ); - - // 使用信号量限制并发的批量传输数量 + tracing::debug!("{} found {} cache nodes: {:?}", log_tag, cache_nodes.len(), cache_nodes); const MAX_CONCURRENT_TRANSFERS: usize = 3; let semaphore = Arc::new(Semaphore::new(MAX_CONCURRENT_TRANSFERS)); - - for (cache_idx, &node_id) in cache_nodes.iter().enumerate() { + + let mut cache_iter = WantIdxIter::new(&GetOrDelDataArgType::All, cache_nodes.len() as u8); + while let Some(cache_idx) = cache_iter.next() { + let node_id = cache_nodes[cache_idx as usize]; let permit = semaphore.clone().acquire_owned().await.unwrap(); - tracing::debug!( - "{} creating cache write task {}/{} for node {}", - log_tag, - cache_idx + 1, - cache_nodes.len(), - node_id - ); - - // 创建批量传输任务 - let unique_id = unique_id.clone(); - let data_item = data_item.clone(); + tracing::debug!("{} creating cache write task {}/{} for node {}", log_tag, cache_idx + 1, cache_nodes.len(), node_id); + let unique_id_clone = unique_id.clone(); + let data_item_cache = data_item.clone(); let view = self.view.clone(); - let task = tokio::spawn(async move { - let _permit = permit; + let _permit = permit; // 持有permit直到任务完成 view.data_general() - .write_data_batch( - &unique_id, - version, - data_item.clone(), - data_item_idx, - node_id, - 1024 * 1024, // 1MB batch size - ) + .write_data_batch(unique_id_clone.clone(), version, data_item_cache, data_item_idx, node_id) .await?; Ok::(proto::WriteOneDataResponse { remote_version: version, @@ -549,13 +604,20 @@ impl DataGeneral { message: String::new(), }) }); - tasks.push(task); + cache_tasks.push(task); } } - // 等待所有写入任务完成 - for task in tasks { - task.await??; + let primary_results = futures::future::join_all(primary_tasks).await; + let cache_results = futures::future::join_all(cache_tasks).await; + + if primary_results.iter().any(|res| res.is_err()) || cache_results.iter().any(|res| res.is_err()) { + let error_msg = format!("主节点或缓存节点数据写入失败"); + tracing::error!("{}", error_msg); + return Err(WSError::WsDataError(WsDataError::WriteDataFailed { + unique_id: unique_id.clone(), + message: error_msg, + })); } } @@ -564,8 +626,8 @@ impl DataGeneral { async fn rpc_handle_write_one_data( &self, - responsor: RPCResponsor, - req: WriteOneDataRequest, + responsor: RPCResponsor, + req: proto::WriteOneDataRequest, ) { tracing::debug!("verify data meta bf write data"); let kv_store_engine = self.view.kv_store_engine(); @@ -716,12 +778,14 @@ impl DataGeneral { for data_with_idx in req.data.into_iter() { let proto::DataItemWithIdx { idx, data } = data_with_idx; let data = data.unwrap(); - let serialize = data.encode_persist(); + let data_source = data.to_data_item_source(); + let data = Arc::new(data_source); + let serialize = data.as_ref().encode_persist(); tracing::debug!( "writing data part uid({:?}) idx({}) item({})", req.unique_id, idx, - data.to_string() + data.to_debug_string() ); if let Err(err) = kv_store_engine.set( KeyTypeDataSetItem { @@ -766,8 +830,14 @@ impl DataGeneral { let key = KeyTypeDataSetMeta(&req.unique_id); let keybytes = key.make_key(); - + + // test only log + #[cfg(test)] + tracing::debug!("rpc_handle_data_meta_update {:?}\n {:?}", req,bincode::deserialize::(&req.serialized_meta)); + // not test log + #[cfg(not(test))] tracing::debug!("rpc_handle_data_meta_update {:?}", req); + let kv_lock = self.view.kv_store_engine().with_rwlock(&keybytes); let _kv_write_lock_guard = kv_lock.write(); @@ -837,7 +907,7 @@ impl DataGeneral { responsor: RPCResponsor, ) -> WSResult<()> { tracing::debug!("rpc_handle_get_data_meta with req({:?})", req); - let meta = self.view.get_data_meta(&req.unique_id, req.delete)?; + let meta = self.view.get_data_meta_local(&req.unique_id, req.delete)?; if meta.is_none() { tracing::debug!("rpc_handle_get_data_meta data meta not found"); } else { @@ -863,9 +933,10 @@ impl DataGeneral { let kv_store_engine = self.view.kv_store_engine(); let _ = self.view - .get_data_meta(&req.unique_id, req.delete) + .get_metadata(&req.unique_id, req.delete) + .await .map_err(|err| { - tracing::warn!("rpc_handle_get_one_data get_data_meta failed: {:?}", err); + tracing::warn!("rpc_handle_get_one_data get_metadata failed: {:?}", err); err })?; @@ -900,7 +971,7 @@ impl DataGeneral { got_or_deleted.push(value); } - let (success, message): (bool, String) = if kv_ope_err.len() > 0 { + let (mut success, mut message): (bool, String) = if kv_ope_err.len() > 0 { (false, { let mut msg = String::from("KvEngine operation failed: "); for e in kv_ope_err.iter() { @@ -919,8 +990,18 @@ impl DataGeneral { if success { for v in got_or_deleted { let decode_res = proto::DataItem::decode_persist(v.unwrap().1); - tracing::debug!("decode_res type: {:?}", decode_res.to_string()); - got_or_deleted_checked.push(decode_res); + match decode_res { + Ok(item) => { + tracing::debug!("decoded data item: {:?}", item.to_string()); + got_or_deleted_checked.push(item); + } + Err(e) => { + tracing::error!("Failed to decode data item: {:?}", e); + success = false; + message = format!("Failed to decode data item: {:?}", e); + break; + } + } } } @@ -934,9 +1015,104 @@ impl DataGeneral { Ok(()) } + + /// 处理批量数据写入请求 + pub async fn rpc_handle_batch_data( + &self, + responsor: RPCResponsor, + req: proto::BatchDataRequest, + ) -> WSResult<()> { + let batch_receive_states = self.batch_receive_states.clone(); + // 预先克隆闭包外需要的字段 + let block_index = req.block_index; + let data = req.data.clone(); + let request_id = req.request_id.clone().unwrap(); + + // 1. 查找或创建状态 + let state = match self.batch_receive_states + .get_or_init(req.request_id.clone().unwrap(), async move { + // 创建任务组和句柄 + let (mut group, handle) = match WriteSplitDataTaskGroup::new( + req.unique_id.clone(), + req.total_size as usize, + req.block_type(), + req.version, + ).await { + Ok((group, handle)) => (group, handle), + Err(e) => { + tracing::error!("Failed to create task group: {:?}", e); + return Err(e); + } + }; + + // 启动process_tasks + let _ = tokio::spawn(async move { + match group.process_tasks().await { + Ok(item) => Ok(item), + Err(e) => { + tracing::error!("Failed to process tasks: {}", e); + Err(e) + } + } + }); + + let state = Arc::new(BatchReceiveState::new(handle, SharedWithBatchHandler::new())); + let state_clone = state.clone(); + + // response task + let _=tokio::spawn(async move { + // 等待所有任务完成 + if let Err(e) = state_clone.handle.wait_all_tasks().await { + tracing::error!("Failed to wait for tasks: {}", e); + return; + } + + // 发送最终响应 + if let Some(final_responsor) = state_clone.shared.get_final_responsor().await { + if let Err(e) = final_responsor.send_resp(proto::BatchDataResponse { + request_id: Some(req.request_id.clone().unwrap()), + success: true, + error_message: String::new(), + version: state_clone.handle.version(), + }).await { + tracing::error!("Failed to send final response: {}", e); + } + } + + // 清理状态 + let _=batch_receive_states.remove(&req.request_id.unwrap()); + }); + + Ok(state) + }) + .await { + Err(e) => return Err(WSError::WsDataError(WsDataError::BatchTransferError { + request_id, + msg: format!("Failed to initialize batch state: {}", e) + })), + Ok(state) => state, + }; + + // 2. 提交分片数据 + let data_item = proto::DataItem { + data_item_dispatch: Some(proto::data_item::DataItemDispatch::RawBytes(data)), + ..Default::default() + }; + + tracing::debug!("submit_split with data split idx: {}, at node: {}", block_index, self.view.p2p().nodes_config.this_node()); + state.handle.submit_split( + block_index as usize * DEFAULT_BLOCK_SIZE, + data_item, + ).await?; + + // 3. 更新响应器 + state.shared.update_responsor(responsor).await; + + Ok(()) + } } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct DataMetaSys { pub cache: i32, pub distribute: i32, @@ -958,9 +1134,7 @@ impl Into for DataMetaSys { } } -/// depracated, latest is v2 -/// the data's all in one meta -/// https://fvd360f8oos.feishu.cn/docx/XoFudWhAgox84MxKC3ccP1TcnUh#share-Tqqkdxubpokwi5xREincb1sFnLc +/// 数据集元信息 #[derive(Serialize, Deserialize)] pub struct DataSetMetaV1 { // unique_id: Vec, @@ -969,21 +1143,20 @@ pub struct DataSetMetaV1 { pub synced_nodes: HashSet, } -pub type CacheMode = u16; - -/// the data's all in one meta +/// 数据集元信息 /// -/// attention: new from `DataSetMetaBuilder` +/// 注意：新建元信息请使用 `DataSetMetaBuilder` /// /// https://fvd360f8oos.feishu.cn/docx/XoFudWhAgox84MxKC3ccP1TcnUh#share-Tqqkdxubpokwi5xREincb1sFnLc -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug,Clone)] pub struct DataSetMetaV2 { // unique_id: Vec, api_version: u8, pub version: u64, - pub cache_mode: Vec, - /// the data splits for each data item, the index is the data item index pub datas_splits: Vec, + pub data_metas: Vec, + pub synced_nodes: HashSet, + pub cache_mode: Vec, } impl DataSetMetaV2 { @@ -1023,8 +1196,8 @@ impl EachNodeSplit { } } -/// the split of one dataitem -/// we need to know the split size for one data +/// 数据项的分片信息 +/// 我们需要知道每个数据项的分片大小 #[derive(Serialize, Deserialize, Debug, Clone)] pub struct DataSplit { pub splits: Vec, @@ -1112,9 +1285,6 @@ impl Into for DataSplit { // uint32 split_size = 1; // repeated uint32 node_ids = 2; -#[derive(Debug, Clone, Copy)] -pub struct CacheModeVisitor(pub u16); - macro_rules! generate_cache_mode_methods { // The macro takes a list of pairs of the form [time, mask] and generates methods. ($(($group:ident, $mode:ident)),*) => { @@ -1201,9 +1371,11 @@ impl DataSetMetaBuilder { Self { building: Some(DataSetMetaV2 { version: 0, - cache_mode: vec![], - api_version: 2, datas_splits: vec![], + data_metas: vec![], + api_version: 2, + synced_nodes: HashSet::new(), + cache_mode: vec![], }), } } @@ -1246,95 +1418,13 @@ impl DataSetMetaBuilder { } } -// impl From for DataSetMetaV2 { -// fn from( -// DataSetMetaV1 { -// version, -// data_metas: _, -// synced_nodes: _, -// }: DataSetMetaV1, -// ) -> Self { -// DataSetMetaBuilder::new() -// .version(version) -// .cache_mode_pos_allnode() -// .build() -// // DataSetMetaV2 { -// // version, -// // data_metas, -// // synced_nodes, -// // } -// } -// } - -mod test { - #[test] - fn test_option_and_vec_serialization_size() { - // 定义一个具体的值 - let value: i32 = 42; - - // 创建 Option 类型的变量 - let some_value: Option = Some(value); - let none_value: Option = None; - - // 创建 Vec 类型的变量 - let empty_vec: Vec = Vec::new(); - let single_element_vec: Vec = vec![value]; - - let some_empty_vec: Option> = Some(vec![]); - let some_one_vec: Option> = Some(vec![value]); - - // 序列化 - let serialized_some = bincode::serialize(&some_value).unwrap(); - let serialized_none = bincode::serialize(&none_value).unwrap(); - let serialized_empty_vec = bincode::serialize(&empty_vec).unwrap(); - let serialized_single_element_vec = bincode::serialize(&single_element_vec).unwrap(); - let serialized_some_empty_vec = bincode::serialize(&some_empty_vec).unwrap(); - let serialized_some_one_vec = bincode::serialize(&some_one_vec).unwrap(); - - // 获取序列化后的字节大小 - let size_some = serialized_some.len(); - let size_none = serialized_none.len(); - let size_empty_vec = serialized_empty_vec.len(); - let size_single_element_vec = serialized_single_element_vec.len(); - let size_some_empty_vec = serialized_some_empty_vec.len(); - let size_some_one_vec = serialized_some_one_vec.len(); - - // 打印结果 - println!("Size of serialized Some(42): {}", size_some); - println!("Size of serialized None: {}", size_none); - println!("Size of serialized empty Vec: {}", size_empty_vec); - println!( - "Size of serialized Vec with one element (42): {}", - size_single_element_vec - ); - println!( - "Size of serialized Some(empty Vec): {}", - size_some_empty_vec - ); - println!( - "Size of serialized Some(one element Vec): {}", - size_some_one_vec - ); - - // 比较大小 - assert!( - size_some > size_none, - "Expected serialized Some to be larger than serialized None" - ); - assert!( - size_single_element_vec > size_empty_vec, - "Expected serialized Vec with one element to be larger than serialized empty Vec" - ); - } -} - pub struct GetOrDelDataArg { pub meta: Option, pub unique_id: Vec, pub ty: GetOrDelDataArgType, } -#[derive(Clone)] +#[derive(Debug, Clone)] pub enum GetOrDelDataArgType { All, Delete, @@ -1343,7 +1433,7 @@ pub enum GetOrDelDataArgType { } impl DataGeneralView { - fn get_data_meta( + fn get_data_meta_local( &self, unique_id: &[u8], delete: bool, @@ -1365,6 +1455,20 @@ impl DataGeneralView { }; Ok(meta_opt) } + + pub async fn get_metadata( + &self, + unique_id: &[u8], + delete: bool, + ) -> WSResult { + // 先尝试从本地获取 + if let Some((_version, meta)) = self.get_data_meta_local(unique_id, delete)? { + return Ok(meta); + } + + // 本地不存在，从 master 获取 + self.data_general().get_or_del_datameta_from_master(unique_id, delete).await + } } impl From for WSError { @@ -1392,8 +1496,9 @@ impl LogicalModule for DataGeneral { rpc_handler_data_meta_update: RPCHandler::new(), rpc_handler_get_data_meta: RPCHandler::new(), rpc_handler_get_data: RPCHandler::new(), - - batch_transfers: DashMap::new(), + + // 批量数据接收状态管理 + batch_receive_states: AsyncInitMap::new(), } } @@ -1412,62 +1517,62 @@ impl LogicalModule for DataGeneral { // register rpc handlers { - let this = self.clone(); + let view = self.view.clone(); self.rpc_handler_write_once_data .regist(p2p, move |responsor, req| { - let this = this.clone(); + let view = view.clone(); let _ = tokio::spawn(async move { - this.rpc_handle_write_one_data(responsor, req).await; + view.data_general().rpc_handle_write_one_data(responsor, req).await; }); Ok(()) }); - let this = self.clone(); + let view = self.view.clone(); self.rpc_handler_batch_data.regist( p2p, - move |responsor: RPCResponsor, - req: proto::sche::BatchDataRequest| { - let this = this.clone(); + move |responsor: RPCResponsor, + req: proto::BatchDataRequest| { + let view = view.clone(); let _ = tokio::spawn(async move { - this.rpc_handle_batch_data(responsor, req).await; + let _ = view.data_general().rpc_handle_batch_data(responsor, req).await; }); Ok(()) }, ); - let this = self.clone(); + let view = self.view.clone(); self.rpc_handler_data_meta_update.regist( p2p, move |responsor: RPCResponsor, req: proto::DataMetaUpdateRequest| { - let this = this.clone(); + let view = view.clone(); let _ = tokio::spawn(async move { - this.rpc_handle_data_meta_update(responsor, req).await + view.data_general().rpc_handle_data_meta_update(responsor, req).await }); Ok(()) }, ); - let this = self.clone(); + let view = self.view.clone(); self.rpc_handler_get_data_meta .regist(p2p, move |responsor, req| { - let this = this.clone(); + let view = view.clone(); let _ = tokio::spawn(async move { - this.rpc_handle_get_data_meta(req, responsor) + view.data_general().rpc_handle_get_data_meta(req, responsor) .await .todo_handle(); }); Ok(()) }); - let this = self.clone(); + let view = self.view.clone(); self.rpc_handler_get_data.regist( p2p, move |responsor: RPCResponsor, req: proto::GetOneDataRequest| { - let this = this.clone(); + let view = view.clone(); let _ = tokio::spawn(async move { - this.rpc_handle_get_one_data(responsor, req).await + view.data_general().rpc_handle_get_one_data(responsor, req).await }); Ok(()) }, @@ -1478,41 +1583,5 @@ impl LogicalModule for DataGeneral { } } -fn flush_the_data( - log_tag: &str, - unique_id: &[u8], - version: u64, - split_size: usize, - view: &DataGeneralView, - one_data_item: &proto::DataItem, - nodeid: NodeID, - offset: usize, - dataitem_idx: usize, - write_source_data_tasks: &mut Vec>>, -) { - let log_tag = log_tag.to_owned(); - let unique_id = unique_id.to_owned(); - let view = view.clone(); - let one_data_item_split = one_data_item.clone_split_range(offset..offset + split_size); - let t = tokio::spawn(async move { - let req = WriteOneDataRequest { - unique_id, - version, - data: vec![proto::DataItemWithIdx { - idx: dataitem_idx as u32, - data: Some(one_data_item_split), - }], - }; - tracing::debug!( - "[{}] write_data flushing, target node: {}, `WriteOneDataRequest` msg_id: {}", - log_tag, - nodeid, - req.msg_id() - ); - view.data_general() - .rpc_call_write_once_data - .call(view.p2p(), nodeid, req, Some(Duration::from_secs(60))) - .await - }); - write_source_data_tasks.push(t); -} +#[derive(Debug, Clone, Copy)] +pub struct CacheModeVisitor(pub u16); \ No newline at end of file diff --git a/src/main/src/general/network/msg_pack.rs b/src/main/src/general/network/msg_pack.rs index 30bf6d7..90c4f82 100644 --- a/src/main/src/general/network/msg_pack.rs +++ b/src/main/src/general/network/msg_pack.rs @@ -133,8 +133,18 @@ define_msg_ids!( } }), (proto::kv::KvLockResponse, _pack, { true }), - (proto::sche::BatchDataRequest, _pack, { true }), - (proto::sche::BatchDataResponse, _pack, { true }) + (proto::BatchDataRequest, _pack, { + // 验证关键字段非空 + // 1. request_id 必须存在，用于请求追踪 + // 2. unique_id 必须存在，标识数据集 + // 3. data 必须存在，实际数据内容 + let req = _pack; + match (req.request_id.is_some(), req.unique_id.is_empty(), req.data.is_empty()) { + (true, false, false) => true, + _ => false + } + }), + (proto::BatchDataResponse, _pack, { true }) ); pub trait RPCReq: MsgPack + Default { @@ -189,8 +199,8 @@ impl RPCReq for proto::kv::KvLockRequest { type Resp = proto::kv::KvLockResponse; } -impl RPCReq for proto::sche::BatchDataRequest { - type Resp = proto::sche::BatchDataResponse; +impl RPCReq for proto::BatchDataRequest { + type Resp = proto::BatchDataResponse; } // impl RPCReq for proto::kv::KvLockWaitAcquireNotifyRequest { diff --git a/src/main/src/general/network/proto_ext.rs b/src/main/src/general/network/proto_ext.rs index 60f64fd..1fbbee4 100644 --- a/src/main/src/general/network/proto_ext.rs +++ b/src/main/src/general/network/proto_ext.rs @@ -1,4 +1,6 @@ use crate::general::app::DataEventTrigger; +use crate::general::data::m_data_general::dataitem::DataItemSource; +use crate::general::data::m_data_general::DataItemIdx; use crate::general::data::m_dist_lock::DistLockOpe; use crate::general::network::proto::sche::distribute_task_req::{ DataEventTriggerNew, DataEventTriggerWrite, Trigger, @@ -7,6 +9,7 @@ use crate::general::network::proto::sche::distribute_task_req::{ use super::proto::{self, kv::KvResponse, FileData}; use std::{ops::Range, path::Path}; +use crate::result::{WSResult, WSError, WsDataError}; pub trait ProtoExtDataItem { fn data_sz_bytes(&self) -> usize; @@ -16,6 +19,7 @@ pub trait ProtoExtDataItem { fn as_raw_bytes<'a>(&'a self) -> Option<&'a [u8]>; fn new_file_data(filepath: impl AsRef, is_dir: bool) -> Self; fn as_file_data(&self) -> Option<&proto::FileData>; + fn to_data_item_source(&self) -> DataItemSource; } impl ProtoExtDataItem for proto::DataItem { @@ -95,6 +99,20 @@ impl ProtoExtDataItem for proto::DataItem { _ => None, } } + + fn to_data_item_source(&self) -> DataItemSource { + match &self.data_item_dispatch { + Some(proto::data_item::DataItemDispatch::RawBytes(bytes)) => DataItemSource::Memory { + data: bytes.clone(), + }, + Some(proto::data_item::DataItemDispatch::File(file_data)) => DataItemSource::File { + path: file_data.file_name_opt.clone().into(), + }, + _ => DataItemSource::Memory { + data: Vec::new(), + }, + } + } } impl AsRef<[u8]> for proto::DataItem { @@ -200,26 +218,43 @@ impl KvRequestExt for proto::kv::KvRequest { } pub trait DataItemExt { - fn decode_persist(data: Vec) -> Self; + fn decode_persist(data: Vec) -> WSResult where Self: Sized; fn encode_persist<'a>(&'a self) -> Vec; } impl DataItemExt for proto::DataItem { - fn decode_persist(data: Vec) -> Self { + fn decode_persist(data: Vec) -> WSResult where Self: Sized { + if data.is_empty() { + return Err(WSError::WsDataError(WsDataError::DataDecodeError { + reason: "Empty data".to_string(), + data_type: "proto::DataItem".to_string(), + })); + } let data_item_dispatch = match data[0] { - 0 => proto::data_item::DataItemDispatch::File(FileData { - file_name_opt: String::new(), - is_dir_opt: false, - file_content: data[1..].to_owned(), - }), - 1 => proto::data_item::DataItemDispatch::RawBytes(data[1..].to_owned()), + 0 => { + let path_str = String::from_utf8(data[1..].to_vec()).map_err(|e| { + WSError::WsDataError(WsDataError::DataDecodeError { + reason: format!("Failed to decode path string: {}", e), + data_type: "proto::DataItem::File".to_string(), + }) + })?; + proto::data_item::DataItemDispatch::File(FileData { + file_name_opt: path_str, + is_dir_opt: false, + file_content: Vec::new(), + }) + }, + 1 => proto::data_item::DataItemDispatch::RawBytes(data[1..].to_vec()), _ => { - panic!("unknown data item type id: {}", data[0]) + return Err(WSError::WsDataError(WsDataError::DataDecodeError { + reason: format!("Unknown data item type id: {}", data[0]), + data_type: "proto::DataItem".to_string(), + })); } }; - Self { + Ok(Self { data_item_dispatch: Some(data_item_dispatch), - } + }) } fn encode_persist<'a>(&'a self) -> Vec { match self.data_item_dispatch.as_ref().unwrap() { @@ -256,6 +291,16 @@ impl ProtoExtDataEventTrigger for DataEventTrigger { } } +pub trait ProtoExtDataScheduleContext { + fn dataitem_cnt(&self) -> DataItemIdx; +} + +impl ProtoExtDataScheduleContext for proto::DataScheduleContext { + fn dataitem_cnt(&self) -> DataItemIdx { + self.each_data_sz_bytes.len() as DataItemIdx + } +} + // Example usage in tests #[cfg(test)] mod tests { diff --git a/src/main/src/general/network/proto_src/data.proto b/src/main/src/general/network/proto_src/data.proto index cb290b2..fdd6fee 100644 --- a/src/main/src/general/network/proto_src/data.proto +++ b/src/main/src/general/network/proto_src/data.proto @@ -169,4 +169,34 @@ message GetOneDataResponse{ bool success=1; repeated DataItem data =2; string message=3; +} + +enum BatchDataBlockType { + MEMORY = 0; // 内存数据块 + FILE = 1; // 文件数据块 +} + +message BatchRequestId { + uint32 node_id = 1; // 节点ID + uint64 sequence = 2; // 原子自增序列号 +} + +message BatchDataRequest { + BatchRequestId request_id = 1; // 请求唯一标识（节点ID + 序列号） + bytes dataset_unique_id = 2; // 数据集唯一标识 + uint32 data_item_idx = 3; // 数据项索引 + BatchDataBlockType block_type = 4; // 数据块类型（文件/内存） + uint32 block_index = 5; // 数据块索引 + bytes data = 6; // 数据块内容 + DataOpeType operation = 7; // 操作类型 + bytes unique_id = 8; // 数据唯一标识 + uint64 version = 9; // 数据版本 + uint64 total_size = 10; // 数据总大小 +} + +message BatchDataResponse { + BatchRequestId request_id = 1; // 对应请求ID + bool success = 2; // 处理状态 + string error_message = 3; // 错误信息 + uint64 version = 4; // 处理后的版本 } \ No newline at end of file diff --git a/src/main/src/general/network/proto_src/sche.proto b/src/main/src/general/network/proto_src/sche.proto index a3cba7d..fdec748 100644 --- a/src/main/src/general/network/proto_src/sche.proto +++ b/src/main/src/general/network/proto_src/sche.proto @@ -47,19 +47,5 @@ message DistributeTaskResp { string err_msg = 2; } -message BatchDataRequest { - bytes unique_id = 1; - uint64 version = 2; - uint32 batch_id = 3; // 当前批次ID - uint32 total_batches = 4; // 总批次数 - bytes data = 5; // 当前批次的数据 - uint32 data_item_idx = 6; // 数据项索引 - bool is_complete = 7; // 是否是最后一个批次 -} -message BatchDataResponse { - bool success = 1; - string error = 2; - uint32 batch_id = 3; -} diff --git a/src/main/src/main.rs b/src/main/src/main.rs index e3b2af3..8a9b56a 100644 --- a/src/main/src/main.rs +++ b/src/main/src/main.rs @@ -65,9 +65,9 @@ pub fn start_tracing() { return false; } if *v.level() == Level::DEBUG { - if mp.contains("wasm_serverless::worker::m_kv_user_client") { - return false; - } + // if mp.contains("wasm_serverless::worker::m_kv_user_client") { + // return false; + // } // if mp.contains("wasm_serverless::general::m_data_general") { // return false; // } diff --git a/src/main/src/master/app/fddg.rs b/src/main/src/master/app/fddg.rs index 3bbad97..31ce5ee 100644 --- a/src/main/src/master/app/fddg.rs +++ b/src/main/src/master/app/fddg.rs @@ -3,14 +3,10 @@ use crate::util::container::sync_trie::SyncedTrie; use crate::{ general::{ app::{AppType, FnMeta}, - data::{self, m_data_general::DataItemIdx}, - network::proto, }, result::WSResult, }; -use dashmap::DashMap; use std::collections::HashMap; -use std::collections::HashSet; // function data dependency graph // - need update when app uploaded diff --git a/src/main/src/master/app/m_app_master.rs b/src/main/src/master/app/m_app_master.rs index 52f2d86..cf25a67 100644 --- a/src/main/src/master/app/m_app_master.rs +++ b/src/main/src/master/app/m_app_master.rs @@ -1,20 +1,13 @@ use crate::general::app::m_executor::Executor; use crate::general::app::AppMetaManager; -use crate::general::app::{AffinityPattern, AffinityRule, AppType, FnMeta, NodeTag}; use crate::general::network::m_p2p::P2PModule; -use crate::general::network::m_p2p::RPCCaller; -use crate::general::network::proto::sche::{self, distribute_task_req::Trigger}; use crate::logical_module_view_impl; use crate::master::app::fddg::FDDGMgmt; -use crate::master::m_master::{FunctionTriggerContext, Master}; -use crate::result::{WSResult, WsFuncError}; -use crate::sys::NodeID; +use crate::master::m_master::{Master}; +use crate::result::{WSResult}; use crate::sys::{LogicalModule, LogicalModuleNewArgs, LogicalModulesRef}; use crate::util::JoinHandleWrapper; use async_trait::async_trait; -use std::collections::{HashMap, HashSet}; -use std::sync::atomic::{AtomicU32, Ordering}; -use std::time::Duration; use ws_derive::LogicalModule; logical_module_view_impl!(MasterAppMgmtView); diff --git a/src/main/src/master/data/m_data_master.rs b/src/main/src/master/data/m_data_master.rs index bd6605c..44a4d70 100644 --- a/src/main/src/master/data/m_data_master.rs +++ b/src/main/src/master/data/m_data_master.rs @@ -1,10 +1,7 @@ -use crate::general::app::m_executor::EventCtx; use crate::general::app::m_executor::Executor; -use crate::general::app::m_executor::FnExeCtxAsync; -use crate::general::app::m_executor::FnExeCtxAsyncAllowedType; use crate::general::app::AppMetaManager; use crate::general::app::DataEventTrigger; -use crate::general::app::{AffinityPattern, AffinityRule, NodeTag}; +use crate::general::data::m_data_general::CacheModeVisitor; use crate::general::network::m_p2p::{P2PModule, RPCCaller, RPCHandler, RPCResponsor}; use crate::general::network::proto::{ self, DataVersionScheduleRequest, DataVersionScheduleResponse, @@ -16,7 +13,7 @@ use crate::util::JoinHandleWrapper; use crate::{ general::data::{ m_data_general::{ - CacheMode, DataGeneral, DataItemIdx, DataSetMeta, DataSetMetaBuilder, DataSplit, + CacheMode, DataGeneral, DataSetMetaBuilder, DataSplit, EachNodeSplit, CACHE_MODE_MAP_COMMON_KV_MASK, CACHE_MODE_TIME_FOREVER_MASK, }, m_kv_store_engine::{KeyType, KeyTypeDataSetMeta, KvAdditionalConf, KvStoreEngine}, @@ -194,14 +191,12 @@ impl DataMaster { // 设置数据分片 let _ = builder.set_data_splits(splits.clone()); - - // 设置缓存模式 - 对所有缓存节点启用永久缓存 - let cache_modes = vec![ - CACHE_MODE_TIME_FOREVER_MASK | CACHE_MODE_MAP_COMMON_KV_MASK; - context.each_data_sz_bytes.len() - ]; - let _ = builder.set_cache_mode_for_all(cache_modes.clone()); - + // 暂时用zui'lzuil + for idx in 0..splits.len() { + let _= builder.cache_mode_time_auto(idx as u8).cache_mode_pos_auto(idx as u8); + } + let cache_modes=builder.build().cache_mode; + tracing::debug!("planned for write data({:?}) cache_modes: {:?}", data_unique_id, cache_modes); Ok((cache_modes, splits, cache_nodes)) } @@ -271,64 +266,68 @@ impl DataMaster { }; // update version peers - let need_notify_nodes = { - let mut need_notify_nodes = HashSet::new(); - for one_data_splits in &new_meta.datas_splits { - for data_split in &one_data_splits.splits { - let _ = need_notify_nodes.insert(data_split.node_id); + { + tracing::debug!("updating meta({:?}) to peers for data({:?})", new_meta, req.unique_id); + let need_notify_nodes = { + let mut need_notify_nodes = HashSet::new(); + for one_data_splits in &new_meta.datas_splits { + for data_split in &one_data_splits.splits { + let _ = need_notify_nodes.insert(data_split.node_id); + } } - } - // TODO: do we need to notify cache nodes? - need_notify_nodes - }; - - for need_notify_node in need_notify_nodes { - let view = self.view.clone(); - let serialized_meta = bincode::serialize(&new_meta).unwrap(); - let unique_id = req.unique_id.clone(); - let version = new_meta.version; - let _ = tokio::spawn(async move { - let p2p = view.p2p(); - let display_id = std::str::from_utf8(&unique_id) - .map_or_else(|_err| format!("{:?}", unique_id), |ok| ok.to_owned()); - tracing::debug!( - "updating version for data({:?}) to node: {}, this_node: {}", - display_id, - need_notify_node, - p2p.nodes_config.this_node() - ); + // TODO: do we need to notify cache nodes? + need_notify_nodes + }; - tracing::debug!( - "async notify `DataMetaUpdateRequest` to node {}", - need_notify_node - ); - let resp = view - .data_master() - .rpc_caller_data_meta_update - .call( - p2p, - need_notify_node, - proto::DataMetaUpdateRequest { - unique_id, - version, - serialized_meta, - }, - Some(Duration::from_secs(60)), - ) - .await; - if let Err(err) = resp { - tracing::error!( - "notify `DataMetaUpdateRequest` to node {} failed: {}", + for need_notify_node in need_notify_nodes { + let view = self.view.clone(); + let serialized_meta = bincode::serialize(&new_meta).unwrap(); + let unique_id = req.unique_id.clone(); + let version = new_meta.version; + let _ = tokio::spawn(async move { + let p2p = view.p2p(); + let display_id = std::str::from_utf8(&unique_id) + .map_or_else(|_err| format!("{:?}", unique_id), |ok| ok.to_owned()); + tracing::debug!( + "updating version for data({:?}) to node: {}, this_node: {}", + display_id, need_notify_node, - err + p2p.nodes_config.this_node() + ); + + tracing::debug!( + "async notify `DataMetaUpdateRequest` to node {}", + need_notify_node ); - } else if let Ok(ok) = resp { - if ok.version != version { - tracing::error!("notify `DataMetaUpdateRequest` to node {} failed: version mismatch, expect: {}, remote: {}", need_notify_node, version, ok.version); + let resp = view + .data_master() + .rpc_caller_data_meta_update + .call( + p2p, + need_notify_node, + proto::DataMetaUpdateRequest { + unique_id, + version, + serialized_meta, + }, + Some(Duration::from_secs(60)), + ) + .await; + if let Err(err) = resp { + tracing::error!( + "notify `DataMetaUpdateRequest` to node {} failed: {}", + need_notify_node, + err + ); + } else if let Ok(ok) = resp { + if ok.version != version { + tracing::error!("notify `DataMetaUpdateRequest` to node {} failed: version mismatch, expect: {}, remote: {}", need_notify_node, version, ok.version); + } } - } - }); + }); + } } + tracing::debug!( "data:{:?} version required({}) and schedule done, caller will do following thing after receive `DataVersionScheduleResponse`", diff --git a/src/main/src/master/m_master.rs b/src/main/src/master/m_master.rs index 92e53f0..5c4849f 100644 --- a/src/main/src/master/m_master.rs +++ b/src/main/src/master/m_master.rs @@ -13,7 +13,7 @@ use ws_derive::LogicalModule; use crate::{ config::NodesConfig, general::{ - app::{AffinityPattern, AffinityRule, AppMetaManager, AppType, DataEventTrigger, FnMeta}, + app::{AppMetaManager, DataEventTrigger}, network::{ m_p2p::{P2PModule, RPCCaller}, proto::{ diff --git a/src/main/src/result.rs b/src/main/src/result.rs index 11f2785..fe823c3 100644 --- a/src/main/src/result.rs +++ b/src/main/src/result.rs @@ -1,4 +1,4 @@ -use std::{fmt::Debug, os::unix::net::SocketAddr, sync::Arc}; +use std::{fmt::Debug, os::unix::net::SocketAddr, sync::Arc, path::PathBuf}; use async_raft::{InitializeError, RaftError}; use camelpaste::paste; @@ -178,6 +178,7 @@ pub enum WsFuncError { #[derive(Debug)] pub enum WsDataError { + InvalidDataType, DataSetNotFound { uniqueid: Vec, }, @@ -199,6 +200,10 @@ pub enum WsDataError { expect: usize, actual: usize, }, + WriteDataFailed { + unique_id: Vec, + message: String, + }, KvDeserializeErr { unique_id: Vec, context: String, @@ -241,6 +246,24 @@ pub enum WsDataError { expect: usize, actual: usize, }, + SplitTaskFailed { + request_id: proto::BatchRequestId, + idx: DataSplitIdx, + }, + BatchTransferTaskFailed { + reason: String, + }, + BatchTransferFailed { + request_id: proto::BatchRequestId, + reason: String, + }, + BatchTransferNotFound { + request_id: proto::BatchRequestId, + }, + BatchTransferError { + request_id: proto::BatchRequestId, + msg: String, + }, UnknownCacheMapMode { mode: u16, }, @@ -255,10 +278,27 @@ pub enum WsDataError { len: u8, }, ItemIdxEmpty, - BatchTransferFailed { - node: NodeID, - batch: u32, + VersionMismatch { + expected: u64, + actual: u64, + }, + SizeMismatch { + expected: usize, // 预期的数据大小 + actual: usize, // 实际的数据大小 + }, + ReadDataFailed { + path: PathBuf, // 读取失败的文件路径 + }, + /// 数据分片任务错误 + DataSplitTaskError { + msg: String, + }, + /// 数据解码错误 + DataDecodeError { + /// 错误原因 reason: String, + /// 数据类型（用于调试） + data_type: String, }, } diff --git a/src/main/src/util/container/async_init_map.rs b/src/main/src/util/container/async_init_map.rs new file mode 100644 index 0000000..953d1d0 --- /dev/null +++ b/src/main/src/util/container/async_init_map.rs @@ -0,0 +1,209 @@ +use std::hash::Hash; +use std::sync::Arc; +use std::ops::Deref; +use dashmap::DashMap; +use tokio::sync::broadcast; +use thiserror::Error; + + +/// AsyncInitMap 的错误类型 +#[derive(Debug, Error)] +pub enum AsyncInitError { + /// 等待初始化完成时发生错误 + #[error("等待初始化完成时发生错误: {0}")] + WaitError(broadcast::error::RecvError), +} + +/// Map 值的包装器,用于异步初始化Map中的值 +#[derive(Clone)] +pub struct AsyncInitMapValue { + inner: ValueState +} + +impl AsyncInitMapValue { + /// 获取就绪值的引用 + pub fn get(&self) -> Option<&V> { + self.inner.as_ready() + } + + fn new_initializing(tx: broadcast::Sender) -> Self { + Self { + inner: ValueState::Initializing(tx) + } + } + + fn new_ready(value: V) -> Self { + Self { + inner: ValueState::Ready(value) + } + } +} + +/// Map 值的状态 +#[derive(Clone)] +enum ValueState { + /// 正在初始化，包含一个通知 channel + Initializing(broadcast::Sender), + /// 初始化完成，包含实际值 + Ready(V), +} + +impl ValueState { + /// 获取就绪值的引用 + fn as_ready(&self) -> Option<&V> { + match self { + Self::Ready(v) => Some(v), + _ => None, + } + } + + /// 获取初始化中的 sender + fn as_initializing(&self) -> Option<&broadcast::Sender> { + match self { + Self::Initializing(tx) => Some(tx), + _ => None, + } + } + + /// 是否已经就绪 + #[allow(dead_code)] + pub(crate) fn is_ready(&self) -> bool { + matches!(self, Self::Ready(_)) + } + + /// 是否正在初始化 + #[allow(dead_code)] + pub(crate) fn is_initializing(&self) -> bool { + matches!(self, Self::Initializing(_)) + } +} + +/// 支持异步初始化的并发 Map +pub struct AsyncInitMap +where + K: Eq + Hash + Clone + Send + Sync + 'static, + V: Clone + Send + Sync+'static, +{ + inner: Arc>>, +} + +impl AsyncInitMap +where + K: Eq + Hash + Clone + Send + Sync + 'static, + V: Clone + Send + Sync+'static, +{ + /// 创建新的异步初始化 Map + pub fn new() -> Self { + Self { + inner: Arc::new(DashMap::new()), + } + } + + /// 获取一个已经初始化的值,如果值不存在或未初始化完成则返回None + pub fn get(&self, key: &K) -> Option { + self.inner.get(key) + .and_then(|entry| entry.value().get().cloned()) + } + + /// 移除一个键值对,返回被移除的值(如果存在且已初始化) + pub fn remove(&self, key: &K) -> Option { + self.inner.remove(key) + .and_then(|(_, value)| value.get().cloned()) + } + + /// 获取或初始化一个值 + /// + /// # 参数 + /// * `key` - 键 + /// * `init_fut` - 初始化 Future + /// + /// # 返回 + /// 返回初始化完成的值，如果初始化失败则返回错误 + pub async fn get_or_init(&self, key: K, init_fut: Fut) -> Result + where + Fut: std::future::Future> + Send + 'static, + FutErr: std::fmt::Debug, + { + // 先尝试只读获取 + if let Some(entry) = self.inner.get(&key) { + match &entry.value().inner { + ValueState::Ready(v) => return Ok(v.clone()), + ValueState::Initializing(tx) => { + let mut rx = tx.subscribe(); + drop(entry); + return Ok(rx.recv().await.map_err(AsyncInitError::WaitError)?); + } + } + } + + // 使用 or_insert_with 进行原子操作并获取 rx + let mut rx = { + let entry = self.inner.entry(key.clone()).or_insert_with(|| { + let (tx, _) = broadcast::channel(1); + let tx_clone = tx.clone(); + + let inner = self.inner.clone(); + let key = key.clone(); + + let _ = tokio::spawn(async move { + match init_fut.await { + Ok(value) => { + // 先通过 channel 发送值 + let _ = tx.send(value.clone()); + // 然后更新状态 + let _ = inner.insert(key, AsyncInitMapValue::new_ready(value)); + } + Err(e) => { + let _ = inner.remove(&key); + tracing::error!("初始化失败: {:?}", e); + drop(tx); // 关闭 channel 通知错误 + } + } + }); + + AsyncInitMapValue::new_initializing(tx_clone) + }); + + entry.value().inner.as_initializing() + .expect("刚插入的值必定处于初始化状态") + .subscribe() + }; + + // 等待值通过 channel 传递 + Ok(rx.recv().await.map_err(AsyncInitError::WaitError)?) + } +} + +impl Default for AsyncInitMap +where + K: Eq + Hash + Clone + Send + Sync + 'static, + V: Clone + Send + Sync+'static, +{ + fn default() -> Self { + Self::new() + } +} + +impl Clone for AsyncInitMap +where + K: Eq + Hash + Clone + Send + Sync + 'static, + V: Clone + Send + Sync+'static, +{ + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +impl Deref for AsyncInitMap +where + K: Eq + Hash + Clone + Send + Sync + 'static, + V: Clone + Send + Sync+'static, +{ + type Target = DashMap>; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} diff --git a/src/main/src/util/container/mod.rs b/src/main/src/util/container/mod.rs index 1c9a676..20198f6 100644 --- a/src/main/src/util/container/mod.rs +++ b/src/main/src/util/container/mod.rs @@ -1,2 +1,4 @@ pub mod map; pub mod sync_trie; + +pub mod async_init_map; diff --git a/src/main/src/util/container/sync_trie.rs b/src/main/src/util/container/sync_trie.rs index a91fae0..2043c35 100644 --- a/src/main/src/util/container/sync_trie.rs +++ b/src/main/src/util/container/sync_trie.rs @@ -1,9 +1,7 @@ -use parking_lot::{RwLock, RwLockReadGuard}; +use parking_lot::{RwLock}; use std::collections::HashMap; use std::ops::{Deref, DerefMut}; use std::sync::Arc; -use std::thread; -use std::time::Duration; pub struct TrieNode { children: HashMap>>>, diff --git a/src/main/src/worker/m_kv_user_client.rs b/src/main/src/worker/m_kv_user_client.rs index c9e97d3..4de9d52 100644 --- a/src/main/src/worker/m_kv_user_client.rs +++ b/src/main/src/worker/m_kv_user_client.rs @@ -235,6 +235,7 @@ impl KvUserClient { _meta: DataSetMetaV2, splits: HashMap, ) -> WSResult> { + tracing::debug!("convert_get_data_res_to_kv_response uid: {:?}, split keys: {:?}", uid, splits.keys().collect::>()); if splits.len() != 1 { return Err(WSError::WsDataError( WsDataError::KvGotWrongSplitCountAndIdx { diff --git a/update_batch_transfer.md b/update_batch_transfer.md new file mode 100644 index 0000000..c78818a --- /dev/null +++ b/update_batch_transfer.md @@ -0,0 +1,70 @@ +# 更新 batch_transfer 函数 + +## 1. 改动目标 +更新 batch_transfer 函数，使其严格遵循设计文档规范。 + +## 2. 相关文件 +1. `/root/prjs/waverless/src/main/src/general/data/m_data_general/mod.rs` + - batch_transfer 函数 + - write_data_batch 函数 + - DataItemSource 结构 + +## 3. 设计文档分析 +1. review.md: + - 保持使用 dyn trait 接口 + - 使用新的错误类型 WsDataError::BatchTransferFailed + - 不删除现有功能代码 + +2. design.canvas: + - batch_sender_group 组件定义了接口规范 + - 使用 DEFAULT_BLOCK_SIZE 常量 (4MB) + - 保持四层架构设计 + +## 4. 改动步骤 +1. 添加块大小常量： + ```rust + /// 默认数据块大小 (4MB) + const DEFAULT_BLOCK_SIZE: usize = 4 * 1024 * 1024; + ``` + +2. 保持 batch_transfer 函数签名： + ```rust + async fn batch_transfer( + unique_id: Vec, + version: u64, + target_node: NodeID, + data: Arc, + view: DataGeneralView, + ) -> WSResult<()> + ``` + +3. 使用正确的错误类型： + ```rust + WsDataError::BatchTransferFailed { + request_id: proto::BatchRequestId { + node_id: target_node as u32, + sequence: block_idx as u32, + }, + reason: String, + } + ``` + +## 5. 改动分析 +1. 符合分层设计： + - 接收层：保持 dyn trait 接口 + - 写入任务层：使用 DEFAULT_BLOCK_SIZE + - 本地存储层：支持文件和内存数据 + - 结果返回层：使用新的错误类型 + +2. 保持兼容性： + - 函数签名不变 + - 错误处理规范化 + - 分块大小标准化 + +## 6. 删除内容分析 +本次改动不涉及删除操作，只是规范化和标准化现有代码。 + +## 7. 后续任务 +1. 添加更多错误处理日志 +2. 更新相关文档 +3. 添加单元测试 diff --git a/update_error_types.md b/update_error_types.md new file mode 100644 index 0000000..a3db43f --- /dev/null +++ b/update_error_types.md @@ -0,0 +1,71 @@ +# 更新错误类型结构 + +## 改动说明 +本次改动主要针对错误类型结构的更新，将在 `src/main/src/result.rs` 中修改 `WsDataError` 枚举。 + +### 1. 修改目标 +- 更新 `WsDataError` 枚举中的错误类型 +- 统一使用 `request_id` 替代之前的节点和批次号 +- 添加新的错误类型以支持分片任务 +- 确保错误信息更加明确和具体 + +### 2. 关联性分析（>500字） +本次错误类型修改与多个部分密切相关： + +1. 与批量传输模块的关联： + - 新的错误类型直接支持 `WriteSplitDataTaskHandle` 和 `WriteSplitDataTaskGroup` 的错误处理 + - 通过 `request_id` 统一标识批量传输任务，替代之前分散的节点和批次号 + - 错误类型的修改为后续删除 `BatchManager` 和 `BatchTransfer` 做准备 + +2. 与四层架构的关联： + - 错误类型覆盖了所有四层的错误场景： + * 接收层：BatchTransferNotFound 用于处理请求接收错误 + * 写入任务层：SplitTaskFailed 用于处理分片任务错误 + * 本地存储层：WriteDataFailed 用于处理写入错误 + * 结果返回层：BatchTransferError 用于处理一般性错误 + +3. 与状态管理的关联： + - 错误类型中包含 version 相关错误，支持版本验证 + - 通过 request_id 可以准确定位出错的任务状态 + - 错误信息包含足够的上下文，便于状态恢复和清理 + +4. 与日志记录的关联： + - 错误类型设计符合 tracing 库的使用规范 + - 每个错误变体都包含足够的信息用于日志记录 + - 错误信息的结构化有助于日志分析和问题定位 + +### 3. 影响分析（>500字） +本次修改将产生以下影响： + +1. 代码结构影响： + - 简化了错误处理逻辑，统一使用 request_id + - 提供了更清晰的错误类型层次 + - 改进了错误信息的可读性和可追踪性 + +2. 功能影响： + - 支持更细粒度的错误处理 + - 提供更准确的错误定位 + - 便于实现错误重试机制 + - 有助于问题诊断和调试 + +3. 性能影响： + - 错误类型的修改不会对性能造成明显影响 + - 结构化的错误信息可能略微增加内存使用 + - 日志记录的信息更加完整，可能略微增加IO开销 + +4. 维护性影响： + - 提高了代码的可维护性 + - 简化了错误处理的代码编写 + - 使错误追踪和修复更加容易 + - 有助于系统监控和问题诊断 + +5. 兼容性影响： + - 需要修改所有使用旧错误类型的代码 + - 需要更新相关的测试用例 + - 可能需要更新错误处理相关的文档 + +### 4. 执行计划 +1. 修改 src/main/src/result.rs 中的 WsDataError 枚举 +2. 更新错误类型的使用位置 +3. 添加必要的注释和文档 +4. 确保与 tracing 日志记录的集成 diff --git a/update_write_data_batch.md b/update_write_data_batch.md new file mode 100644 index 0000000..5c59e5b --- /dev/null +++ b/update_write_data_batch.md @@ -0,0 +1,172 @@ +# 更新写入数据批处理函数 + +## 1. 删除代码分析（>500字） + +我们需要删除以下代码： + +```rust +// 在 src/main/src/general/data/m_data_general/mod.rs 中 +async fn transfer_data( + &self, + node_id: NodeID, + unique_id: Vec, + version: u64, + data: proto::DataItem, + data_item_idx: usize, + batch_size: usize, +) -> WSResult<()> +``` + +删除原因分析： +1. 功能重叠：transfer_data 函数与设计文档中的 batch_transfer 函数功能重叠，但实现不符合规范 +2. 参数不一致： + - transfer_data 使用了 data_item_idx 和 batch_size 参数，这在设计中并不需要 + - 缺少了 DataSource trait 的抽象 +3. 错误处理： + - 原实现的错误处理不符合四层架构的要求 + - 缺少对版本号的验证 +4. 并发控制： + - 原实现使用了固定的信号量大小(10) + - 新设计中使用32作为并发限制 +5. 代码组织： + - 原实现将所有逻辑放在一个函数中 + - 新设计通过 DataSource trait 实现更好的抽象 +6. 资源管理： + - 原实现没有很好地管理资源生命周期 + - 新设计通过 Arc 更好地管理资源 + +删除这段代码不会影响其他功能，因为： +1. write_data_batch 函数会调用新的 batch_transfer 函数 +2. 错误处理逻辑会更加完善 +3. 并发控制更加合理 +4. 代码结构更加清晰 + +## 2. 新增代码 + +### 2.1 DataSource Trait +```rust +/// 数据源接口 +#[async_trait] +pub trait DataSource: Send + Sync + 'static { + /// 获取数据总大小 + async fn size(&self) -> WSResult; + /// 读取指定范围的数据 + async fn read_chunk(&self, offset: usize, size: usize) -> WSResult>; + /// 获取数据块类型 + fn block_type(&self) -> BatchDataBlockType; +} +``` + +### 2.2 批量传输函数 +```rust +/// 批量传输数据 +pub async fn batch_transfer( + unique_id: Vec, + version: u64, + target_node: NodeID, + data: Arc, + view: DataGeneralView, +) -> WSResult<()> { + let total_size = data.size().await?; + let total_blocks = (total_size + DEFAULT_BLOCK_SIZE - 1) / DEFAULT_BLOCK_SIZE; + let semaphore = Arc::new(Semaphore::new(32)); + let mut handles = Vec::new(); + + // 发送所有数据块 + for block_idx in 0..total_blocks { + // 获取信号量许可 + let permit = semaphore.clone().acquire_owned().await.unwrap(); + + let offset = block_idx as usize * DEFAULT_BLOCK_SIZE; + let size = DEFAULT_BLOCK_SIZE.min(total_size - offset); + + // 读取数据块 + let block_data = data.read_chunk(offset, size).await?; + + // 构造请求 + let request = proto::BatchDataRequest { + request_id: Some(proto::BatchRequestId { + node_id: target_node as u32, + sequence: block_idx as u32, + }), + block_type: data.block_type() as i32, + block_index: block_idx as u32, + data: block_data, + operation: proto::DataOpeType::Write as i32, + unique_id: unique_id.clone(), + version, + }; + + // 发送请求 + let view = view.clone(); + let handle = tokio::spawn(async move { + let _permit = permit; // 持有permit直到任务完成 + let resp = view.data_general().rpc_call_batch_data.call( + view.p2p(), + target_node, + request, + Some(Duration::from_secs(30)), + ).await?; + + if !resp.success { + return Err(WsDataError::BatchTransferFailed { + node: target_node, + batch: block_idx as u32, + reason: resp.error_message, + }.into()); + } + + Ok(()) + }); + + handles.push(handle); + } + + // 等待所有请求完成 + for handle in handles { + handle.await??; + } + + Ok(()) +} +``` + +### 2.3 更新 write_data_batch 函数 +```rust +pub async fn write_data_batch( + &self, + unique_id: &[u8], + version: u64, + data: proto::DataItem, + data_item_idx: usize, + node_id: NodeID, + batch_size: usize, +) -> WSResult<()> { + // 创建 DataSource + let data_source = Arc::new(DataItemSource::new(data)); + + // 调用 batch_transfer 函数处理数据传输 + batch_transfer( + unique_id.to_vec(), + version, + node_id, + data_source, + self.view.clone(), + ).await +} +``` + +## 3. 实现说明 + +1. 严格按照设计文档实现 +2. 保持四层架构设计 +3. 遵循错误处理规范 +4. 使用规范中定义的数据类型 +5. 保持代码清晰可维护 + +## 4. 下一步计划 + +1. 实现 DataItemSource 结构体 +2. 添加必要的单元测试 +3. 完善错误处理 +4. 添加详细的文档注释