diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index 55b553c..0000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,5 +0,0 @@ -[env] -CXXFLAGS = "-I/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/c++/v1 -stdlib=libc++" - -[build] -target = "aarch64-apple-darwin" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..450628e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,159 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + fmt: + name: Format Check + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + + - name: Check formatting + run: cargo fmt --all -- --check + + clippy: + name: Clippy + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler libclang-dev + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - name: Cache cargo registry + uses: Swatinem/rust-cache@v2 + with: + shared-key: "clippy" + + - name: Run clippy + run: cargo clippy --workspace --all-targets --all-features -- -D warnings + + test: + name: Test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-latest] + steps: + - uses: actions/checkout@v4 + + - name: Install system dependencies (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler libclang-dev + + - name: Install system dependencies (macOS) + if: runner.os == 'macOS' + run: | + brew install protobuf llvm + echo "LIBCLANG_PATH=$(brew --prefix llvm)/lib" >> $GITHUB_ENV + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: Swatinem/rust-cache@v2 + with: + shared-key: "test-${{ matrix.os }}" + + - name: Run tests + run: cargo test --workspace --all-features + + build: + name: Build (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-latest] + steps: + - uses: actions/checkout@v4 + + - name: Install system dependencies (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler libclang-dev + + - name: Install system dependencies (macOS) + if: runner.os == 'macOS' + run: | + brew install protobuf llvm + echo "LIBCLANG_PATH=$(brew --prefix llvm)/lib" >> $GITHUB_ENV + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: Swatinem/rust-cache@v2 + with: + shared-key: "build-${{ matrix.os }}" + + - name: Build release + run: cargo build --release --workspace + + doc: + name: Documentation + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler libclang-dev + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: Swatinem/rust-cache@v2 + with: + shared-key: "doc" + + - name: Build documentation + env: + RUSTDOCFLAGS: "-D warnings" + run: cargo doc --no-deps --workspace --all-features + + # Summary job that depends on all other jobs + ci-success: + name: CI Success + needs: [fmt, clippy, test, build, doc] + runs-on: ubuntu-24.04 + if: always() + steps: + - name: Check all jobs passed + run: | + if [[ "${{ needs.fmt.result }}" != "success" ]] || \ + [[ "${{ needs.clippy.result }}" != "success" ]] || \ + [[ "${{ needs.test.result }}" != "success" ]] || \ + [[ "${{ needs.build.result }}" != "success" ]] || \ + [[ "${{ needs.doc.result }}" != "success" ]]; then + echo "One or more jobs failed" + exit 1 + fi + echo "All CI jobs passed!" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..ed23edf --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,200 @@ +name: Release + +on: + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + workflow_dispatch: + inputs: + version: + description: 'Version to release (e.g., 0.2.0)' + required: true + type: string + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + name: Build (${{ matrix.name }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu + os: ubuntu-latest + name: linux-x86_64 + cross: false + - target: aarch64-unknown-linux-gnu + os: ubuntu-latest + name: linux-aarch64 + cross: true + - target: x86_64-apple-darwin + os: macos-13 + name: macos-x86_64 + cross: false + - target: aarch64-apple-darwin + os: macos-14 + name: macos-aarch64 + cross: false + - target: x86_64-pc-windows-msvc + os: windows-latest + name: windows-x86_64 + cross: false + + steps: + - uses: actions/checkout@v4 + + - name: Get version + id: version + shell: bash + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "version=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT + else + echo "version=${GITHUB_REF_NAME#v}" >> $GITHUB_OUTPUT + fi + + - name: Install system dependencies (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler libclang-dev + + - name: Install system dependencies (macOS) + if: runner.os == 'macOS' + run: | + brew install protobuf llvm + echo "LIBCLANG_PATH=$(brew --prefix llvm)/lib" >> $GITHUB_ENV + + - name: Install system dependencies (Windows) + if: runner.os == 'Windows' + run: | + choco install protoc llvm -y + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install cross + if: matrix.cross + run: cargo install cross --git https://github.com/cross-rs/cross + + - name: Cache cargo registry + uses: Swatinem/rust-cache@v2 + with: + shared-key: "release-${{ matrix.name }}" + + - name: Build (native) + if: "!matrix.cross" + run: cargo build --release --target ${{ matrix.target }} + + - name: Build (cross) + if: matrix.cross + run: cross build --release --target ${{ matrix.target }} + + - name: Create archive directory + shell: bash + run: | + VERSION=${{ steps.version.outputs.version }} + ARCHIVE_DIR="memory-daemon-${VERSION}-${{ matrix.name }}" + mkdir -p "dist/${ARCHIVE_DIR}" + + # Copy binaries + if [[ "${{ runner.os }}" == "Windows" ]]; then + cp target/${{ matrix.target }}/release/memory-daemon.exe "dist/${ARCHIVE_DIR}/" + cp target/${{ matrix.target }}/release/memory-ingest.exe "dist/${ARCHIVE_DIR}/" || true + else + cp target/${{ matrix.target }}/release/memory-daemon "dist/${ARCHIVE_DIR}/" + cp target/${{ matrix.target }}/release/memory-ingest "dist/${ARCHIVE_DIR}/" || true + fi + + # Copy documentation + cp LICENSE "dist/${ARCHIVE_DIR}/" || true + cp README.md "dist/${ARCHIVE_DIR}/" || true + + - name: Create archive (Unix) + if: runner.os != 'Windows' + shell: bash + run: | + VERSION=${{ steps.version.outputs.version }} + ARCHIVE_DIR="memory-daemon-${VERSION}-${{ matrix.name }}" + cd dist + tar -czvf "${ARCHIVE_DIR}.tar.gz" "${ARCHIVE_DIR}" + rm -rf "${ARCHIVE_DIR}" + + - name: Create archive (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $VERSION = "${{ steps.version.outputs.version }}" + $ARCHIVE_DIR = "memory-daemon-${VERSION}-${{ matrix.name }}" + cd dist + Compress-Archive -Path $ARCHIVE_DIR -DestinationPath "${ARCHIVE_DIR}.zip" + Remove-Item -Recurse -Force $ARCHIVE_DIR + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: release-${{ matrix.name }} + path: dist/* + retention-days: 1 + + release: + name: Create Release + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - name: Get version + id: version + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "version=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT + echo "tag=v${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT + else + echo "version=${GITHUB_REF_NAME#v}" >> $GITHUB_OUTPUT + echo "tag=${GITHUB_REF_NAME}" >> $GITHUB_OUTPUT + fi + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + pattern: release-* + merge-multiple: true + + - name: List artifacts + run: ls -laR artifacts/ + + - name: Generate checksums + run: | + cd artifacts + sha256sum *.tar.gz *.zip > SHA256SUMS.txt + cat SHA256SUMS.txt + + - name: Create tag (workflow_dispatch only) + if: github.event_name == 'workflow_dispatch' + run: | + git config user.name "GitHub Actions" + git config user.email "actions@github.com" + git tag -a ${{ steps.version.outputs.tag }} -m "Release ${{ steps.version.outputs.tag }}" + git push origin ${{ steps.version.outputs.tag }} + + - name: Create GitHub Release + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ steps.version.outputs.tag }} + name: Release ${{ steps.version.outputs.version }} + draft: false + prerelease: ${{ contains(steps.version.outputs.version, '-') }} + generate_release_notes: true + files: | + artifacts/*.tar.gz + artifacts/*.zip + artifacts/SHA256SUMS.txt diff --git a/.gitignore b/.gitignore index 01c52df..715d4fb 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ coverage/ # Override global gitignore for project-specific config !CLAUDE.md !AGENTS.md + +# Local Cargo configuration (platform-specific) +.cargo/ diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index e8a9150..dbebf66 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -12,11 +12,12 @@ Phases are grouped by the cognitive layer they implement: |-------|--------|------------|--------| | **Foundation** (0-1) | 1-6 | Events + TOC hierarchy | Complete | | **Integration** | 7-10 | Plugins, hooks, scheduler | Complete | -| **Agentic Navigation** (2) | 10.5 | Index-free search (always works) | Planned | -| **Keyword Acceleration** (3) | 11 | BM25/Tantivy teleport | Planned | -| **Semantic Acceleration** (4) | 12 | Vector/HNSW teleport | Planned | -| **Index Lifecycle** | 13 | Outbox-driven index updates | Planned | -| **Conceptual Enrichment** (5) | 14 | Topic graph discovery | Planned | +| **Agentic Navigation** (2) | 10.5 | Index-free search (always works) | Complete | +| **Keyword Acceleration** (3) | 11 | BM25/Tantivy teleport | Complete | +| **Semantic Acceleration** (4) | 12 | Vector/HNSW teleport | Complete | +| **Index Lifecycle** | 13 | Outbox-driven index updates | Complete | +| **Conceptual Enrichment** (5) | 14 | Topic graph discovery | Complete | +| **Configuration UX** | 15 | Interactive wizard skills | Planned | **See:** [Cognitive Architecture Manifesto](../docs/COGNITIVE_ARCHITECTURE.md) @@ -36,11 +37,12 @@ Phases are grouped by the cognitive layer they implement: - [x] **Phase 8: CCH Hook Integration** - Automatic event capture via CCH hooks - [x] **Phase 9: Setup & Installer Plugin** - Interactive setup wizard plugin with commands and agents - [x] **Phase 10: Background Scheduler** - In-process Tokio cron scheduler for TOC rollups and periodic jobs -- [ ] **Phase 10.5: Agentic TOC Search** - Index-free search using TOC navigation with progressive disclosure (INSERTED) -- [ ] **Phase 11: BM25 Teleport (Tantivy)** - Full-text search index for keyword-based teleportation to relevant TOC nodes -- [ ] **Phase 12: Vector Teleport (HNSW)** - Semantic similarity search via local HNSW vector index -- [ ] **Phase 13: Outbox Index Ingestion** - Event-driven index updates from outbox for rebuildable search indexes -- [ ] **Phase 14: Topic Graph Memory** - Semantic topic extraction, time-decayed importance, topic relationships for conceptual discovery +- [x] **Phase 10.5: Agentic TOC Search** - Index-free search using TOC navigation with progressive disclosure (INSERTED) +- [x] **Phase 11: BM25 Teleport (Tantivy)** - Full-text search index for keyword-based teleportation to relevant TOC nodes +- [x] **Phase 12: Vector Teleport (HNSW)** - Semantic similarity search via local HNSW vector index +- [x] **Phase 13: Outbox Index Ingestion** - Event-driven index updates from outbox for rebuildable search indexes +- [x] **Phase 14: Topic Graph Memory** - Semantic topic extraction, time-decayed importance, topic relationships for conceptual discovery +- [ ] **Phase 15: Configuration Wizard Skills** - Interactive AskUserQuestion-based configuration wizards for storage, LLM, and multi-agent settings ## Phase Details @@ -268,9 +270,9 @@ Plans: **Plans**: 3 plans in 3 waves Plans: -- [ ] 10.5-01-PLAN.md — Core search logic (search_node function, term overlap scoring, unit tests) -- [ ] 10.5-02-PLAN.md — gRPC integration (SearchNode/SearchChildren RPCs, integration tests) -- [ ] 10.5-03-PLAN.md — CLI and agent (search command, navigator agent updates, documentation) +- [x] 10.5-01-PLAN.md — Core search logic (search_node function, term overlap scoring, unit tests) +- [x] 10.5-02-PLAN.md — gRPC integration (SearchNode/SearchChildren RPCs, integration tests) +- [x] 10.5-03-PLAN.md — CLI and agent (search command, navigator agent updates, documentation) **Documentation:** - Technical Plan: docs/plans/phase-10.5-agentic-toc-search.md @@ -293,10 +295,10 @@ Plans: - Research: .planning/phases/11-bm25-teleport-tantivy/11-RESEARCH.md Plans: -- [ ] 11-01-PLAN.md — Tantivy integration (memory-search crate, schema, index setup) -- [ ] 11-02-PLAN.md — Indexing pipeline (TOC node and grip text extraction, document mapping) -- [ ] 11-03-PLAN.md — Search API (gRPC TeleportSearch RPC, BM25 scoring) -- [ ] 11-04-PLAN.md — CLI and testing (teleport command, background commit job) +- [x] 11-01-PLAN.md — Tantivy integration (memory-search crate, schema, index setup) +- [x] 11-02-PLAN.md — Indexing pipeline (TOC node and grip text extraction, document mapping) +- [x] 11-03-PLAN.md — Search API (gRPC TeleportSearch RPC, BM25 scoring) +- [x] 11-04-PLAN.md — CLI and testing (teleport command, background commit job) ### Phase 12: Vector Teleport (HNSW) **Goal**: Enable semantic similarity search for conceptually related content even when keywords don't match @@ -318,11 +320,11 @@ Plans: - Research: .planning/phases/12-vector-teleport-hnsw/12-RESEARCH.md Plans: -- [ ] 12-01-PLAN.md — Embedding infrastructure (memory-embeddings crate, Candle model, caching) -- [ ] 12-02-PLAN.md — Vector index (memory-vector crate, usearch HNSW, metadata storage) -- [ ] 12-02b-PLAN.md — Vector indexing pipeline (outbox consumer, checkpoint recovery, admin commands) -- [ ] 12-03-PLAN.md — gRPC integration (VectorTeleport, HybridSearch, GetVectorIndexStatus RPCs) -- [ ] 12-04-PLAN.md — CLI and documentation (teleport commands, user guide) +- [x] 12-01-PLAN.md — Embedding infrastructure (memory-embeddings crate, Candle model, caching) +- [x] 12-02-PLAN.md — Vector index (memory-vector crate, usearch HNSW, metadata storage) +- [x] 12-02b-PLAN.md — Vector indexing pipeline (outbox consumer, checkpoint recovery, admin commands) +- [x] 12-03-PLAN.md — gRPC integration (VectorTeleport, HybridSearch, GetVectorIndexStatus RPCs) +- [x] 12-04-PLAN.md — CLI and documentation (teleport commands, user guide) ### Phase 13: Outbox Index Ingestion **Goal**: Drive index updates from the existing outbox pattern for rebuildable, crash-safe search indexes @@ -334,13 +336,16 @@ Plans: 3. Full index rebuild from storage is supported via admin command 4. Index state is independent of primary storage (can be deleted and rebuilt) 5. Indexing is async and doesn't block event ingestion -**Plans**: TBD +**Plans**: 4 plans in 3 waves + +**Documentation:** +- Research: .planning/phases/13-outbox-index-ingestion/13-RESEARCH.md Plans: -- [ ] 13-01: Outbox consumer for indexing (checkpoint tracking) -- [ ] 13-02: Incremental index updates (add/update documents) -- [ ] 13-03: Full rebuild command (admin rebuild-indexes) -- [ ] 13-04: Async indexing pipeline (scheduled via Phase 10) +- [x] 13-01-PLAN.md — Outbox consumer infrastructure (memory-indexing crate, outbox reading, checkpoint tracking) +- [x] 13-02-PLAN.md — Incremental index updates (IndexingPipeline, dispatch logic, mock tests) +- [x] 13-03-PLAN.md — Full rebuild command (admin rebuild-indexes, dry-run support) +- [x] 13-04-PLAN.md — Scheduler integration (background job, GetIndexingStatus RPC) ### Phase 14: Topic Graph Memory **Goal**: Enable conceptual discovery through semantic topics extracted from TOC summaries with time-decayed importance scoring @@ -362,17 +367,40 @@ Plans: - Technical Plan: docs/plans/topic-graph-memory.md Plans: -- [ ] 14-01-PLAN.md — Topic extraction (memory-topics crate, CF_TOPICS, HDBSCAN clustering, cosine similarity) -- [ ] 14-02-PLAN.md — Topic labeling (LLM integration with keyword fallback, stopword filtering) -- [ ] 14-03-PLAN.md — Importance scoring (exponential time decay with configurable half-life) -- [ ] 14-04-PLAN.md — Topic relationships (similarity detection, parent/child hierarchy, cycle prevention) -- [ ] 14-05-PLAN.md — Navigation RPCs (5 gRPC endpoints: status, query, nodes, top, related) -- [ ] 14-06-PLAN.md — Lifecycle management (pruning, resurrection, scheduler jobs, CLI commands) +- [x] 14-01-PLAN.md — Topic extraction (memory-topics crate, CF_TOPICS, HDBSCAN clustering, cosine similarity) +- [x] 14-02-PLAN.md — Topic labeling (LLM integration with keyword fallback, stopword filtering) +- [x] 14-03-PLAN.md — Importance scoring (exponential time decay with configurable half-life) +- [x] 14-04-PLAN.md — Topic relationships (similarity detection, parent/child hierarchy, cycle prevention) +- [x] 14-05-PLAN.md — Navigation RPCs (5 gRPC endpoints: status, query, nodes, top, related) +- [x] 14-06-PLAN.md — Lifecycle management (pruning, resurrection, scheduler jobs, CLI commands) + +### Phase 15: Configuration Wizard Skills +**Goal**: Create interactive AskUserQuestion-based configuration wizard skills for advanced storage, LLM, and multi-agent configuration +**Depends on**: Phase 9 (Setup & Installer Plugin) +**Requirements**: CONFIG-01, CONFIG-02, CONFIG-03, CONFIG-04 +**Success Criteria** (what must be TRUE): + 1. `/memory-storage` skill provides interactive wizard for storage paths, retention, cleanup, GDPR mode + 2. `/memory-llm` skill provides interactive wizard for LLM provider, model discovery, cost estimation, API testing + 3. `/memory-agents` skill provides interactive wizard for multi-agent mode, agent tagging, cross-agent queries + 4. All 29 config options are addressable through wizard skills (coverage verified) + 5. State detection skips already-configured options + 6. Skills follow existing memory-setup patterns (--minimal, --advanced, --fresh flags) +**Plans**: 5 plans in 3 waves + +**Documentation:** +- Technical Plan: docs/plans/configuration-wizard-skills-plan.md + +Plans: +- [ ] 15-01-PLAN.md — memory-storage skill (storage paths, retention, cleanup, GDPR, performance tuning) +- [ ] 15-02-PLAN.md — memory-llm skill (provider, model discovery, API testing, cost estimation, budget) +- [ ] 15-03-PLAN.md — memory-agents skill (multi-agent mode, agent ID, query scope, team settings) +- [ ] 15-04-PLAN.md — Reference documentation (retention-policies.md, provider-comparison.md, storage-strategies.md) +- [ ] 15-05-PLAN.md — Plugin integration and memory-setup updates (marketplace.json, gap resolution) ## Progress **Execution Order:** -Phases execute in numeric order: 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7 -> 8 -> 9 -> 10 -> 10.5 -> 11 -> 12 -> 13 -> 14 +Phases execute in numeric order: 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7 -> 8 -> 9 -> 10 -> 10.5 -> 11 -> 12 -> 13 -> 14 -> 15 | Phase | Plans Complete | Status | Completed | |-------|----------------|--------|-----------| @@ -386,11 +414,12 @@ Phases execute in numeric order: 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7 -> 8 -> 9 -> 10 | 8. CCH Hook Integration | 1/1 | Complete | 2026-01-30 | | 9. Setup & Installer Plugin | 4/4 | Complete | 2026-01-31 | | 10. Background Scheduler | 4/4 | Complete | 2026-01-31 | -| 10.5. Agentic TOC Search | 0/3 | Planned | - | -| 11. BM25 Teleport (Tantivy) | 0/4 | Planned | - | -| 12. Vector Teleport (HNSW) | 0/5 | Planned | - | -| 13. Outbox Index Ingestion | 0/4 | Planned | - | -| 14. Topic Graph Memory | 0/6 | Planned | - | +| 10.5. Agentic TOC Search | 3/3 | Complete | 2026-02-02 | +| 11. BM25 Teleport (Tantivy) | 4/4 | Complete | 2026-02-02 | +| 12. Vector Teleport (HNSW) | 5/5 | Complete | 2026-02-02 | +| 13. Outbox Index Ingestion | 4/4 | Complete | 2026-02-02 | +| 14. Topic Graph Memory | 6/6 | Complete | 2026-02-02 | +| 15. Configuration Wizard Skills | 0/5 | Planned | - | --- *Roadmap created: 2026-01-29* @@ -405,3 +434,10 @@ Phases execute in numeric order: 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7 -> 8 -> 9 -> 10 *Phase 14 added: 2026-02-01 (Topic Graph Memory - conceptual enrichment layer)* *Total plans: 48 across 15 phases (22 v1.0 + 26 v2.0)* *Phase 12 plans created: 2026-02-01 (5 plans including outbox indexing pipeline)* +*Phase 15 added: 2026-02-01 (Configuration Wizard Skills - AskUserQuestion-based config wizards)* +*Total plans: 53 across 16 phases (22 v1.0 + 31 v2.0)* +*Phase 10.5 completed: 2026-02-02 (Agentic TOC Search - 3 plans)* +*Phase 11 completed: 2026-02-02 (BM25 Teleport - 4 plans)* +*Phase 12 completed: 2026-02-02 (Vector Teleport - 5 plans)* +*Phase 13 completed: 2026-02-02 (Outbox Index Ingestion - 4 plans)* +*Phase 14 completed: 2026-02-02 (Topic Graph Memory - 6 plans)* diff --git a/.planning/STATE.md b/.planning/STATE.md index 5e6b9ea..8b3defc 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -5,17 +5,21 @@ See: .planning/PROJECT.md (updated 2026-01-30) **Core value:** Agent can answer "what were we talking about last week?" without scanning everything -**Current focus:** v2.0 in progress - Phase 11 & 12 PLANNED - Ready for execution +**Current focus:** v2.0 in progress - Phases 10.5-14 COMPLETE - Phase 15 ready for execution ## Current Position Milestone: v2.0 Scheduler+Teleport (in progress) -Current: Phase 11 - BM25 Teleport (Tantivy) + Phase 12 - Vector Teleport (HNSW) -Status: All plans ready for execution -Last activity: 2026-02-01 -- Created Phase 12 RESEARCH.md and 4 PLAN.md files +Current: Phase 15 - Configuration Wizard Skills (planning complete) +Status: Phases 10.5-14 complete, Phase 15 plans ready for execution +Last activity: 2026-02-02 -- Completed Phases 10.5, 11, 12, 13, and 14 -Progress Phase 11: [ ] 0% (0/4 plans) -Progress Phase 12: [ ] 0% (0/4 plans) +Progress Phase 10.5: [====================] 100% (3/3 plans) +Progress Phase 11: [====================] 100% (4/4 plans) +Progress Phase 12: [====================] 100% (5/5 plans) +Progress Phase 13: [====================] 100% (4/4 plans) +Progress Phase 14: [====================] 100% (6/6 plans) +Progress Phase 15: [ ] 0% (0/5 plans) ## Performance Metrics @@ -200,6 +204,10 @@ Recent decisions affecting current work: - Timestamps formatted as local time for human readability in CLI - SchedulerGrpcService delegates from MemoryServiceImpl when scheduler is configured +### Roadmap Evolution + +- Phase 15 added: Configuration Wizard Skills (AskUserQuestion-based interactive config wizards for storage, LLM, multi-agent) + ### Pending Todos None yet. @@ -210,8 +218,8 @@ None yet. ## Session Continuity -Last session: 2026-01-31 -Stopped at: Completed Phase 11 planning (11-RESEARCH.md + 4 PLAN.md files) +Last session: 2026-02-02 +Stopped at: Completed Phases 10.5, 11, 12, 13, and 14 execution Resume file: None ## Milestone History @@ -291,40 +299,59 @@ See: .planning/MILESTONES.md for complete history | 10-03 | 2 | TOC rollup jobs (wire existing rollups to scheduler) | Complete | | 10-04 | 3 | Job observability (status RPC, CLI, metrics) | Complete | +## Phase 10.5 Plans (v2.0 Agentic TOC Search) + +| Plan | Wave | Description | Status | +|------|------|-------------|--------| +| 10.5-01 | 1 | Core search algorithm (memory-toc/src/search.rs) | Complete | +| 10.5-02 | 2 | gRPC integration (SearchNode/SearchChildren RPCs) | Complete | +| 10.5-03 | 3 | CLI search command | Complete | + ## Phase 11 Plans (v2.0 Teleport - BM25) | Plan | Wave | Description | Status | |------|------|-------------|--------| -| 11-01 | 1 | Tantivy integration (memory-search crate, schema, index) | Planned | -| 11-02 | 2 | Indexing pipeline (TOC node and grip document mapping) | Planned | -| 11-03 | 2 | Search API (gRPC TeleportSearch RPC, BM25 scoring) | Planned | -| 11-04 | 3 | CLI and testing (teleport command, commit job) | Planned | +| 11-01 | 1 | Tantivy integration (memory-search crate, schema, index) | Complete | +| 11-02 | 2 | Indexing pipeline (TOC node and grip document mapping) | Complete | +| 11-03 | 2 | Search API (gRPC TeleportSearch RPC, BM25 scoring) | Complete | +| 11-04 | 3 | CLI and testing (teleport command, commit job) | Complete | ## Phase 12 Plans (v2.0 Teleport - Vector) | Plan | Wave | Description | Status | |------|------|-------------|--------| -| 12-01 | 1 | HNSW index setup (usearch or hnsw-rs integration) | Planned | -| 12-02 | 1 | Local embedding model (sentence-transformers or candle) | Planned | -| 12-03 | 2 | Vector search API (gRPC VectorTeleport RPC) | Planned | -| 12-04 | 3 | Hybrid ranking (BM25 + vector fusion) | Planned | +| 12-01 | 1 | Embedding infrastructure (memory-embeddings crate, Candle all-MiniLM-L6-v2) | Complete | +| 12-02 | 1 | Vector index (memory-vector crate, usearch HNSW) | Complete | +| 12-03 | 2 | Vector indexing pipeline | Complete | +| 12-04 | 3 | gRPC integration (VectorTeleport/HybridSearch RPCs) | Complete | +| 12-05 | 4 | CLI and documentation (vector teleport commands) | Complete | ## Phase 13 Plans (v2.0 Teleport - Outbox) | Plan | Wave | Description | Status | |------|------|-------------|--------| -| 13-01 | 1 | Outbox consumer for indexing (checkpoint tracking) | Planned | -| 13-02 | 1 | Incremental index updates (add/update documents) | Planned | -| 13-03 | 2 | Full rebuild command (admin rebuild-indexes) | Planned | -| 13-04 | 3 | Async indexing pipeline (scheduled via Phase 10) | Planned | +| 13-01 | 1 | Outbox consumer for indexing (checkpoint tracking) | Complete | +| 13-02 | 1 | Incremental index updates (add/update documents) | Complete | +| 13-03 | 2 | Full rebuild command (admin rebuild-indexes) | Complete | +| 13-04 | 3 | Async indexing pipeline (scheduled via Phase 10) | Complete | ## Phase 14 Plans (Topic Graph Memory) | Plan | Wave | Description | Status | |------|------|-------------|--------| -| 14-01 | 1 | Topic extraction (CF_TOPICS, embedding clustering) | Planned | -| 14-02 | 2 | Topic labeling (LLM integration with keyword fallback) | Planned | -| 14-03 | 3 | Importance scoring (time decay with configurable half-life) | Planned | -| 14-04 | 4 | Topic relationships (similarity, hierarchy discovery) | Planned | -| 14-05 | 5 | Navigation RPCs (GetTopicsByQuery, GetTocNodesForTopic) | Planned | -| 14-06 | 6 | Lifecycle management (pruning, resurrection, CLI) | Planned | +| 14-01 | 1 | Topic extraction (memory-topics crate, HDBSCAN clustering) | Complete | +| 14-02 | 2 | Topic labeling (LLM integration with keyword fallback) | Complete | +| 14-03 | 3 | Importance scoring (time decay with configurable half-life) | Complete | +| 14-04 | 4 | Topic relationships (similarity, hierarchy discovery) | Complete | +| 14-05 | 5 | Navigation RPCs (topic gRPC endpoints) | Complete | +| 14-06 | 6 | Lifecycle management (pruning, resurrection, CLI) | Complete | + +## Phase 15 Plans (Configuration Wizard Skills) + +| Plan | Wave | Description | Status | +|------|------|-------------|--------| +| 15-01 | 1 | memory-storage skill (storage, retention, cleanup, GDPR) | Ready | +| 15-02 | 1 | memory-llm skill (provider, model discovery, cost, API test) | Ready | +| 15-03 | 2 | memory-agents skill (multi-agent, tagging, query scope) | Ready | +| 15-04 | 2 | Reference documentation (all reference/*.md files) | Ready | +| 15-05 | 3 | Plugin integration (marketplace.json, memory-setup updates) | Ready | diff --git a/.planning/phases/10.5-agentic-toc-search/10.5-02-SUMMARY.md b/.planning/phases/10.5-agentic-toc-search/10.5-02-SUMMARY.md new file mode 100644 index 0000000..111aab4 --- /dev/null +++ b/.planning/phases/10.5-agentic-toc-search/10.5-02-SUMMARY.md @@ -0,0 +1,76 @@ +# 10.5-02 Summary: gRPC SearchNode/SearchChildren RPCs + +## Status: COMPLETE + +## What Was Done + +### 1. Proto Definitions (proto/memory.proto) + +Added search RPC definitions and messages: + +- **SearchField enum**: `SEARCH_FIELD_UNSPECIFIED`, `TITLE`, `SUMMARY`, `BULLETS`, `KEYWORDS` +- **SearchNodeRequest/Response**: Search within a single node +- **SearchChildrenRequest/Response**: Search children of a parent node +- **SearchMatch**: Individual match with field, text, grip_ids, score +- **SearchNodeResult**: Node result with matches and relevance score +- **RPCs**: `SearchNode` and `SearchChildren` added to MemoryService + +### 2. Search Service Implementation (crates/memory-service/src/search_service.rs) + +Created new module with: + +- **proto_to_domain_field()**: Converts proto SearchField to domain SearchField +- **domain_to_proto_field()**: Converts domain SearchField to proto SearchField +- **domain_to_proto_match()**: Converts domain SearchMatch to proto SearchMatch +- **domain_to_proto_level()**: Converts domain TocLevel to proto TocLevel +- **search_node()**: RPC handler that loads node, calls core search, returns matches +- **search_children()**: RPC handler that searches all children of parent, sorts by relevance + +### 3. Service Wiring + +- Added `pub mod search_service;` to `crates/memory-service/src/lib.rs` +- Added `memory-toc` dependency to `crates/memory-service/Cargo.toml` +- Wired `search_node` and `search_children` RPCs in `MemoryServiceImpl` + +### 4. Tests Added + +- test_search_node_not_found +- test_search_node_empty_node_id +- test_search_node_empty_query +- test_search_node_whitespace_query +- test_search_children_empty_query +- test_search_children_empty_results +- test_proto_to_domain_field_* (title, summary, bullets, keywords, unspecified, invalid) +- test_domain_to_proto_field_roundtrip +- test_domain_to_proto_level + +## Files Modified + +| File | Changes | +|------|---------| +| `proto/memory.proto` | Added SearchField enum, search messages, SearchNode/SearchChildren RPCs | +| `crates/memory-service/src/search_service.rs` | NEW - RPC handlers and conversion helpers | +| `crates/memory-service/src/lib.rs` | Added search_service module export | +| `crates/memory-service/src/ingest.rs` | Added search RPC methods to MemoryServiceImpl | +| `crates/memory-service/Cargo.toml` | Added memory-toc dependency | +| `Cargo.toml` (workspace) | memory-toc already present in workspace deps | + +## Verification + +```bash +cargo build -p memory-service # Compiles successfully +cargo test -p memory-service # All 43 tests pass (includes 16 new search_service tests) +cargo check # Full workspace compiles +``` + +## Key Design Decisions + +1. **Follows existing RPC patterns**: search_service.rs follows query.rs patterns +2. **Proto field naming**: Uses `SEARCH_FIELD_` prefix like `EVENT_ROLE_` +3. **Empty fields = all fields**: When fields array is empty, searches all fields +4. **Relevance sorting**: SearchChildren sorts results by average match score descending +5. **Limit defaults**: Default limit of 10 for both RPCs + +## Next Steps + +Phase 10.5-03: Add CLI commands (`mem search-node`, `mem search-children`) that call these RPCs. diff --git a/.planning/phases/10.5-agentic-toc-search/10.5-03-SUMMARY.md b/.planning/phases/10.5-agentic-toc-search/10.5-03-SUMMARY.md new file mode 100644 index 0000000..1410f61 --- /dev/null +++ b/.planning/phases/10.5-agentic-toc-search/10.5-03-SUMMARY.md @@ -0,0 +1,82 @@ +--- +phase: 10.5-agentic-toc-search +plan: 03 +status: complete +completed_at: 2026-02-01 +--- + +# 10.5-03 Summary: CLI Search Command and Navigator Updates + +## Completed Tasks + +### Task 1: Add Search command to CLI +**Files Modified:** `crates/memory-daemon/src/cli.rs` + +Added `Search` variant to `QueryCommands` enum with the following options: +- `--query, -q` (required): Search terms (space-separated) +- `--node`: Search within a specific node (mutually exclusive with --parent) +- `--parent`: Search children of a parent node (empty for root level) +- `--fields`: Fields to search: title, summary, bullets, keywords (comma-separated) +- `--limit`: Maximum results to return (default: 10) + +Added 3 new CLI tests: +- `test_cli_query_search` - Basic search parsing +- `test_cli_query_search_with_node` - Search with --node option +- `test_cli_query_search_with_parent` - Search with --parent and --fields options + +### Task 2: Implement search command handler +**Files Modified:** `crates/memory-daemon/src/commands.rs` + +Added `handle_search()` async function that: +- Parses comma-separated fields string into `Vec` enum values +- For `--node` option: calls `SearchNodeRequest` RPC, displays matches with field names, scores, and grip IDs +- For `--parent` option (or root): calls `SearchChildrenRequest` RPC, displays matching nodes with relevance scores +- Properly formats output with match details, truncated text, and pagination hints + +Also added imports for `SearchChildrenRequest`, `SearchField`, and `SearchNodeRequest` from `memory_service::pb`. + +### Task 3: Update SKILL.md with search documentation +**Files Modified:** +- `plugins/memory-query-plugin/skills/memory-query/SKILL.md` +- `plugins/memory-query-plugin/skills/memory-query/references/command-reference.md` + +Added to SKILL.md: +- New "Search-Based Navigation" section with workflow documentation +- Search workflow steps (root level -> drill down -> segment level -> expand grip) +- Search command reference examples +- Agent navigation loop documentation with example path + +Added to command-reference.md: +- New "Search Commands" section with `search` command documentation +- Usage syntax, options table, and field descriptions +- Example commands for various search scenarios +- Sample output format + +## Verification + +1. `cargo build -p memory-daemon` - Compiles successfully +2. `cargo run -p memory-daemon -- query search --help` - Shows correct help: + ``` + Search TOC nodes for matching content + + Usage: memory-daemon query search [OPTIONS] --query + + Options: + -q, --query Search query terms (space-separated) + --node Search within a specific node + --parent Search children of a parent node + --fields Fields to search + --limit Maximum results to return [default: 10] + ``` +3. `cargo test -p memory-daemon` - All 22 unit tests and 13 integration tests pass + +## Notes + +- Removed `-l` short option from `--limit` to avoid conflict with global `--log-level` option +- The search handler uses the gRPC client directly (`MemoryServiceClient`) rather than the `MemoryClient` wrapper +- Fixed vector RPC stub implementations in `memory-service/src/ingest.rs` to unblock build + +## Dependencies + +- Depends on 10.5-02 (search gRPC RPCs) +- Required by: Agent workflows that use search-based navigation diff --git a/.planning/phases/11-bm25-teleport-tantivy/11-04-SUMMARY.md b/.planning/phases/11-bm25-teleport-tantivy/11-04-SUMMARY.md new file mode 100644 index 0000000..0dfaba4 --- /dev/null +++ b/.planning/phases/11-bm25-teleport-tantivy/11-04-SUMMARY.md @@ -0,0 +1,77 @@ +--- +phase: 11-bm25-teleport-tantivy +plan: 04 +type: summary +status: complete +completed: 2026-02-01 +--- + +# Summary: Teleport CLI and Index Commit Job (11.04) + +## What Was Done + +### Task 1: Added Teleport CLI Subcommand +- Added `TeleportCommand` enum to `crates/memory-daemon/src/cli.rs` with three subcommands: + - `search` - Search for TOC nodes or grips by keyword + - `stats` - Show index statistics + - `rebuild` - Trigger index rebuild (placeholder for Phase 13) +- Added `Teleport` variant to `Commands` enum +- Exported `TeleportCommand` from `lib.rs` + +### Task 2: Implemented Teleport Command Handlers +- Added `handle_teleport_command()` function to `crates/memory-daemon/src/commands.rs` +- Implemented: + - `teleport_search()` - Executes search via gRPC, displays results with scores + - `teleport_stats()` - Shows index statistics + - `teleport_rebuild()` - Placeholder for Phase 13 +- Added `teleport_search()` method to `MemoryClient` in `crates/memory-client/src/client.rs` +- Updated `main.rs` to handle `Commands::Teleport` variant + +### Task 3: Added Index Commit Scheduled Job +- Created `crates/memory-scheduler/src/jobs/search.rs` with: + - `IndexCommitJobConfig` - Configuration with cron schedule (default: every minute) + - `create_index_commit_job()` - Registers job that commits search index periodically +- Updated `crates/memory-scheduler/src/jobs/mod.rs` to include search module +- Updated `crates/memory-scheduler/src/lib.rs` to export job types +- Added `memory-search` as optional dependency in scheduler's `Cargo.toml` + +## Files Modified + +1. `crates/memory-daemon/src/cli.rs` - Added TeleportCommand enum and Teleport variant +2. `crates/memory-daemon/src/commands.rs` - Added handle_teleport_command and helpers +3. `crates/memory-daemon/src/lib.rs` - Exported TeleportCommand and handle_teleport_command +4. `crates/memory-daemon/src/main.rs` - Added Teleport command handling +5. `crates/memory-client/src/client.rs` - Added teleport_search method to MemoryClient +6. `crates/memory-scheduler/src/jobs/search.rs` - New file with index commit job +7. `crates/memory-scheduler/src/jobs/mod.rs` - Added search module export +8. `crates/memory-scheduler/src/lib.rs` - Exported IndexCommitJobConfig and create_index_commit_job +9. `crates/memory-scheduler/Cargo.toml` - Added memory-search dependency + +## Verification + +All tests pass: +- `cargo test -p memory-daemon` - 19 unit tests, 13 integration tests +- `cargo test -p memory-scheduler` - 60 unit tests +- `cargo test -p memory-search` - 36 unit tests + +CLI commands work: +``` +$ memory-daemon teleport --help +$ memory-daemon teleport search --help +$ memory-daemon teleport stats --help +``` + +Clippy clean: +- `cargo clippy -p memory-daemon -p memory-scheduler -p memory-search -- -D warnings` + +## Success Criteria Met + +- [x] `memory-daemon teleport search ` command added +- [x] Search results display with doc_id, type, and score +- [x] `memory-daemon teleport stats` shows index statistics +- [x] IndexCommitJobConfig has sensible defaults (every minute) +- [x] create_index_commit_job registers with scheduler +- [x] memory-daemon depends on memory-search (via memory-service) +- [x] memory-scheduler depends on memory-search +- [x] All tests pass +- [x] No clippy warnings diff --git a/.planning/phases/12-vector-teleport-hnsw/12-03-SUMMARY.md b/.planning/phases/12-vector-teleport-hnsw/12-03-SUMMARY.md new file mode 100644 index 0000000..a14ba0c --- /dev/null +++ b/.planning/phases/12-vector-teleport-hnsw/12-03-SUMMARY.md @@ -0,0 +1,106 @@ +--- +phase: 12-vector-teleport-hnsw +plan: 03 +status: complete +completed: 2026-02-01 +--- + +# Phase 12.03 Summary: VectorTeleport/HybridSearch gRPC RPCs + +## Objective + +Added gRPC RPCs for vector search and hybrid BM25+vector fusion to expose vector search capabilities to agents. + +## Deliverables + +### Proto Definitions (proto/memory.proto) + +Added new RPC methods to MemoryService: +- `VectorTeleport` - Semantic similarity search using HNSW index +- `HybridSearch` - Combined BM25 + vector search using RRF fusion +- `GetVectorIndexStatus` - Returns index availability and statistics + +Added new enums and messages: +- `VectorTargetType` - Filter by TOC nodes, grips, or all +- `HybridMode` - Vector only, BM25 only, or hybrid fusion +- `TimeRange` - Time range filter for searches +- `VectorTeleportRequest/Response` - Vector search request/response +- `VectorMatch` - Individual search result with score and metadata +- `HybridSearchRequest/Response` - Hybrid search request/response +- `VectorIndexStatus` - Index stats (available, vector_count, dimension, etc.) + +### VectorTeleportHandler (crates/memory-service/src/vector.rs) + +Implements vector semantic search: +- Uses `spawn_blocking` for CPU-bound embedding operations +- Searches HNSW index and looks up metadata +- Supports target type filtering (TOC nodes, grips, all) +- Supports time range filtering +- Supports minimum score threshold +- Returns VectorMatch with doc_id, score, text_preview, timestamp + +### HybridSearchHandler (crates/memory-service/src/hybrid.rs) + +Implements hybrid search with RRF fusion: +- Uses standard RRF constant k=60 (from original paper) +- Supports three modes: VECTOR_ONLY, BM25_ONLY, HYBRID +- Graceful fallback when only one index is available +- Weighted fusion with configurable bm25_weight and vector_weight +- RRF formula: score(doc) = sum(weight_i / (k + rank_i(doc))) + +### Service Integration (crates/memory-service/src/ingest.rs) + +Updated MemoryServiceImpl: +- Added `vector_service` and `hybrid_service` optional fields +- Added `with_vector()` and `with_all_services()` constructors +- Connected RPC handlers to service implementations +- Graceful "unavailable" response when vector service not configured + +### Dependencies (crates/memory-service/Cargo.toml) + +Added: +- `memory-embeddings` - For CandleEmbedder +- `memory-vector` - For HnswIndex and VectorMetadata + +## Verification + +```bash +# Proto compiles and generates code +cargo check -p memory-service # PASSED + +# All tests pass (45 tests) +cargo test -p memory-service # PASSED + +# Clippy clean +cargo clippy -p memory-service --no-deps -- -D warnings # PASSED +``` + +## Success Criteria Met + +- [x] VectorTeleport RPC defined in proto with request/response messages +- [x] HybridSearch RPC defined in proto with mode enum +- [x] GetVectorIndexStatus RPC defined in proto +- [x] VectorTeleportHandler embeds query via spawn_blocking +- [x] VectorTeleportHandler searches HNSW and looks up metadata +- [x] VectorMatch includes doc_id, score, text_preview, timestamp +- [x] HybridSearchHandler supports VECTOR_ONLY, BM25_ONLY, HYBRID modes +- [x] RRF fusion uses k=60 constant +- [x] Graceful fallback when only one index available +- [x] Time and target type filters work correctly +- [x] All tests pass (45 tests) +- [x] No clippy warnings + +## Files Modified + +- `proto/memory.proto` - Added vector RPC definitions +- `crates/memory-service/Cargo.toml` - Added memory-embeddings and memory-vector deps +- `crates/memory-service/src/lib.rs` - Added vector and hybrid modules +- `crates/memory-service/src/vector.rs` - NEW: VectorTeleportHandler +- `crates/memory-service/src/hybrid.rs` - NEW: HybridSearchHandler with RRF +- `crates/memory-service/src/ingest.rs` - Added vector/hybrid service integration + +## Notes + +- BM25 integration in HybridSearch is stubbed pending Phase 11 completion +- Integration tests for vector search require embedding model download +- VectorTeleportHandler uses std::sync::RwLock matching HnswIndex implementation diff --git a/.planning/phases/13-outbox-index-ingestion/13-01-PLAN.md b/.planning/phases/13-outbox-index-ingestion/13-01-PLAN.md new file mode 100644 index 0000000..8aa9d80 --- /dev/null +++ b/.planning/phases/13-outbox-index-ingestion/13-01-PLAN.md @@ -0,0 +1,179 @@ +--- +phase: 13-outbox-index-ingestion +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - crates/memory-indexing/Cargo.toml + - crates/memory-indexing/src/lib.rs + - crates/memory-indexing/src/error.rs + - crates/memory-indexing/src/checkpoint.rs + - crates/memory-storage/src/db.rs + - Cargo.toml +autonomous: true + +must_haves: + truths: + - "Outbox entries can be read in sequence order starting from any sequence number" + - "Checkpoint tracks last processed sequence per index type" + - "Checkpoint persists across daemon restarts" + artifacts: + - path: "crates/memory-indexing/Cargo.toml" + provides: "Crate manifest for indexing pipeline" + contains: "[package]" + - path: "crates/memory-indexing/src/lib.rs" + provides: "Public API exports" + exports: ["IndexCheckpoint", "IndexType", "IndexingError"] + - path: "crates/memory-indexing/src/checkpoint.rs" + provides: "Checkpoint type and persistence" + contains: "IndexCheckpoint" + - path: "crates/memory-indexing/src/error.rs" + provides: "Error types for indexing" + contains: "IndexingError" + key_links: + - from: "crates/memory-indexing/src/checkpoint.rs" + to: "crates/memory-storage/src/db.rs" + via: "Storage::put_checkpoint/get_checkpoint" + pattern: "storage\\.(put|get)_checkpoint" + - from: "crates/memory-storage/src/db.rs" + to: "CF_OUTBOX" + via: "get_outbox_entries" + pattern: "iterator_cf.*CF_OUTBOX" +--- + + +Create the memory-indexing crate with outbox reading and checkpoint tracking infrastructure. + +Purpose: Establish the foundation for consuming outbox entries and tracking indexing progress with crash recovery. +Output: New memory-indexing crate with checkpoint persistence and outbox entry reading APIs in storage. + + + +@/Users/richardhightower/.claude/get-shit-done/workflows/execute-plan.md +@/Users/richardhightower/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/13-outbox-index-ingestion/13-RESEARCH.md +@crates/memory-storage/src/db.rs +@crates/memory-storage/src/keys.rs +@crates/memory-types/src/outbox.rs + + + + + + Task 1: Add memory-indexing crate to workspace + Cargo.toml, crates/memory-indexing/Cargo.toml + + 1. Add "memory-indexing" to workspace members in root Cargo.toml + 2. Create crates/memory-indexing/Cargo.toml with: + - Package name: memory-indexing + - Version: 0.1.0 + - Edition: 2021 + - Dependencies: + - memory-storage = { workspace = true } + - memory-types = { workspace = true } + - tokio = { workspace = true, features = ["rt-multi-thread", "sync"] } + - tracing = { workspace = true } + - serde = { workspace = true, features = ["derive"] } + - serde_json = { workspace = true } + - chrono = { workspace = true, features = ["serde"] } + - thiserror = { workspace = true } + - Dev dependencies: tempfile (for tests) + 3. Create src/ directory structure + + Do NOT add memory-search or memory-vector dependencies yet - those will be added in Plan 13-02. + + cargo check -p memory-indexing compiles without errors + memory-indexing crate exists in workspace and compiles + + + + Task 2: Implement outbox entry reading in Storage + crates/memory-storage/src/db.rs + + Add two new methods to Storage impl: + + 1. `get_outbox_entries(start_sequence: u64, limit: usize) -> Result, StorageError>` + - Get CF_OUTBOX handle + - Create OutboxKey::new(start_sequence) for starting position + - Use iterator_cf with IteratorMode::From(start_key, Direction::Forward) + - Take up to `limit` entries + - For each item, decode OutboxKey and OutboxEntry + - Return Vec of (sequence, entry) tuples + + 2. `delete_outbox_entries(up_to_sequence: u64) -> Result` + - Iterate from start to up_to_sequence + - Batch delete all entries with WriteBatch + - Return count of deleted entries + + Follow existing patterns from get_events_in_range for iteration. + Import OutboxEntry from memory_types in the db.rs file. + Add unit tests for both methods. + + cargo test -p memory-storage -- outbox passes + Storage has get_outbox_entries and delete_outbox_entries methods with tests + + + + Task 3: Create IndexCheckpoint and error types + + crates/memory-indexing/src/lib.rs, + crates/memory-indexing/src/error.rs, + crates/memory-indexing/src/checkpoint.rs + + + 1. Create error.rs with IndexingError enum: + - Storage(StorageError) - wrap storage errors + - Checkpoint(String) - checkpoint load/save issues + - Serialization(String) - JSON encoding errors + - Index(String) - generic index operation errors + Use thiserror derive macro. + + 2. Create checkpoint.rs with: + - IndexType enum: Bm25, Vector, Combined (with Serialize/Deserialize) + - IndexCheckpoint struct with fields: + - index_type: IndexType + - last_sequence: u64 (last outbox sequence processed) + - last_processed_time: DateTime (use chrono::serde::ts_milliseconds) + - processed_count: u64 (total items processed) + - created_at: DateTime + - Implement checkpoint_key() returning "index_bm25", "index_vector", or "index_combined" + - Implement to_bytes() and from_bytes() using serde_json + - Add unit tests for serialization roundtrip + + 3. Create lib.rs that: + - Declares modules: pub mod error; pub mod checkpoint; + - Re-exports: IndexingError, IndexCheckpoint, IndexType + + Follow patterns from memory-types/src/outbox.rs for serialization. + + cargo test -p memory-indexing passes + IndexCheckpoint, IndexType, and IndexingError types exist with serialization + + + + + +1. `cargo build -p memory-indexing` compiles without warnings +2. `cargo test -p memory-indexing` all tests pass +3. `cargo test -p memory-storage -- outbox` all outbox tests pass +4. `cargo clippy -p memory-indexing -- -D warnings` no lint errors + + + +- memory-indexing crate exists and compiles +- Storage has get_outbox_entries() and delete_outbox_entries() methods +- IndexCheckpoint can be serialized/deserialized to/from JSON +- IndexType enum distinguishes BM25, Vector, and Combined indexing +- All unit tests pass + + + +After completion, create `.planning/phases/13-outbox-index-ingestion/13-01-SUMMARY.md` + diff --git a/.planning/phases/13-outbox-index-ingestion/13-01-SUMMARY.md b/.planning/phases/13-outbox-index-ingestion/13-01-SUMMARY.md new file mode 100644 index 0000000..63cde3e --- /dev/null +++ b/.planning/phases/13-outbox-index-ingestion/13-01-SUMMARY.md @@ -0,0 +1,78 @@ +# 13-01 Summary: Outbox Consumer Foundation + +## Completed + +### Task 1: Add memory-indexing crate to workspace +- Added `memory-indexing` to workspace members in root `Cargo.toml` +- Created `crates/memory-indexing/Cargo.toml` with dependencies: + - memory-storage (workspace) + - memory-types (workspace) + - tokio (workspace, features: rt-multi-thread, sync) + - tracing (workspace) + - serde (workspace, features: derive) + - serde_json (workspace) + - chrono (workspace, features: serde) + - thiserror (workspace) + - tempfile (dev-dependency) +- Added `memory-indexing` to workspace dependencies + +### Task 2: Implement outbox entry reading in Storage +- Added `get_outbox_entries(start_sequence: u64, limit: usize)` method + - Returns `Vec<(u64, OutboxEntry)>` tuples in sequence order + - Uses iterator with `IteratorMode::From` for efficient range scans + - Supports pagination via start_sequence and limit +- Added `delete_outbox_entries(up_to_sequence: u64)` method + - Uses `WriteBatch` for atomic deletion + - Returns count of deleted entries + - Includes up_to_sequence (inclusive) +- Added 7 new tests for outbox operations: + - `test_get_outbox_entries_empty` + - `test_get_outbox_entries_after_event` + - `test_get_outbox_entries_with_limit` + - `test_get_outbox_entries_from_offset` + - `test_delete_outbox_entries` + - `test_delete_outbox_entries_none` + - `test_delete_outbox_entries_all` + +### Task 3: Create IndexCheckpoint and error types +- Created `error.rs` with `IndexingError` enum: + - `Storage(StorageError)` - wraps storage errors + - `Checkpoint(String)` - checkpoint load/save issues + - `Serialization(String)` - JSON encoding errors + - `Index(String)` - generic index operation errors +- Created `checkpoint.rs` with: + - `IndexType` enum: `Bm25`, `Vector`, `Combined` + - `IndexCheckpoint` struct with: + - `index_type: IndexType` + - `last_sequence: u64` + - `last_processed_time: DateTime` (serialized as milliseconds) + - `processed_count: u64` + - `created_at: DateTime` (serialized as milliseconds) + - Methods: `new()`, `with_sequence()`, `checkpoint_key()`, `update()`, `to_bytes()`, `from_bytes()` +- Created `lib.rs` with public exports + +## Files Modified +- `/Cargo.toml` - Added memory-indexing to workspace members and dependencies +- `/crates/memory-storage/src/db.rs` - Added get_outbox_entries, delete_outbox_entries, and tests +- `/crates/memory-indexing/Cargo.toml` - New file +- `/crates/memory-indexing/src/lib.rs` - New file +- `/crates/memory-indexing/src/error.rs` - New file +- `/crates/memory-indexing/src/checkpoint.rs` - New file + +## Verification +- `cargo build -p memory-indexing` - Compiles without warnings +- `cargo test -p memory-indexing` - 10 tests pass +- `cargo test -p memory-storage -- outbox` - 8 tests pass +- `cargo test -p memory-storage` - All 25 tests pass + +## Key Decisions +- Used `chrono::serde::ts_milliseconds` for timestamp serialization (JSON-compatible) +- Checkpoint keys follow pattern: `index_bm25`, `index_vector`, `index_combined` +- Outbox iteration uses `IteratorMode::From` with forward direction for efficient range scans +- Delete operations use `WriteBatch` for atomicity + +## Ready for Next Phase +The foundation is now in place for Plan 13-02 which will add: +- BM25 index integration +- Vector index integration +- The actual `IndexingPipeline` that consumes entries and updates indexes diff --git a/.planning/phases/13-outbox-index-ingestion/13-02-PLAN.md b/.planning/phases/13-outbox-index-ingestion/13-02-PLAN.md new file mode 100644 index 0000000..ab2b029 --- /dev/null +++ b/.planning/phases/13-outbox-index-ingestion/13-02-PLAN.md @@ -0,0 +1,194 @@ +--- +phase: 13-outbox-index-ingestion +plan: 02 +type: execute +wave: 2 +depends_on: ["13-01"] +files_modified: + - crates/memory-indexing/Cargo.toml + - crates/memory-indexing/src/lib.rs + - crates/memory-indexing/src/pipeline.rs + - crates/memory-indexing/src/consumer.rs +autonomous: true + +must_haves: + truths: + - "IndexingPipeline processes outbox entries in batches" + - "Each entry triggers appropriate index update based on action type" + - "Checkpoint is saved after successful batch processing" + - "Index operations are idempotent (delete-then-add pattern)" + artifacts: + - path: "crates/memory-indexing/src/pipeline.rs" + provides: "Main indexing pipeline orchestration" + contains: "IndexingPipeline" + min_lines: 100 + - path: "crates/memory-indexing/src/consumer.rs" + provides: "Outbox entry dispatch logic" + contains: "process_batch" + key_links: + - from: "crates/memory-indexing/src/pipeline.rs" + to: "crates/memory-storage/src/db.rs" + via: "Storage for checkpoint and outbox" + pattern: "storage\\.get_outbox_entries" + - from: "crates/memory-indexing/src/pipeline.rs" + to: "checkpoint.rs" + via: "IndexCheckpoint for progress tracking" + pattern: "IndexCheckpoint" +--- + + +Implement the IndexingPipeline for processing outbox entries and dispatching to indexers. + +Purpose: Create the core orchestration that reads outbox entries, dispatches to BM25/vector indexers, and tracks progress. +Output: IndexingPipeline with process_batch() method that handles incremental index updates. + + + +@/Users/richardhightower/.claude/get-shit-done/workflows/execute-plan.md +@/Users/richardhightower/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/13-outbox-index-ingestion/13-RESEARCH.md +@.planning/phases/13-outbox-index-ingestion/13-01-SUMMARY.md +@crates/memory-indexing/src/lib.rs +@crates/memory-indexing/src/checkpoint.rs +@crates/memory-storage/src/db.rs + + + + + + Task 1: Create indexer trait definitions + crates/memory-indexing/src/pipeline.rs + + Create pipeline.rs with trait definitions for pluggable indexers: + + 1. Define Bm25Indexer trait (Send + Sync): + - async fn index_toc_node(&self, node: &TocNode) -> Result<(), IndexingError> + - async fn index_grip(&self, grip: &Grip) -> Result<(), IndexingError> + - async fn delete_document(&self, doc_id: &str) -> Result<(), IndexingError> + - async fn commit(&self) -> Result<(), IndexingError> + + 2. Define VectorIndexer trait (Send + Sync): + - async fn index_toc_node(&self, node: &TocNode) -> Result<(), IndexingError> + - async fn index_grip(&self, grip: &Grip) -> Result<(), IndexingError> + - async fn delete_document(&self, doc_id: &str) -> Result<(), IndexingError> + - async fn commit(&self) -> Result<(), IndexingError> + + 3. Create IndexingStats struct: + - bm25_indexed: u64 + - vector_indexed: u64 + - toc_updates: u64 + - skipped: u64 + - errors: u64 + - Implement total() -> u64 method + + 4. Create IndexingPipelineConfig struct: + - batch_size: usize (default 100) + - enable_bm25: bool (default true) + - enable_vector: bool (default true) + - checkpoint_name: String (default "index_combined") + - Implement Default trait + + These traits will be implemented by memory-search (Bm25Indexer) and memory-vector (VectorIndexer) in later phases. + For now, the pipeline uses Option> to allow either or both to be None. + + cargo check -p memory-indexing compiles + Bm25Indexer and VectorIndexer traits defined with IndexingStats and config + + + + Task 2: Implement IndexingPipeline struct + crates/memory-indexing/src/pipeline.rs, crates/memory-indexing/src/consumer.rs + + Create IndexingPipeline struct in pipeline.rs: + + ```rust + pub struct IndexingPipeline { + storage: Arc, + bm25_indexer: Option>, + vector_indexer: Option>, + config: IndexingPipelineConfig, + } + ``` + + Implement: + 1. new() constructor taking storage, optional indexers, and config + 2. load_checkpoint() -> Result, IndexingError> + - Use storage.get_checkpoint() with config.checkpoint_name + - Deserialize with IndexCheckpoint::from_bytes() + 3. save_checkpoint(last_sequence: u64, stats: &IndexingStats) -> Result<(), IndexingError> + - Create new IndexCheckpoint with current time + - Serialize and save via storage.put_checkpoint() + + Create consumer.rs with process_batch() implementation: + 1. Load checkpoint to get starting sequence (or start at 0) + 2. Call storage.get_outbox_entries(start_seq, batch_size) + 3. For each entry, dispatch based on entry.action: + - OutboxAction::IndexEvent: Index for both BM25 and vector if enabled + - OutboxAction::UpdateToc: Re-index the TOC node + 4. After processing batch, save checkpoint + 5. Return IndexingStats + + Handle idempotent updates: + - Use delete_document() before indexing to handle re-indexing + - Log warnings for errors but continue processing (don't fail entire batch) + + Add tracing for observability (info! for batch completion, warn! for errors). + + cargo check -p memory-indexing compiles + IndexingPipeline struct exists with checkpoint load/save and process_batch + + + + Task 3: Add unit tests with mock indexers + crates/memory-indexing/src/pipeline.rs + + Create comprehensive tests in pipeline.rs #[cfg(test)] module: + + 1. Create MockBm25Indexer that: + - Tracks indexed items in Arc>> + - Implements all Bm25Indexer trait methods + - Returns Ok(()) for all operations + + 2. Create MockVectorIndexer similarly + + 3. Write tests: + - test_empty_outbox_returns_zero_stats: Empty outbox returns stats with all zeros + - test_process_batch_indexes_entries: Add outbox entries, verify mock indexers receive them + - test_checkpoint_persists_progress: Process batch, verify checkpoint saved + - test_checkpoint_resume_from_last: Process partial, verify next batch starts from checkpoint + - test_skips_when_no_indexers: Both indexers None, entries still processed (checkpoint advanced) + - test_error_handling_continues: One indexer fails, other continues + + Use tempfile for Storage instances in tests. + Add helper function to create test outbox entries. + + cargo test -p memory-indexing -- pipeline passes + Unit tests pass for IndexingPipeline with mock indexers + + + + + +1. `cargo build -p memory-indexing` compiles without warnings +2. `cargo test -p memory-indexing` all tests pass +3. `cargo clippy -p memory-indexing -- -D warnings` no lint errors +4. IndexingPipeline correctly dispatches to both indexer types + + + +- Bm25Indexer and VectorIndexer traits defined for pluggable indexers +- IndexingPipeline processes outbox entries in configurable batches +- Checkpoint is saved after each batch with last sequence number +- process_batch() returns accurate IndexingStats +- Mock indexer tests verify correct dispatch behavior + + + +After completion, create `.planning/phases/13-outbox-index-ingestion/13-02-SUMMARY.md` + diff --git a/.planning/phases/13-outbox-index-ingestion/13-03-PLAN.md b/.planning/phases/13-outbox-index-ingestion/13-03-PLAN.md new file mode 100644 index 0000000..60d9947 --- /dev/null +++ b/.planning/phases/13-outbox-index-ingestion/13-03-PLAN.md @@ -0,0 +1,224 @@ +--- +phase: 13-outbox-index-ingestion +plan: 03 +type: execute +wave: 2 +depends_on: ["13-01"] +files_modified: + - crates/memory-indexing/src/lib.rs + - crates/memory-indexing/src/rebuild.rs + - crates/memory-daemon/src/commands.rs + - crates/memory-daemon/src/cli.rs +autonomous: true + +must_haves: + truths: + - "Admin can rebuild all indexes from scratch via CLI command" + - "Rebuild iterates all TOC nodes and grips from storage" + - "Rebuild clears existing indexes before re-indexing" + - "Progress is reported during rebuild" + artifacts: + - path: "crates/memory-indexing/src/rebuild.rs" + provides: "Full index rebuild logic" + contains: "rebuild_all" + min_lines: 80 + - path: "crates/memory-daemon/src/commands.rs" + provides: "CLI command handler for rebuild-indexes" + contains: "rebuild_indexes" + key_links: + - from: "crates/memory-indexing/src/rebuild.rs" + to: "crates/memory-storage/src/db.rs" + via: "Storage for TOC node and grip iteration" + pattern: "storage\\.get_toc_nodes_by_level" + - from: "crates/memory-daemon/src/commands.rs" + to: "crates/memory-indexing/src/rebuild.rs" + via: "rebuild_all" + pattern: "rebuild_all" +--- + + +Implement full index rebuild command for admin recovery scenarios. + +Purpose: Enable administrators to rebuild search indexes from primary storage when indexes are corrupted or need regeneration. +Output: Admin CLI command `rebuild-indexes` with options for selective or full rebuild. + + + +@/Users/richardhightower/.claude/get-shit-done/workflows/execute-plan.md +@/Users/richardhightower/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/13-outbox-index-ingestion/13-RESEARCH.md +@.planning/phases/13-outbox-index-ingestion/13-01-SUMMARY.md +@crates/memory-daemon/src/commands.rs +@crates/memory-daemon/src/cli.rs +@crates/memory-storage/src/db.rs + + + + + + Task 1: Implement rebuild logic in memory-indexing + crates/memory-indexing/src/rebuild.rs, crates/memory-indexing/src/lib.rs + + Create rebuild.rs with RebuildStats struct and rebuild_all() method: + + 1. RebuildStats struct: + - nodes_indexed: u64 (count of TOC nodes processed) + - grips_indexed: u64 (count of grips processed) + - duration: std::time::Duration + - bm25_rebuilt: bool + - vector_rebuilt: bool + + 2. Implement rebuild_all() on IndexingPipeline: + ```rust + pub async fn rebuild_all(&self) -> Result + ``` + Logic: + a. Log "Starting full index rebuild" + b. Track start time + c. If bm25_indexer is Some, call clear() trait method (add to trait in pipeline.rs) + d. If vector_indexer is Some, call clear() trait method + e. Iterate all TOC levels in order: Year, Month, Week, Day, Segment + - For each level, call storage.get_toc_nodes_by_level(level, None, None) + - For each node, call index_toc_node() on both indexers + - Log progress every 100 nodes + f. Get all grips (add storage.get_all_grips() if needed) + - For each grip, call index_grip() on both indexers + g. Commit both indexers + h. Reset checkpoint to sequence 0 (or delete checkpoint) + i. Log completion with stats + j. Return RebuildStats + + 3. Add clear() method to Bm25Indexer and VectorIndexer traits: + ```rust + async fn clear(&self) -> Result<(), IndexingError>; + ``` + + 4. Update lib.rs to export rebuild module and RebuildStats + + Add storage.get_all_grips() if it doesn't exist - iterate CF_GRIPS from start, + filter out index entries (skip keys starting with "node:"). + + cargo check -p memory-indexing compiles + rebuild_all() method exists with TOC and grip iteration logic + + + + Task 2: Add get_all_grips to Storage if needed + crates/memory-storage/src/db.rs + + Check if Storage::get_all_grips() exists. If not, add it: + + ```rust + /// Get all grips from storage (excludes index entries). + pub fn get_all_grips(&self) -> Result, StorageError> { + let grips_cf = self.db.cf_handle(CF_GRIPS) + .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_GRIPS.to_string()))?; + + let mut grips = Vec::new(); + let iter = self.db.iterator_cf(&grips_cf, IteratorMode::Start); + + for item in iter { + let (key, value) = item?; + let key_str = String::from_utf8_lossy(&key); + + // Skip index entries (node:xxx:yyy) + if key_str.starts_with("node:") { + continue; + } + + // Skip if not a grip entry (starts with "grip:") + if !key_str.starts_with("grip:") { + continue; + } + + let grip = memory_types::Grip::from_bytes(&value) + .map_err(|e| StorageError::Serialization(e.to_string()))?; + grips.push(grip); + } + + Ok(grips) + } + ``` + + Add unit test for get_all_grips(): + - Create test storage + - Add 3 grips (some with node links, some without) + - Call get_all_grips() + - Verify all 3 grips returned (not index entries) + + cargo test -p memory-storage -- get_all_grips passes + Storage has get_all_grips() method with test + + + + Task 3: Add rebuild-indexes CLI command + crates/memory-daemon/src/commands.rs, crates/memory-daemon/src/cli.rs + + 1. Add rebuild-indexes subcommand to admin CLI in cli.rs: + ```rust + #[derive(Parser)] + pub struct RebuildIndexesCmd { + /// Only rebuild BM25 index + #[arg(long)] + bm25_only: bool, + + /// Only rebuild vector index + #[arg(long)] + vector_only: bool, + + /// Show what would be rebuilt without actually rebuilding + #[arg(long)] + dry_run: bool, + } + ``` + Add to AdminCommands enum. + + 2. Implement handler in commands.rs: + - Open storage directly (like other admin commands) + - If dry_run, print stats about what would be rebuilt: + - Count TOC nodes by level + - Count grips + - Show which indexes would be rebuilt + - If not dry_run: + - Note: For now, print "Index rebuilding requires indexers to be configured" + - The actual rebuild will work once Phase 11/12 provide real indexers + - Store the stats struct and print summary + + 3. Wire up in cli.rs admin match arm + + Note: The actual rebuild execution will print a placeholder message until + memory-search and memory-vector crates provide real indexer implementations. + The dry-run mode will work fully since it only reads storage stats. + + cargo build -p memory-daemon && target/debug/memory-daemon admin rebuild-indexes --dry-run works + CLI has rebuild-indexes command with --dry-run, --bm25-only, --vector-only flags + + + + + +1. `cargo build -p memory-indexing` compiles without warnings +2. `cargo build -p memory-daemon` compiles without warnings +3. `cargo test -p memory-indexing` all tests pass +4. `cargo test -p memory-storage` all tests pass +5. `target/debug/memory-daemon admin rebuild-indexes --help` shows correct options +6. `target/debug/memory-daemon admin rebuild-indexes --dry-run` runs without error + + + +- rebuild_all() method iterates all TOC nodes and grips +- Storage has get_all_grips() if it didn't exist +- CLI command `admin rebuild-indexes` is available +- Dry run mode shows what would be rebuilt +- Progress logging during rebuild + + + +After completion, create `.planning/phases/13-outbox-index-ingestion/13-03-SUMMARY.md` + diff --git a/.planning/phases/13-outbox-index-ingestion/13-04-PLAN.md b/.planning/phases/13-outbox-index-ingestion/13-04-PLAN.md new file mode 100644 index 0000000..7c72808 --- /dev/null +++ b/.planning/phases/13-outbox-index-ingestion/13-04-PLAN.md @@ -0,0 +1,205 @@ +--- +phase: 13-outbox-index-ingestion +plan: 04 +type: execute +wave: 3 +depends_on: ["13-02"] +files_modified: + - crates/memory-indexing/src/lib.rs + - crates/memory-indexing/src/job.rs + - crates/memory-scheduler/src/jobs/mod.rs + - crates/memory-scheduler/src/jobs/indexing.rs + - crates/memory-daemon/src/lib.rs + - proto/memory.proto + - crates/memory-service/src/lib.rs +autonomous: true + +must_haves: + truths: + - "Indexing job runs on configurable cron schedule" + - "Indexing job processes outbox entries without blocking ingestion" + - "GetIndexingStatus RPC returns pipeline health and checkpoint info" + - "Job integrates with existing scheduler infrastructure" + artifacts: + - path: "crates/memory-scheduler/src/jobs/indexing.rs" + provides: "Indexing job creation and scheduling" + contains: "create_indexing_job" + min_lines: 50 + - path: "proto/memory.proto" + provides: "GetIndexingStatus RPC definition" + contains: "GetIndexingStatus" + key_links: + - from: "crates/memory-scheduler/src/jobs/indexing.rs" + to: "crates/memory-indexing/src/pipeline.rs" + via: "IndexingPipeline.process_batch()" + pattern: "pipeline\\.process_batch" + - from: "crates/memory-daemon/src/lib.rs" + to: "crates/memory-scheduler/src/jobs/indexing.rs" + via: "create_indexing_job" + pattern: "create_indexing_job" +--- + + +Integrate indexing pipeline with scheduler and add observability RPC. + +Purpose: Enable background async indexing that doesn't block event ingestion, with status visibility for monitoring. +Output: Scheduled indexing job and GetIndexingStatus gRPC endpoint. + + + +@/Users/richardhightower/.claude/get-shit-done/workflows/execute-plan.md +@/Users/richardhightower/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/13-outbox-index-ingestion/13-RESEARCH.md +@.planning/phases/13-outbox-index-ingestion/13-02-SUMMARY.md +@crates/memory-scheduler/src/jobs/rollup.rs +@crates/memory-scheduler/src/lib.rs +@crates/memory-daemon/src/lib.rs +@proto/memory.proto + + + + + + Task 1: Create IndexingJobConfig and job module + crates/memory-scheduler/src/jobs/mod.rs, crates/memory-scheduler/src/jobs/indexing.rs, crates/memory-scheduler/Cargo.toml + + 1. Add memory-indexing dependency to memory-scheduler Cargo.toml: + ```toml + memory-indexing = { workspace = true } + ``` + + 2. Create indexing.rs following the rollup.rs pattern: + - IndexingJobConfig struct: + - cron: String (default: "*/30 * * * * *" = every 30 seconds) + - timezone: String (default: "UTC") + - jitter_secs: u64 (default: 5) + - batch_size: usize (default: 100) + - Implement Default trait + + - create_indexing_job() async function: + ```rust + pub async fn create_indexing_job( + scheduler: &SchedulerService, + pipeline: Arc, + config: IndexingJobConfig, + ) -> Result<(), SchedulerError> + ``` + Logic: + - Register job named "index_outbox_consumer" + - Use OverlapPolicy::Skip (only one consumer at a time) + - Job closure: + a. Call pipeline.process_batch().await + b. On success with stats.total() > 0, log info + c. On error, log warning, return Err(e.to_string()) + + 3. Update jobs/mod.rs to add: + ```rust + pub mod indexing; + pub use indexing::{IndexingJobConfig, create_indexing_job}; + ``` + + cargo check -p memory-scheduler compiles + IndexingJobConfig and create_indexing_job exist with scheduler integration + + + + Task 2: Add GetIndexingStatus RPC to proto + proto/memory.proto, crates/memory-service/build.rs + + Add to memory.proto: + + 1. IndexingStatus message: + ```protobuf + message IndexingStatus { + bool available = 1; // Is indexing enabled + bool bm25_enabled = 2; // Is BM25 indexer configured + bool vector_enabled = 3; // Is vector indexer configured + uint64 last_sequence = 4; // Last outbox sequence processed + string last_processed_time = 5; // ISO8601 timestamp + uint64 total_processed = 6; // Total items processed + uint64 pending_entries = 7; // Outbox entries waiting to be processed + } + ``` + + 2. GetIndexingStatusRequest (empty): + ```protobuf + message GetIndexingStatusRequest {} + ``` + + 3. GetIndexingStatusResponse: + ```protobuf + message GetIndexingStatusResponse { + IndexingStatus status = 1; + } + ``` + + 4. Add RPC to MemoryService: + ```protobuf + rpc GetIndexingStatus(GetIndexingStatusRequest) returns (GetIndexingStatusResponse); + ``` + + Run cargo build to regenerate proto code. + + cargo build -p memory-service compiles with new proto + Proto has GetIndexingStatus RPC with IndexingStatus message + + + + Task 3: Implement GetIndexingStatus handler and wire daemon + crates/memory-service/src/lib.rs, crates/memory-daemon/src/lib.rs + + 1. Add indexing status handler to memory-service: + - Store Option> in MemoryServiceImpl + - Add with_indexing_pipeline() builder method + - Implement get_indexing_status() handler: + a. If pipeline is None, return status with available=false + b. If pipeline is Some: + - Load checkpoint from pipeline + - Get outbox count from storage stats + - Calculate pending_entries = outbox_count - last_sequence + - Return populated IndexingStatus + + 2. Wire indexing job in daemon startup (memory-daemon/src/lib.rs): + - In run_server_with_scheduler or equivalent: + a. Create IndexingPipeline with storage (no actual indexers yet - pass None for both) + b. Call create_indexing_job() with scheduler and pipeline + c. Pass pipeline to MemoryServiceImpl::with_indexing_pipeline() + - Note: Pipeline will process entries but skip actual indexing until Phase 11/12 provide indexers + + 3. Add integration test: + - Start daemon with scheduler + - Call GetIndexingStatus RPC + - Verify available=true, bm25_enabled=false, vector_enabled=false (no indexers) + + cargo test -p memory-daemon -- indexing passes + GetIndexingStatus RPC returns pipeline status, indexing job registered at startup + + + + + +1. `cargo build -p memory-scheduler` compiles without warnings +2. `cargo build -p memory-service` compiles with proto changes +3. `cargo build -p memory-daemon` compiles with indexing wiring +4. `cargo test -p memory-scheduler` all tests pass +5. `cargo test -p memory-daemon` all tests pass +6. gRPC GetIndexingStatus returns valid response + + + +- Indexing job runs every 30 seconds by default +- Job uses OverlapPolicy::Skip to prevent concurrent execution +- GetIndexingStatus RPC returns checkpoint info and pending count +- Daemon starts with indexing job registered +- Indexing is non-blocking for event ingestion + + + +After completion, create `.planning/phases/13-outbox-index-ingestion/13-04-SUMMARY.md` + diff --git a/.planning/phases/13-outbox-index-ingestion/13-RESEARCH.md b/.planning/phases/13-outbox-index-ingestion/13-RESEARCH.md new file mode 100644 index 0000000..5b4798a --- /dev/null +++ b/.planning/phases/13-outbox-index-ingestion/13-RESEARCH.md @@ -0,0 +1,638 @@ +# Phase 13: Outbox Index Ingestion - Research + +**Researched:** 2026-02-01 +**Domain:** Event-driven index ingestion, outbox pattern, checkpoint-based crash recovery, async indexing +**Confidence:** HIGH + +## Summary + +Phase 13 implements the Index Lifecycle layer of the Cognitive Architecture, connecting the outbox pattern (already established in memory-storage) to both BM25/Tantivy (Phase 11) and Vector/HNSW (Phase 12) indexes. The outbox column family (CF_OUTBOX) already stores entries atomically with events via `put_event()`. This phase builds the consumer side that reads outbox entries, dispatches them to appropriate indexers, and tracks progress via checkpoints for crash recovery. + +The research confirms the existing codebase has all necessary infrastructure: outbox entries are written with monotonic sequence keys, checkpoint storage is implemented in CF_CHECKPOINTS, and the scheduler (Phase 10) provides the execution framework for background jobs. The implementation follows the transactional outbox pattern with at-least-once delivery semantics, requiring idempotent index operations. + +**Primary recommendation:** Create a unified `IndexingPipeline` in a new `memory-indexing` crate that consumes outbox entries, dispatches to BM25/vector indexers based on entry type, tracks checkpoint per index type, and integrates with the scheduler for periodic processing. Support full rebuild via admin command that iterates all TOC nodes and grips from RocksDB. + +## Standard Stack + +The established libraries/tools for this domain: + +### Core +| Library | Version | Purpose | Why Standard | +|---------|---------|---------|--------------| +| memory-storage | workspace | Outbox reading, checkpoint persistence | Already has outbox/checkpoint APIs | +| memory-scheduler | workspace | Background job execution | Already has cron scheduling, overlap policy | +| memory-search (Phase 11) | workspace | BM25/Tantivy indexing | Provides SearchIndexer for text indexing | +| memory-vector (Phase 12) | workspace | HNSW vector indexing | Provides VectorIndexPipeline for embeddings | + +### Supporting +| Library | Version | Purpose | When to Use | +|---------|---------|---------|-------------| +| tokio | 1.43 | Async runtime, spawn_blocking | Already in workspace | +| tracing | 0.1 | Logging | Already in workspace | +| serde | 1.0 | Checkpoint serialization | Already in workspace | +| chrono | 0.4 | Timestamp handling | Already in workspace | + +### Alternatives Considered +| Instead of | Could Use | Tradeoff | +|------------|-----------|----------| +| Poll-based consumer | Push-based (channel) | Poll simpler for batch processing, push adds complexity | +| Single checkpoint | Per-index checkpoints | Per-index allows independent recovery | +| Unified pipeline | Separate pipelines | Unified simpler, shares outbox iteration | + +**Installation:** +```toml +# New crate in workspace +[package] +name = "memory-indexing" +version = "0.1.0" +edition = "2021" + +[dependencies] +memory-storage = { workspace = true } +memory-types = { workspace = true } +memory-search = { workspace = true } +memory-vector = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "sync"] } +tracing = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +chrono = { workspace = true, features = ["serde"] } +thiserror = { workspace = true } +``` + +## Architecture Patterns + +### Recommended Project Structure +``` +crates/ + memory-indexing/ # NEW crate for index lifecycle + src/ + lib.rs # Public API exports + pipeline.rs # IndexingPipeline: unified outbox consumer + consumer.rs # OutboxConsumer: reads and dispatches entries + checkpoint.rs # IndexCheckpoint: per-index progress tracking + rebuild.rs # Full rebuild from storage + error.rs # Error types + Cargo.toml +``` + +### Pattern 1: Unified Outbox Consumer with Dispatch +**What:** Single consumer reads outbox, dispatches to appropriate indexer based on action type +**When to use:** When multiple indexes need updates from same source +**Example:** +```rust +// Source: Derived from existing memory-toc/rollup.rs patterns +pub struct IndexingPipeline { + storage: Arc, + bm25_indexer: Option>, + vector_indexer: Option>, + batch_size: usize, +} + +impl IndexingPipeline { + pub async fn process_batch(&self) -> Result { + // Load checkpoint to get last processed sequence + let checkpoint = self.load_checkpoint()?; + let start_seq = checkpoint.map(|c| c.last_sequence + 1).unwrap_or(0); + + // Read batch of outbox entries + let entries = self.storage.get_outbox_entries(start_seq, self.batch_size)?; + + let mut stats = IndexingStats::default(); + for entry in entries { + match entry.action { + OutboxAction::IndexEvent => { + // Index for both BM25 and vector if available + if let Some(ref bm25) = self.bm25_indexer { + self.index_for_bm25(bm25, &entry).await?; + stats.bm25_indexed += 1; + } + if let Some(ref vector) = self.vector_indexer { + self.index_for_vector(vector, &entry).await?; + stats.vector_indexed += 1; + } + } + OutboxAction::UpdateToc => { + // TOC updates trigger index of new TOC node + self.index_toc_update(&entry).await?; + stats.toc_updates += 1; + } + } + + // Update checkpoint after each entry (crash recovery) + self.save_checkpoint(entry.sequence)?; + } + + Ok(stats) + } +} +``` + +### Pattern 2: Checkpoint Per Index Type +**What:** Track separate checkpoints for BM25 and vector indexes +**When to use:** When indexes may have different processing speeds or failures +**Example:** +```rust +// Source: Derived from memory-toc/rollup.rs RollupCheckpoint +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IndexCheckpoint { + pub index_type: IndexType, // BM25 or Vector + pub last_sequence: u64, // Last outbox sequence processed + #[serde(with = "chrono::serde::ts_milliseconds")] + pub last_processed_time: DateTime, + pub processed_count: u64, + #[serde(with = "chrono::serde::ts_milliseconds")] + pub created_at: DateTime, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum IndexType { + Bm25, + Vector, + Combined, // For unified processing +} + +impl IndexCheckpoint { + pub fn checkpoint_key(index_type: &IndexType) -> String { + match index_type { + IndexType::Bm25 => "index_bm25".to_string(), + IndexType::Vector => "index_vector".to_string(), + IndexType::Combined => "index_combined".to_string(), + } + } +} +``` + +### Pattern 3: Idempotent Index Updates +**What:** Use delete-then-add pattern for updates; skip if already indexed +**When to use:** Always - at-least-once delivery requires idempotency +**Example:** +```rust +// Source: Phase 11 research - Tantivy delete/add pattern +impl IndexingPipeline { + async fn index_toc_node(&self, node: &TocNode) -> Result<(), IndexingError> { + if let Some(ref bm25) = self.bm25_indexer { + // Delete existing document first (idempotent) + bm25.delete_document(&node.node_id)?; + // Add new version + bm25.index_toc_node(node)?; + } + + if let Some(ref vector) = self.vector_indexer { + // Vector index also uses delete-then-add + vector.update_toc_node(node).await?; + } + + Ok(()) + } +} +``` + +### Pattern 4: Full Rebuild from Storage +**What:** Admin command iterates all TOC nodes and grips, re-indexes completely +**When to use:** After corruption, model upgrade, or initial setup +**Example:** +```rust +// Source: Derived from docs/prds/hierarchical-vector-indexing-prd.md +impl IndexingPipeline { + pub async fn rebuild_all(&self) -> Result { + info!("Starting full index rebuild"); + + // Clear existing indexes + if let Some(ref bm25) = self.bm25_indexer { + bm25.clear()?; + } + if let Some(ref vector) = self.vector_indexer { + vector.clear()?; + } + + let mut stats = RebuildStats::default(); + + // Iterate all TOC nodes by level + for level in [TocLevel::Year, TocLevel::Month, TocLevel::Week, + TocLevel::Day, TocLevel::Segment] { + let nodes = self.storage.get_toc_nodes_by_level(level, None, None)?; + for node in nodes { + self.index_toc_node(&node).await?; + stats.nodes_indexed += 1; + + // Progress reporting + if stats.nodes_indexed % 100 == 0 { + info!(count = stats.nodes_indexed, "Rebuild progress"); + } + } + } + + // Index all grips + let grips = self.storage.get_all_grips()?; + for grip in grips { + self.index_grip(&grip).await?; + stats.grips_indexed += 1; + } + + // Commit indexes + if let Some(ref bm25) = self.bm25_indexer { + bm25.commit()?; + } + if let Some(ref vector) = self.vector_indexer { + vector.commit()?; + } + + // Reset checkpoint to current position + self.reset_checkpoint()?; + + info!(stats = ?stats, "Full rebuild complete"); + Ok(stats) + } +} +``` + +### Pattern 5: Scheduler Integration +**What:** Register indexing job with scheduler using existing patterns +**When to use:** For periodic background processing +**Example:** +```rust +// Source: Derived from memory-scheduler/jobs/rollup.rs +pub async fn create_indexing_job( + scheduler: &SchedulerService, + pipeline: Arc, + config: IndexingJobConfig, +) -> Result<(), SchedulerError> { + scheduler.register_job( + "index_outbox_consumer", + &config.cron, // e.g., "*/30 * * * * *" (every 30 seconds) + Some(&config.timezone), + OverlapPolicy::Skip, // Don't overlap - one consumer at a time + JitterConfig::new(config.jitter_secs), + move || { + let pipeline = pipeline.clone(); + async move { + match pipeline.process_batch().await { + Ok(stats) => { + if stats.total() > 0 { + info!(stats = ?stats, "Indexing batch complete"); + } + Ok(()) + } + Err(e) => { + warn!(error = %e, "Indexing batch failed"); + Err(e.to_string()) + } + } + } + }, + ).await +} +``` + +### Anti-Patterns to Avoid +- **Processing outbox synchronously during ingestion:** Never block event ingestion on indexing. Always async via background job. +- **Single checkpoint for multiple indexes:** If one index fails, the other shouldn't re-process. Use per-index checkpoints. +- **Committing after every document:** Batch commits for efficiency. Commit after batch or periodically. +- **No idempotency:** Must handle duplicate processing due to at-least-once semantics. +- **Blocking the async runtime:** Use `spawn_blocking` for Tantivy operations (they're synchronous). + +## Don't Hand-Roll + +Problems that look simple but have existing solutions: + +| Problem | Don't Build | Use Instead | Why | +|---------|-------------|-------------|-----| +| Outbox reading | Custom iterator | Existing storage.get_outbox_entries() | Already implemented with FIFO semantics | +| Checkpoint persistence | File-based | storage.put_checkpoint()/get_checkpoint() | Already in CF_CHECKPOINTS | +| Cron scheduling | Custom timer | memory-scheduler | Has overlap policy, jitter, timezone | +| BM25 indexing | Custom full-text | memory-search/SearchIndexer | Phase 11 provides complete solution | +| Vector indexing | Custom embedding | memory-vector/VectorIndexPipeline | Phase 12 handles embedding + HNSW | +| Batch iteration | Manual loop | Iterator with batch_size | Rust iterators compose well | + +**Key insight:** The existing codebase already has 80% of the infrastructure. Phase 13 is primarily orchestration code that connects existing components. + +## Common Pitfalls + +### Pitfall 1: Outbox Entries Not Being Deleted +**What goes wrong:** Outbox grows unboundedly +**Why it happens:** Forgetting to delete processed entries or relying on FIFO compaction alone +**How to avoid:** Explicitly delete entries after successful indexing, or mark as processed +**Warning signs:** CF_OUTBOX entry count in stats grows continuously + +### Pitfall 2: Checkpoint Saved Before Index Commit +**What goes wrong:** Crash loses indexed documents +**Why it happens:** Checkpoint saved before Tantivy/HNSW commit completes +**How to avoid:** Save checkpoint AFTER index commit succeeds +**Warning signs:** After restart, search returns fewer results than expected + +### Pitfall 3: Blocking Async Runtime with Tantivy +**What goes wrong:** gRPC requests timeout during indexing +**Why it happens:** Tantivy operations are synchronous, block tokio threads +**How to avoid:** Use `tokio::task::spawn_blocking` for all Tantivy calls +**Warning signs:** High latency on unrelated gRPC calls during indexing bursts + +### Pitfall 4: No Progress on Persistent Failure +**What goes wrong:** Same entry fails repeatedly, pipeline stuck +**Why it happens:** Entry causes error, checkpoint not advanced, retry forever +**How to avoid:** Implement dead-letter handling or skip-after-N-retries +**Warning signs:** Same error message in logs repeatedly + +### Pitfall 5: Index Drift from Storage +**What goes wrong:** Index contains stale or missing data +**Why it happens:** Outbox entries lost, index not rebuilt after issue +**How to avoid:** Periodic consistency check, admin rebuild command +**Warning signs:** Search returns results for deleted items or misses recent items + +## Code Examples + +Verified patterns from official sources and existing codebase: + +### Outbox Entry Reading (Add to Storage) +```rust +// Source: Derived from existing memory-storage/db.rs patterns +impl Storage { + /// Get outbox entries starting from a sequence number. + /// Returns up to `limit` entries, ordered by sequence. + pub fn get_outbox_entries( + &self, + start_sequence: u64, + limit: usize, + ) -> Result, StorageError> { + let outbox_cf = self.db.cf_handle(CF_OUTBOX) + .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_OUTBOX.to_string()))?; + + let start_key = OutboxKey::new(start_sequence); + let mut results = Vec::with_capacity(limit); + + let iter = self.db.iterator_cf( + &outbox_cf, + IteratorMode::From(&start_key.to_bytes(), Direction::Forward), + ); + + for item in iter.take(limit) { + let (key, value) = item?; + let outbox_key = OutboxKey::from_bytes(&key)?; + let entry = OutboxEntry::from_bytes(&value) + .map_err(|e| StorageError::Serialization(e.to_string()))?; + results.push((outbox_key.sequence, entry)); + } + + Ok(results) + } + + /// Delete outbox entries up to and including the given sequence. + pub fn delete_outbox_entries(&self, up_to_sequence: u64) -> Result { + let outbox_cf = self.db.cf_handle(CF_OUTBOX) + .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_OUTBOX.to_string()))?; + + let mut count = 0; + let mut batch = WriteBatch::default(); + + let iter = self.db.iterator_cf(&outbox_cf, IteratorMode::Start); + for item in iter { + let (key, _) = item?; + let outbox_key = OutboxKey::from_bytes(&key)?; + if outbox_key.sequence > up_to_sequence { + break; + } + batch.delete_cf(&outbox_cf, &key); + count += 1; + } + + self.db.write(batch)?; + Ok(count) + } +} +``` + +### Extended OutboxEntry for Index Types +```rust +// Source: Derived from memory-types/outbox.rs +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OutboxAction { + /// Index this event for BM25/vector search (existing) + IndexEvent, + /// Update TOC node with new event (existing) + UpdateToc, + /// Index a newly created TOC node + IndexTocNode { node_id: String }, + /// Index a newly created grip + IndexGrip { grip_id: String }, +} + +impl OutboxEntry { + /// Create entry for TOC node indexing + pub fn for_toc_node(event_id: String, timestamp_ms: i64, node_id: String) -> Self { + Self { + event_id, + timestamp_ms, + action: OutboxAction::IndexTocNode { node_id }, + } + } + + /// Create entry for grip indexing + pub fn for_grip(event_id: String, timestamp_ms: i64, grip_id: String) -> Self { + Self { + event_id, + timestamp_ms, + action: OutboxAction::IndexGrip { grip_id }, + } + } +} +``` + +### IndexingJobConfig +```rust +// Source: Derived from memory-scheduler/jobs/rollup.rs RollupJobConfig +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IndexingJobConfig { + /// Cron expression (default: "*/30 * * * * *" = every 30 seconds) + pub cron: String, + + /// Timezone for scheduling (default: "UTC") + pub timezone: String, + + /// Max jitter in seconds (default: 5) + pub jitter_secs: u64, + + /// Batch size per run (default: 100) + pub batch_size: usize, + + /// Whether to enable BM25 indexing + pub enable_bm25: bool, + + /// Whether to enable vector indexing + pub enable_vector: bool, +} + +impl Default for IndexingJobConfig { + fn default() -> Self { + Self { + cron: "*/30 * * * * *".to_string(), + timezone: "UTC".to_string(), + jitter_secs: 5, + batch_size: 100, + enable_bm25: true, + enable_vector: true, + } + } +} +``` + +### Admin Rebuild Command Pattern +```rust +// Source: Derived from memory-daemon/src/commands.rs admin patterns +#[derive(Parser)] +pub struct RebuildIndexesCmd { + /// Only rebuild BM25 index + #[arg(long)] + bm25_only: bool, + + /// Only rebuild vector index + #[arg(long)] + vector_only: bool, + + /// Show what would be rebuilt without rebuilding + #[arg(long)] + dry_run: bool, +} + +pub async fn rebuild_indexes( + storage: Arc, + bm25: Option>, + vector: Option>, + cmd: RebuildIndexesCmd, +) -> Result<(), MemoryError> { + let stats = storage.get_stats()?; + + if cmd.dry_run { + println!("Would rebuild indexes:"); + println!(" TOC nodes: {}", stats.toc_node_count); + println!(" Grips: {}", stats.grip_count); + if !cmd.vector_only { + println!(" BM25: {}", if bm25.is_some() { "enabled" } else { "disabled" }); + } + if !cmd.bm25_only { + println!(" Vector: {}", if vector.is_some() { "enabled" } else { "disabled" }); + } + return Ok(()); + } + + let pipeline = IndexingPipeline::new( + storage, + if cmd.vector_only { None } else { bm25 }, + if cmd.bm25_only { None } else { vector }, + ); + + println!("Starting index rebuild..."); + let rebuild_stats = pipeline.rebuild_all().await?; + + println!("Rebuild complete:"); + println!(" TOC nodes indexed: {}", rebuild_stats.nodes_indexed); + println!(" Grips indexed: {}", rebuild_stats.grips_indexed); + println!(" Duration: {:?}", rebuild_stats.duration); + + Ok(()) +} +``` + +## State of the Art + +| Old Approach | Current Approach | When Changed | Impact | +|--------------|------------------|--------------|--------| +| Synchronous indexing | Async via outbox | Phase 11+ design | Non-blocking ingestion | +| Full rebuild only | Incremental + rebuild | This phase | Efficient updates | +| Single index | Multi-index dispatch | Phase 11+12 | BM25 + Vector combined | + +**Deprecated/outdated:** +- Direct indexing during event ingestion: Use outbox pattern instead +- Shared checkpoint for all indexes: Use per-index checkpoints + +## Open Questions + +Things that couldn't be fully resolved: + +1. **Outbox entry cleanup timing** + - What we know: Entries should be deleted after successful indexing + - What's unclear: Should delete happen immediately or in batch? What about FIFO compaction interaction? + - Recommendation: Delete in batch after each consumer run; FIFO compaction handles any stragglers + +2. **Dead-letter handling for persistent failures** + - What we know: Some entries may fail repeatedly (e.g., invalid data) + - What's unclear: Should there be a dead-letter queue or just skip after N retries? + - Recommendation: Skip after 3 retries, log error, advance checkpoint. User can rebuild if needed. + +3. **Consistency verification** + - What we know: Index can drift from storage due to bugs or partial failures + - What's unclear: How often to check? What metrics to track? + - Recommendation: Expose IndexStatus RPC with document counts; let admin compare with storage stats + +## Sources + +### Primary (HIGH confidence) +- [Existing memory-storage/db.rs](crates/memory-storage/src/db.rs) - Outbox and checkpoint APIs +- [Existing memory-types/outbox.rs](crates/memory-types/src/outbox.rs) - OutboxEntry type +- [Existing memory-scheduler](crates/memory-scheduler/src/) - Job scheduling patterns +- [Phase 11 Research](../11-bm25-teleport-tantivy/11-RESEARCH.md) - Tantivy patterns +- [Phase 12 Research](../12-vector-teleport-hnsw/12-RESEARCH.md) - Vector indexing patterns +- [Tantivy docs.rs](https://docs.rs/tantivy/latest/tantivy/) - IndexWriter, commit, delete_term +- [USearch Rust README](https://github.com/unum-cloud/usearch/blob/main/rust/README.md) - add, save, load + +### Secondary (MEDIUM confidence) +- [Transactional Outbox Pattern](https://microservices.io/patterns/data/transactional-outbox.html) - Pattern fundamentals +- [Outbox Patterns Explained](https://event-driven.io/en/outbox_inbox_patterns_and_delivery_guarantees_explained/) - At-least-once semantics +- [tokio-cron-scheduler](https://github.com/mvniekerk/tokio-cron-scheduler) - Cron job patterns + +### Tertiary (LOW confidence) +- Web search results on idempotent processing - General patterns, not Rust-specific + +## Metadata + +**Confidence breakdown:** +- Standard stack: HIGH - All components exist in workspace +- Architecture: HIGH - Derived from existing patterns in codebase +- Pitfalls: MEDIUM - Based on general distributed systems knowledge +- Code examples: HIGH - Derived from existing codebase patterns + +**Research date:** 2026-02-01 +**Valid until:** 2026-03-01 (30 days - stable domain, existing infrastructure) + +--- + +## Recommended Plan Breakdown + +Based on this research, Phase 13 should be split into 4 plans: + +### Plan 13-01: Outbox Consumer Infrastructure +**Focus:** Create `memory-indexing` crate, outbox reading, checkpoint tracking +**Tasks:** +- Add memory-indexing crate to workspace +- Implement Storage::get_outbox_entries() and delete_outbox_entries() +- Create IndexCheckpoint type with per-index tracking +- Implement checkpoint load/save using existing CF_CHECKPOINTS +- Unit tests for outbox reading and checkpoint persistence + +### Plan 13-02: Incremental Index Updates +**Focus:** IndexingPipeline for dispatching to BM25 and vector indexers +**Tasks:** +- Implement IndexingPipeline with bm25/vector indexer injection +- Add index_toc_node() and index_grip() with idempotent update pattern +- Implement process_batch() for outbox consumption +- Add spawn_blocking wrapper for Tantivy operations +- Integration tests with mock indexers + +### Plan 13-03: Full Rebuild Command +**Focus:** Admin command for complete index rebuild from storage +**Tasks:** +- Implement rebuild_all() method in IndexingPipeline +- Add RebuildIndexesCmd to CLI (rebuild-indexes subcommand) +- Support --bm25-only, --vector-only, --dry-run flags +- Progress reporting during rebuild +- Integration test for rebuild functionality + +### Plan 13-04: Scheduler Integration +**Focus:** Background job for periodic outbox processing +**Tasks:** +- Create IndexingJobConfig for job configuration +- Implement create_indexing_job() following rollup job pattern +- Wire indexing job into daemon startup +- Add GetIndexingStatus RPC for observability +- End-to-end tests verifying async indexing flow +- Update documentation diff --git a/.planning/phases/15-configuration-wizard-skills/15-01-PLAN.md b/.planning/phases/15-configuration-wizard-skills/15-01-PLAN.md new file mode 100644 index 0000000..60d70e1 --- /dev/null +++ b/.planning/phases/15-configuration-wizard-skills/15-01-PLAN.md @@ -0,0 +1,282 @@ +--- +id: 15-01 +name: memory-storage skill +phase: 15 +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - plugins/memory-setup-plugin/skills/memory-storage/SKILL.md + - plugins/memory-setup-plugin/skills/memory-storage/references/retention-policies.md + - plugins/memory-setup-plugin/skills/memory-storage/references/gdpr-compliance.md + - plugins/memory-setup-plugin/skills/memory-storage/references/archive-strategies.md +autonomous: true +estimated_effort: M + +must_haves: + truths: + - "User can configure storage path via interactive wizard" + - "User can set retention policy (forever, 90 days, 30 days, 7 days)" + - "User can configure cleanup schedule (cron-based)" + - "User can enable/disable GDPR mode" + - "User can tune performance parameters (buffer size, background jobs)" + - "State detection skips already-configured options" + artifacts: + - path: "plugins/memory-setup-plugin/skills/memory-storage/SKILL.md" + provides: "Interactive storage wizard skill" + min_lines: 200 + - path: "plugins/memory-setup-plugin/skills/memory-storage/references/retention-policies.md" + provides: "Retention policy documentation" + min_lines: 50 + - path: "plugins/memory-setup-plugin/skills/memory-storage/references/gdpr-compliance.md" + provides: "GDPR mode documentation" + min_lines: 40 + - path: "plugins/memory-setup-plugin/skills/memory-storage/references/archive-strategies.md" + provides: "Archive strategy documentation" + min_lines: 40 + key_links: + - from: "plugins/memory-setup-plugin/skills/memory-storage/SKILL.md" + to: "~/.config/memory-daemon/config.toml" + via: "config generation heredoc" + pattern: "\\[storage\\]|\\[retention\\]" +--- + + +Create the `/memory-storage` interactive wizard skill for configuring storage paths, data retention policies, cleanup schedules, GDPR compliance mode, and performance tuning. + +Purpose: Provide users with guided configuration for storage-related settings that are too advanced for the basic `/memory-setup` wizard but essential for production deployments. + +Output: SKILL.md file with 6-step wizard flow plus 3 reference documentation files. + + + +@/Users/richardhightower/.claude/get-shit-done/workflows/execute-plan.md +@/Users/richardhightower/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/15-configuration-wizard-skills/15-RESEARCH.md +@docs/plans/configuration-wizard-skills-plan.md +@plugins/memory-setup-plugin/skills/memory-setup/SKILL.md +@plugins/memory-setup-plugin/skills/memory-setup/references/wizard-questions.md + + + + + + Task 1: Create memory-storage SKILL.md with 6-step wizard + plugins/memory-setup-plugin/skills/memory-storage/SKILL.md + +Create the SKILL.md file following the exact pattern from memory-setup/SKILL.md. + +**YAML Frontmatter:** +```yaml +--- +name: memory-storage +description: | + This skill should be used when the user asks to "configure storage", + "set up retention policies", "configure GDPR mode", "tune memory performance", + "change storage path", or "configure data cleanup". Provides interactive wizard + for storage configuration with state detection. +license: MIT +metadata: + version: 1.0.0 + author: SpillwaveSolutions +--- +``` + +**Content Structure:** +1. Header section with purpose and when not to use +2. Quick Start table with commands: `/memory-storage`, `/memory-storage --minimal`, `/memory-storage --advanced` +3. Question Flow diagram (ASCII art showing 6 steps) +4. State Detection section with bash commands: + - Check storage path: `grep -A5 '\[storage\]' ~/.config/memory-daemon/config.toml` + - Check retention: `grep retention ~/.config/memory-daemon/config.toml` + - Check disk usage: `du -sh ~/.memory-store && df -h ~/.memory-store | tail -1` + - Check archive: `ls ~/.memory-archive` + +5. **Six wizard steps with AskUserQuestion format:** + +**Step 1: Storage Path** +- question: "Where should agent-memory store conversation data?" +- header: "Storage" (max 12 chars) +- options: ~/.memory-store (Recommended), ~/.local/share/agent-memory/db (XDG), Custom path +- Skip if: path configured AND not --fresh + +**Step 2: Retention Policy** +- question: "How long should conversation data be retained?" +- header: "Retention" +- options: Forever (Recommended), 90 days, 30 days, 7 days +- Skip if: retention configured AND not --fresh + +**Step 3: Cleanup Schedule** (--advanced only) +- question: "When should automatic cleanup run?" +- header: "Schedule" +- options: Daily at 3 AM (Recommended), Weekly on Sunday, Disabled, Custom cron +- Skip in --minimal mode + +**Step 4: Archive Strategy** (--advanced only) +- question: "How should old data be archived before deletion?" +- header: "Archive" +- options: Compress to archive (Recommended), Export to JSON, No archive +- Skip in --minimal mode + +**Step 5: GDPR Mode** +- question: "Enable GDPR-compliant deletion mode?" +- header: "GDPR" +- options: No (Recommended), Yes +- Show if: EU locale detected OR --advanced flag + +**Step 6: Performance Tuning** (--advanced only) +- question: "Configure storage performance parameters?" +- header: "Performance" +- options: Balanced (64MB buffer, 4 jobs), Low memory (16MB, 1 job), High performance (128MB, 8 jobs), Custom +- Skip in --minimal mode + +6. Config Generation section showing heredoc for config.toml updates +7. Validation section (path writable, 100MB minimum space, cron valid) +8. Output Formatting with [check]/[x] indicators +9. Reference Files links +10. Cross-skill navigation hints to /memory-llm and /memory-agents + + +```bash +ls -la plugins/memory-setup-plugin/skills/memory-storage/SKILL.md +wc -l plugins/memory-setup-plugin/skills/memory-storage/SKILL.md +grep -c "AskUserQuestion\|question:" plugins/memory-setup-plugin/skills/memory-storage/SKILL.md +``` +File exists, has 200+ lines, contains question definitions + + SKILL.md exists with 6-step wizard flow, state detection, AskUserQuestion format, all three flag modes documented + + + + Task 2: Create retention-policies.md reference documentation + plugins/memory-setup-plugin/skills/memory-storage/references/retention-policies.md + +Create reference documentation for retention policies. + +**Content:** +1. Overview of retention concept and why it matters +2. Policy options table: + | Policy | Value | Storage Impact | Use Case | + |--------|-------|----------------|----------| + | Forever | `forever` | Grows unbounded | Maximum historical context | + | 90 Days | `days:90` | ~3 months data | Balance of history and storage | + | 30 Days | `days:30` | ~1 month data | Lower storage needs | + | 7 Days | `days:7` | ~1 week data | Short-term memory only | + +3. Cleanup Schedule section: + - Default: `0 3 * * *` (daily at 3 AM) + - Options with cron examples + - Cron format explanation + +4. Data Lifecycle diagram (ASCII): + - Event ingested -> Active storage -> Retention period elapsed -> Archived/Deleted + +5. Storage estimation table: + - Typical usage patterns and size estimates + - Formula: ~10KB per conversation turn average + +6. Configuration example with config.toml snippet + + +```bash +ls -la plugins/memory-setup-plugin/skills/memory-storage/references/retention-policies.md +grep -c "Policy\|forever\|days:" plugins/memory-setup-plugin/skills/memory-storage/references/retention-policies.md +``` +File exists, contains policy options + + retention-policies.md documents all retention options with storage impact estimates + + + + Task 3: Create gdpr-compliance.md and archive-strategies.md references + +plugins/memory-setup-plugin/skills/memory-storage/references/gdpr-compliance.md +plugins/memory-setup-plugin/skills/memory-storage/references/archive-strategies.md + + +**Create gdpr-compliance.md:** +1. What GDPR mode enables: + - Complete data removal (no tombstones) + - Audit logging of deletions + - Export-before-delete option + - Right to erasure support + +2. When to enable: + - EU users or EU data subjects + - Compliance requirements + - Privacy-first deployments + +3. Trade-offs table: + | Aspect | Standard Mode | GDPR Mode | + |--------|---------------|-----------| + | Deletion | Tombstones | Complete removal | + | Audit | Optional | Required | + | Recovery | Possible | Not possible | + +4. Configuration example + +**Create archive-strategies.md:** +1. Archive strategy options: + - Compress: gzip to ~/.memory-archive/ + - JSON export: human-readable backup + - No archive: direct deletion (irreversible) + +2. Archive location and format +3. Recovery procedures for each strategy +4. Disk space considerations +5. Configuration examples + + +```bash +ls -la plugins/memory-setup-plugin/skills/memory-storage/references/ +wc -l plugins/memory-setup-plugin/skills/memory-storage/references/*.md +``` +Both files exist with appropriate content + + gdpr-compliance.md and archive-strategies.md created with detailed documentation + + + + + +Run these commands to verify the plan is complete: + +```bash +# Verify directory structure +ls -la plugins/memory-setup-plugin/skills/memory-storage/ +ls -la plugins/memory-setup-plugin/skills/memory-storage/references/ + +# Verify SKILL.md structure +grep -E "^---$|^name:|question:|header:|options:" plugins/memory-setup-plugin/skills/memory-storage/SKILL.md | head -30 + +# Verify reference docs +head -20 plugins/memory-setup-plugin/skills/memory-storage/references/retention-policies.md +head -20 plugins/memory-setup-plugin/skills/memory-storage/references/gdpr-compliance.md +head -20 plugins/memory-setup-plugin/skills/memory-storage/references/archive-strategies.md + +# Count lines +wc -l plugins/memory-setup-plugin/skills/memory-storage/SKILL.md +wc -l plugins/memory-setup-plugin/skills/memory-storage/references/*.md +``` + + + +1. SKILL.md has valid YAML frontmatter with name: memory-storage +2. Six wizard steps documented with AskUserQuestion format +3. State detection commands for checking existing config +4. Three flag modes (--minimal, --advanced, --fresh) documented +5. Three reference files created (retention, gdpr, archive) +6. Output formatting matches memory-setup patterns +7. Cross-skill navigation hints included + + + +After completion, create `.planning/phases/15-configuration-wizard-skills/15-01-SUMMARY.md` + diff --git a/.planning/phases/15-configuration-wizard-skills/15-02-PLAN.md b/.planning/phases/15-configuration-wizard-skills/15-02-PLAN.md new file mode 100644 index 0000000..9b92a41 --- /dev/null +++ b/.planning/phases/15-configuration-wizard-skills/15-02-PLAN.md @@ -0,0 +1,327 @@ +--- +id: 15-02 +name: memory-llm skill +phase: 15 +plan: 02 +type: execute +wave: 1 +depends_on: [] +files_modified: + - plugins/memory-setup-plugin/skills/memory-llm/SKILL.md + - plugins/memory-setup-plugin/skills/memory-llm/references/provider-comparison.md + - plugins/memory-setup-plugin/skills/memory-llm/references/model-selection.md + - plugins/memory-setup-plugin/skills/memory-llm/references/cost-estimation.md + - plugins/memory-setup-plugin/skills/memory-llm/references/custom-endpoints.md +autonomous: true +estimated_effort: M + +must_haves: + truths: + - "User can select LLM provider via interactive wizard" + - "User can discover available models for selected provider" + - "User can test API connection before saving config" + - "User can see cost estimation for selected model" + - "User can configure quality/latency tradeoffs" + - "User can set token budget optimization mode" + artifacts: + - path: "plugins/memory-setup-plugin/skills/memory-llm/SKILL.md" + provides: "Interactive LLM wizard skill" + min_lines: 250 + - path: "plugins/memory-setup-plugin/skills/memory-llm/references/provider-comparison.md" + provides: "Provider comparison documentation" + min_lines: 60 + - path: "plugins/memory-setup-plugin/skills/memory-llm/references/model-selection.md" + provides: "Model selection guide" + min_lines: 50 + - path: "plugins/memory-setup-plugin/skills/memory-llm/references/cost-estimation.md" + provides: "Cost estimation documentation" + min_lines: 40 + - path: "plugins/memory-setup-plugin/skills/memory-llm/references/custom-endpoints.md" + provides: "Custom endpoint configuration" + min_lines: 30 + key_links: + - from: "plugins/memory-setup-plugin/skills/memory-llm/SKILL.md" + to: "~/.config/memory-daemon/config.toml" + via: "config generation for [summarizer] section" + pattern: "\\[summarizer\\]" + - from: "plugins/memory-setup-plugin/skills/memory-llm/SKILL.md" + to: "OpenAI/Anthropic/Ollama APIs" + via: "API test calls" + pattern: "curl.*api\\.openai\\.com|api\\.anthropic\\.com|localhost:11434" +--- + + +Create the `/memory-llm` interactive wizard skill for deep LLM configuration including provider selection, model discovery, API testing, cost estimation, and quality/budget tuning. + +Purpose: Provide users with guided LLM configuration that goes beyond the basic provider selection in `/memory-setup`, enabling model discovery, live API testing, and cost-aware decisions. + +Output: SKILL.md file with 7-step wizard flow plus 4 reference documentation files. + + + +@/Users/richardhightower/.claude/get-shit-done/workflows/execute-plan.md +@/Users/richardhightower/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/15-configuration-wizard-skills/15-RESEARCH.md +@docs/plans/configuration-wizard-skills-plan.md +@plugins/memory-setup-plugin/skills/memory-setup/SKILL.md +@plugins/memory-setup-plugin/skills/memory-setup/references/wizard-questions.md + + + + + + Task 1: Create memory-llm SKILL.md with 7-step wizard + plugins/memory-setup-plugin/skills/memory-llm/SKILL.md + +Create the SKILL.md file following the memory-setup/SKILL.md pattern. + +**YAML Frontmatter:** +```yaml +--- +name: memory-llm +description: | + This skill should be used when the user asks to "configure LLM", + "change summarizer provider", "test API connection", "estimate LLM costs", + "discover models", or "tune summarization quality". Provides interactive wizard + for LLM provider configuration with model discovery and API testing. +license: MIT +metadata: + version: 1.0.0 + author: SpillwaveSolutions +--- +``` + +**Content Structure:** +1. Header section with purpose and when not to use +2. Quick Start table with commands: + - `/memory-llm` - Interactive LLM wizard + - `/memory-llm --test` - Test current API key only + - `/memory-llm --discover` - List available models + - `/memory-llm --estimate` - Show cost estimation + +3. Question Flow diagram (ASCII art showing 7 steps) + +4. State Detection section with bash commands: + ```bash + # API keys set? + [ -n "$OPENAI_API_KEY" ] && echo "OPENAI: set" || echo "OPENAI: not set" + [ -n "$ANTHROPIC_API_KEY" ] && echo "ANTHROPIC: set" || echo "ANTHROPIC: not set" + + # Current config + grep -A10 '\[summarizer\]' ~/.config/memory-daemon/config.toml + + # Test connectivity + curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $OPENAI_API_KEY" https://api.openai.com/v1/models + + # Check Ollama + curl -s http://localhost:11434/api/tags + ``` + +5. **Seven wizard steps with AskUserQuestion format:** + +**Step 1: Provider Selection** +- question: "Which LLM provider should generate summaries?" +- header: "Provider" +- options: OpenAI (Recommended), Anthropic, Ollama (Local), None +- Always ask (core decision) unless --minimal + +**Step 2: Model Discovery** +- question: "Which model should be used for summarization?" +- header: "Model" +- options: Dynamic based on provider + - OpenAI: gpt-4o-mini (Recommended, $0.15/1M), gpt-4o ($5/1M), gpt-4-turbo ($10/1M) + - Anthropic: claude-3-5-haiku-latest (Recommended), claude-3-5-sonnet-latest + - Ollama: List from `curl localhost:11434/api/tags` +- Show discovered models with pricing + +**Step 3: API Key** +- question: "How should the API key be configured?" +- header: "API Key" +- options: Use existing environment variable (Recommended), Enter new key, Test existing key +- Skip if: env var set AND provider matches AND not --fresh + +**Step 4: Test Connection** +- Live API test (not a question) +- Show result: [check] Connected or [x] Failed with error +- Offer retry or skip + +**Step 5: Cost Estimation** +- Informational display (not a question) +- Show: "Based on typical usage (~1000 events/day), estimated cost: $X.XX/month" +- Include token calculation + +**Step 6: Quality/Latency Tradeoffs** (--advanced only) +- question: "Configure quality vs latency tradeoff?" +- header: "Quality" +- options: Balanced (temp=0.3, max_tokens=512), Deterministic (temp=0.0), Creative (temp=0.7), Custom +- Skip in --minimal mode + +**Step 7: Token Budget** (--advanced only) +- question: "Configure token budget optimization?" +- header: "Budget" +- options: Balanced (~$0.02/month), Economical (shorter summaries), Detailed (longer summaries), Custom +- Skip in --minimal mode + +6. Config Generation section for [summarizer] config.toml +7. Validation section (key format, live test, model available) +8. Output Formatting with success/error displays +9. Reference Files links +10. Cross-skill navigation hints + + +```bash +ls -la plugins/memory-setup-plugin/skills/memory-llm/SKILL.md +wc -l plugins/memory-setup-plugin/skills/memory-llm/SKILL.md +grep -c "Provider\|Model\|API Key" plugins/memory-setup-plugin/skills/memory-llm/SKILL.md +``` +File exists, has 250+ lines, contains wizard steps + + SKILL.md exists with 7-step wizard flow including model discovery and API testing + + + + Task 2: Create provider-comparison.md and model-selection.md references + +plugins/memory-setup-plugin/skills/memory-llm/references/provider-comparison.md +plugins/memory-setup-plugin/skills/memory-llm/references/model-selection.md + + +**Create provider-comparison.md:** +1. Provider overview table: + | Provider | Cost | Quality | Latency | Privacy | + |----------|------|---------|---------|---------| + | OpenAI | $$ | High | Fast | Cloud | + | Anthropic | $$$ | Highest | Medium | Cloud | + | Ollama | Free | Variable| Slow | Local | + | None | Free | N/A | N/A | N/A | + +2. Detailed provider sections: + - **OpenAI:** GPT models, API key from platform.openai.com, fast and reliable + - **Anthropic:** Claude models, higher quality, API key from console.anthropic.com + - **Ollama:** Local models, no API costs, requires local resources + - **None:** Disables summarization, TOC only mode + +3. When to choose each provider +4. API key management recommendations + +**Create model-selection.md:** +1. Model comparison by provider +2. OpenAI models table: + | Model | Price/1M tokens | Context | Best For | + |-------|-----------------|---------|----------| + | gpt-4o-mini | $0.15 | 128k | Most users (recommended) | + | gpt-4o | $5.00 | 128k | Highest quality | + | gpt-4-turbo | $10.00 | 128k | Legacy support | + +3. Anthropic models table +4. Ollama models (common ones: llama3.2:3b, mistral, phi) +5. Model discovery commands +6. Quality vs cost tradeoff guidance + + +```bash +ls -la plugins/memory-setup-plugin/skills/memory-llm/references/provider-comparison.md +ls -la plugins/memory-setup-plugin/skills/memory-llm/references/model-selection.md +grep -c "OpenAI\|Anthropic\|Ollama" plugins/memory-setup-plugin/skills/memory-llm/references/*.md +``` +Both files exist with provider documentation + + provider-comparison.md and model-selection.md created with detailed comparisons + + + + Task 3: Create cost-estimation.md and custom-endpoints.md references + +plugins/memory-setup-plugin/skills/memory-llm/references/cost-estimation.md +plugins/memory-setup-plugin/skills/memory-llm/references/custom-endpoints.md + + +**Create cost-estimation.md:** +1. Cost calculation formula: + - Tokens per summary: ~200-500 + - Summaries per day: depends on conversation volume + - Monthly cost = (tokens/summary * summaries/day * 30) / 1,000,000 * price_per_1M + +2. Usage tiers table: + | Usage | Events/Day | Summaries | Monthly Cost (gpt-4o-mini) | + |-------|------------|-----------|---------------------------| + | Light | 100 | ~5 | $0.01 | + | Medium | 1000 | ~50 | $0.10 | + | Heavy | 5000 | ~250 | $0.50 | + +3. Budget optimization modes +4. Token counting explanation +5. Monitoring usage section + +**Create custom-endpoints.md:** +1. When to use custom endpoints: + - OpenAI-compatible APIs (Azure OpenAI, LocalAI, LM Studio) + - Private deployments + - Proxy servers + +2. Configuration example: + ```toml + [summarizer] + provider = "openai" # Use OpenAI-compatible protocol + api_endpoint = "https://your-custom-endpoint/v1" + model = "your-model" + ``` + +3. Azure OpenAI specific configuration +4. Local API server setup (LocalAI, LM Studio) +5. Testing custom endpoints + + +```bash +ls -la plugins/memory-setup-plugin/skills/memory-llm/references/cost-estimation.md +ls -la plugins/memory-setup-plugin/skills/memory-llm/references/custom-endpoints.md +wc -l plugins/memory-setup-plugin/skills/memory-llm/references/*.md +``` +All four reference files exist + + cost-estimation.md and custom-endpoints.md created with usage and configuration guidance + + + + + +Run these commands to verify the plan is complete: + +```bash +# Verify directory structure +ls -la plugins/memory-setup-plugin/skills/memory-llm/ +ls -la plugins/memory-setup-plugin/skills/memory-llm/references/ + +# Verify SKILL.md structure +grep -E "^---$|^name:|question:|header:|options:" plugins/memory-setup-plugin/skills/memory-llm/SKILL.md | head -40 + +# Verify reference docs +for f in plugins/memory-setup-plugin/skills/memory-llm/references/*.md; do + echo "=== $f ===" && head -10 "$f" +done + +# Count lines +wc -l plugins/memory-setup-plugin/skills/memory-llm/SKILL.md +wc -l plugins/memory-setup-plugin/skills/memory-llm/references/*.md +``` + + + +1. SKILL.md has valid YAML frontmatter with name: memory-llm +2. Seven wizard steps documented with AskUserQuestion format +3. Model discovery logic for each provider +4. API testing commands included +5. Four reference files created (provider, model, cost, endpoints) +6. Cost estimation formulas and tables +7. Custom endpoint configuration documented + + + +After completion, create `.planning/phases/15-configuration-wizard-skills/15-02-SUMMARY.md` + diff --git a/.planning/phases/15-configuration-wizard-skills/15-03-PLAN.md b/.planning/phases/15-configuration-wizard-skills/15-03-PLAN.md new file mode 100644 index 0000000..d2bb925 --- /dev/null +++ b/.planning/phases/15-configuration-wizard-skills/15-03-PLAN.md @@ -0,0 +1,323 @@ +--- +id: 15-03 +name: memory-agents skill +phase: 15 +plan: 03 +type: execute +wave: 2 +depends_on: ["15-01", "15-02"] +files_modified: + - plugins/memory-setup-plugin/skills/memory-agents/SKILL.md + - plugins/memory-setup-plugin/skills/memory-agents/references/storage-strategies.md + - plugins/memory-setup-plugin/skills/memory-agents/references/team-setup.md + - plugins/memory-setup-plugin/skills/memory-agents/references/agent-identifiers.md +autonomous: true +estimated_effort: M + +must_haves: + truths: + - "User can select usage mode (single user, multi-agent, team)" + - "User can configure storage strategy (unified vs separate)" + - "User can set agent identifier for tagging" + - "User can configure cross-agent query scope" + - "User can set up team mode settings" + - "State detection identifies existing agent configuration" + artifacts: + - path: "plugins/memory-setup-plugin/skills/memory-agents/SKILL.md" + provides: "Interactive multi-agent wizard skill" + min_lines: 200 + - path: "plugins/memory-setup-plugin/skills/memory-agents/references/storage-strategies.md" + provides: "Storage strategy documentation" + min_lines: 50 + - path: "plugins/memory-setup-plugin/skills/memory-agents/references/team-setup.md" + provides: "Team mode documentation" + min_lines: 40 + - path: "plugins/memory-setup-plugin/skills/memory-agents/references/agent-identifiers.md" + provides: "Agent ID documentation" + min_lines: 40 + key_links: + - from: "plugins/memory-setup-plugin/skills/memory-agents/SKILL.md" + to: "~/.config/memory-daemon/config.toml" + via: "config generation for [agents] section" + pattern: "\\[agents\\]|\\[team\\]" +--- + + +Create the `/memory-agents` interactive wizard skill for configuring multi-agent memory settings including store isolation, agent tagging, cross-agent query permissions, and team settings. + +Purpose: Enable users who run multiple AI agents (Claude Code, Cursor, etc.) or work in teams to configure how memory is shared or isolated between agents. + +Output: SKILL.md file with 6-step wizard flow plus 3 reference documentation files. + + + +@/Users/richardhightower/.claude/get-shit-done/workflows/execute-plan.md +@/Users/richardhightower/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/15-configuration-wizard-skills/15-RESEARCH.md +@docs/plans/configuration-wizard-skills-plan.md +@plugins/memory-setup-plugin/skills/memory-setup/SKILL.md + + + + + + Task 1: Create memory-agents SKILL.md with 6-step wizard + plugins/memory-setup-plugin/skills/memory-agents/SKILL.md + +Create the SKILL.md file following the memory-setup/SKILL.md pattern. + +**YAML Frontmatter:** +```yaml +--- +name: memory-agents +description: | + This skill should be used when the user asks to "configure multi-agent memory", + "set up team memory", "configure agent isolation", "set agent ID", + "share memory between agents", or "configure cross-agent queries". + Provides interactive wizard for multi-agent configuration. +license: MIT +metadata: + version: 1.0.0 + author: SpillwaveSolutions +--- +``` + +**Content Structure:** +1. Header section with purpose and when not to use +2. Quick Start table with commands: + - `/memory-agents` - Interactive multi-agent wizard + - `/memory-agents --single` - Configure for single user mode + - `/memory-agents --team` - Configure for team use + +3. Question Flow diagram (ASCII art showing 6 steps with conditional branches) + +4. State Detection section with bash commands: + ```bash + # Current multi-agent config + grep -A5 'agents' ~/.config/memory-daemon/config.toml + + # Current agent_id + grep 'agent_id' ~/.config/memory-daemon/config.toml + + # Detect other agents + ls ~/.memory-store/agents/ + + # Hostname and user + hostname + whoami + ``` + +5. **Six wizard steps with AskUserQuestion format:** + +**Step 1: Usage Mode** +- question: "How will agent-memory be used?" +- header: "Mode" +- options: + - Single user (Recommended): One person, one agent (Claude Code) + - Single user, multiple agents: One person using Claude Code, Cursor, etc. + - Team mode: Multiple users sharing memory on a team +- Always ask (core decision) + +**Step 2: Storage Strategy** (if not single user) +- question: "How should agent data be stored?" +- header: "Storage" +- options: + - Unified store with tags (Recommended): Single database, agents identified by tag, easy cross-query + - Separate stores per agent: Complete isolation, cannot query across agents +- Skip if: Step 1 selected "Single user" + +**Step 3: Agent Identifier** +- question: "Choose your agent identifier (tags all events from this instance):" +- header: "Agent ID" +- options: + - claude-code (Recommended): Standard identifier for Claude Code + - claude-code-{hostname}: Unique per machine for multi-machine setups + - {username}-claude: User-specific for shared machines + - Custom: Specify a custom identifier +- Always ask (important for tracking) + +**Step 4: Cross-Agent Query Permissions** (if unified store) +- question: "What data should queries return?" +- header: "Query Scope" +- options: + - Own events only (Recommended): Query only this agent's data + - All agents: Query all agents' data (read-only) + - Specified agents: Query specific agents' data +- Skip if: Storage strategy is "separate" + +**Step 5: Storage Organization** (--advanced, if separate stores) +- question: "How should separate stores be organized?" +- header: "Organization" +- options: + - ~/.memory-store/{agent_id}/: Agent-specific subdirectories + - Custom paths: Specify custom storage paths +- Skip unless: --advanced AND separate storage selected + +**Step 6: Team Settings** (if team mode) +- question: "Configure team sharing settings?" +- header: "Team" +- options: + - Read-only sharing: See team events, write to own store + - Full sharing: All team members read/write to shared store + - Custom permissions: Configure per-agent permissions +- Skip unless: Step 1 selected "Team mode" + +6. Config Generation section showing [agents] and [team] sections +7. Validation section (agent ID format, path writable, unique ID) +8. Output Formatting +9. Reference Files links +10. Cross-skill navigation hints + + +```bash +ls -la plugins/memory-setup-plugin/skills/memory-agents/SKILL.md +wc -l plugins/memory-setup-plugin/skills/memory-agents/SKILL.md +grep -c "Mode\|Storage\|Agent ID\|Query Scope" plugins/memory-setup-plugin/skills/memory-agents/SKILL.md +``` +File exists, has 200+ lines, contains wizard steps + + SKILL.md exists with 6-step wizard flow for multi-agent configuration + + + + Task 2: Create storage-strategies.md reference documentation + plugins/memory-setup-plugin/skills/memory-agents/references/storage-strategies.md + +Create reference documentation for multi-agent storage strategies. + +**Content:** +1. Overview of multi-agent storage concepts + +2. Strategy comparison table: + | Strategy | Isolation | Cross-Query | Complexity | Use Case | + |----------|-----------|-------------|------------|----------| + | Unified with tags | Logical | Yes (configurable) | Simple | Most multi-agent setups | + | Separate stores | Physical | No | Higher | Strict isolation needed | + +3. **Unified Store with Tags:** + - Single database at ~/.memory-store + - Events tagged with agent_id field + - Query filtering by agent_id + - Pros: Easy cross-agent search, single backup + - Cons: All data in one place + +4. **Separate Stores:** + - Directory per agent: ~/.memory-store/{agent_id}/ + - Complete isolation + - Pros: Maximum privacy, independent backups + - Cons: No cross-agent queries, more disk usage + +5. Migration between strategies +6. Configuration examples +7. Decision tree for choosing strategy + + +```bash +ls -la plugins/memory-setup-plugin/skills/memory-agents/references/storage-strategies.md +grep -c "Unified\|Separate\|agent_id" plugins/memory-setup-plugin/skills/memory-agents/references/storage-strategies.md +``` +File exists with strategy documentation + + storage-strategies.md documents unified vs separate storage with tradeoffs + + + + Task 3: Create team-setup.md and agent-identifiers.md references + +plugins/memory-setup-plugin/skills/memory-agents/references/team-setup.md +plugins/memory-setup-plugin/skills/memory-agents/references/agent-identifiers.md + + +**Create team-setup.md:** +1. Team mode overview and use cases +2. Team configuration options: + - Team name + - Shared storage path + - Permission levels + +3. Permission models table: + | Mode | Read | Write | Use Case | + |------|------|-------|----------| + | Read-only | All team | Own only | Team visibility | + | Full sharing | All team | All team | Collaborative work | + | Custom | Configurable | Configurable | Enterprise | + +4. Setup steps for team mode +5. Network considerations (shared storage, NFS, etc.) +6. Configuration example + +**Create agent-identifiers.md:** +1. What is an agent identifier: + - Tags all events from this agent instance + - Used for filtering and attribution + - Persists across sessions + +2. Identifier patterns table: + | Pattern | Example | Use Case | + |---------|---------|----------| + | Simple | `claude-code` | Single user, single machine | + | Host-specific | `claude-code-macbook` | Multi-machine | + | User-specific | `alice-claude` | Shared machines | + | Project-specific | `project-x-claude` | Per-project isolation | + +3. Identifier requirements: + - 3-50 characters + - Alphanumeric, hyphens, underscores + - Must be unique in unified store + +4. Changing identifiers (migration considerations) +5. Environment variable override + + +```bash +ls -la plugins/memory-setup-plugin/skills/memory-agents/references/team-setup.md +ls -la plugins/memory-setup-plugin/skills/memory-agents/references/agent-identifiers.md +wc -l plugins/memory-setup-plugin/skills/memory-agents/references/*.md +``` +Both files exist with appropriate content + + team-setup.md and agent-identifiers.md created with team and ID documentation + + + + + +Run these commands to verify the plan is complete: + +```bash +# Verify directory structure +ls -la plugins/memory-setup-plugin/skills/memory-agents/ +ls -la plugins/memory-setup-plugin/skills/memory-agents/references/ + +# Verify SKILL.md structure +grep -E "^---$|^name:|question:|header:|options:" plugins/memory-setup-plugin/skills/memory-agents/SKILL.md | head -30 + +# Verify reference docs +for f in plugins/memory-setup-plugin/skills/memory-agents/references/*.md; do + echo "=== $f ===" && head -10 "$f" +done + +# Count lines +wc -l plugins/memory-setup-plugin/skills/memory-agents/SKILL.md +wc -l plugins/memory-setup-plugin/skills/memory-agents/references/*.md +``` + + + +1. SKILL.md has valid YAML frontmatter with name: memory-agents +2. Six wizard steps documented with conditional branching +3. All three modes covered (single, multi-agent, team) +4. Three reference files created (strategies, team, identifiers) +5. Agent identifier validation rules documented +6. Cross-agent query permissions explained + + + +After completion, create `.planning/phases/15-configuration-wizard-skills/15-03-SUMMARY.md` + diff --git a/.planning/phases/15-configuration-wizard-skills/15-04-PLAN.md b/.planning/phases/15-configuration-wizard-skills/15-04-PLAN.md new file mode 100644 index 0000000..f8d8dc9 --- /dev/null +++ b/.planning/phases/15-configuration-wizard-skills/15-04-PLAN.md @@ -0,0 +1,265 @@ +--- +id: 15-04 +name: Reference documentation consolidation +phase: 15 +plan: 04 +type: execute +wave: 2 +depends_on: ["15-01", "15-02"] +files_modified: + - plugins/memory-setup-plugin/skills/memory-storage/references/performance-tuning.md + - plugins/memory-setup-plugin/skills/memory-llm/references/api-testing.md + - plugins/memory-setup-plugin/skills/memory-setup/references/advanced-options.md +autonomous: true +estimated_effort: S + +must_haves: + truths: + - "Performance tuning options fully documented" + - "API testing procedures documented for all providers" + - "Advanced options document covers gap config options" + - "All reference docs link back to parent SKILL.md" + artifacts: + - path: "plugins/memory-setup-plugin/skills/memory-storage/references/performance-tuning.md" + provides: "Performance tuning documentation" + min_lines: 50 + - path: "plugins/memory-setup-plugin/skills/memory-llm/references/api-testing.md" + provides: "API testing procedures" + min_lines: 40 + - path: "plugins/memory-setup-plugin/skills/memory-setup/references/advanced-options.md" + provides: "Gap config options documentation" + min_lines: 60 + key_links: + - from: "plugins/memory-setup-plugin/skills/memory-setup/references/advanced-options.md" + to: "~/.config/memory-daemon/config.toml" + via: "config examples for gap options" + pattern: "timeout_secs|overlap_tokens|logging" +--- + + +Create additional reference documentation files that support the wizard skills and document gap configuration options that need to be covered. + +Purpose: Ensure complete documentation coverage for all configuration options and provide detailed reference materials for advanced users. + +Output: Three additional reference documentation files covering performance tuning, API testing, and gap config options. + + + +@/Users/richardhightower/.claude/get-shit-done/workflows/execute-plan.md +@/Users/richardhightower/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/15-configuration-wizard-skills/15-RESEARCH.md +@docs/plans/configuration-wizard-skills-plan.md + + + + + + Task 1: Create performance-tuning.md reference + plugins/memory-setup-plugin/skills/memory-storage/references/performance-tuning.md + +Create comprehensive performance tuning documentation. + +**Content:** +1. Overview of performance-related settings + +2. Write Buffer Size (`write_buffer_size_mb`): + | Setting | Value | Memory | Write Throughput | Use Case | + |---------|-------|--------|------------------|----------| + | Low | 16 MB | ~20 MB | Lower | Constrained systems | + | Balanced | 64 MB | ~80 MB | Good | Most users | + | High | 128 MB | ~160 MB | Best | Heavy usage | + +3. Background Jobs (`max_background_jobs`): + | Setting | Jobs | CPU Impact | Compaction Speed | + |---------|------|------------|------------------| + | Minimal | 1 | Low | Slower | + | Balanced | 4 | Moderate | Good | + | Aggressive | 8 | Higher | Fastest | + +4. When to tune: + - High write volume + - Limited memory systems + - SSD vs HDD considerations + +5. Monitoring performance: + - Check compaction lag + - Monitor memory usage + - Watch disk I/O + +6. RocksDB-specific tuning (advanced): + - Block cache size + - Bloom filters + - Compression options + +7. Configuration examples for each profile + + +```bash +ls -la plugins/memory-setup-plugin/skills/memory-storage/references/performance-tuning.md +grep -c "write_buffer\|background_jobs\|Balanced" plugins/memory-setup-plugin/skills/memory-storage/references/performance-tuning.md +``` +File exists with performance documentation + + performance-tuning.md created with tuning profiles and guidance + + + + Task 2: Create api-testing.md reference + plugins/memory-setup-plugin/skills/memory-llm/references/api-testing.md + +Create API testing procedures documentation. + +**Content:** +1. Why test API connections: + - Verify key is valid + - Confirm model access + - Check rate limits + +2. OpenAI API Testing: + ```bash + # List models (verifies key) + curl -s -H "Authorization: Bearer $OPENAI_API_KEY" \ + https://api.openai.com/v1/models | head -20 + + # Test completion (verifies model access) + curl -s -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + https://api.openai.com/v1/chat/completions \ + -d '{"model":"gpt-4o-mini","messages":[{"role":"user","content":"Hi"}],"max_tokens":10}' + ``` + +3. Anthropic API Testing: + ```bash + # Test message (verifies key and model) + curl -s -H "x-api-key: $ANTHROPIC_API_KEY" \ + -H "anthropic-version: 2023-06-01" \ + -H "Content-Type: application/json" \ + https://api.anthropic.com/v1/messages \ + -d '{"model":"claude-3-5-haiku-latest","max_tokens":10,"messages":[{"role":"user","content":"Hi"}]}' + ``` + +4. Ollama API Testing: + ```bash + # Check if running + curl -s http://localhost:11434/api/tags + + # Test generation + curl -s http://localhost:11434/api/generate \ + -d '{"model":"llama3.2:3b","prompt":"Hi","stream":false}' + ``` + +5. Common error codes and meanings: + | Code | Provider | Meaning | Resolution | + |------|----------|---------|------------| + | 401 | OpenAI/Anthropic | Invalid key | Check key format | + | 403 | OpenAI | No access | Check billing | + | 429 | All | Rate limited | Wait and retry | + | 500 | All | Server error | Try later | + +6. Troubleshooting guide + + +```bash +ls -la plugins/memory-setup-plugin/skills/memory-llm/references/api-testing.md +grep -c "curl\|API\|401\|429" plugins/memory-setup-plugin/skills/memory-llm/references/api-testing.md +``` +File exists with testing procedures + + api-testing.md created with provider-specific test commands + + + + Task 3: Create advanced-options.md for gap config options + plugins/memory-setup-plugin/skills/memory-setup/references/advanced-options.md + +Create documentation for configuration options that were identified as gaps in the coverage matrix. + +**Content:** +1. Overview: These options are available in --advanced mode of memory-setup + +2. **Server Options:** + ```toml + [server] + host = "[::1]" # Already covered + port = 50051 # Already covered + timeout_secs = 30 # GAP - request timeout + ``` + - `timeout_secs`: gRPC request timeout in seconds (default: 30) + +3. **TOC Segmentation Options:** + ```toml + [toc] + segment_min_tokens = 500 # Already covered + segment_max_tokens = 4000 # Already covered + time_gap_minutes = 30 # Already covered + overlap_tokens = 500 # GAP - token overlap for context + overlap_minutes = 5 # GAP - time overlap for context + ``` + - `overlap_tokens`: Tokens to include from previous segment (default: 500) + - `overlap_minutes`: Minutes to include from previous segment (default: 5) + +4. **Logging Options:** + ```toml + [logging] + level = "info" # GAP - trace, debug, info, warn, error + format = "pretty" # GAP - pretty, json, compact + file = "" # GAP - log file path (empty = stderr) + ``` + - `level`: Minimum log level to output + - `format`: Log output format + - `file`: Optional file path for log output + +5. When to use each option +6. Configuration examples +7. Troubleshooting logging issues + + +```bash +ls -la plugins/memory-setup-plugin/skills/memory-setup/references/advanced-options.md +grep -c "timeout_secs\|overlap_tokens\|logging" plugins/memory-setup-plugin/skills/memory-setup/references/advanced-options.md +``` +File exists with gap options documentation + + advanced-options.md created documenting all gap config options + + + + + +Run these commands to verify the plan is complete: + +```bash +# Verify all reference files exist +ls -la plugins/memory-setup-plugin/skills/memory-storage/references/performance-tuning.md +ls -la plugins/memory-setup-plugin/skills/memory-llm/references/api-testing.md +ls -la plugins/memory-setup-plugin/skills/memory-setup/references/advanced-options.md + +# Verify content +grep -l "write_buffer\|background_jobs" plugins/memory-setup-plugin/skills/memory-storage/references/*.md +grep -l "curl\|Authorization" plugins/memory-setup-plugin/skills/memory-llm/references/*.md +grep -l "timeout_secs\|overlap\|logging" plugins/memory-setup-plugin/skills/memory-setup/references/*.md + +# Count lines +wc -l plugins/memory-setup-plugin/skills/memory-storage/references/performance-tuning.md +wc -l plugins/memory-setup-plugin/skills/memory-llm/references/api-testing.md +wc -l plugins/memory-setup-plugin/skills/memory-setup/references/advanced-options.md +``` + + + +1. performance-tuning.md documents all RocksDB tuning options +2. api-testing.md has test commands for all three providers +3. advanced-options.md covers all gap config options (timeout, overlap, logging) +4. All files have clear examples and use cases +5. Files link back to parent skills + + + +After completion, create `.planning/phases/15-configuration-wizard-skills/15-04-SUMMARY.md` + diff --git a/.planning/phases/15-configuration-wizard-skills/15-05-PLAN.md b/.planning/phases/15-configuration-wizard-skills/15-05-PLAN.md new file mode 100644 index 0000000..c74783e --- /dev/null +++ b/.planning/phases/15-configuration-wizard-skills/15-05-PLAN.md @@ -0,0 +1,365 @@ +--- +id: 15-05 +name: Plugin integration and memory-setup updates +phase: 15 +plan: 05 +type: execute +wave: 3 +depends_on: ["15-01", "15-02", "15-03", "15-04"] +files_modified: + - plugins/memory-setup-plugin/.claude-plugin/marketplace.json + - plugins/memory-setup-plugin/skills/memory-setup/SKILL.md + - plugins/memory-setup-plugin/skills/memory-setup/references/wizard-questions.md +autonomous: true +estimated_effort: S + +must_haves: + truths: + - "marketplace.json includes all three new skills" + - "memory-setup SKILL.md updated with gap options in --advanced mode" + - "wizard-questions.md updated with new advanced questions" + - "All 29 config options are addressable through wizard skills" + - "Skills are discoverable via Claude Code plugin system" + artifacts: + - path: "plugins/memory-setup-plugin/.claude-plugin/marketplace.json" + provides: "Plugin manifest with all skills" + contains: "memory-storage" + - path: "plugins/memory-setup-plugin/skills/memory-setup/SKILL.md" + provides: "Updated memory-setup skill with gap options" + contains: "timeout_secs" + - path: "plugins/memory-setup-plugin/skills/memory-setup/references/wizard-questions.md" + provides: "Updated wizard questions with advanced options" + contains: "overlap_tokens" + key_links: + - from: "plugins/memory-setup-plugin/.claude-plugin/marketplace.json" + to: "plugins/memory-setup-plugin/skills/memory-storage" + via: "skills array reference" + pattern: "./skills/memory-storage" +--- + + +Integrate all new skills into the plugin manifest and update memory-setup to cover gap configuration options, ensuring 100% coverage of all 29 config options. + +Purpose: Complete the phase by wiring up all new skills for discovery and closing configuration coverage gaps. + +Output: Updated marketplace.json, updated memory-setup SKILL.md and wizard-questions.md with gap options. + + + +@/Users/richardhightower/.claude/get-shit-done/workflows/execute-plan.md +@/Users/richardhightower/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/15-configuration-wizard-skills/15-RESEARCH.md +@docs/plans/configuration-wizard-skills-plan.md +@plugins/memory-setup-plugin/.claude-plugin/marketplace.json +@plugins/memory-setup-plugin/skills/memory-setup/SKILL.md +@plugins/memory-setup-plugin/skills/memory-setup/references/wizard-questions.md + + + + + + Task 1: Update marketplace.json with new skills + plugins/memory-setup-plugin/.claude-plugin/marketplace.json + +Update the marketplace.json to include the three new skills. + +**Current structure has:** +```json +{ + "plugins": [{ + "skills": ["./skills/memory-setup"], + "commands": [...], + "agents": [...] + }] +} +``` + +**Update to:** +```json +{ + "name": "memory-setup-agentic-plugin", + "owner": { + "name": "SpillwaveSolutions", + "email": "rick@spillwave.com" + }, + "metadata": { + "description": "Setup, configure, and troubleshoot agent-memory installation with specialized wizards for storage, LLM, and multi-agent configuration", + "version": "1.1.0" + }, + "plugins": [ + { + "name": "memory-setup", + "description": "Setup, configure, and troubleshoot agent-memory installation. Use when asked to 'install agent-memory', 'setup memory', 'check memory status', 'configure memory', 'fix memory daemon', or 'troubleshoot memory'. Provides /memory-setup (interactive wizard), /memory-status (health check), /memory-config (configuration management). For advanced configuration, use /memory-storage, /memory-llm, or /memory-agents.", + "source": "./", + "strict": false, + "skills": [ + "./skills/memory-setup", + "./skills/memory-storage", + "./skills/memory-llm", + "./skills/memory-agents" + ], + "commands": [ + "./commands/memory-setup.md", + "./commands/memory-status.md", + "./commands/memory-config.md" + ], + "agents": [ + "./agents/setup-troubleshooter.md" + ] + } + ] +} +``` + +Key changes: +- Version bump to 1.1.0 +- Updated description mentioning specialized wizards +- Added three new skill paths +- Updated plugin description with cross-references + + +```bash +cat plugins/memory-setup-plugin/.claude-plugin/marketplace.json | grep -E "memory-storage|memory-llm|memory-agents" +cat plugins/memory-setup-plugin/.claude-plugin/marketplace.json | grep "version" +``` +All three new skills listed, version is 1.1.0 + + marketplace.json updated with all four skills and version 1.1.0 + + + + Task 2: Update memory-setup SKILL.md with gap options + plugins/memory-setup-plugin/skills/memory-setup/SKILL.md + +Update memory-setup SKILL.md to add the gap configuration options in --advanced mode. + +**Add new section after existing Advanced Mode section:** + +```markdown +### Advanced Configuration Options + +These options are available in `--advanced` mode: + +#### Server Timeout + +In --advanced mode, after server configuration, ask: + +``` +Configure server timeout? + +1. 30 seconds (Default) +2. 60 seconds (for slow connections) +3. Custom +``` + +Config: `[server] timeout_secs = 30` + +#### TOC Overlap Settings + +In --advanced mode, after segmentation tuning, ask: + +``` +Configure segment overlap for context continuity? + +1. Standard (500 tokens, 5 minutes) - Recommended +2. Minimal (100 tokens, 1 minute) - Less context +3. Maximum (1000 tokens, 10 minutes) - More context +4. Custom +``` + +Config: +```toml +[toc] +overlap_tokens = 500 +overlap_minutes = 5 +``` + +#### Logging Configuration + +In --advanced mode, add logging step: + +``` +Configure logging output? + +1. Info to stderr (Default) +2. Debug to stderr (verbose) +3. Debug to file +4. Custom +``` + +Config: +```toml +[logging] +level = "info" # trace, debug, info, warn, error +format = "pretty" # pretty, json, compact +file = "" # empty = stderr, or path like ~/.memory-daemon.log +``` +``` + +**Also update the Reference Files section** to add: +```markdown +- [Advanced Options](references/advanced-options.md) - Server, TOC, and logging options +``` + +**Add cross-skill navigation** at the end: +```markdown +## Related Skills + +For specialized configuration: +- `/memory-storage` - Storage paths, retention, cleanup, GDPR +- `/memory-llm` - LLM provider, model discovery, cost estimation +- `/memory-agents` - Multi-agent mode, team settings +``` + + +```bash +grep -c "timeout_secs\|overlap_tokens\|logging" plugins/memory-setup-plugin/skills/memory-setup/SKILL.md +grep "memory-storage\|memory-llm\|memory-agents" plugins/memory-setup-plugin/skills/memory-setup/SKILL.md +``` +Gap options added, cross-skill references added + + memory-setup SKILL.md updated with gap options and cross-skill navigation + + + + Task 3: Update wizard-questions.md with gap questions + plugins/memory-setup-plugin/skills/memory-setup/references/wizard-questions.md + +Update wizard-questions.md to include the new advanced questions. + +**Add after Advanced Step 3c (Segmentation Tuning):** + +```markdown +### Advanced Step 3d: Server Timeout + +**Condition:** `--advanced` flag + +``` +Configure gRPC request timeout: + +1. 30 seconds (Default) - Standard timeout +2. 60 seconds - For slow networks +3. 120 seconds - For very slow connections +4. Custom - Specify seconds +``` + +### Advanced Step 3e: Segment Overlap + +**Condition:** `--advanced` flag + +``` +Configure segment overlap for context continuity: + +1. Standard (Recommended) - 500 tokens, 5 minutes overlap +2. Minimal - 100 tokens, 1 minute overlap +3. Maximum - 1000 tokens, 10 minutes overlap +4. Custom - Specify overlap_tokens and overlap_minutes +``` + +### Advanced Step 3f: Logging + +**Condition:** `--advanced` flag + +``` +Configure logging output: + +1. Info to stderr (Default) - Standard logging +2. Debug to stderr - Verbose for troubleshooting +3. Info to file - Log to ~/.memory-daemon.log +4. Debug to file - Verbose logging to file +5. Custom - Specify level, format, and file +``` + +Options: + +| Option | level | format | file | +|--------|-------|--------|------| +| Info stderr | info | pretty | (empty) | +| Debug stderr | debug | pretty | (empty) | +| Info file | info | json | ~/.memory-daemon.log | +| Debug file | debug | json | ~/.memory-daemon.log | +``` + +**Update the question dependencies diagram** to show the new steps. + +**Add section at end:** + +```markdown +## Configuration Coverage Verification + +All 29 configuration options are now covered: + +| Section | Options | Covered By | +|---------|---------|------------| +| [storage] | path, write_buffer_size_mb, max_background_jobs | memory-setup, memory-storage | +| [server] | host, port, timeout_secs | memory-setup | +| [summarizer] | provider, model, api_key, api_endpoint, max_tokens, temperature | memory-setup, memory-llm | +| [toc] | segment_min/max_tokens, time_gap_minutes, overlap_tokens/minutes | memory-setup | +| [rollup] | min_age_hours, schedule | memory-storage | +| [logging] | level, format, file | memory-setup | +| [agents] | mode, storage_strategy, agent_id, query_scope | memory-agents | +| [retention] | policy, cleanup_schedule, archive_strategy, gdpr_mode | memory-storage | +| [team] | name, storage_path, shared | memory-agents | +``` + + +```bash +grep -c "timeout\|overlap\|logging" plugins/memory-setup-plugin/skills/memory-setup/references/wizard-questions.md +grep "Coverage\|29" plugins/memory-setup-plugin/skills/memory-setup/references/wizard-questions.md +``` +Gap questions added, coverage matrix added + + wizard-questions.md updated with gap questions and coverage verification matrix + + + + + +Run these commands to verify the plan is complete: + +```bash +# Verify marketplace.json +cat plugins/memory-setup-plugin/.claude-plugin/marketplace.json | python3 -c "import sys,json; json.load(sys.stdin); print('Valid JSON')" +grep -c "memory-storage\|memory-llm\|memory-agents" plugins/memory-setup-plugin/.claude-plugin/marketplace.json + +# Verify SKILL.md updates +grep -c "timeout_secs\|overlap\|logging" plugins/memory-setup-plugin/skills/memory-setup/SKILL.md +grep "Related Skills" plugins/memory-setup-plugin/skills/memory-setup/SKILL.md + +# Verify wizard-questions.md +grep -c "3d\|3e\|3f" plugins/memory-setup-plugin/skills/memory-setup/references/wizard-questions.md +grep "Coverage" plugins/memory-setup-plugin/skills/memory-setup/references/wizard-questions.md + +# Final directory structure verification +find plugins/memory-setup-plugin/skills -name "*.md" | wc -l +``` + +Expected: 4+ skills in marketplace.json, all gap options documented, 15+ .md files total + + + +1. marketplace.json valid JSON with all four skills listed +2. Version bumped to 1.1.0 +3. memory-setup SKILL.md has gap options section +4. Cross-skill navigation added to memory-setup +5. wizard-questions.md has three new advanced steps (3d, 3e, 3f) +6. Coverage verification matrix shows all 29 options covered + + + +After completion, create `.planning/phases/15-configuration-wizard-skills/15-05-SUMMARY.md` + +Then verify complete phase coverage by running: +```bash +# Count all config options covered +grep -r "config.toml\|[storage]\|[server]\|[summarizer]\|[toc]\|[rollup]\|[logging]\|[agents]\|[retention]\|[team]" \ + plugins/memory-setup-plugin/skills/*/SKILL.md \ + plugins/memory-setup-plugin/skills/*/references/*.md | wc -l +``` + diff --git a/.planning/phases/15-configuration-wizard-skills/15-RESEARCH.md b/.planning/phases/15-configuration-wizard-skills/15-RESEARCH.md new file mode 100644 index 0000000..2d4c917 --- /dev/null +++ b/.planning/phases/15-configuration-wizard-skills/15-RESEARCH.md @@ -0,0 +1,434 @@ +# Phase 15: Configuration Wizard Skills - Research + +**Researched:** 2026-02-01 +**Domain:** Claude Code Skills with AskUserQuestion Interactive Wizards +**Confidence:** HIGH + +## Summary + +This phase creates three interactive configuration wizard skills (`/memory-storage`, `/memory-llm`, `/memory-agents`) that extend the existing `/memory-setup` plugin. The skills use Claude Code's `AskUserQuestion` tool to guide users through advanced configuration scenarios with multi-step question flows, state detection, and conditional skip logic. + +Research confirms that the existing `memory-setup` skill provides a robust pattern to follow. The AskUserQuestion tool is well-documented and supports single/multi-select questions with headers, labels, and descriptions. Skills are defined in SKILL.md files with YAML frontmatter and markdown instructions. + +**Primary recommendation:** Follow the existing memory-setup skill structure exactly, using AskUserQuestion for interactive prompts with proper state detection before each question to skip already-configured options. + +## Standard Stack + +### Core + +| Component | Version | Purpose | Why Standard | +|-----------|---------|---------|--------------| +| Claude Code Skills | Current | Interactive wizard implementation | Native Claude Code capability | +| SKILL.md | - | Skill definition format | Standard Claude Code skill format | +| AskUserQuestion | - | Interactive user prompts | Built-in tool for multi-option selection | +| Bash | - | State detection and config generation | Standard for file/system checks | + +### Supporting + +| Component | Purpose | When to Use | +|-----------|---------|-------------| +| Write tool | Generate config files | After gathering user choices | +| Read tool | Load existing config | State detection phase | +| Glob tool | Find configuration files | State detection phase | + +### Alternatives Considered + +| Instead of | Could Use | Tradeoff | +|------------|-----------|----------| +| AskUserQuestion | Multiple single-line prompts | AskUserQuestion better for structured options with descriptions | +| SKILL.md | Command files only | Skills provide richer context and reference docs | +| Bash state detection | gRPC calls | Bash is simpler, gRPC requires daemon running | + +## Architecture Patterns + +### Recommended Project Structure + +``` +plugins/memory-setup-plugin/skills/ +├── memory-setup/ # Existing (Phase 9) +│ ├── SKILL.md +│ └── references/ +├── memory-storage/ # NEW - Phase 15 +│ ├── SKILL.md +│ └── references/ +│ ├── retention-policies.md +│ ├── gdpr-compliance.md +│ └── archive-strategies.md +├── memory-llm/ # NEW - Phase 15 +│ ├── SKILL.md +│ └── references/ +│ ├── provider-comparison.md +│ ├── model-selection.md +│ ├── cost-estimation.md +│ └── custom-endpoints.md +└── memory-agents/ # NEW - Phase 15 + ├── SKILL.md + └── references/ + ├── storage-strategies.md + ├── team-setup.md + └── agent-identifiers.md +``` + +### Pattern 1: SKILL.md Frontmatter Structure + +**What:** Standard YAML frontmatter for skill metadata +**When to use:** Every skill file + +**Example:** +```yaml +--- +name: memory-storage +description: | + This skill should be used when the user asks to "configure storage", + "set up retention policies", "configure GDPR mode", "tune memory performance", + or "change storage path". Provides interactive wizard for storage configuration. +license: MIT +metadata: + version: 1.0.0 + author: SpillwaveSolutions +--- +``` + +Source: Verified from existing `/plugins/memory-setup-plugin/skills/memory-setup/SKILL.md` + +### Pattern 2: AskUserQuestion Tool Usage + +**What:** Structured questions with header, options, and descriptions +**When to use:** Each wizard step requiring user choice + +**Example:** +```typescript +{ + questions: [ + { + question: "How long should conversation data be retained?", + header: "Retention", // max 12 chars + multiSelect: false, + options: [ + { + label: "Forever (Recommended)", + description: "Keep all data permanently for maximum historical context" + }, + { + label: "90 days", + description: "Quarter retention, good balance of history and storage" + }, + { + label: "30 days", + description: "One month retention, lower storage usage" + } + ] + } + ] +} +``` + +Source: Context7 - Claude Code documentation `/anthropics/claude-code` + +### Pattern 3: State Detection Before Questions + +**What:** Check existing configuration before asking questions +**When to use:** At start of each wizard and before each step + +**Example:** +```bash +# State detection commands +grep -A5 '\[storage\]' ~/.config/memory-daemon/config.toml 2>/dev/null | grep path +grep retention ~/.config/memory-daemon/config.toml 2>/dev/null +ls ~/.memory-archive 2>/dev/null +``` + +Source: Existing `memory-setup/SKILL.md` State Detection section + +### Pattern 4: Flag Modes (--minimal, --advanced, --fresh) + +**What:** Three execution modes for wizard behavior +**When to use:** Command invocation + +| Flag | Behavior | +|------|----------| +| (none) | Standard mode - skip completed steps, show core options | +| `--minimal` | Use defaults for everything possible, minimal questions | +| `--advanced` | Show all options including expert settings | +| `--fresh` | Ignore existing config, ask all questions, backup before overwrite | + +Source: Existing `memory-setup/references/wizard-questions.md` + +### Pattern 5: Output Formatting Consistency + +**What:** Standard visual formatting for wizard output +**When to use:** All wizard steps and status messages + +**Status Indicators:** +| Symbol | Meaning | +|--------|---------| +| `[check]` | Success/Complete | +| `[x]` | Missing/Failed | +| `[!]` | Warning | +| `[?]` | Unknown | +| `[>]` | In Progress | + +**Step Headers:** +``` +Step N of 6: Step Name +---------------------- +[Question or action content] +``` + +Source: Existing `memory-setup/SKILL.md` Output Formatting section + +### Anti-Patterns to Avoid + +- **Asking already-answered questions:** Always detect state first and skip configured options +- **Hardcoded paths:** Use platform detection for correct paths (macOS/Linux/Windows) +- **Missing backup on --fresh:** Always backup before overwriting config +- **Unclear option descriptions:** Each option must have clear, actionable description +- **Overly long headers:** AskUserQuestion headers max 12 characters +- **Too many options:** Keep to 2-4 options per question when possible + +## Don't Hand-Roll + +Problems that look simple but have existing solutions: + +| Problem | Don't Build | Use Instead | Why | +|---------|-------------|-------------|-----| +| Interactive prompts | Custom text parsing | AskUserQuestion tool | Built-in, structured, reliable | +| Config file generation | String concatenation | Heredoc with Write tool | Cleaner, proper escaping | +| Platform detection | Manual path checks | Existing platform-specifics.md patterns | Already solved | +| API key validation | Regex only | Live API test calls | Format valid but key might be revoked | +| Cron validation | Custom parser | Use memory-daemon's built-in validation | Already implemented | + +**Key insight:** The existing memory-setup skill has solved most UX patterns. Follow those patterns rather than inventing new ones. + +## Common Pitfalls + +### Pitfall 1: Forgetting State Detection + +**What goes wrong:** Wizard asks about already-configured options +**Why it happens:** Jumping to questions without checking config first +**How to avoid:** Every question block must start with state detection +**Warning signs:** User complains about redundant questions + +### Pitfall 2: Breaking Existing Config + +**What goes wrong:** --fresh flag overwrites config without backup +**Why it happens:** Missing backup step in execution flow +**How to avoid:** Always create `.bak` file before overwriting +**Warning signs:** User loses previous configuration + +### Pitfall 3: Platform Path Confusion + +**What goes wrong:** Wrong paths on different platforms +**Why it happens:** Hardcoded macOS paths used on Linux/Windows +**How to avoid:** Use platform detection at start, reference platform-specifics.md +**Warning signs:** "file not found" errors on non-macOS + +### Pitfall 4: Incomplete Config Coverage + +**What goes wrong:** Some config options not addressable through any wizard +**Why it happens:** Options added to daemon but not to wizard skills +**How to avoid:** Maintain coverage matrix (exists in docs/plans/configuration-wizard-skills-plan.md) +**Warning signs:** Users must manually edit config.toml for some options + +### Pitfall 5: AskUserQuestion Header Too Long + +**What goes wrong:** Headers get truncated or display incorrectly +**Why it happens:** Headers longer than 12 characters +**How to avoid:** Keep headers short: "Storage", "Retention", "Provider" +**Warning signs:** Truncated text in Claude Code UI + +### Pitfall 6: Missing Validation After Execution + +**What goes wrong:** Config written but not verified as working +**Why it happens:** Wizard ends immediately after writing config +**How to avoid:** Add verification step with daemon restart if needed +**Warning signs:** Config written but daemon doesn't use new values + +## Code Examples + +Verified patterns from existing implementation: + +### State Detection Flow + +```bash +# 1. Check if config file exists +CONFIG_PATH="~/.config/memory-daemon/config.toml" +ls $CONFIG_PATH 2>/dev/null && echo "CONFIG_EXISTS" || echo "NO_CONFIG" + +# 2. Check specific section +grep -A5 '\[retention\]' $CONFIG_PATH 2>/dev/null + +# 3. Check environment variables +[ -n "$OPENAI_API_KEY" ] && echo "OPENAI_KEY_SET" || echo "OPENAI_KEY_MISSING" + +# 4. Check storage usage +du -sh ~/.memory-store 2>/dev/null +df -h ~/.memory-store 2>/dev/null | tail -1 +``` + +Source: `memory-setup/SKILL.md` State Detection section + +### Config File Generation + +```bash +# Create config directory +mkdir -p ~/.config/memory-daemon + +# Generate config.toml with heredoc +cat > ~/.config/memory-daemon/config.toml << 'EOF' +[storage] +path = "~/.memory-store" +write_buffer_size_mb = 64 +max_background_jobs = 4 + +[retention] +policy = "forever" +cleanup_schedule = "0 3 * * *" +archive_strategy = "compress" +gdpr_mode = false +EOF +``` + +Source: `memory-setup/references/configuration-options.md` + +### Success Display Format + +``` +================================================== + Storage Configuration Complete! +================================================== + +[check] Storage path: ~/.memory-store (2.3 GB used) +[check] Retention policy: 90 days +[check] Cleanup schedule: Daily at 3 AM +[check] GDPR mode: Disabled + +Next steps: + * Run /memory-llm to configure summarization + * Run /memory-agents for multi-agent setup +``` + +Source: `memory-setup/SKILL.md` Output Formatting section + +### Reference File Structure + +```markdown +# Retention Policies + +## Policy Options + +| Policy | Description | Storage Impact | +|--------|-------------|----------------| +| forever | Keep all data permanently | Grows unbounded | +| days:N | Delete data older than N days | Bounded growth | + +## Cleanup Schedule + +Cleanup runs on a cron schedule... +``` + +Source: Pattern from existing `references/` files in memory-setup + +## State of the Art + +| Old Approach | Current Approach | When Changed | Impact | +|--------------|------------------|--------------|--------| +| Manual config.toml editing | Interactive wizards | Phase 9 (2026-01-31) | UX improvement | +| Single monolithic wizard | Specialized skill per domain | Phase 15 | Better modularity | +| No validation | Live API testing | Phase 9 | Catch errors early | + +**Deprecated/outdated:** +- Direct config editing: Still possible but wizards preferred for UX + +## Open Questions + +Things that couldn't be fully resolved: + +1. **GDPR Mode Implementation Details** + - What we know: GDPR mode flag exists in plan, enables complete data removal + - What's unclear: Exact implementation in memory-daemon (may not exist yet) + - Recommendation: Document as "coming feature" if not implemented, or verify with codebase search + +2. **Multi-Agent Storage Strategy** + - What we know: Plan documents unified vs separate storage + - What's unclear: Whether memory-daemon currently supports `[agents]` config section + - Recommendation: Verify with codebase grep, may need daemon updates + +3. **Budget Optimization for LLM** + - What we know: Plan mentions cost estimation and budget modes + - What's unclear: How memory-daemon tracks/enforces token budgets + - Recommendation: May be advisory only (show estimates, don't enforce) + +## Sources + +### Primary (HIGH confidence) + +- Context7 `/anthropics/claude-code` - AskUserQuestion tool documentation +- Context7 `/websites/code_claude` - Skill and command patterns +- Existing `plugins/memory-setup-plugin/skills/memory-setup/SKILL.md` - Proven patterns +- Existing `plugins/memory-setup-plugin/skills/memory-setup/references/` - Reference doc patterns +- `docs/plans/configuration-wizard-skills-plan.md` - Detailed question flows + +### Secondary (MEDIUM confidence) + +- Context7 `/affaan-m/everything-claude-code` - Additional skill patterns +- `.planning/ROADMAP.md` - Phase dependencies and success criteria + +### Tertiary (LOW confidence) + +- N/A - All critical patterns verified with primary sources + +## Metadata + +**Confidence breakdown:** +- Standard stack: HIGH - AskUserQuestion and SKILL.md patterns well-documented +- Architecture: HIGH - Follows existing memory-setup patterns exactly +- Pitfalls: HIGH - Based on actual Claude Code documentation and existing implementation + +**Research date:** 2026-02-01 +**Valid until:** 2026-03-01 (30 days - stable pattern) + +## Implementation Recommendations + +Based on research, the implementation should: + +1. **Create three skill directories** following exact structure of memory-setup +2. **Reuse marketplace.json pattern** - add new skills to existing plugin +3. **Follow question flows** from `docs/plans/configuration-wizard-skills-plan.md` +4. **Create reference docs** for each skill with detailed option explanations +5. **Update memory-setup** to add gap options (timeout_secs, overlap_*, logging.*) +6. **Verify coverage** with matrix at end of phase + +### Skills Summary + +| Skill | Config Sections | Key Questions | +|-------|-----------------|---------------| +| `/memory-storage` | `[storage]`, `[retention]`, `[rollup]` | Path, retention, cleanup, GDPR, performance | +| `/memory-llm` | `[summarizer]` | Provider, model, API key, quality, budget | +| `/memory-agents` | `[agents]`, `[team]` | Mode, storage strategy, agent ID, query scope | + +### Files to Create + +| File | Purpose | +|------|---------| +| `skills/memory-storage/SKILL.md` | Storage wizard skill | +| `skills/memory-storage/references/retention-policies.md` | Retention reference | +| `skills/memory-storage/references/gdpr-compliance.md` | GDPR reference | +| `skills/memory-storage/references/archive-strategies.md` | Archive reference | +| `skills/memory-llm/SKILL.md` | LLM wizard skill | +| `skills/memory-llm/references/provider-comparison.md` | Provider reference | +| `skills/memory-llm/references/model-selection.md` | Model reference | +| `skills/memory-llm/references/cost-estimation.md` | Cost reference | +| `skills/memory-llm/references/custom-endpoints.md` | Endpoint reference | +| `skills/memory-agents/SKILL.md` | Agent wizard skill | +| `skills/memory-agents/references/storage-strategies.md` | Storage reference | +| `skills/memory-agents/references/team-setup.md` | Team reference | +| `skills/memory-agents/references/agent-identifiers.md` | ID reference | + +### Files to Modify + +| File | Change | +|------|--------| +| `.claude-plugin/marketplace.json` | Add new skill paths | +| `skills/memory-setup/SKILL.md` | Add missing advanced options | +| `skills/memory-setup/references/wizard-questions.md` | Add missing questions | diff --git a/.rustfmt.toml b/.rustfmt.toml new file mode 100644 index 0000000..0274b1e --- /dev/null +++ b/.rustfmt.toml @@ -0,0 +1,4 @@ +edition = "2021" +max_width = 100 +tab_spaces = 4 +newline_style = "Unix" diff --git a/CLAUDE.md b/CLAUDE.md index d148633..c478a30 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,12 +29,63 @@ cargo build cargo test # Check with clippy -cargo clippy -- -D warnings +cargo clippy --workspace --all-targets --all-features -- -D warnings # Build specific crate cargo build -p memory-daemon + +# Full QA check (format + clippy + test + doc) +cargo fmt --all -- --check && \ +cargo clippy --workspace --all-targets --all-features -- -D warnings && \ +cargo test --workspace --all-features && \ +RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --workspace --all-features +``` + +## Local Skills + +Project-specific skills in `.claude/skills/`: + +| Skill | Purpose | +|-------|---------| +| `modern-rust-expert` | Rust 2024 patterns, clippy compliance, functional-but-pragmatic philosophy | +| `rust-testing` | Test patterns, assertions, parameterized tests | +| `rust-cargo-assistant` | Cargo commands and dependency management | +| `releasing-rust` | Cross-platform release workflow, versioning, artifact naming | + +## Local Agents + +Project-specific agents in `.claude/agents/`: + +| Agent | Purpose | +|-------|---------| +| `qa-rust-agent` | Enforces code quality after Rust file changes (format, clippy, test, doc) | + +## CI/CD Workflows + +GitHub Actions workflows in `.github/workflows/`: + +| Workflow | Trigger | Purpose | +|----------|---------|---------| +| `ci.yml` | Push to main, PRs | Format, clippy, test, build, doc checks | +| `release.yml` | Tags `v*.*.*`, manual | Multi-platform release builds | + +### Release Process + +```bash +# Bump version +cargo set-version 0.2.0 + +# Commit and tag +git add -A && git commit -m "chore: release v0.2.0" +git tag -a v0.2.0 -m "Release v0.2.0" +git push origin main --tags ``` +The release workflow automatically builds for: +- Linux x86_64 / ARM64 +- macOS Intel / Apple Silicon +- Windows x86_64 + ## GSD Workflow This project uses the Get Shit Done (GSD) workflow: diff --git a/Cargo.toml b/Cargo.toml index 1de4f6c..ad9f714 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,11 @@ members = [ "crates/memory-client", "crates/memory-ingest", "crates/memory-scheduler", + "crates/memory-search", + "crates/memory-embeddings", + "crates/memory-vector", + "crates/memory-indexing", + "crates/memory-topics", ] [workspace.package] @@ -23,7 +28,13 @@ memory-types = { path = "crates/memory-types" } memory-storage = { path = "crates/memory-storage" } memory-service = { path = "crates/memory-service" } memory-client = { path = "crates/memory-client" } +memory-toc = { path = "crates/memory-toc" } memory-scheduler = { path = "crates/memory-scheduler" } +memory-search = { path = "crates/memory-search" } +memory-embeddings = { path = "crates/memory-embeddings" } +memory-vector = { path = "crates/memory-vector" } +memory-indexing = { path = "crates/memory-indexing" } +memory-topics = { path = "crates/memory-topics" } # Async runtime tokio = { version = "1.43", features = ["full"] } @@ -51,6 +62,9 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } # Storage (to be used in later phases) rocksdb = "0.22" +# Full-text search +tantivy = "0.25" + # ID generation ulid = "1.1" @@ -82,3 +96,23 @@ backoff = { version = "0.4", features = ["tokio"] } # Secret handling secrecy = { version = "0.10", features = ["serde"] } + +# Candle ML framework +candle-core = "0.8" +candle-nn = "0.8" +candle-transformers = "0.8" + +# Tokenization +tokenizers = "0.20" + +# Model downloading +hf-hub = "0.3" + +# Directory utilities +dirs = "5" + +# Vector search +usearch = "2" + +# Clustering +hdbscan = "0.12" diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 0000000..938980f --- /dev/null +++ b/clippy.toml @@ -0,0 +1,3 @@ +cognitive-complexity-threshold = 25 +too-many-lines-threshold = 200 +too-many-arguments-threshold = 7 diff --git a/crates/memory-client/src/client.rs b/crates/memory-client/src/client.rs index 6d262c5..ebf2e5c 100644 --- a/crates/memory-client/src/client.rs +++ b/crates/memory-client/src/client.rs @@ -6,18 +6,13 @@ use tonic::transport::Channel; use tracing::{debug, info}; use memory_service::pb::{ - memory_service_client::MemoryServiceClient, - BrowseTocRequest, - Event as ProtoEvent, - EventRole as ProtoEventRole, - EventType as ProtoEventType, - ExpandGripRequest, - GetEventsRequest, - GetNodeRequest, - GetTocRootRequest, - IngestEventRequest, - TocNode as ProtoTocNode, - Grip as ProtoGrip, + memory_service_client::MemoryServiceClient, BrowseTocRequest, Event as ProtoEvent, + EventRole as ProtoEventRole, EventType as ProtoEventType, ExpandGripRequest, GetEventsRequest, + GetNodeRequest, GetRelatedTopicsRequest, GetTocRootRequest, GetTopTopicsRequest, + GetTopicGraphStatusRequest, GetTopicsByQueryRequest, GetVectorIndexStatusRequest, + Grip as ProtoGrip, HybridSearchRequest, HybridSearchResponse, IngestEventRequest, + TeleportSearchRequest, TeleportSearchResponse, TocNode as ProtoTocNode, Topic as ProtoTopic, + VectorIndexStatus, VectorTeleportRequest, VectorTeleportResponse, }; use memory_types::{Event, EventRole, EventType}; @@ -36,7 +31,7 @@ impl MemoryClient { /// /// # Arguments /// - /// * `endpoint` - The gRPC endpoint (e.g., "http://[::1]:50051") + /// * `endpoint` - The gRPC endpoint (e.g., `http://localhost:50051`) /// /// # Errors /// @@ -154,7 +149,10 @@ impl MemoryClient { to_timestamp_ms: i64, limit: u32, ) -> Result { - debug!("GetEvents request: from={} to={} limit={}", from_timestamp_ms, to_timestamp_ms, limit); + debug!( + "GetEvents request: from={} to={} limit={}", + from_timestamp_ms, to_timestamp_ms, limit + ); let request = tonic::Request::new(GetEventsRequest { from_timestamp_ms, to_timestamp_ms, @@ -192,6 +190,203 @@ impl MemoryClient { events_after: resp.events_after, }) } + + // ===== Teleport Search Methods ===== + + /// Search for TOC nodes or grips using BM25 keyword search. + /// + /// Per TEL-02: BM25 search returns ranked results. + pub async fn teleport_search( + &mut self, + query: &str, + doc_type: i32, + limit: i32, + ) -> Result { + debug!("TeleportSearch request: query={}", query); + let request = tonic::Request::new(TeleportSearchRequest { + query: query.to_string(), + doc_type, + limit, + }); + let response = self.inner.teleport_search(request).await?; + Ok(response.into_inner()) + } + + // ===== Vector Search Methods ===== + + /// Search for TOC nodes or grips using vector semantic search. + /// + /// Per VEC-01: Vector similarity search using HNSW index. + /// + /// # Arguments + /// + /// * `query` - Query text to embed and search + /// * `top_k` - Number of results to return + /// * `min_score` - Minimum similarity score (0.0-1.0) + /// * `target` - Target type filter (0=unspecified, 1=toc, 2=grip, 3=all) + pub async fn vector_teleport( + &mut self, + query: &str, + top_k: i32, + min_score: f32, + target: i32, + ) -> Result { + debug!("VectorTeleport request: query={}", query); + let request = tonic::Request::new(VectorTeleportRequest { + query: query.to_string(), + top_k, + min_score, + time_filter: None, + target, + }); + let response = self.inner.vector_teleport(request).await?; + Ok(response.into_inner()) + } + + /// Hybrid BM25 + vector search using RRF fusion. + /// + /// Per VEC-02: Combines keyword and semantic matching. + /// + /// # Arguments + /// + /// * `query` - Search query + /// * `top_k` - Number of results to return + /// * `mode` - Search mode (0=unspecified, 1=vector-only, 2=bm25-only, 3=hybrid) + /// * `bm25_weight` - Weight for BM25 in fusion (0.0-1.0) + /// * `vector_weight` - Weight for vector in fusion (0.0-1.0) + /// * `target` - Target type filter (0=unspecified, 1=toc, 2=grip, 3=all) + pub async fn hybrid_search( + &mut self, + query: &str, + top_k: i32, + mode: i32, + bm25_weight: f32, + vector_weight: f32, + target: i32, + ) -> Result { + debug!("HybridSearch request: query={}, mode={}", query, mode); + let request = tonic::Request::new(HybridSearchRequest { + query: query.to_string(), + top_k, + mode, + bm25_weight, + vector_weight, + time_filter: None, + target, + }); + let response = self.inner.hybrid_search(request).await?; + Ok(response.into_inner()) + } + + /// Get vector index status and statistics. + /// + /// Per VEC-03: Observable index health and stats. + pub async fn get_vector_index_status(&mut self) -> Result { + debug!("GetVectorIndexStatus request"); + let request = tonic::Request::new(GetVectorIndexStatusRequest {}); + let response = self.inner.get_vector_index_status(request).await?; + Ok(response.into_inner()) + } + + // ===== Topic Graph Methods (Phase 14) ===== + + /// Get topic graph status and statistics. + /// + /// Per TOPIC-08: Topic graph discovery. + pub async fn get_topic_graph_status(&mut self) -> Result { + debug!("GetTopicGraphStatus request"); + let request = tonic::Request::new(GetTopicGraphStatusRequest {}); + let response = self.inner.get_topic_graph_status(request).await?; + let resp = response.into_inner(); + Ok(TopicGraphStatus { + topic_count: resp.topic_count, + relationship_count: resp.relationship_count, + last_updated: resp.last_updated, + available: resp.available, + }) + } + + /// Get topics matching a query. + /// + /// Searches topic labels and keywords for matches. + /// + /// # Arguments + /// + /// * `query` - Search query (keywords to match) + /// * `limit` - Maximum results to return + pub async fn get_topics_by_query( + &mut self, + query: &str, + limit: u32, + ) -> Result, ClientError> { + debug!("GetTopicsByQuery request: query={}", query); + let request = tonic::Request::new(GetTopicsByQueryRequest { + query: query.to_string(), + limit, + }); + let response = self.inner.get_topics_by_query(request).await?; + Ok(response.into_inner().topics) + } + + /// Get topics related to a specific topic. + /// + /// # Arguments + /// + /// * `topic_id` - Topic to find related topics for + /// * `rel_type` - Optional relationship type filter ("co-occurrence", "semantic", "hierarchical") + /// * `limit` - Maximum results to return + pub async fn get_related_topics( + &mut self, + topic_id: &str, + rel_type: Option<&str>, + limit: u32, + ) -> Result { + debug!("GetRelatedTopics request: topic_id={}", topic_id); + let request = tonic::Request::new(GetRelatedTopicsRequest { + topic_id: topic_id.to_string(), + relationship_type: rel_type.unwrap_or("").to_string(), + limit, + }); + let response = self.inner.get_related_topics(request).await?; + let resp = response.into_inner(); + Ok(RelatedTopicsResult { + related_topics: resp.related_topics, + relationships: resp.relationships, + }) + } + + /// Get top topics by importance score. + /// + /// # Arguments + /// + /// * `limit` - Maximum results to return + /// * `days` - Look back window in days for importance calculation + pub async fn get_top_topics( + &mut self, + limit: u32, + days: u32, + ) -> Result, ClientError> { + debug!("GetTopTopics request: limit={}, days={}", limit, days); + let request = tonic::Request::new(GetTopTopicsRequest { limit, days }); + let response = self.inner.get_top_topics(request).await?; + Ok(response.into_inner().topics) + } +} + +/// Topic graph status. +#[derive(Debug)] +pub struct TopicGraphStatus { + pub topic_count: u64, + pub relationship_count: u64, + pub last_updated: String, + pub available: bool, +} + +/// Result of get_related_topics operation. +#[derive(Debug)] +pub struct RelatedTopicsResult { + pub related_topics: Vec, + pub relationships: Vec, } /// Result of browse_toc operation. @@ -287,7 +482,8 @@ mod tests { EventType::ToolResult, EventRole::Tool, "Result".to_string(), - ).with_metadata(metadata); + ) + .with_metadata(metadata); let proto = event_to_proto(event); @@ -300,7 +496,10 @@ mod tests { let types = vec![ (EventType::SessionStart, ProtoEventType::SessionStart), (EventType::UserMessage, ProtoEventType::UserMessage), - (EventType::AssistantMessage, ProtoEventType::AssistantMessage), + ( + EventType::AssistantMessage, + ProtoEventType::AssistantMessage, + ), (EventType::ToolResult, ProtoEventType::ToolResult), (EventType::AssistantStop, ProtoEventType::AssistantStop), (EventType::SubagentStart, ProtoEventType::SubagentStart), diff --git a/crates/memory-client/src/hook_mapping.rs b/crates/memory-client/src/hook_mapping.rs index 870ff50..dd28c2e 100644 --- a/crates/memory-client/src/hook_mapping.rs +++ b/crates/memory-client/src/hook_mapping.rs @@ -46,7 +46,11 @@ pub struct HookEvent { impl HookEvent { /// Create a new hook event. - pub fn new(session_id: impl Into, event_type: HookEventType, content: impl Into) -> Self { + pub fn new( + session_id: impl Into, + event_type: HookEventType, + content: impl Into, + ) -> Self { Self { session_id: session_id.into(), event_type, diff --git a/crates/memory-client/src/lib.rs b/crates/memory-client/src/lib.rs index ac71bb0..0d3a3fc 100644 --- a/crates/memory-client/src/lib.rs +++ b/crates/memory-client/src/lib.rs @@ -12,7 +12,7 @@ //! #[tokio::main] //! async fn main() -> Result<(), Box> { //! // Connect to daemon -//! let mut client = MemoryClient::connect("http://[::1]:50051").await?; +//! let mut client = MemoryClient::connect("http://localhost:50051").await?; //! //! // Create a hook event //! let hook = HookEvent::new("session-1", HookEventType::UserPromptSubmit, "Hello!"); @@ -36,11 +36,15 @@ pub mod error; pub mod hook_mapping; pub use client::{ - BrowseTocResult, ExpandGripResult, GetEventsResult, - MemoryClient, DEFAULT_ENDPOINT, + BrowseTocResult, ExpandGripResult, GetEventsResult, MemoryClient, DEFAULT_ENDPOINT, }; + +// Re-export vector search response types for convenience pub use error::ClientError; pub use hook_mapping::{map_hook_event, HookEvent, HookEventType}; +pub use memory_service::pb::{ + HybridSearchResponse, VectorIndexStatus, VectorMatch, VectorTeleportResponse, +}; // Re-export Event type for convenience pub use memory_types::Event; diff --git a/crates/memory-daemon/Cargo.toml b/crates/memory-daemon/Cargo.toml index 6e4984a..6b7bd8e 100644 --- a/crates/memory-daemon/Cargo.toml +++ b/crates/memory-daemon/Cargo.toml @@ -15,6 +15,11 @@ memory-service = { workspace = true } memory-client = { workspace = true } memory-scheduler = { workspace = true } memory-toc = { path = "../memory-toc" } +memory-indexing = { workspace = true } +memory-search = { workspace = true } +memory-vector = { workspace = true } +memory-embeddings = { workspace = true } +memory-topics = { workspace = true } tokio = { workspace = true } tonic = { workspace = true } clap = { workspace = true } diff --git a/crates/memory-daemon/examples/ingest_demo.rs b/crates/memory-daemon/examples/ingest_demo.rs index 1e170bf..ec01824 100644 --- a/crates/memory-daemon/examples/ingest_demo.rs +++ b/crates/memory-daemon/examples/ingest_demo.rs @@ -17,8 +17,8 @@ async fn main() -> Result<(), Box> { // Initialize tracing tracing_subscriber::fmt::init(); - let endpoint = std::env::var("MEMORY_ENDPOINT") - .unwrap_or_else(|_| "http://[::1]:50051".to_string()); + let endpoint = + std::env::var("MEMORY_ENDPOINT").unwrap_or_else(|_| "http://[::1]:50051".to_string()); println!("Connecting to memory daemon at {}...", endpoint); @@ -54,11 +54,11 @@ async fn main() -> Result<(), Box> { Each grip contains an excerpt and pointers to the original events, \ providing provenance for summary claims.", ), - (HookEventType::UserPromptSubmit, "Can you show me a tool use?"), ( - HookEventType::ToolUse, - "Reading file /tmp/example.txt...", + HookEventType::UserPromptSubmit, + "Can you show me a tool use?", ), + (HookEventType::ToolUse, "Reading file /tmp/example.txt..."), ( HookEventType::ToolResult, "File contents: Hello from the example file!", @@ -78,8 +78,10 @@ async fn main() -> Result<(), Box> { let hook_event = HookEvent::new(session_id.clone(), event_type.clone(), *content); // Add tool name for tool events - let hook_event = if matches!(event_type, HookEventType::ToolUse | HookEventType::ToolResult) - { + let hook_event = if matches!( + event_type, + HookEventType::ToolUse | HookEventType::ToolResult + ) { hook_event.with_tool_name("Read") } else { hook_event @@ -109,7 +111,10 @@ async fn main() -> Result<(), Box> { ); println!(); println!("You can now query the events using:"); - println!(" cargo run --bin memory-daemon -- query --endpoint {} root", endpoint); + println!( + " cargo run --bin memory-daemon -- query --endpoint {} root", + endpoint + ); println!(" cargo run --bin memory-daemon -- query --endpoint {} events --from --to ", endpoint); Ok(()) diff --git a/crates/memory-daemon/src/cli.rs b/crates/memory-daemon/src/cli.rs index 8d985cc..b01061c 100644 --- a/crates/memory-daemon/src/cli.rs +++ b/crates/memory-daemon/src/cli.rs @@ -50,7 +50,7 @@ pub enum Commands { /// Query the memory system Query { - /// gRPC endpoint (default: http://[::1]:50051) + /// gRPC endpoint (default: `http://[::1]:50051`) #[arg(short, long, default_value = "http://[::1]:50051")] endpoint: String, @@ -70,13 +70,21 @@ pub enum Commands { /// Scheduler management commands Scheduler { - /// gRPC endpoint (default: http://[::1]:50051) + /// gRPC endpoint (default: `http://[::1]:50051`) #[arg(short, long, default_value = "http://[::1]:50051")] endpoint: String, #[command(subcommand)] command: SchedulerCommands, }, + + /// Teleport (BM25 keyword search) commands + #[command(subcommand)] + Teleport(TeleportCommand), + + /// Topic graph management commands + #[command(subcommand)] + Topics(TopicsCommand), } /// Query subcommands @@ -133,6 +141,29 @@ pub enum QueryCommands { #[arg(long, default_value = "3")] after: u32, }, + + /// Search TOC nodes for matching content + Search { + /// Search query terms (space-separated) + #[arg(short, long)] + query: String, + + /// Search within a specific node (mutually exclusive with --parent) + #[arg(long, conflicts_with = "parent")] + node: Option, + + /// Search children of a parent node (empty for root level) + #[arg(long, conflicts_with = "node")] + parent: Option, + + /// Fields to search: title, summary, bullets, keywords (comma-separated) + #[arg(long)] + fields: Option, + + /// Maximum results to return + #[arg(long, default_value = "10")] + limit: u32, + }, } /// Admin subcommands @@ -158,6 +189,59 @@ pub enum AdminCommands { #[arg(long)] dry_run: bool, }, + + /// Rebuild search indexes from storage + RebuildIndexes { + /// Which index to rebuild: bm25, vector, or all + #[arg(long, default_value = "all")] + index: String, + + /// Batch size for processing (progress reported after each batch) + #[arg(long, default_value = "100")] + batch_size: usize, + + /// Skip confirmation prompt + #[arg(long)] + force: bool, + + /// Path to search index directory (default from config) + #[arg(long)] + search_path: Option, + + /// Path to vector index directory (default from config) + #[arg(long)] + vector_path: Option, + }, + + /// Show search index statistics + IndexStats { + /// Path to search index directory (default from config) + #[arg(long)] + search_path: Option, + + /// Path to vector index directory (default from config) + #[arg(long)] + vector_path: Option, + }, + + /// Clear and reset a search index + ClearIndex { + /// Which index to clear: bm25, vector, or all + #[arg(long)] + index: String, + + /// Skip confirmation prompt + #[arg(long)] + force: bool, + + /// Path to search index directory (default from config) + #[arg(long)] + search_path: Option, + + /// Path to vector index directory (default from config) + #[arg(long)] + vector_path: Option, + }, } /// Scheduler subcommands @@ -179,6 +263,183 @@ pub enum SchedulerCommands { }, } +/// Teleport (BM25 search) commands +#[derive(Subcommand, Debug, Clone)] +pub enum TeleportCommand { + /// Search for TOC nodes or grips by keyword (BM25) + Search { + /// Search query (keywords) + query: String, + + /// Filter by document type: all, toc, grip + #[arg(long, short = 't', default_value = "all")] + doc_type: String, + + /// Maximum results to return + #[arg(long, short = 'n', default_value = "10")] + limit: usize, + + /// gRPC server address + #[arg(long, default_value = "http://[::1]:50051")] + addr: String, + }, + + /// Semantic similarity search using vector embeddings + VectorSearch { + /// Search query text + #[arg(short, long)] + query: String, + + /// Number of results to return + #[arg(long, default_value = "10")] + top_k: i32, + + /// Minimum similarity score (0.0-1.0) + #[arg(long, default_value = "0.0")] + min_score: f32, + + /// Filter by target type: all, toc, grip + #[arg(long, default_value = "all")] + target: String, + + /// gRPC server address + #[arg(long, default_value = "http://[::1]:50051")] + addr: String, + }, + + /// Combined BM25 + vector search using RRF fusion + HybridSearch { + /// Search query text + #[arg(short, long)] + query: String, + + /// Number of results to return + #[arg(long, default_value = "10")] + top_k: i32, + + /// Search mode: hybrid, vector-only, bm25-only + #[arg(long, default_value = "hybrid")] + mode: String, + + /// Weight for BM25 in fusion (0.0-1.0) + #[arg(long, default_value = "0.5")] + bm25_weight: f32, + + /// Weight for vector in fusion (0.0-1.0) + #[arg(long, default_value = "0.5")] + vector_weight: f32, + + /// Filter by target type: all, toc, grip + #[arg(long, default_value = "all")] + target: String, + + /// gRPC server address + #[arg(long, default_value = "http://[::1]:50051")] + addr: String, + }, + + /// Show BM25 index statistics + Stats { + /// gRPC server address + #[arg(long, default_value = "http://[::1]:50051")] + addr: String, + }, + + /// Show vector index statistics + VectorStats { + /// gRPC server address + #[arg(long, default_value = "http://[::1]:50051")] + addr: String, + }, + + /// Rebuild the search index from storage + Rebuild { + /// gRPC server address (for triggering rebuild) + #[arg(long, default_value = "http://[::1]:50051")] + addr: String, + }, +} + +/// Topics (topic graph) commands +#[derive(Subcommand, Debug, Clone)] +pub enum TopicsCommand { + /// Show topic graph status and lifecycle stats + Status { + /// gRPC server address + #[arg(long, default_value = "http://[::1]:50051")] + addr: String, + }, + + /// List topics matching a query + Explore { + /// Search query (keywords to match against topic labels/keywords) + query: String, + + /// Maximum results to return + #[arg(long, short = 'n', default_value = "10")] + limit: u32, + + /// gRPC server address + #[arg(long, default_value = "http://[::1]:50051")] + addr: String, + }, + + /// Show related topics for a given topic + Related { + /// Topic ID to find related topics for + topic_id: String, + + /// Filter by relationship type (co-occurrence, semantic, hierarchical) + #[arg(long, short = 't')] + rel_type: Option, + + /// Maximum results to return + #[arg(long, short = 'n', default_value = "10")] + limit: u32, + + /// gRPC server address + #[arg(long, default_value = "http://[::1]:50051")] + addr: String, + }, + + /// Show top topics by importance score + Top { + /// Maximum results to return + #[arg(long, short = 'n', default_value = "10")] + limit: u32, + + /// Look back window in days (default: 30) + #[arg(long, default_value = "30")] + days: u32, + + /// gRPC server address + #[arg(long, default_value = "http://[::1]:50051")] + addr: String, + }, + + /// Trigger importance score refresh + RefreshScores { + /// Database path (default from config) + #[arg(long)] + db_path: Option, + }, + + /// Archive stale topics not mentioned in N days + Prune { + /// Days of inactivity before pruning (default: 90) + #[arg(long, default_value = "90")] + days: u32, + + /// Skip confirmation prompt + #[arg(long)] + force: bool, + + /// Database path (default from config) + #[arg(long)] + db_path: Option, + }, +} + impl Cli { /// Parse CLI arguments pub fn parse_args() -> Self { @@ -296,4 +557,630 @@ mod tests { _ => panic!("Expected Scheduler command"), } } + + #[test] + fn test_cli_teleport_search() { + let cli = Cli::parse_from(["memory-daemon", "teleport", "search", "rust memory"]); + match cli.command { + Commands::Teleport(TeleportCommand::Search { + query, + doc_type, + limit, + .. + }) => { + assert_eq!(query, "rust memory"); + assert_eq!(doc_type, "all"); + assert_eq!(limit, 10); + } + _ => panic!("Expected Teleport Search command"), + } + } + + #[test] + fn test_cli_teleport_search_with_options() { + let cli = Cli::parse_from([ + "memory-daemon", + "teleport", + "search", + "rust memory", + "-t", + "toc", + "-n", + "5", + "--addr", + "http://localhost:9999", + ]); + match cli.command { + Commands::Teleport(TeleportCommand::Search { + query, + doc_type, + limit, + addr, + }) => { + assert_eq!(query, "rust memory"); + assert_eq!(doc_type, "toc"); + assert_eq!(limit, 5); + assert_eq!(addr, "http://localhost:9999"); + } + _ => panic!("Expected Teleport Search command"), + } + } + + #[test] + fn test_cli_teleport_stats() { + let cli = Cli::parse_from(["memory-daemon", "teleport", "stats"]); + match cli.command { + Commands::Teleport(TeleportCommand::Stats { addr }) => { + assert_eq!(addr, "http://[::1]:50051"); + } + _ => panic!("Expected Teleport Stats command"), + } + } + + #[test] + fn test_cli_teleport_rebuild() { + let cli = Cli::parse_from([ + "memory-daemon", + "teleport", + "rebuild", + "--addr", + "http://localhost:9999", + ]); + match cli.command { + Commands::Teleport(TeleportCommand::Rebuild { addr }) => { + assert_eq!(addr, "http://localhost:9999"); + } + _ => panic!("Expected Teleport Rebuild command"), + } + } + + #[test] + fn test_cli_teleport_vector_search() { + let cli = Cli::parse_from([ + "memory-daemon", + "teleport", + "vector-search", + "--query", + "authentication patterns", + ]); + match cli.command { + Commands::Teleport(TeleportCommand::VectorSearch { + query, + top_k, + min_score, + target, + .. + }) => { + assert_eq!(query, "authentication patterns"); + assert_eq!(top_k, 10); + assert!((min_score - 0.0).abs() < f32::EPSILON); + assert_eq!(target, "all"); + } + _ => panic!("Expected Teleport VectorSearch command"), + } + } + + #[test] + fn test_cli_teleport_vector_search_with_options() { + let cli = Cli::parse_from([ + "memory-daemon", + "teleport", + "vector-search", + "-q", + "rust patterns", + "--top-k", + "5", + "--min-score", + "0.7", + "--target", + "toc", + "--addr", + "http://localhost:9999", + ]); + match cli.command { + Commands::Teleport(TeleportCommand::VectorSearch { + query, + top_k, + min_score, + target, + addr, + }) => { + assert_eq!(query, "rust patterns"); + assert_eq!(top_k, 5); + assert!((min_score - 0.7).abs() < f32::EPSILON); + assert_eq!(target, "toc"); + assert_eq!(addr, "http://localhost:9999"); + } + _ => panic!("Expected Teleport VectorSearch command"), + } + } + + #[test] + fn test_cli_teleport_hybrid_search() { + let cli = Cli::parse_from([ + "memory-daemon", + "teleport", + "hybrid-search", + "--query", + "memory systems", + ]); + match cli.command { + Commands::Teleport(TeleportCommand::HybridSearch { + query, + top_k, + mode, + bm25_weight, + vector_weight, + target, + .. + }) => { + assert_eq!(query, "memory systems"); + assert_eq!(top_k, 10); + assert_eq!(mode, "hybrid"); + assert!((bm25_weight - 0.5).abs() < f32::EPSILON); + assert!((vector_weight - 0.5).abs() < f32::EPSILON); + assert_eq!(target, "all"); + } + _ => panic!("Expected Teleport HybridSearch command"), + } + } + + #[test] + fn test_cli_teleport_hybrid_search_with_options() { + let cli = Cli::parse_from([ + "memory-daemon", + "teleport", + "hybrid-search", + "-q", + "debugging", + "--top-k", + "20", + "--mode", + "vector-only", + "--bm25-weight", + "0.3", + "--vector-weight", + "0.7", + "--target", + "grip", + ]); + match cli.command { + Commands::Teleport(TeleportCommand::HybridSearch { + query, + top_k, + mode, + bm25_weight, + vector_weight, + target, + .. + }) => { + assert_eq!(query, "debugging"); + assert_eq!(top_k, 20); + assert_eq!(mode, "vector-only"); + assert!((bm25_weight - 0.3).abs() < f32::EPSILON); + assert!((vector_weight - 0.7).abs() < f32::EPSILON); + assert_eq!(target, "grip"); + } + _ => panic!("Expected Teleport HybridSearch command"), + } + } + + #[test] + fn test_cli_teleport_vector_stats() { + let cli = Cli::parse_from(["memory-daemon", "teleport", "vector-stats"]); + match cli.command { + Commands::Teleport(TeleportCommand::VectorStats { addr }) => { + assert_eq!(addr, "http://[::1]:50051"); + } + _ => panic!("Expected Teleport VectorStats command"), + } + } + + #[test] + fn test_cli_query_search() { + let cli = Cli::parse_from([ + "memory-daemon", + "query", + "search", + "--query", + "JWT authentication", + ]); + match cli.command { + Commands::Query { command, .. } => match command { + QueryCommands::Search { + query, + node, + parent, + fields, + limit, + } => { + assert_eq!(query, "JWT authentication"); + assert!(node.is_none()); + assert!(parent.is_none()); + assert!(fields.is_none()); + assert_eq!(limit, 10); + } + _ => panic!("Expected Search command"), + }, + _ => panic!("Expected Query command"), + } + } + + #[test] + fn test_cli_query_search_with_node() { + let cli = Cli::parse_from([ + "memory-daemon", + "query", + "search", + "--query", + "debugging", + "--node", + "toc:month:2026-01", + ]); + match cli.command { + Commands::Query { command, .. } => match command { + QueryCommands::Search { + query, + node, + parent, + .. + } => { + assert_eq!(query, "debugging"); + assert_eq!(node, Some("toc:month:2026-01".to_string())); + assert!(parent.is_none()); + } + _ => panic!("Expected Search command"), + }, + _ => panic!("Expected Query command"), + } + } + + #[test] + fn test_cli_query_search_with_parent() { + let cli = Cli::parse_from([ + "memory-daemon", + "query", + "search", + "--query", + "token", + "--parent", + "toc:week:2026-W04", + "--fields", + "title,bullets", + "--limit", + "20", + ]); + match cli.command { + Commands::Query { command, .. } => match command { + QueryCommands::Search { + query, + node, + parent, + fields, + limit, + } => { + assert_eq!(query, "token"); + assert!(node.is_none()); + assert_eq!(parent, Some("toc:week:2026-W04".to_string())); + assert_eq!(fields, Some("title,bullets".to_string())); + assert_eq!(limit, 20); + } + _ => panic!("Expected Search command"), + }, + _ => panic!("Expected Query command"), + } + } + + #[test] + fn test_cli_admin_rebuild_indexes_defaults() { + let cli = Cli::parse_from(["memory-daemon", "admin", "rebuild-indexes"]); + match cli.command { + Commands::Admin { command, .. } => match command { + AdminCommands::RebuildIndexes { + index, + batch_size, + force, + search_path, + vector_path, + } => { + assert_eq!(index, "all"); + assert_eq!(batch_size, 100); + assert!(!force); + assert!(search_path.is_none()); + assert!(vector_path.is_none()); + } + _ => panic!("Expected RebuildIndexes command"), + }, + _ => panic!("Expected Admin command"), + } + } + + #[test] + fn test_cli_admin_rebuild_indexes_with_options() { + let cli = Cli::parse_from([ + "memory-daemon", + "admin", + "rebuild-indexes", + "--index", + "bm25", + "--batch-size", + "50", + "--force", + "--search-path", + "/custom/search", + ]); + match cli.command { + Commands::Admin { command, .. } => match command { + AdminCommands::RebuildIndexes { + index, + batch_size, + force, + search_path, + vector_path, + } => { + assert_eq!(index, "bm25"); + assert_eq!(batch_size, 50); + assert!(force); + assert_eq!(search_path, Some("/custom/search".to_string())); + assert!(vector_path.is_none()); + } + _ => panic!("Expected RebuildIndexes command"), + }, + _ => panic!("Expected Admin command"), + } + } + + #[test] + fn test_cli_admin_index_stats() { + let cli = Cli::parse_from(["memory-daemon", "admin", "index-stats"]); + match cli.command { + Commands::Admin { command, .. } => match command { + AdminCommands::IndexStats { + search_path, + vector_path, + } => { + assert!(search_path.is_none()); + assert!(vector_path.is_none()); + } + _ => panic!("Expected IndexStats command"), + }, + _ => panic!("Expected Admin command"), + } + } + + #[test] + fn test_cli_admin_clear_index() { + let cli = Cli::parse_from([ + "memory-daemon", + "admin", + "clear-index", + "--index", + "vector", + "--force", + ]); + match cli.command { + Commands::Admin { command, .. } => match command { + AdminCommands::ClearIndex { + index, + force, + search_path, + vector_path, + } => { + assert_eq!(index, "vector"); + assert!(force); + assert!(search_path.is_none()); + assert!(vector_path.is_none()); + } + _ => panic!("Expected ClearIndex command"), + }, + _ => panic!("Expected Admin command"), + } + } + + #[test] + fn test_cli_topics_status() { + let cli = Cli::parse_from(["memory-daemon", "topics", "status"]); + match cli.command { + Commands::Topics(TopicsCommand::Status { addr }) => { + assert_eq!(addr, "http://[::1]:50051"); + } + _ => panic!("Expected Topics Status command"), + } + } + + #[test] + fn test_cli_topics_status_with_addr() { + let cli = Cli::parse_from([ + "memory-daemon", + "topics", + "status", + "--addr", + "http://localhost:9999", + ]); + match cli.command { + Commands::Topics(TopicsCommand::Status { addr }) => { + assert_eq!(addr, "http://localhost:9999"); + } + _ => panic!("Expected Topics Status command"), + } + } + + #[test] + fn test_cli_topics_explore() { + let cli = Cli::parse_from(["memory-daemon", "topics", "explore", "rust memory"]); + match cli.command { + Commands::Topics(TopicsCommand::Explore { + query, limit, addr, .. + }) => { + assert_eq!(query, "rust memory"); + assert_eq!(limit, 10); + assert_eq!(addr, "http://[::1]:50051"); + } + _ => panic!("Expected Topics Explore command"), + } + } + + #[test] + fn test_cli_topics_explore_with_options() { + let cli = Cli::parse_from([ + "memory-daemon", + "topics", + "explore", + "authentication", + "-n", + "5", + "--addr", + "http://localhost:9999", + ]); + match cli.command { + Commands::Topics(TopicsCommand::Explore { query, limit, addr }) => { + assert_eq!(query, "authentication"); + assert_eq!(limit, 5); + assert_eq!(addr, "http://localhost:9999"); + } + _ => panic!("Expected Topics Explore command"), + } + } + + #[test] + fn test_cli_topics_related() { + let cli = Cli::parse_from(["memory-daemon", "topics", "related", "topic-123"]); + match cli.command { + Commands::Topics(TopicsCommand::Related { + topic_id, + rel_type, + limit, + addr, + }) => { + assert_eq!(topic_id, "topic-123"); + assert!(rel_type.is_none()); + assert_eq!(limit, 10); + assert_eq!(addr, "http://[::1]:50051"); + } + _ => panic!("Expected Topics Related command"), + } + } + + #[test] + fn test_cli_topics_related_with_options() { + let cli = Cli::parse_from([ + "memory-daemon", + "topics", + "related", + "topic-abc", + "-t", + "semantic", + "-n", + "20", + ]); + match cli.command { + Commands::Topics(TopicsCommand::Related { + topic_id, + rel_type, + limit, + .. + }) => { + assert_eq!(topic_id, "topic-abc"); + assert_eq!(rel_type, Some("semantic".to_string())); + assert_eq!(limit, 20); + } + _ => panic!("Expected Topics Related command"), + } + } + + #[test] + fn test_cli_topics_top() { + let cli = Cli::parse_from(["memory-daemon", "topics", "top"]); + match cli.command { + Commands::Topics(TopicsCommand::Top { limit, days, addr }) => { + assert_eq!(limit, 10); + assert_eq!(days, 30); + assert_eq!(addr, "http://[::1]:50051"); + } + _ => panic!("Expected Topics Top command"), + } + } + + #[test] + fn test_cli_topics_top_with_options() { + let cli = Cli::parse_from(["memory-daemon", "topics", "top", "-n", "25", "--days", "7"]); + match cli.command { + Commands::Topics(TopicsCommand::Top { limit, days, .. }) => { + assert_eq!(limit, 25); + assert_eq!(days, 7); + } + _ => panic!("Expected Topics Top command"), + } + } + + #[test] + fn test_cli_topics_refresh_scores() { + let cli = Cli::parse_from(["memory-daemon", "topics", "refresh-scores"]); + match cli.command { + Commands::Topics(TopicsCommand::RefreshScores { db_path }) => { + assert!(db_path.is_none()); + } + _ => panic!("Expected Topics RefreshScores command"), + } + } + + #[test] + fn test_cli_topics_refresh_scores_with_path() { + let cli = Cli::parse_from([ + "memory-daemon", + "topics", + "refresh-scores", + "--db-path", + "/custom/db", + ]); + match cli.command { + Commands::Topics(TopicsCommand::RefreshScores { db_path }) => { + assert_eq!(db_path, Some("/custom/db".to_string())); + } + _ => panic!("Expected Topics RefreshScores command"), + } + } + + #[test] + fn test_cli_topics_prune() { + let cli = Cli::parse_from(["memory-daemon", "topics", "prune"]); + match cli.command { + Commands::Topics(TopicsCommand::Prune { + days, + force, + db_path, + }) => { + assert_eq!(days, 90); + assert!(!force); + assert!(db_path.is_none()); + } + _ => panic!("Expected Topics Prune command"), + } + } + + #[test] + fn test_cli_topics_prune_with_options() { + let cli = Cli::parse_from([ + "memory-daemon", + "topics", + "prune", + "--days", + "60", + "--force", + "--db-path", + "/custom/db", + ]); + match cli.command { + Commands::Topics(TopicsCommand::Prune { + days, + force, + db_path, + }) => { + assert_eq!(days, 60); + assert!(force); + assert_eq!(db_path, Some("/custom/db".to_string())); + } + _ => panic!("Expected Topics Prune command"), + } + } } diff --git a/crates/memory-daemon/src/commands.rs b/crates/memory-daemon/src/commands.rs index 5be7d81..f9053b9 100644 --- a/crates/memory-daemon/src/commands.rs +++ b/crates/memory-daemon/src/commands.rs @@ -6,9 +6,11 @@ //! - status: Check if daemon is running use std::fs; +use std::io::{self, Write}; use std::net::SocketAddr; -use std::path::PathBuf; -use std::sync::Arc; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, RwLock}; +use std::time::Instant; use anyhow::{Context, Result}; use chrono::TimeZone; @@ -17,11 +19,12 @@ use tracing::{info, warn}; use memory_client::MemoryClient; use memory_scheduler::{ - create_compaction_job, create_rollup_jobs, CompactionJobConfig, RollupJobConfig, - SchedulerConfig, SchedulerService, + create_compaction_job, create_indexing_job, create_rollup_jobs, CompactionJobConfig, + IndexingJobConfig, RollupJobConfig, SchedulerConfig, SchedulerService, }; use memory_service::pb::{ GetSchedulerStatusRequest, JobResultStatus, PauseJobRequest, ResumeJobRequest, + SearchChildrenRequest, SearchField as ProtoSearchField, SearchNodeRequest, TocLevel as ProtoTocLevel, }; use memory_service::run_server_with_scheduler; @@ -29,7 +32,7 @@ use memory_storage::Storage; use memory_toc::summarizer::MockSummarizer; use memory_types::Settings; -use crate::cli::{AdminCommands, QueryCommands, SchedulerCommands}; +use crate::cli::{AdminCommands, QueryCommands, SchedulerCommands, TeleportCommand, TopicsCommand}; /// Get the PID file path fn pid_file_path() -> PathBuf { @@ -97,6 +100,57 @@ fn is_process_running(_pid: u32) -> bool { true } +/// Register the indexing job if search indexes are available. +/// +/// This function attempts to: +/// 1. Open the BM25 search index (required) +/// 2. Create an indexing pipeline with the BM25 updater +/// 3. Register the pipeline with the scheduler +/// +/// If the search index doesn't exist, returns an error. Users should +/// run `rebuild-indexes` first to initialize the search index. +async fn register_indexing_job( + scheduler: &SchedulerService, + storage: Arc, + db_path: &Path, +) -> Result<()> { + use memory_indexing::{Bm25IndexUpdater, IndexingPipeline, PipelineConfig}; + use memory_search::{SearchIndex, SearchIndexConfig, SearchIndexer}; + + // Check if search index exists + let search_dir = db_path.join("search"); + if !search_dir.exists() { + anyhow::bail!("Search index directory not found at {:?}", search_dir); + } + + // Open search index + let search_config = SearchIndexConfig::new(&search_dir); + let search_index = + SearchIndex::open_or_create(search_config).context("Failed to open search index")?; + let indexer = + Arc::new(SearchIndexer::new(&search_index).context("Failed to create search indexer")?); + + // Create BM25 updater + let bm25_updater = Bm25IndexUpdater::new(indexer, storage.clone()); + + // Create indexing pipeline with BM25 updater + let mut pipeline = IndexingPipeline::new(storage.clone(), PipelineConfig::default()); + pipeline.add_updater(Box::new(bm25_updater)); + pipeline + .load_checkpoints() + .context("Failed to load indexing checkpoints")?; + + let pipeline = Arc::new(tokio::sync::Mutex::new(pipeline)); + + // Register with scheduler + create_indexing_job(scheduler, pipeline, IndexingJobConfig::default()) + .await + .context("Failed to register indexing job")?; + + info!("Indexing job registered with BM25 updater"); + Ok(()) +} + /// Start the memory daemon. /// /// 1. Load configuration (CFG-01: defaults -> file -> env -> CLI) @@ -168,8 +222,7 @@ pub async fn start_daemon( // Create summarizer for rollup jobs // TODO: Load from config - use ApiSummarizer if OPENAI_API_KEY or ANTHROPIC_API_KEY set - let summarizer: Arc = - Arc::new(MockSummarizer::new()); + let summarizer: Arc = Arc::new(MockSummarizer::new()); // Register rollup jobs (day/week/month) create_rollup_jobs( @@ -186,6 +239,13 @@ pub async fn start_daemon( .await .context("Failed to register compaction job")?; + // Register indexing job if search index exists + // The indexing pipeline processes outbox entries into search indexes + if let Err(e) = register_indexing_job(&scheduler, storage.clone(), &db_path).await { + warn!("Indexing job not registered: {}", e); + info!("Run 'rebuild-indexes' to initialize the search index"); + } + info!( "Scheduler initialized with {} jobs", scheduler.registry().job_count() @@ -299,7 +359,10 @@ pub async fn handle_query(endpoint: &str, command: QueryCommands) -> Result<()> match command { QueryCommands::Root => { - let nodes = client.get_toc_root().await.context("Failed to get TOC root")?; + let nodes = client + .get_toc_root() + .await + .context("Failed to get TOC root")?; if nodes.is_empty() { println!("No TOC nodes found."); } else { @@ -315,7 +378,11 @@ pub async fn handle_query(endpoint: &str, command: QueryCommands) -> Result<()> } QueryCommands::Node { node_id } => { - match client.get_node(&node_id).await.context("Failed to get node")? { + match client + .get_node(&node_id) + .await + .context("Failed to get node")? + { Some(node) => { print_node_details(&node); } @@ -325,15 +392,24 @@ pub async fn handle_query(endpoint: &str, command: QueryCommands) -> Result<()> } } - QueryCommands::Browse { parent_id, limit, token } => { - let result = client.browse_toc(&parent_id, limit, token) + QueryCommands::Browse { + parent_id, + limit, + token, + } => { + let result = client + .browse_toc(&parent_id, limit, token) .await .context("Failed to browse TOC")?; if result.children.is_empty() { println!("No children found for: {}", parent_id); } else { - println!("Children of {} ({} found):\n", parent_id, result.children.len()); + println!( + "Children of {} ({} found):\n", + parent_id, + result.children.len() + ); for child in result.children { let level = level_to_string(child.level); println!(" {} [{}]", child.title, level); @@ -349,7 +425,8 @@ pub async fn handle_query(endpoint: &str, command: QueryCommands) -> Result<()> } QueryCommands::Events { from, to, limit } => { - let result = client.get_events(from, to, limit) + let result = client + .get_events(from, to, limit) .await .context("Failed to get events")?; @@ -379,8 +456,13 @@ pub async fn handle_query(endpoint: &str, command: QueryCommands) -> Result<()> } } - QueryCommands::Expand { grip_id, before, after } => { - let result = client.expand_grip(&grip_id, Some(before), Some(after)) + QueryCommands::Expand { + grip_id, + before, + after, + } => { + let result = client + .expand_grip(&grip_id, Some(before), Some(after)) .await .context("Failed to expand grip")?; @@ -417,6 +499,14 @@ pub async fn handle_query(endpoint: &str, command: QueryCommands) -> Result<()> } } } + + QueryCommands::Search { + query, + node, + parent, + fields, + limit, + } => handle_search(endpoint, query, node, parent, fields, limit).await?, } Ok(()) @@ -439,7 +529,10 @@ fn print_node_details(node: &memory_service::pb::TocNode) { println!(" ID: {}", node.node_id); println!(" Level: {}", level); println!(" Version: {}", node.version); - println!(" Time Range: {} - {}", node.start_time_ms, node.end_time_ms); + println!( + " Time Range: {} - {}", + node.start_time_ms, node.end_time_ms + ); if let Some(summary) = &node.summary { println!("\nSummary: {}", summary); @@ -479,9 +572,138 @@ fn truncate_text(text: &str, max_len: usize) -> String { } } +/// Handle search command. +/// +/// Per SEARCH-01, SEARCH-02: Search TOC nodes for matching content. +async fn handle_search( + endpoint: &str, + query: String, + node: Option, + parent: Option, + fields_str: Option, + limit: u32, +) -> Result<()> { + use memory_service::pb::memory_service_client::MemoryServiceClient; + + let mut client = MemoryServiceClient::connect(endpoint.to_string()) + .await + .context("Failed to connect to daemon")?; + + // Parse fields from comma-separated string + let fields: Vec = fields_str + .map(|s| { + s.split(',') + .filter_map(|f| match f.trim().to_lowercase().as_str() { + "title" => Some(ProtoSearchField::Title as i32), + "summary" => Some(ProtoSearchField::Summary as i32), + "bullets" => Some(ProtoSearchField::Bullets as i32), + "keywords" => Some(ProtoSearchField::Keywords as i32), + _ => None, + }) + .collect() + }) + .unwrap_or_default(); + + if let Some(node_id) = node { + // Search within single node + let response = client + .search_node(SearchNodeRequest { + node_id: node_id.clone(), + query: query.clone(), + fields: fields.clone(), + limit: limit as i32, + token_budget: 0, + }) + .await + .context("SearchNode RPC failed")?; + + let resp = response.into_inner(); + println!("Search Results for node: {}", node_id); + println!("Query: \"{}\"", query); + println!("Matched: {}", resp.matched); + println!(); + + if resp.matches.is_empty() { + println!("No matches found."); + } else { + for (i, m) in resp.matches.iter().enumerate() { + let field_name = match ProtoSearchField::try_from(m.field) { + Ok(ProtoSearchField::Title) => "title", + Ok(ProtoSearchField::Summary) => "summary", + Ok(ProtoSearchField::Bullets) => "bullets", + Ok(ProtoSearchField::Keywords) => "keywords", + _ => "unknown", + }; + println!("{}. [{}] score={:.2}", i + 1, field_name, m.score); + println!(" Text: {}", truncate_text(&m.text, 100)); + if !m.grip_ids.is_empty() { + println!(" Grips: {}", m.grip_ids.join(", ")); + } + } + } + } else { + // Search children of parent (or root if no parent) + let parent_id = parent.unwrap_or_default(); + let response = client + .search_children(SearchChildrenRequest { + parent_id: parent_id.clone(), + query: query.clone(), + child_level: 0, // Ignored when parent_id is provided + fields: fields.clone(), + limit: limit as i32, + token_budget: 0, + }) + .await + .context("SearchChildren RPC failed")?; + + let resp = response.into_inner(); + let scope = if parent_id.is_empty() { + "root level".to_string() + } else { + format!("children of {}", parent_id) + }; + println!("Search Results for {}", scope); + println!("Query: \"{}\"", query); + println!("Found: {} nodes", resp.results.len()); + if resp.has_more { + println!("(more results available, increase --limit)"); + } + println!(); + + if resp.results.is_empty() { + println!("No matching nodes found."); + } else { + for result in resp.results { + println!( + "Node: {} (score={:.2})", + result.node_id, result.relevance_score + ); + println!(" Title: {}", result.title); + println!(" Matches:"); + for m in result.matches.iter().take(3) { + let field_name = match ProtoSearchField::try_from(m.field) { + Ok(ProtoSearchField::Title) => "title", + Ok(ProtoSearchField::Summary) => "summary", + Ok(ProtoSearchField::Bullets) => "bullets", + Ok(ProtoSearchField::Keywords) => "keywords", + _ => "unknown", + }; + println!(" - [{}] {}", field_name, truncate_text(&m.text, 80)); + } + if result.matches.len() > 3 { + println!(" ... and {} more matches", result.matches.len() - 3); + } + println!(); + } + } + } + + Ok(()) +} + /// Handle admin commands. /// -/// Per CLI-03: Admin commands include rebuild-toc, compact, status. +/// Per CLI-03: Admin commands include rebuild-toc, compact, status, rebuild-indexes. pub fn handle_admin(db_path: Option, command: AdminCommands) -> Result<()> { // Load settings to get default db_path if not provided let settings = Settings::load(None).context("Failed to load configuration")?; @@ -491,6 +713,7 @@ pub fn handle_admin(db_path: Option, command: AdminCommands) -> Result<( // Open storage directly (not via gRPC) let storage = Storage::open(std::path::Path::new(&expanded_path)) .context(format!("Failed to open storage at {}", expanded_path))?; + let storage = Arc::new(storage); match command { AdminCommands::Stats => { @@ -508,21 +731,20 @@ pub fn handle_admin(db_path: Option, command: AdminCommands) -> Result<( println!("Disk Usage: {:>10}", format_bytes(stats.disk_usage_bytes)); } - AdminCommands::Compact { cf } => { - match cf { - Some(cf_name) => { - println!("Compacting column family: {}", cf_name); - storage.compact_cf(&cf_name) - .context(format!("Failed to compact {}", cf_name))?; - println!("Compaction complete."); - } - None => { - println!("Compacting all column families..."); - storage.compact().context("Failed to compact")?; - println!("Compaction complete."); - } + AdminCommands::Compact { cf } => match cf { + Some(cf_name) => { + println!("Compacting column family: {}", cf_name); + storage + .compact_cf(&cf_name) + .context(format!("Failed to compact {}", cf_name))?; + println!("Compaction complete."); } - } + None => { + println!("Compacting all column families..."); + storage.compact().context("Failed to compact")?; + println!("Compaction complete."); + } + }, AdminCommands::RebuildToc { from_date, dry_run } => { if dry_run { @@ -544,7 +766,8 @@ pub fn handle_admin(db_path: Option, command: AdminCommands) -> Result<( let start_ms = from_timestamp.unwrap_or(0); let end_ms = chrono::Utc::now().timestamp_millis(); - let events = storage.get_events_in_range(start_ms, end_ms) + let events = storage + .get_events_in_range(start_ms, end_ms) .context("Failed to query events")?; println!("Found {} events to process", events.len()); @@ -557,8 +780,14 @@ pub fn handle_admin(db_path: Option, command: AdminCommands) -> Result<( if dry_run { println!(); println!("Would process events from {} to {}", start_ms, end_ms); - println!("First event timestamp: {}", events.first().map(|(k, _)| k.timestamp_ms).unwrap_or(0)); - println!("Last event timestamp: {}", events.last().map(|(k, _)| k.timestamp_ms).unwrap_or(0)); + println!( + "First event timestamp: {}", + events.first().map(|(k, _)| k.timestamp_ms).unwrap_or(0) + ); + println!( + "Last event timestamp: {}", + events.last().map(|(k, _)| k.timestamp_ms).unwrap_or(0) + ); println!(); println!("To actually rebuild, run without --dry-run"); } else { @@ -570,11 +799,387 @@ pub fn handle_admin(db_path: Option, command: AdminCommands) -> Result<( println!("Events are intact and can be manually processed."); } } + + AdminCommands::RebuildIndexes { + index, + batch_size, + force, + search_path, + vector_path, + } => { + handle_rebuild_indexes( + storage, + &expanded_path, + &index, + batch_size, + force, + search_path, + vector_path, + )?; + } + + AdminCommands::IndexStats { + search_path, + vector_path, + } => { + handle_index_stats(&expanded_path, search_path, vector_path)?; + } + + AdminCommands::ClearIndex { + index, + force, + search_path, + vector_path, + } => { + handle_clear_index(&index, force, search_path, vector_path, &expanded_path)?; + } } Ok(()) } +/// Handle the rebuild-indexes command. +fn handle_rebuild_indexes( + storage: Arc, + db_path: &str, + index: &str, + batch_size: usize, + force: bool, + search_path: Option, + vector_path: Option, +) -> Result<()> { + use memory_embeddings::EmbeddingModel; + use memory_indexing::{ + rebuild_bm25_index, rebuild_vector_index, Bm25IndexUpdater, RebuildConfig, + VectorIndexUpdater, + }; + use memory_search::{SearchIndex, SearchIndexConfig, SearchIndexer}; + use memory_vector::{HnswConfig, HnswIndex, VectorMetadata}; + + // Determine which indexes to rebuild + let rebuild_bm25 = index == "all" || index == "bm25"; + let rebuild_vector = index == "all" || index == "vector"; + + if !rebuild_bm25 && !rebuild_vector { + anyhow::bail!("Invalid index type: {}. Use bm25, vector, or all.", index); + } + + // Count documents to process + let stats = storage.get_stats().context("Failed to get stats")?; + let total_docs = stats.toc_node_count + stats.grip_count; + + if total_docs == 0 { + println!("No documents found in storage to index."); + return Ok(()); + } + + println!("Index Rebuild"); + println!("============="); + println!("Storage path: {}", db_path); + println!("Index type: {}", index); + println!( + "Documents: {} ({} TOC nodes, {} grips)", + total_docs, stats.toc_node_count, stats.grip_count + ); + println!("Batch size: {}", batch_size); + println!(); + + // Confirmation prompt + if !force { + print!("This will rebuild the index from scratch. Continue? [y/N] "); + io::stdout().flush()?; + let mut input = String::new(); + io::stdin().read_line(&mut input)?; + if !input.trim().eq_ignore_ascii_case("y") { + println!("Aborted."); + return Ok(()); + } + } + + let start_time = Instant::now(); + let config = RebuildConfig::default().with_batch_size(batch_size); + + // Progress callback that prints to console + let progress_callback = ConsoleProgressCallback::new(batch_size); + + // Rebuild BM25 index + if rebuild_bm25 { + let search_dir = search_path + .clone() + .unwrap_or_else(|| format!("{}/search", db_path)); + let search_dir = shellexpand::tilde(&search_dir).to_string(); + let search_path = Path::new(&search_dir); + + println!("Rebuilding BM25 index at: {}", search_dir); + + // Create search directory if needed + std::fs::create_dir_all(search_path).context("Failed to create search index directory")?; + + // Open or create search index + let search_config = SearchIndexConfig::new(search_path); + let search_index = + SearchIndex::open_or_create(search_config).context("Failed to open search index")?; + let indexer = + Arc::new(SearchIndexer::new(&search_index).context("Failed to create search indexer")?); + + let updater = Bm25IndexUpdater::new(indexer, storage.clone()); + + let progress = rebuild_bm25_index(storage.clone(), &updater, &config, &progress_callback) + .map_err(|e| anyhow::anyhow!("BM25 rebuild failed: {}", e))?; + + println!(); + println!("BM25 index rebuilt:"); + println!(" TOC nodes: {}", progress.toc_nodes_indexed); + println!(" Grips: {}", progress.grips_indexed); + println!(" Errors: {}", progress.errors); + } + + // Rebuild vector index + if rebuild_vector { + let vector_dir = vector_path + .clone() + .unwrap_or_else(|| format!("{}/vector", db_path)); + let vector_dir = shellexpand::tilde(&vector_dir).to_string(); + let vector_path = Path::new(&vector_dir); + + println!("Rebuilding vector index at: {}", vector_dir); + + // Create vector directory if needed + std::fs::create_dir_all(vector_path).context("Failed to create vector index directory")?; + + // Create embedder + let embedder = Arc::new( + memory_embeddings::CandleEmbedder::load_default() + .context("Failed to create embedder")?, + ); + + // Open or create HNSW index + let hnsw_config = HnswConfig::new(embedder.info().dimension, vector_path); + let hnsw_index = Arc::new(RwLock::new( + HnswIndex::open_or_create(hnsw_config).context("Failed to open HNSW index")?, + )); + + // Open metadata store + let metadata_path = vector_path.join("metadata"); + std::fs::create_dir_all(&metadata_path).context("Failed to create metadata directory")?; + let metadata = Arc::new( + VectorMetadata::open(&metadata_path).context("Failed to open vector metadata")?, + ); + + let updater = VectorIndexUpdater::new(hnsw_index, embedder, metadata, storage.clone()); + + let progress = rebuild_vector_index(storage.clone(), &updater, &config, &progress_callback) + .map_err(|e| anyhow::anyhow!("Vector rebuild failed: {}", e))?; + + println!(); + println!("Vector index rebuilt:"); + println!(" TOC nodes: {}", progress.toc_nodes_indexed); + println!(" Grips: {}", progress.grips_indexed); + println!(" Skipped: {}", progress.skipped); + println!(" Errors: {}", progress.errors); + } + + let elapsed = start_time.elapsed(); + println!(); + println!("Rebuild complete in {:.2}s", elapsed.as_secs_f64()); + + Ok(()) +} + +/// Handle the index-stats command. +fn handle_index_stats( + db_path: &str, + search_path: Option, + vector_path: Option, +) -> Result<()> { + use memory_search::{SearchIndex, SearchIndexConfig}; + use memory_vector::{HnswConfig, HnswIndex, VectorIndex, VectorMetadata}; + + println!("Search Index Statistics"); + println!("======================="); + println!(); + + // BM25 index stats + let search_dir = search_path.unwrap_or_else(|| format!("{}/search", db_path)); + let search_dir = shellexpand::tilde(&search_dir).to_string(); + let search_path = Path::new(&search_dir); + + println!("BM25 Index:"); + println!(" Path: {}", search_dir); + + if search_path.exists() { + match SearchIndex::open_or_create(SearchIndexConfig::new(search_path)) { + Ok(index) => { + // Create a searcher to get doc count + match memory_search::TeleportSearcher::new(&index) { + Ok(searcher) => { + let doc_count = searcher.num_docs(); + println!(" Documents: {}", doc_count); + println!(" Status: Available"); + } + Err(e) => { + println!(" Status: Error creating searcher - {}", e); + } + } + } + Err(e) => { + println!(" Status: Error - {}", e); + } + } + } else { + println!(" Status: Not found"); + } + + println!(); + + // Vector index stats + let vector_dir = vector_path.unwrap_or_else(|| format!("{}/vector", db_path)); + let vector_dir = shellexpand::tilde(&vector_dir).to_string(); + let vector_path = Path::new(&vector_dir); + + println!("Vector Index:"); + println!(" Path: {}", vector_dir); + + if vector_path.exists() { + // Try to get dimension from an existing index + // Default to 384 for all-MiniLM-L6-v2 + let dimension = 384; + let hnsw_config = HnswConfig::new(dimension, vector_path); + + match HnswIndex::open_or_create(hnsw_config) { + Ok(index) => { + println!(" Vectors: {}", index.len()); + println!(" Dimension: {}", dimension); + println!(" Status: Available"); + } + Err(e) => { + println!(" Status: Error - {}", e); + } + } + + // Metadata stats + let metadata_path = vector_path.join("metadata"); + if metadata_path.exists() { + match VectorMetadata::open(&metadata_path) { + Ok(metadata) => { + println!(" Metadata: Available"); + if let Ok(count) = metadata.count() { + println!(" Entries: {}", count); + } + } + Err(e) => { + println!(" Metadata: Error - {}", e); + } + } + } + } else { + println!(" Status: Not found"); + } + + Ok(()) +} + +/// Handle the clear-index command. +fn handle_clear_index( + index: &str, + force: bool, + search_path: Option, + vector_path: Option, + db_path: &str, +) -> Result<()> { + let clear_bm25 = index == "all" || index == "bm25"; + let clear_vector = index == "all" || index == "vector"; + + if !clear_bm25 && !clear_vector { + anyhow::bail!("Invalid index type: {}. Use bm25, vector, or all.", index); + } + + println!("Clear Index"); + println!("==========="); + + // Confirmation prompt + if !force { + print!( + "This will PERMANENTLY DELETE the {} index. Continue? [y/N] ", + index + ); + io::stdout().flush()?; + let mut input = String::new(); + io::stdin().read_line(&mut input)?; + if !input.trim().eq_ignore_ascii_case("y") { + println!("Aborted."); + return Ok(()); + } + } + + // Clear BM25 index + if clear_bm25 { + let search_dir = search_path + .clone() + .unwrap_or_else(|| format!("{}/search", db_path)); + let search_dir = shellexpand::tilde(&search_dir).to_string(); + let search_path = Path::new(&search_dir); + + if search_path.exists() { + println!("Removing BM25 index at: {}", search_dir); + std::fs::remove_dir_all(search_path) + .context("Failed to remove BM25 index directory")?; + println!("BM25 index cleared."); + } else { + println!("BM25 index not found at: {}", search_dir); + } + } + + // Clear vector index + if clear_vector { + let vector_dir = vector_path + .clone() + .unwrap_or_else(|| format!("{}/vector", db_path)); + let vector_dir = shellexpand::tilde(&vector_dir).to_string(); + let vector_path = Path::new(&vector_dir); + + if vector_path.exists() { + println!("Removing vector index at: {}", vector_dir); + std::fs::remove_dir_all(vector_path) + .context("Failed to remove vector index directory")?; + println!("Vector index cleared."); + } else { + println!("Vector index not found at: {}", vector_dir); + } + } + + Ok(()) +} + +/// Console progress callback for rebuild operations. +struct ConsoleProgressCallback { + batch_size: usize, +} + +impl ConsoleProgressCallback { + fn new(batch_size: usize) -> Self { + Self { batch_size } + } +} + +impl memory_indexing::ProgressCallback for ConsoleProgressCallback { + fn on_progress(&self, progress: &memory_indexing::RebuildProgress) { + if progress + .total_processed + .is_multiple_of(self.batch_size as u64) + && progress.total_processed > 0 + { + println!( + " Progress: {} documents ({} TOC nodes, {} grips, {} errors)", + progress.total_processed, + progress.toc_nodes_indexed, + progress.grips_indexed, + progress.errors + ); + } + } +} + fn format_bytes(bytes: u64) -> String { const KB: u64 = 1024; const MB: u64 = KB * 1024; @@ -716,6 +1321,600 @@ fn format_timestamp(ms: i64) -> String { .unwrap_or_else(|| "Invalid".to_string()) } +/// Handle teleport commands. +/// +/// Per TEL-01 through TEL-04: BM25 keyword search for teleporting to content. +/// Per VEC-01 through VEC-03: Vector semantic search for teleporting to content. +pub async fn handle_teleport_command(cmd: TeleportCommand) -> Result<()> { + match cmd { + TeleportCommand::Search { + query, + doc_type, + limit, + addr, + } => teleport_search(&query, &doc_type, limit, &addr).await, + TeleportCommand::VectorSearch { + query, + top_k, + min_score, + target, + addr, + } => vector_search(&query, top_k, min_score, &target, &addr).await, + TeleportCommand::HybridSearch { + query, + top_k, + mode, + bm25_weight, + vector_weight, + target, + addr, + } => { + hybrid_search( + &query, + top_k, + &mode, + bm25_weight, + vector_weight, + &target, + &addr, + ) + .await + } + TeleportCommand::Stats { addr } => teleport_stats(&addr).await, + TeleportCommand::VectorStats { addr } => vector_stats(&addr).await, + TeleportCommand::Rebuild { addr } => teleport_rebuild(&addr).await, + } +} + +/// Execute teleport search via gRPC. +async fn teleport_search(query: &str, doc_type: &str, limit: usize, addr: &str) -> Result<()> { + println!("Searching for: \"{}\"", query); + println!("Filter: {}, Limit: {}", doc_type, limit); + println!(); + + let mut client = MemoryClient::connect(addr) + .await + .context("Failed to connect to daemon")?; + + // Map doc_type string to enum value + let doc_type_value = match doc_type.to_lowercase().as_str() { + "toc" | "toc_node" => 1, // TeleportDocType::TocNode + "grip" | "grips" => 2, // TeleportDocType::Grip + _ => 0, // TeleportDocType::Unspecified (all) + }; + + let response = client + .teleport_search(query, doc_type_value, limit as i32) + .await + .context("Teleport search failed")?; + + if response.results.is_empty() { + println!("No results found."); + return Ok(()); + } + + println!("Found {} results:", response.results.len()); + println!("{:-<60}", ""); + + for (i, result) in response.results.iter().enumerate() { + let type_str = match result.doc_type { + 1 => "TOC", + 2 => "Grip", + _ => "?", + }; + + println!( + "{}. [{}] {} (score: {:.4})", + i + 1, + type_str, + result.doc_id, + result.score + ); + + if let Some(ref keywords) = result.keywords { + if !keywords.is_empty() { + println!(" Keywords: {}", keywords); + } + } + } + + println!("{:-<60}", ""); + println!("Total documents in index: {}", response.total_docs); + + Ok(()) +} + +/// Show teleport index statistics. +async fn teleport_stats(addr: &str) -> Result<()> { + let mut client = MemoryClient::connect(addr) + .await + .context("Failed to connect to daemon")?; + + // Use empty search to get total_docs + let response = client + .teleport_search("", 0, 0) + .await + .context("Failed to get index stats")?; + + println!("Teleport Index Statistics"); + println!("{:-<40}", ""); + println!("Total documents: {}", response.total_docs); + + Ok(()) +} + +/// Trigger index rebuild (placeholder - will be implemented in Phase 13). +async fn teleport_rebuild(_addr: &str) -> Result<()> { + println!("Index rebuild not yet implemented."); + println!("This will be available in Phase 13 (Outbox Index Ingestion)."); + Ok(()) +} + +/// Execute vector semantic search via gRPC. +async fn vector_search( + query: &str, + top_k: i32, + min_score: f32, + target: &str, + addr: &str, +) -> Result<()> { + println!("Vector Search: \"{}\"", query); + println!( + "Top-K: {}, Min Score: {:.2}, Target: {}", + top_k, min_score, target + ); + println!(); + + let mut client = MemoryClient::connect(addr) + .await + .context("Failed to connect to daemon")?; + + // Map target string to enum value + let target_value = match target.to_lowercase().as_str() { + "toc" | "toc_node" => 1, // VectorTargetType::TocNode + "grip" | "grips" => 2, // VectorTargetType::Grip + "all" => 3, // VectorTargetType::All + _ => 0, // VectorTargetType::Unspecified + }; + + let response = client + .vector_teleport(query, top_k, min_score, target_value) + .await + .context("Vector search failed")?; + + if response.matches.is_empty() { + println!("No results found."); + return Ok(()); + } + + println!("Found {} results:", response.matches.len()); + println!("{:-<70}", ""); + + for (i, m) in response.matches.iter().enumerate() { + println!( + "{}. [{}] {} (score: {:.4})", + i + 1, + m.doc_type, + m.doc_id, + m.score + ); + + // Show text preview (truncated) + let preview = truncate_text(&m.text_preview, 80); + println!(" {}", preview); + + // Show timestamp if available + if m.timestamp_ms > 0 { + println!(" Time: {}", format_timestamp(m.timestamp_ms)); + } + + println!(); + } + + // Show index status if available + if let Some(status) = &response.index_status { + println!("{:-<70}", ""); + println!( + "Index: {} vectors, dim={}, last updated: {}", + status.vector_count, status.dimension, status.last_indexed + ); + } + + Ok(()) +} + +/// Execute hybrid BM25 + vector search via gRPC. +async fn hybrid_search( + query: &str, + top_k: i32, + mode: &str, + bm25_weight: f32, + vector_weight: f32, + target: &str, + addr: &str, +) -> Result<()> { + println!("Hybrid Search: \"{}\"", query); + println!( + "Mode: {}, BM25 Weight: {:.2}, Vector Weight: {:.2}", + mode, bm25_weight, vector_weight + ); + println!(); + + let mut client = MemoryClient::connect(addr) + .await + .context("Failed to connect to daemon")?; + + // Map mode string to enum value + let mode_value = match mode.to_lowercase().as_str() { + "vector-only" | "vector" => 1, // HybridMode::VectorOnly + "bm25-only" | "bm25" => 2, // HybridMode::Bm25Only + "hybrid" => 3, // HybridMode::Hybrid + _ => 0, // HybridMode::Unspecified + }; + + // Map target string to enum value + let target_value = match target.to_lowercase().as_str() { + "toc" | "toc_node" => 1, // VectorTargetType::TocNode + "grip" | "grips" => 2, // VectorTargetType::Grip + "all" => 3, // VectorTargetType::All + _ => 0, // VectorTargetType::Unspecified + }; + + let response = client + .hybrid_search( + query, + top_k, + mode_value, + bm25_weight, + vector_weight, + target_value, + ) + .await + .context("Hybrid search failed")?; + + // Show mode used and availability + let mode_used = match response.mode_used { + 1 => "vector-only", + 2 => "bm25-only", + 3 => "hybrid", + _ => "unknown", + }; + println!( + "Mode used: {} (BM25: {}, Vector: {})", + mode_used, + if response.bm25_available { "yes" } else { "no" }, + if response.vector_available { + "yes" + } else { + "no" + } + ); + println!(); + + if response.matches.is_empty() { + println!("No results found."); + return Ok(()); + } + + println!("Found {} results:", response.matches.len()); + println!("{:-<70}", ""); + + for (i, m) in response.matches.iter().enumerate() { + println!( + "{}. [{}] {} (score: {:.4})", + i + 1, + m.doc_type, + m.doc_id, + m.score + ); + + // Show text preview (truncated) + let preview = truncate_text(&m.text_preview, 80); + println!(" {}", preview); + + // Show timestamp if available + if m.timestamp_ms > 0 { + println!(" Time: {}", format_timestamp(m.timestamp_ms)); + } + + println!(); + } + + Ok(()) +} + +/// Show vector index statistics. +async fn vector_stats(addr: &str) -> Result<()> { + let mut client = MemoryClient::connect(addr) + .await + .context("Failed to connect to daemon")?; + + let status = client + .get_vector_index_status() + .await + .context("Failed to get vector index status")?; + + println!("Vector Index Statistics"); + println!("{:-<40}", ""); + println!( + "Status: {}", + if status.available { + "Available" + } else { + "Unavailable" + } + ); + println!("Vectors: {}", status.vector_count); + println!("Dimension: {}", status.dimension); + println!("Last Indexed: {}", status.last_indexed); + println!("Index Path: {}", status.index_path); + println!("Index Size: {}", format_bytes(status.size_bytes as u64)); + + Ok(()) +} + +/// Handle topics commands. +/// +/// Per TOPIC-08: Topic graph discovery and navigation. +pub async fn handle_topics_command(cmd: TopicsCommand) -> Result<()> { + match cmd { + TopicsCommand::Status { addr } => topics_status(&addr).await, + TopicsCommand::Explore { query, limit, addr } => topics_explore(&query, limit, &addr).await, + TopicsCommand::Related { + topic_id, + rel_type, + limit, + addr, + } => topics_related(&topic_id, rel_type.as_deref(), limit, &addr).await, + TopicsCommand::Top { limit, days, addr } => topics_top(limit, days, &addr).await, + TopicsCommand::RefreshScores { db_path } => topics_refresh_scores(db_path).await, + TopicsCommand::Prune { + days, + force, + db_path, + } => topics_prune(days, force, db_path).await, + } +} + +/// Show topic graph status. +async fn topics_status(addr: &str) -> Result<()> { + let mut client = MemoryClient::connect(addr) + .await + .context("Failed to connect to daemon")?; + + let status = client + .get_topic_graph_status() + .await + .context("Failed to get topic graph status")?; + + println!("Topic Graph Status"); + println!("{:-<40}", ""); + println!( + "Status: {}", + if status.available { + "Available" + } else { + "Unavailable" + } + ); + println!("Topics: {}", status.topic_count); + println!("Relationships: {}", status.relationship_count); + println!("Last Updated: {}", status.last_updated); + + Ok(()) +} + +/// Explore topics matching a query. +async fn topics_explore(query: &str, limit: u32, addr: &str) -> Result<()> { + let mut client = MemoryClient::connect(addr) + .await + .context("Failed to connect to daemon")?; + + println!("Searching for topics: \"{}\"", query); + println!(); + + let topics = client + .get_topics_by_query(query, limit) + .await + .context("Failed to search topics")?; + + if topics.is_empty() { + println!("No topics found matching query."); + return Ok(()); + } + + println!("Found {} topics:", topics.len()); + println!("{:-<70}", ""); + + for (i, topic) in topics.iter().enumerate() { + println!( + "{}. {} (importance: {:.4})", + i + 1, + topic.label, + topic.importance_score + ); + println!(" ID: {}", topic.id); + if !topic.keywords.is_empty() { + println!(" Keywords: {}", topic.keywords.join(", ")); + } + println!(" Created: {}", topic.created_at); + println!(" Last Mention: {}", topic.last_mention); + println!(); + } + + Ok(()) +} + +/// Show related topics. +async fn topics_related( + topic_id: &str, + rel_type: Option<&str>, + limit: u32, + addr: &str, +) -> Result<()> { + let mut client = MemoryClient::connect(addr) + .await + .context("Failed to connect to daemon")?; + + println!("Finding topics related to: {}", topic_id); + if let Some(rt) = rel_type { + println!("Filtering by relationship type: {}", rt); + } + println!(); + + let response = client + .get_related_topics(topic_id, rel_type, limit) + .await + .context("Failed to get related topics")?; + + if response.related_topics.is_empty() { + println!("No related topics found."); + return Ok(()); + } + + println!("Found {} related topics:", response.related_topics.len()); + println!("{:-<70}", ""); + + for (i, topic) in response.related_topics.iter().enumerate() { + // Find the relationship for this topic + let rel = response + .relationships + .iter() + .find(|r| r.target_id == topic.id); + + let rel_info = rel + .map(|r| format!("{} (strength: {:.2})", r.relationship_type, r.strength)) + .unwrap_or_default(); + + println!( + "{}. {} (importance: {:.4})", + i + 1, + topic.label, + topic.importance_score + ); + println!(" ID: {}", topic.id); + if !rel_info.is_empty() { + println!(" Relationship: {}", rel_info); + } + println!(); + } + + Ok(()) +} + +/// Show top topics by importance. +async fn topics_top(limit: u32, days: u32, addr: &str) -> Result<()> { + let mut client = MemoryClient::connect(addr) + .await + .context("Failed to connect to daemon")?; + + println!("Top {} topics (last {} days):", limit, days); + println!(); + + let topics = client + .get_top_topics(limit, days) + .await + .context("Failed to get top topics")?; + + if topics.is_empty() { + println!("No topics found."); + return Ok(()); + } + + println!("{:-<70}", ""); + + for (i, topic) in topics.iter().enumerate() { + println!( + "{}. {} (importance: {:.4})", + i + 1, + topic.label, + topic.importance_score + ); + println!(" ID: {}", topic.id); + if !topic.keywords.is_empty() { + println!(" Keywords: {}", topic.keywords.join(", ")); + } + println!(" Last Mention: {}", topic.last_mention); + println!(); + } + + Ok(()) +} + +/// Refresh topic importance scores. +async fn topics_refresh_scores(db_path: Option) -> Result<()> { + use memory_topics::{config::ImportanceConfig, ImportanceScorer, TopicStorage}; + + // Load settings to get default db_path if not provided + let settings = Settings::load(None).context("Failed to load configuration")?; + let db_path = db_path.unwrap_or_else(|| settings.db_path.clone()); + let expanded_path = shellexpand::tilde(&db_path).to_string(); + + println!("Refreshing topic importance scores..."); + println!("Database: {}", expanded_path); + println!(); + + // Open storage directly + let storage = Storage::open(std::path::Path::new(&expanded_path)) + .context(format!("Failed to open storage at {}", expanded_path))?; + let storage = Arc::new(storage); + let topic_storage = TopicStorage::new(storage); + + let scorer = ImportanceScorer::new(ImportanceConfig::default()); + let updated = topic_storage + .refresh_importance_scores(&scorer) + .context("Failed to refresh importance scores")?; + + println!("Refreshed {} topic importance scores.", updated); + + Ok(()) +} + +/// Prune stale topics. +async fn topics_prune(days: u32, force: bool, db_path: Option) -> Result<()> { + use memory_topics::{TopicLifecycleManager, TopicStorage}; + + // Load settings to get default db_path if not provided + let settings = Settings::load(None).context("Failed to load configuration")?; + let db_path = db_path.unwrap_or_else(|| settings.db_path.clone()); + let expanded_path = shellexpand::tilde(&db_path).to_string(); + + println!("Pruning stale topics..."); + println!("Database: {}", expanded_path); + println!("Inactivity threshold: {} days", days); + println!(); + + // Confirmation prompt + if !force { + print!( + "This will archive topics not mentioned in {} days. Continue? [y/N] ", + days + ); + io::stdout().flush()?; + let mut input = String::new(); + io::stdin().read_line(&mut input)?; + if !input.trim().eq_ignore_ascii_case("y") { + println!("Aborted."); + return Ok(()); + } + } + + // Open storage directly + let storage = Storage::open(std::path::Path::new(&expanded_path)) + .context(format!("Failed to open storage at {}", expanded_path))?; + let storage = Arc::new(storage); + let topic_storage = TopicStorage::new(storage); + + let mut manager = TopicLifecycleManager::new(&topic_storage); + let pruned = manager + .prune_stale_topics(days) + .context("Failed to prune topics")?; + + println!("Pruned {} stale topics.", pruned); + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/memory-daemon/src/lib.rs b/crates/memory-daemon/src/lib.rs index bc196d6..9de46b2 100644 --- a/crates/memory-daemon/src/lib.rs +++ b/crates/memory-daemon/src/lib.rs @@ -10,7 +10,10 @@ pub mod cli; pub mod commands; -pub use cli::{AdminCommands, Cli, Commands, QueryCommands, SchedulerCommands}; +pub use cli::{ + AdminCommands, Cli, Commands, QueryCommands, SchedulerCommands, TeleportCommand, TopicsCommand, +}; pub use commands::{ - handle_admin, handle_query, handle_scheduler, show_status, start_daemon, stop_daemon, + handle_admin, handle_query, handle_scheduler, handle_teleport_command, handle_topics_command, + show_status, start_daemon, stop_daemon, }; diff --git a/crates/memory-daemon/src/main.rs b/crates/memory-daemon/src/main.rs index 4b3ecc0..0152d26 100644 --- a/crates/memory-daemon/src/main.rs +++ b/crates/memory-daemon/src/main.rs @@ -22,8 +22,8 @@ use anyhow::Result; use clap::Parser; use memory_daemon::{ - handle_admin, handle_query, handle_scheduler, show_status, start_daemon, stop_daemon, Cli, - Commands, + handle_admin, handle_query, handle_scheduler, handle_teleport_command, handle_topics_command, + show_status, start_daemon, stop_daemon, Cli, Commands, }; #[tokio::main] @@ -60,6 +60,12 @@ async fn main() -> Result<()> { Commands::Scheduler { endpoint, command } => { handle_scheduler(&endpoint, command).await?; } + Commands::Teleport(cmd) => { + handle_teleport_command(cmd).await?; + } + Commands::Topics(cmd) => { + handle_topics_command(cmd).await?; + } } Ok(()) diff --git a/crates/memory-daemon/tests/integration_test.rs b/crates/memory-daemon/tests/integration_test.rs index fe0ee19..a07da23 100644 --- a/crates/memory-daemon/tests/integration_test.rs +++ b/crates/memory-daemon/tests/integration_test.rs @@ -35,13 +35,9 @@ impl TestHarness { let service_storage = storage.clone(); let server_handle = tokio::spawn(async move { - run_server_with_shutdown( - addr, - service_storage, - async { - shutdown_rx.await.ok(); - }, - ) + run_server_with_shutdown(addr, service_storage, async { + shutdown_rx.await.ok(); + }) .await }); @@ -91,11 +87,7 @@ async fn test_event_ingestion_lifecycle() { let session_id = "test-session-123"; let events = vec![ HookEvent::new(session_id, HookEventType::SessionStart, "Session started"), - HookEvent::new( - session_id, - HookEventType::UserPromptSubmit, - "What is Rust?", - ), + HookEvent::new(session_id, HookEventType::UserPromptSubmit, "What is Rust?"), HookEvent::new( session_id, HookEventType::AssistantResponse, @@ -157,10 +149,13 @@ async fn test_event_with_metadata() { metadata.insert("tool_name".to_string(), "Read".to_string()); metadata.insert("file_path".to_string(), "/tmp/test.rs".to_string()); - let hook_event = - HookEvent::new("session-789", HookEventType::ToolResult, "File contents here") - .with_tool_name("Read") - .with_metadata(metadata); + let hook_event = HookEvent::new( + "session-789", + HookEventType::ToolResult, + "File contents here", + ) + .with_tool_name("Read") + .with_metadata(metadata); let event = map_hook_event(hook_event); let (event_id, created) = client.ingest(event).await.unwrap(); @@ -195,10 +190,7 @@ async fn test_browse_toc_empty() { let harness = TestHarness::new(50105).await; let mut client = harness.client().await; - let result = client - .browse_toc("toc:year:2026", 10, None) - .await - .unwrap(); + let result = client.browse_toc("toc:year:2026", 10, None).await.unwrap(); assert!(result.children.is_empty()); assert!(!result.has_more); } diff --git a/crates/memory-embeddings/Cargo.toml b/crates/memory-embeddings/Cargo.toml new file mode 100644 index 0000000..9f28065 --- /dev/null +++ b/crates/memory-embeddings/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "memory-embeddings" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Local embedding generation for Agent Memory using Candle" + +[dependencies] +# Candle ML framework +candle-core = { workspace = true } +candle-nn = { workspace = true } +candle-transformers = { workspace = true } + +# Tokenization +tokenizers = { workspace = true } + +# Model downloading +hf-hub = { workspace = true } + +# Internal crates +memory-types = { workspace = true } + +# Async runtime +tokio = { workspace = true } + +# Logging +tracing = { workspace = true } + +# Error handling +thiserror = { workspace = true } + +# Serialization +serde = { workspace = true } +serde_json = { workspace = true } + +# Directory utilities +dirs = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +tokio = { workspace = true, features = ["test-util", "macros", "rt-multi-thread"] } diff --git a/crates/memory-embeddings/src/cache.rs b/crates/memory-embeddings/src/cache.rs new file mode 100644 index 0000000..d0c49b3 --- /dev/null +++ b/crates/memory-embeddings/src/cache.rs @@ -0,0 +1,134 @@ +//! Model file caching. +//! +//! Downloads and caches model files from HuggingFace Hub. + +use std::path::PathBuf; +use tracing::{debug, info}; + +use crate::error::EmbeddingError; + +/// Default model repository on HuggingFace +pub const DEFAULT_MODEL_REPO: &str = "sentence-transformers/all-MiniLM-L6-v2"; + +/// Required model files +pub const MODEL_FILES: &[&str] = &["config.json", "tokenizer.json", "model.safetensors"]; + +/// Model cache configuration +#[derive(Debug, Clone)] +pub struct ModelCache { + /// Cache directory path + pub cache_dir: PathBuf, + /// Model repository ID + pub repo_id: String, +} + +impl Default for ModelCache { + fn default() -> Self { + let cache_dir = dirs::cache_dir() + .unwrap_or_else(|| PathBuf::from(".cache")) + .join("agent-memory") + .join("models"); + + Self { + cache_dir, + repo_id: DEFAULT_MODEL_REPO.to_string(), + } + } +} + +impl ModelCache { + /// Create a new model cache with custom settings + pub fn new(cache_dir: impl Into, repo_id: impl Into) -> Self { + Self { + cache_dir: cache_dir.into(), + repo_id: repo_id.into(), + } + } + + /// Get the model directory path + pub fn model_dir(&self) -> PathBuf { + self.cache_dir.join(self.repo_id.replace('/', "_")) + } + + /// Check if all model files are cached + pub fn is_cached(&self) -> bool { + let model_dir = self.model_dir(); + MODEL_FILES.iter().all(|f| model_dir.join(f).exists()) + } + + /// Get path to a specific model file + pub fn file_path(&self, filename: &str) -> PathBuf { + self.model_dir().join(filename) + } +} + +/// Paths to model files +#[derive(Debug, Clone)] +pub struct ModelPaths { + pub config: PathBuf, + pub tokenizer: PathBuf, + pub weights: PathBuf, +} + +/// Get or download model files. +/// +/// Returns paths to config.json, tokenizer.json, and model.safetensors. +pub fn get_or_download_model(cache: &ModelCache) -> Result { + let model_dir = cache.model_dir(); + + if cache.is_cached() { + debug!(path = ?model_dir, "Using cached model"); + } else { + info!(repo = %cache.repo_id, "Downloading model files..."); + download_model_files(cache)?; + } + + Ok(ModelPaths { + config: model_dir.join("config.json"), + tokenizer: model_dir.join("tokenizer.json"), + weights: model_dir.join("model.safetensors"), + }) +} + +/// Download model files from HuggingFace Hub +fn download_model_files(cache: &ModelCache) -> Result<(), EmbeddingError> { + use hf_hub::api::sync::Api; + + let api = Api::new().map_err(|e| EmbeddingError::Download(e.to_string()))?; + let repo = api.model(cache.repo_id.clone()); + + std::fs::create_dir_all(cache.model_dir())?; + + for filename in MODEL_FILES { + info!(file = filename, "Downloading..."); + let source_path = repo + .get(filename) + .map_err(|e| EmbeddingError::Download(format!("{}: {}", filename, e)))?; + + let dest_path = cache.file_path(filename); + std::fs::copy(&source_path, &dest_path)?; + debug!(file = filename, "Downloaded to {:?}", dest_path); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_cache_default() { + let cache = ModelCache::default(); + assert!(cache.cache_dir.to_string_lossy().contains("agent-memory")); + assert_eq!(cache.repo_id, DEFAULT_MODEL_REPO); + } + + #[test] + fn test_is_cached_empty() { + let temp = TempDir::new().unwrap(); + let cache = ModelCache::new(temp.path(), "test/model"); + assert!(!cache.is_cached()); + } +} diff --git a/crates/memory-embeddings/src/candle.rs b/crates/memory-embeddings/src/candle.rs new file mode 100644 index 0000000..4d09866 --- /dev/null +++ b/crates/memory-embeddings/src/candle.rs @@ -0,0 +1,249 @@ +//! Candle-based embedding implementation. +//! +//! Uses all-MiniLM-L6-v2 for 384-dimensional embeddings. + +use candle_core::{DType, Device, Tensor}; +use candle_nn::VarBuilder; +use candle_transformers::models::bert::{BertModel, Config as BertConfig}; +use tokenizers::Tokenizer; +use tracing::{debug, info}; + +use crate::cache::{get_or_download_model, ModelCache}; +use crate::error::EmbeddingError; +use crate::model::{Embedding, EmbeddingModel, ModelInfo}; + +/// Embedding dimension for all-MiniLM-L6-v2 +pub const EMBEDDING_DIM: usize = 384; + +/// Maximum sequence length +pub const MAX_SEQ_LENGTH: usize = 256; + +/// Default batch size for embedding +pub const DEFAULT_BATCH_SIZE: usize = 32; + +/// Candle-based embedder using all-MiniLM-L6-v2. +pub struct CandleEmbedder { + model: BertModel, + tokenizer: Tokenizer, + device: Device, + info: ModelInfo, +} + +impl CandleEmbedder { + /// Load the embedding model from cache (downloading if needed). + pub fn load(cache: &ModelCache) -> Result { + let paths = get_or_download_model(cache)?; + Self::load_from_paths(&paths.config, &paths.tokenizer, &paths.weights) + } + + /// Load with default cache settings + pub fn load_default() -> Result { + let cache = ModelCache::default(); + Self::load(&cache) + } + + /// Load from explicit file paths + pub fn load_from_paths( + config_path: &std::path::Path, + tokenizer_path: &std::path::Path, + weights_path: &std::path::Path, + ) -> Result { + info!("Loading embedding model..."); + + // Use CPU device (GPU support can be added later with feature flags) + let device = Device::Cpu; + + // Load config + let config_str = std::fs::read_to_string(config_path)?; + let config: BertConfig = serde_json::from_str(&config_str) + .map_err(|e| EmbeddingError::ModelNotFound(format!("Invalid config: {}", e)))?; + + // Load tokenizer + let tokenizer = Tokenizer::from_file(tokenizer_path) + .map_err(|e| EmbeddingError::Tokenizer(e.to_string()))?; + + // Load model weights + let vb = unsafe { + VarBuilder::from_mmaped_safetensors(&[weights_path.to_path_buf()], DType::F32, &device)? + }; + + let model = BertModel::load(vb, &config)?; + + info!( + dim = EMBEDDING_DIM, + max_seq = MAX_SEQ_LENGTH, + "Model loaded successfully" + ); + + Ok(Self { + model, + tokenizer, + device, + info: ModelInfo { + name: "all-MiniLM-L6-v2".to_string(), + dimension: EMBEDDING_DIM, + max_sequence_length: MAX_SEQ_LENGTH, + }, + }) + } + + /// Mean pooling over token embeddings (excluding padding) + fn mean_pooling( + &self, + embeddings: &Tensor, + attention_mask: &Tensor, + ) -> Result { + // Expand attention mask to embedding dimension + let mask = attention_mask + .unsqueeze(2)? + .broadcast_as(embeddings.shape())?; + let mask_f32 = mask.to_dtype(DType::F32)?; + + // Masked sum + let masked = embeddings.broadcast_mul(&mask_f32)?; + let sum = masked.sum(1)?; + + // Divide by sum of mask (number of real tokens) + let mask_sum = mask_f32.sum(1)?; + let mask_sum = mask_sum.clamp(1e-9, f64::MAX)?; // Avoid division by zero + + let mean = sum.broadcast_div(&mask_sum)?; + Ok(mean) + } +} + +impl EmbeddingModel for CandleEmbedder { + fn info(&self) -> &ModelInfo { + &self.info + } + + fn embed(&self, text: &str) -> Result { + let embeddings = self.embed_batch(&[text])?; + Ok(embeddings.into_iter().next().unwrap()) + } + + fn embed_batch(&self, texts: &[&str]) -> Result, EmbeddingError> { + if texts.is_empty() { + return Ok(vec![]); + } + + debug!(count = texts.len(), "Embedding batch"); + + // Tokenize all texts + let encodings = self + .tokenizer + .encode_batch(texts.to_vec(), true) + .map_err(|e| EmbeddingError::Tokenizer(e.to_string()))?; + + // Pad to same length + let max_len = encodings + .iter() + .map(|e| e.get_ids().len()) + .max() + .unwrap_or(0) + .min(MAX_SEQ_LENGTH); + + let mut input_ids: Vec> = Vec::new(); + let mut attention_masks: Vec> = Vec::new(); + + for encoding in &encodings { + let ids = encoding.get_ids(); + let mask = encoding.get_attention_mask(); + + let truncated_len = ids.len().min(max_len); + let mut padded_ids = ids[..truncated_len].to_vec(); + let mut padded_mask = mask[..truncated_len].to_vec(); + + // Pad to max_len + padded_ids.resize(max_len, 0); + padded_mask.resize(max_len, 0); + + input_ids.push(padded_ids); + attention_masks.push(padded_mask); + } + + // Convert to tensors + let batch_size = texts.len(); + let input_ids_flat: Vec = input_ids.into_iter().flatten().collect(); + let mask_flat: Vec = attention_masks.into_iter().flatten().collect(); + + let input_ids = Tensor::from_vec(input_ids_flat, (batch_size, max_len), &self.device)?; + let attention_mask = Tensor::from_vec(mask_flat, (batch_size, max_len), &self.device)?; + let token_type_ids = Tensor::zeros_like(&input_ids)?; + + // Forward pass + let output = self + .model + .forward(&input_ids, &token_type_ids, Some(&attention_mask))?; + + // Mean pooling + let pooled = self.mean_pooling(&output, &attention_mask)?; + + // Convert to embeddings + let pooled_vec: Vec> = pooled.to_vec2()?; + + let embeddings: Vec = pooled_vec + .into_iter() + .map(Embedding::new) // Normalizes the vector + .collect(); + + debug!( + count = embeddings.len(), + dim = EMBEDDING_DIM, + "Batch complete" + ); + + Ok(embeddings) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Note: Integration tests require model download, run with: + // cargo test -p memory-embeddings --features integration -- --ignored + + #[test] + #[ignore = "requires model download"] + fn test_load_model() { + let embedder = CandleEmbedder::load_default().unwrap(); + assert_eq!(embedder.info().dimension, EMBEDDING_DIM); + } + + #[test] + #[ignore = "requires model download"] + fn test_embed_single() { + let embedder = CandleEmbedder::load_default().unwrap(); + let emb = embedder.embed("Hello, world!").unwrap(); + assert_eq!(emb.dimension(), EMBEDDING_DIM); + } + + #[test] + #[ignore = "requires model download"] + fn test_embed_batch() { + let embedder = CandleEmbedder::load_default().unwrap(); + let texts = vec!["Hello", "World", "Test"]; + let embeddings = embedder.embed_batch(&texts).unwrap(); + assert_eq!(embeddings.len(), 3); + for emb in &embeddings { + assert_eq!(emb.dimension(), EMBEDDING_DIM); + } + } + + #[test] + #[ignore = "requires model download"] + fn test_similar_texts_high_similarity() { + let embedder = CandleEmbedder::load_default().unwrap(); + let emb1 = embedder.embed("The cat sat on the mat").unwrap(); + let emb2 = embedder.embed("A cat is sitting on a mat").unwrap(); + let emb3 = embedder.embed("Python programming language").unwrap(); + + let sim_similar = emb1.cosine_similarity(&emb2); + let sim_different = emb1.cosine_similarity(&emb3); + + // Similar sentences should have higher similarity + assert!(sim_similar > sim_different); + assert!(sim_similar > 0.7); // Should be quite similar + } +} diff --git a/crates/memory-embeddings/src/error.rs b/crates/memory-embeddings/src/error.rs new file mode 100644 index 0000000..88d9f0f --- /dev/null +++ b/crates/memory-embeddings/src/error.rs @@ -0,0 +1,43 @@ +//! Embedding error types. + +use thiserror::Error; + +/// Errors that can occur during embedding operations. +#[derive(Debug, Error)] +pub enum EmbeddingError { + /// Candle model error + #[error("Candle error: {0}")] + Candle(#[from] candle_core::Error), + + /// Tokenizer error + #[error("Tokenizer error: {0}")] + Tokenizer(String), + + /// Model file not found + #[error("Model file not found: {0}")] + ModelNotFound(String), + + /// Download error + #[error("Failed to download model: {0}")] + Download(String), + + /// IO error + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + + /// Cache error + #[error("Cache error: {0}")] + Cache(String), + + /// Invalid input + #[error("Invalid input: {0}")] + InvalidInput(String), + + /// Dimension mismatch + #[error("Dimension mismatch: expected {expected}, got {actual}")] + DimensionMismatch { expected: usize, actual: usize }, + + /// Serialization error + #[error("Serialization error: {0}")] + Serialization(String), +} diff --git a/crates/memory-embeddings/src/lib.rs b/crates/memory-embeddings/src/lib.rs new file mode 100644 index 0000000..fb6068f --- /dev/null +++ b/crates/memory-embeddings/src/lib.rs @@ -0,0 +1,27 @@ +//! # memory-embeddings +//! +//! Local embedding generation for Agent Memory using Candle. +//! +//! This crate provides semantic vector embeddings for TOC summaries and grip +//! excerpts, enabling similarity search without external API calls. +//! +//! ## Features +//! - Local inference via Candle (no Python, no API) +//! - all-MiniLM-L6-v2 model (384 dimensions) +//! - Automatic model file caching +//! - Batch embedding for efficiency +//! +//! ## Requirements +//! - FR-01: Local embedding via Candle +//! - No external API dependencies +//! - Works offline after initial model download + +pub mod cache; +pub mod candle; +pub mod error; +pub mod model; + +pub use crate::candle::CandleEmbedder; +pub use cache::{get_or_download_model, ModelCache, ModelPaths, DEFAULT_MODEL_REPO, MODEL_FILES}; +pub use error::EmbeddingError; +pub use model::{Embedding, EmbeddingModel, ModelInfo}; diff --git a/crates/memory-embeddings/src/model.rs b/crates/memory-embeddings/src/model.rs new file mode 100644 index 0000000..377d784 --- /dev/null +++ b/crates/memory-embeddings/src/model.rs @@ -0,0 +1,118 @@ +//! Embedding model trait and types. +//! +//! Defines the interface for generating vector embeddings from text. + +use crate::error::EmbeddingError; + +/// Vector embedding - a normalized float array. +#[derive(Debug, Clone)] +pub struct Embedding { + /// The embedding vector (normalized to unit length) + pub values: Vec, +} + +impl Embedding { + /// Create a new embedding from a vector. + /// Normalizes the vector to unit length. + pub fn new(values: Vec) -> Self { + let norm: f32 = values.iter().map(|x| x * x).sum::().sqrt(); + let normalized = if norm > 0.0 { + values.iter().map(|x| x / norm).collect() + } else { + values + }; + Self { values: normalized } + } + + /// Create embedding without normalization (for pre-normalized vectors) + pub fn from_normalized(values: Vec) -> Self { + Self { values } + } + + /// Get the embedding dimension + pub fn dimension(&self) -> usize { + self.values.len() + } + + /// Compute cosine similarity with another embedding. + /// Returns value in [-1, 1] range (1 = identical). + pub fn cosine_similarity(&self, other: &Embedding) -> f32 { + if self.values.len() != other.values.len() { + return 0.0; + } + // Since both are normalized, dot product = cosine similarity + self.values + .iter() + .zip(other.values.iter()) + .map(|(a, b)| a * b) + .sum() + } +} + +/// Model information +#[derive(Debug, Clone)] +pub struct ModelInfo { + /// Model name (e.g., "all-MiniLM-L6-v2") + pub name: String, + /// Embedding dimension + pub dimension: usize, + /// Maximum sequence length in tokens + pub max_sequence_length: usize, +} + +/// Trait for embedding models. +/// +/// Implementations must be thread-safe (Send + Sync) for concurrent use. +pub trait EmbeddingModel: Send + Sync { + /// Get model information + fn info(&self) -> &ModelInfo; + + /// Generate embedding for a single text. + fn embed(&self, text: &str) -> Result; + + /// Generate embeddings for multiple texts (batch). + /// Default implementation calls embed() for each text. + fn embed_batch(&self, texts: &[&str]) -> Result, EmbeddingError> { + texts.iter().map(|text| self.embed(text)).collect() + } + + /// Generate embeddings for multiple owned strings. + fn embed_texts(&self, texts: &[String]) -> Result, EmbeddingError> { + let refs: Vec<&str> = texts.iter().map(|s| s.as_str()).collect(); + self.embed_batch(&refs) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_embedding_normalization() { + let emb = Embedding::new(vec![3.0, 4.0]); + // 3-4-5 triangle: normalized should be [0.6, 0.8] + assert!((emb.values[0] - 0.6).abs() < 0.001); + assert!((emb.values[1] - 0.8).abs() < 0.001); + } + + #[test] + fn test_cosine_similarity_identical() { + let emb1 = Embedding::new(vec![1.0, 0.0, 0.0]); + let emb2 = Embedding::new(vec![1.0, 0.0, 0.0]); + assert!((emb1.cosine_similarity(&emb2) - 1.0).abs() < 0.001); + } + + #[test] + fn test_cosine_similarity_orthogonal() { + let emb1 = Embedding::new(vec![1.0, 0.0]); + let emb2 = Embedding::new(vec![0.0, 1.0]); + assert!(emb1.cosine_similarity(&emb2).abs() < 0.001); + } + + #[test] + fn test_cosine_similarity_opposite() { + let emb1 = Embedding::new(vec![1.0, 0.0]); + let emb2 = Embedding::new(vec![-1.0, 0.0]); + assert!((emb1.cosine_similarity(&emb2) + 1.0).abs() < 0.001); + } +} diff --git a/crates/memory-indexing/Cargo.toml b/crates/memory-indexing/Cargo.toml new file mode 100644 index 0000000..3d569d3 --- /dev/null +++ b/crates/memory-indexing/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "memory-indexing" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +memory-embeddings = { workspace = true } +memory-search = { workspace = true } +memory-storage = { workspace = true } +memory-types = { workspace = true } +memory-vector = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "sync"] } +tracing = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +chrono = { workspace = true, features = ["serde"] } +thiserror = { workspace = true } + +[dev-dependencies] +tempfile = "3" +rand = "0.9" +ulid = "1" diff --git a/crates/memory-indexing/src/bm25_updater.rs b/crates/memory-indexing/src/bm25_updater.rs new file mode 100644 index 0000000..bfb31b1 --- /dev/null +++ b/crates/memory-indexing/src/bm25_updater.rs @@ -0,0 +1,332 @@ +//! BM25 index updater for Tantivy-based full-text search. +//! +//! Wraps SearchIndexer from memory-search to handle outbox-driven updates. +//! Converts OutboxEntry references to searchable documents. + +use std::sync::Arc; + +use tracing::{debug, warn}; + +use memory_search::SearchIndexer; +use memory_storage::Storage; +use memory_types::{Grip, OutboxAction, OutboxEntry, TocNode}; + +use crate::checkpoint::IndexType; +use crate::error::IndexingError; +use crate::updater::{IndexUpdater, UpdateResult}; + +/// BM25 index updater using Tantivy. +/// +/// Indexes TOC nodes and grips for full-text BM25 search. +/// Consumes outbox entries and fetches the corresponding +/// data from storage for indexing. +pub struct Bm25IndexUpdater { + indexer: Arc, + storage: Arc, +} + +impl Bm25IndexUpdater { + /// Create a new BM25 updater. + pub fn new(indexer: Arc, storage: Arc) -> Self { + Self { indexer, storage } + } + + /// Index a TOC node. + fn index_toc_node(&self, node: &TocNode) -> Result<(), IndexingError> { + self.indexer + .index_toc_node(node) + .map_err(|e| IndexingError::Index(format!("BM25 index error: {}", e))) + } + + /// Index a grip. + fn index_grip(&self, grip: &Grip) -> Result<(), IndexingError> { + self.indexer + .index_grip(grip) + .map_err(|e| IndexingError::Index(format!("BM25 index error: {}", e))) + } + + /// Process an outbox entry by fetching the event and related data. + /// + /// For IndexEvent actions, we need to determine if this event + /// is associated with a TOC node or grip and index accordingly. + fn process_entry(&self, entry: &OutboxEntry) -> Result { + match entry.action { + OutboxAction::IndexEvent => { + // The event_id in the outbox entry points to the event + // We need to check if there's a corresponding TOC node or grip + // For now, we'll try to find and index any related content + + // Check if there's a TOC node associated with this timestamp + // This is a simplified approach - in practice, you might have + // more sophisticated event-to-document mapping + debug!(event_id = %entry.event_id, "Processing index event for BM25"); + + // Try to find TOC nodes that might reference this event + // The event_id format is typically a ULID + // We could look up grips that span this event + if let Some(grip) = self.find_grip_for_event(&entry.event_id)? { + self.index_grip(&grip)?; + return Ok(true); + } + + // If no direct match, the event will be indexed when + // the summarizer creates TOC nodes/grips + debug!(event_id = %entry.event_id, "No grip found for event, skipping"); + Ok(false) + } + OutboxAction::UpdateToc => { + // For TOC updates, we'd need additional context about which + // TOC node was updated. For now, skip these as they're + // typically handled by the TOC expansion logic. + debug!(event_id = %entry.event_id, "Skipping TOC update action"); + Ok(false) + } + } + } + + /// Find a grip that references this event. + fn find_grip_for_event(&self, event_id: &str) -> Result, IndexingError> { + // This is a simplified lookup - in a full implementation, + // you might have an index from event_id to grip_id + // For now, we'll return None and rely on explicit grip indexing + debug!(event_id = %event_id, "Looking up grip for event"); + Ok(None) + } + + /// Process a batch of outbox entries. + pub fn process_batch( + &self, + entries: &[(u64, OutboxEntry)], + ) -> Result { + let mut result = UpdateResult::new(); + + for (sequence, entry) in entries { + match self.process_entry(entry) { + Ok(true) => { + result.record_success(); + } + Ok(false) => { + result.record_skip(); + } + Err(e) => { + warn!( + sequence = sequence, + event_id = %entry.event_id, + error = %e, + "Failed to process entry for BM25" + ); + result.record_error(); + } + } + result.set_sequence(*sequence); + } + + Ok(result) + } + + /// Index a TOC node directly (for bulk indexing). + pub fn index_node(&self, node: &TocNode) -> Result<(), IndexingError> { + self.index_toc_node(node) + } + + /// Index a grip directly (for bulk indexing). + pub fn index_grip_direct(&self, grip: &Grip) -> Result<(), IndexingError> { + self.index_grip(grip) + } + + /// Get the underlying storage reference. + pub fn storage(&self) -> &Arc { + &self.storage + } +} + +impl IndexUpdater for Bm25IndexUpdater { + fn index_document(&self, entry: &OutboxEntry) -> Result<(), IndexingError> { + match self.process_entry(entry)? { + true => Ok(()), + false => Ok(()), // Skipped entries are not errors + } + } + + fn remove_document(&self, doc_id: &str) -> Result<(), IndexingError> { + self.indexer + .delete_document(doc_id) + .map_err(|e| IndexingError::Index(format!("BM25 delete error: {}", e))) + } + + fn commit(&self) -> Result<(), IndexingError> { + self.indexer + .commit() + .map_err(|e| IndexingError::Index(format!("BM25 commit error: {}", e)))?; + Ok(()) + } + + fn index_type(&self) -> IndexType { + IndexType::Bm25 + } + + fn name(&self) -> &str { + "bm25" + } +} + +#[cfg(test)] +mod tests { + use super::*; + use memory_search::{SearchIndex, SearchIndexConfig}; + use tempfile::TempDir; + + fn create_test_storage() -> (Arc, TempDir) { + let temp_dir = TempDir::new().unwrap(); + let storage = Storage::open(temp_dir.path()).unwrap(); + (Arc::new(storage), temp_dir) + } + + fn create_test_indexer(path: &std::path::Path) -> Arc { + let config = SearchIndexConfig::new(path); + let index = SearchIndex::open_or_create(config).unwrap(); + Arc::new(SearchIndexer::new(&index).unwrap()) + } + + #[test] + fn test_bm25_updater_creation() { + let (storage, temp_dir) = create_test_storage(); + let search_path = temp_dir.path().join("search"); + std::fs::create_dir_all(&search_path).unwrap(); + let indexer = create_test_indexer(&search_path); + + let updater = Bm25IndexUpdater::new(indexer, storage); + assert_eq!(updater.index_type(), IndexType::Bm25); + assert_eq!(updater.name(), "bm25"); + } + + #[test] + fn test_process_index_event_no_grip() { + let (storage, temp_dir) = create_test_storage(); + let search_path = temp_dir.path().join("search"); + std::fs::create_dir_all(&search_path).unwrap(); + let indexer = create_test_indexer(&search_path); + + let updater = Bm25IndexUpdater::new(indexer, storage); + + let entry = OutboxEntry::for_index("event-123".to_string(), 1706540400000); + let result = updater.process_entry(&entry).unwrap(); + + // Should return false since no grip found + assert!(!result); + } + + #[test] + fn test_process_batch_empty() { + let (storage, temp_dir) = create_test_storage(); + let search_path = temp_dir.path().join("search"); + std::fs::create_dir_all(&search_path).unwrap(); + let indexer = create_test_indexer(&search_path); + + let updater = Bm25IndexUpdater::new(indexer, storage); + + let entries: Vec<(u64, OutboxEntry)> = vec![]; + let result = updater.process_batch(&entries).unwrap(); + + assert_eq!(result.processed, 0); + assert_eq!(result.skipped, 0); + assert_eq!(result.errors, 0); + } + + #[test] + fn test_process_batch_with_entries() { + let (storage, temp_dir) = create_test_storage(); + let search_path = temp_dir.path().join("search"); + std::fs::create_dir_all(&search_path).unwrap(); + let indexer = create_test_indexer(&search_path); + + let updater = Bm25IndexUpdater::new(indexer, storage); + + let entries = vec![ + (0, OutboxEntry::for_index("event-1".to_string(), 1000)), + (1, OutboxEntry::for_index("event-2".to_string(), 2000)), + (2, OutboxEntry::for_toc("event-3".to_string(), 3000)), + ]; + + let result = updater.process_batch(&entries).unwrap(); + + // All should be skipped (no grips found, TOC updates skipped) + assert_eq!(result.skipped, 3); + assert_eq!(result.last_sequence, 2); + } + + #[test] + fn test_index_toc_node_direct() { + use chrono::Utc; + use memory_types::TocLevel; + + let (storage, temp_dir) = create_test_storage(); + let search_path = temp_dir.path().join("search"); + std::fs::create_dir_all(&search_path).unwrap(); + let indexer = create_test_indexer(&search_path); + + let updater = Bm25IndexUpdater::new(indexer, storage); + + let node = TocNode::new( + "toc:day:2024-01-15".to_string(), + TocLevel::Day, + "Monday, January 15".to_string(), + Utc::now(), + Utc::now(), + ); + + updater.index_node(&node).unwrap(); + updater.commit().unwrap(); + } + + #[test] + fn test_index_grip_direct() { + use chrono::Utc; + + let (storage, temp_dir) = create_test_storage(); + let search_path = temp_dir.path().join("search"); + std::fs::create_dir_all(&search_path).unwrap(); + let indexer = create_test_indexer(&search_path); + + let updater = Bm25IndexUpdater::new(indexer, storage); + + let grip = Grip::new( + "grip:12345".to_string(), + "User asked about authentication".to_string(), + "event-001".to_string(), + "event-003".to_string(), + Utc::now(), + "test".to_string(), + ); + + updater.index_grip_direct(&grip).unwrap(); + updater.commit().unwrap(); + } + + #[test] + fn test_remove_document() { + use chrono::Utc; + use memory_types::TocLevel; + + let (storage, temp_dir) = create_test_storage(); + let search_path = temp_dir.path().join("search"); + std::fs::create_dir_all(&search_path).unwrap(); + let indexer = create_test_indexer(&search_path); + + let updater = Bm25IndexUpdater::new(indexer, storage); + + let node = TocNode::new( + "toc:day:2024-01-15".to_string(), + TocLevel::Day, + "Monday, January 15".to_string(), + Utc::now(), + Utc::now(), + ); + + updater.index_node(&node).unwrap(); + updater.commit().unwrap(); + + updater.remove_document("toc:day:2024-01-15").unwrap(); + updater.commit().unwrap(); + } +} diff --git a/crates/memory-indexing/src/checkpoint.rs b/crates/memory-indexing/src/checkpoint.rs new file mode 100644 index 0000000..e3057a6 --- /dev/null +++ b/crates/memory-indexing/src/checkpoint.rs @@ -0,0 +1,222 @@ +//! Checkpoint tracking for indexing pipelines. +//! +//! Checkpoints track the last processed outbox sequence number for each +//! index type, enabling crash recovery and resumable indexing. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::error::IndexingError; + +/// Type of index being tracked +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IndexType { + /// BM25 full-text search index + Bm25, + /// Vector similarity search index + Vector, + /// Combined index (both BM25 and vector) + Combined, +} + +impl IndexType { + /// Get the checkpoint key for this index type + pub fn checkpoint_key(&self) -> &'static str { + match self { + IndexType::Bm25 => "index_bm25", + IndexType::Vector => "index_vector", + IndexType::Combined => "index_combined", + } + } +} + +impl std::fmt::Display for IndexType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + IndexType::Bm25 => write!(f, "bm25"), + IndexType::Vector => write!(f, "vector"), + IndexType::Combined => write!(f, "combined"), + } + } +} + +/// Checkpoint for tracking indexing progress. +/// +/// Persisted to storage to enable crash recovery. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IndexCheckpoint { + /// Type of index this checkpoint is for + pub index_type: IndexType, + + /// Last outbox sequence number processed + pub last_sequence: u64, + + /// Timestamp of last processing (milliseconds since epoch for JSON compatibility) + #[serde(with = "chrono::serde::ts_milliseconds")] + pub last_processed_time: DateTime, + + /// Total items processed since checkpoint creation + pub processed_count: u64, + + /// When this checkpoint was first created (milliseconds since epoch) + #[serde(with = "chrono::serde::ts_milliseconds")] + pub created_at: DateTime, +} + +impl IndexCheckpoint { + /// Create a new checkpoint for the given index type + pub fn new(index_type: IndexType) -> Self { + let now = Utc::now(); + Self { + index_type, + last_sequence: 0, + last_processed_time: now, + processed_count: 0, + created_at: now, + } + } + + /// Create a checkpoint with a specific starting sequence + pub fn with_sequence(index_type: IndexType, sequence: u64) -> Self { + let now = Utc::now(); + Self { + index_type, + last_sequence: sequence, + last_processed_time: now, + processed_count: 0, + created_at: now, + } + } + + /// Get the checkpoint key for storage + pub fn checkpoint_key(&self) -> &'static str { + self.index_type.checkpoint_key() + } + + /// Update checkpoint after processing entries + pub fn update(&mut self, new_sequence: u64, items_processed: u64) { + self.last_sequence = new_sequence; + self.last_processed_time = Utc::now(); + self.processed_count += items_processed; + } + + /// Serialize to JSON bytes for storage + pub fn to_bytes(&self) -> Result, IndexingError> { + serde_json::to_vec(self).map_err(IndexingError::from) + } + + /// Deserialize from JSON bytes + pub fn from_bytes(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(IndexingError::from) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_index_type_checkpoint_keys() { + assert_eq!(IndexType::Bm25.checkpoint_key(), "index_bm25"); + assert_eq!(IndexType::Vector.checkpoint_key(), "index_vector"); + assert_eq!(IndexType::Combined.checkpoint_key(), "index_combined"); + } + + #[test] + fn test_index_type_display() { + assert_eq!(IndexType::Bm25.to_string(), "bm25"); + assert_eq!(IndexType::Vector.to_string(), "vector"); + assert_eq!(IndexType::Combined.to_string(), "combined"); + } + + #[test] + fn test_checkpoint_new() { + let checkpoint = IndexCheckpoint::new(IndexType::Bm25); + assert_eq!(checkpoint.index_type, IndexType::Bm25); + assert_eq!(checkpoint.last_sequence, 0); + assert_eq!(checkpoint.processed_count, 0); + assert_eq!(checkpoint.checkpoint_key(), "index_bm25"); + } + + #[test] + fn test_checkpoint_with_sequence() { + let checkpoint = IndexCheckpoint::with_sequence(IndexType::Vector, 100); + assert_eq!(checkpoint.index_type, IndexType::Vector); + assert_eq!(checkpoint.last_sequence, 100); + assert_eq!(checkpoint.processed_count, 0); + } + + #[test] + fn test_checkpoint_update() { + let mut checkpoint = IndexCheckpoint::new(IndexType::Bm25); + let original_time = checkpoint.last_processed_time; + + // Small sleep to ensure time difference + std::thread::sleep(std::time::Duration::from_millis(10)); + + checkpoint.update(50, 10); + assert_eq!(checkpoint.last_sequence, 50); + assert_eq!(checkpoint.processed_count, 10); + assert!(checkpoint.last_processed_time >= original_time); + + checkpoint.update(100, 5); + assert_eq!(checkpoint.last_sequence, 100); + assert_eq!(checkpoint.processed_count, 15); + } + + #[test] + fn test_checkpoint_serialization_roundtrip() { + let checkpoint = IndexCheckpoint::with_sequence(IndexType::Combined, 42); + let bytes = checkpoint.to_bytes().unwrap(); + let decoded = IndexCheckpoint::from_bytes(&bytes).unwrap(); + + assert_eq!(checkpoint.index_type, decoded.index_type); + assert_eq!(checkpoint.last_sequence, decoded.last_sequence); + assert_eq!(checkpoint.processed_count, decoded.processed_count); + // Compare timestamps at millisecond precision (JSON serializes to ms) + assert_eq!( + checkpoint.created_at.timestamp_millis(), + decoded.created_at.timestamp_millis() + ); + assert_eq!( + checkpoint.last_processed_time.timestamp_millis(), + decoded.last_processed_time.timestamp_millis() + ); + } + + #[test] + fn test_checkpoint_json_format() { + let checkpoint = IndexCheckpoint::new(IndexType::Bm25); + let bytes = checkpoint.to_bytes().unwrap(); + let json_str = String::from_utf8(bytes).unwrap(); + + // Verify JSON contains expected fields + assert!(json_str.contains("\"index_type\":\"bm25\"")); + assert!(json_str.contains("\"last_sequence\":0")); + assert!(json_str.contains("\"processed_count\":0")); + assert!(json_str.contains("\"last_processed_time\":")); + assert!(json_str.contains("\"created_at\":")); + } + + #[test] + fn test_index_type_serialization() { + // Test all variants serialize correctly + let bm25 = serde_json::to_string(&IndexType::Bm25).unwrap(); + let vector = serde_json::to_string(&IndexType::Vector).unwrap(); + let combined = serde_json::to_string(&IndexType::Combined).unwrap(); + + assert_eq!(bm25, "\"bm25\""); + assert_eq!(vector, "\"vector\""); + assert_eq!(combined, "\"combined\""); + + // Test deserialization + let bm25: IndexType = serde_json::from_str("\"bm25\"").unwrap(); + let vector: IndexType = serde_json::from_str("\"vector\"").unwrap(); + let combined: IndexType = serde_json::from_str("\"combined\"").unwrap(); + + assert_eq!(bm25, IndexType::Bm25); + assert_eq!(vector, IndexType::Vector); + assert_eq!(combined, IndexType::Combined); + } +} diff --git a/crates/memory-indexing/src/error.rs b/crates/memory-indexing/src/error.rs new file mode 100644 index 0000000..a4fc1c2 --- /dev/null +++ b/crates/memory-indexing/src/error.rs @@ -0,0 +1,69 @@ +//! Error types for the indexing pipeline. + +use memory_embeddings::EmbeddingError; +use memory_search::SearchError; +use memory_storage::StorageError; +use memory_vector::VectorError; +use thiserror::Error; + +/// Errors that can occur in the indexing pipeline +#[derive(Error, Debug)] +pub enum IndexingError { + /// Storage operation failed + #[error("Storage error: {0}")] + Storage(#[from] StorageError), + + /// Checkpoint load/save issues + #[error("Checkpoint error: {0}")] + Checkpoint(String), + + /// JSON encoding/decoding errors + #[error("Serialization error: {0}")] + Serialization(String), + + /// Generic index operation error + #[error("Index error: {0}")] + Index(String), + + /// BM25 search index error + #[error("Search error: {0}")] + Search(#[from] SearchError), + + /// Vector index error + #[error("Vector error: {0}")] + Vector(#[from] VectorError), + + /// Embedding generation error + #[error("Embedding error: {0}")] + Embedding(#[from] EmbeddingError), +} + +impl From for IndexingError { + fn from(err: serde_json::Error) -> Self { + IndexingError::Serialization(err.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_error_display() { + let err = IndexingError::Checkpoint("failed to load".to_string()); + assert_eq!(err.to_string(), "Checkpoint error: failed to load"); + + let err = IndexingError::Serialization("invalid json".to_string()); + assert_eq!(err.to_string(), "Serialization error: invalid json"); + + let err = IndexingError::Index("index corrupted".to_string()); + assert_eq!(err.to_string(), "Index error: index corrupted"); + } + + #[test] + fn test_from_serde_error() { + let json_err = serde_json::from_str::("not a number").unwrap_err(); + let indexing_err: IndexingError = json_err.into(); + assert!(matches!(indexing_err, IndexingError::Serialization(_))); + } +} diff --git a/crates/memory-indexing/src/lib.rs b/crates/memory-indexing/src/lib.rs new file mode 100644 index 0000000..9c09597 --- /dev/null +++ b/crates/memory-indexing/src/lib.rs @@ -0,0 +1,60 @@ +//! Indexing pipeline for agent-memory system. +//! +//! This crate provides the infrastructure for consuming outbox entries +//! and updating search indexes (BM25 and vector). +//! +//! ## Key Components +//! +//! - [`IndexCheckpoint`]: Tracks indexing progress for crash recovery +//! - [`IndexType`]: Distinguishes between BM25, vector, and combined indexes +//! - [`IndexingError`]: Error types for indexing operations +//! - [`IndexUpdater`]: Trait for index-specific update operations +//! - [`Bm25IndexUpdater`]: BM25 full-text search updater using Tantivy +//! - [`VectorIndexUpdater`]: Vector similarity search updater using HNSW +//! - [`IndexingPipeline`]: Coordinates multiple updaters with checkpointing +//! +//! ## Architecture +//! +//! The indexing pipeline follows the outbox pattern: +//! 1. Events are written with outbox entries atomically +//! 2. This crate consumes outbox entries in sequence order +//! 3. Each [`IndexUpdater`] processes entries for its index type +//! 4. Checkpoints track progress for crash recovery +//! 5. After processing, outbox entries can be cleaned up +//! +//! ## Example +//! +//! ```ignore +//! use memory_indexing::{IndexingPipeline, PipelineConfig, Bm25IndexUpdater, VectorIndexUpdater}; +//! +//! let mut pipeline = IndexingPipeline::new(storage, PipelineConfig::default()); +//! pipeline.add_updater(Box::new(bm25_updater)); +//! pipeline.add_updater(Box::new(vector_updater)); +//! pipeline.load_checkpoints()?; +//! +//! // Process until caught up +//! let result = pipeline.process_until_caught_up(100)?; +//! +//! // Clean up processed entries +//! pipeline.cleanup_outbox()?; +//! ``` + +pub mod bm25_updater; +pub mod checkpoint; +pub mod error; +pub mod pipeline; +pub mod rebuild; +pub mod updater; +pub mod vector_updater; + +pub use bm25_updater::Bm25IndexUpdater; +pub use checkpoint::{IndexCheckpoint, IndexType}; +pub use error::IndexingError; +pub use pipeline::{IndexingPipeline, PipelineConfig, ProcessResult}; +pub use rebuild::{ + iter_all_grips, iter_all_toc_nodes, rebuild_bm25_index, rebuild_vector_index, + LoggingProgressCallback, NoOpProgressCallback, ProgressCallback, RebuildConfig, + RebuildProgress, RebuildResult, +}; +pub use updater::{IndexUpdater, UpdateResult}; +pub use vector_updater::VectorIndexUpdater; diff --git a/crates/memory-indexing/src/pipeline.rs b/crates/memory-indexing/src/pipeline.rs new file mode 100644 index 0000000..286eec5 --- /dev/null +++ b/crates/memory-indexing/src/pipeline.rs @@ -0,0 +1,667 @@ +//! Indexing pipeline for processing outbox entries. +//! +//! Coordinates multiple index updaters, manages checkpoints, +//! and handles batch processing with crash recovery. + +use std::collections::HashMap; +use std::sync::Arc; + +use tracing::{debug, info, warn}; + +use memory_storage::Storage; + +use crate::checkpoint::{IndexCheckpoint, IndexType}; +use crate::error::IndexingError; +use crate::updater::{IndexUpdater, UpdateResult}; + +/// Result of processing a batch of outbox entries. +#[derive(Debug, Default)] +pub struct ProcessResult { + /// Results per index type + pub by_index: HashMap, + /// Total entries processed across all indexes + pub total_processed: usize, + /// The last sequence number processed + pub last_sequence: Option, + /// Whether all indexes were successfully committed + pub committed: bool, +} + +impl ProcessResult { + /// Create a new empty result. + pub fn new() -> Self { + Self::default() + } + + /// Add a result for an index type. + pub fn add_result(&mut self, index_type: IndexType, result: UpdateResult) { + self.total_processed += result.processed; + if let Some(last_seq) = self.last_sequence { + if result.last_sequence > last_seq { + self.last_sequence = Some(result.last_sequence); + } + } else if result.last_sequence > 0 { + self.last_sequence = Some(result.last_sequence); + } + self.by_index.insert(index_type, result); + } + + /// Check if any entries were processed. + pub fn has_updates(&self) -> bool { + self.total_processed > 0 + } +} + +/// Configuration for the indexing pipeline. +#[derive(Debug, Clone)] +pub struct PipelineConfig { + /// Maximum entries to process per batch + pub batch_size: usize, + /// Whether to continue processing on individual entry errors + pub continue_on_error: bool, + /// Whether to commit after each batch + pub commit_after_batch: bool, +} + +impl Default for PipelineConfig { + fn default() -> Self { + Self { + batch_size: 100, + continue_on_error: true, + commit_after_batch: true, + } + } +} + +impl PipelineConfig { + /// Create a new config with the given batch size. + pub fn with_batch_size(mut self, size: usize) -> Self { + self.batch_size = size; + self + } + + /// Set whether to continue on errors. + pub fn with_continue_on_error(mut self, continue_on_error: bool) -> Self { + self.continue_on_error = continue_on_error; + self + } + + /// Set whether to commit after each batch. + pub fn with_commit_after_batch(mut self, commit: bool) -> Self { + self.commit_after_batch = commit; + self + } +} + +/// Indexing pipeline that coordinates multiple index updaters. +/// +/// Processes outbox entries in sequence order and updates +/// all registered indexes with checkpoint tracking. +pub struct IndexingPipeline { + storage: Arc, + updaters: Vec>, + checkpoints: HashMap, + config: PipelineConfig, +} + +impl IndexingPipeline { + /// Create a new indexing pipeline. + pub fn new(storage: Arc, config: PipelineConfig) -> Self { + Self { + storage, + updaters: Vec::new(), + checkpoints: HashMap::new(), + config, + } + } + + /// Add an index updater to the pipeline. + pub fn add_updater(&mut self, updater: Box) { + let index_type = updater.index_type(); + self.updaters.push(updater); + + // Initialize checkpoint if not already loaded + self.checkpoints + .entry(index_type) + .or_insert_with(|| IndexCheckpoint::new(index_type)); + } + + /// Load checkpoints from storage. + pub fn load_checkpoints(&mut self) -> Result<(), IndexingError> { + for updater in &self.updaters { + let index_type = updater.index_type(); + let key = index_type.checkpoint_key(); + + if let Some(bytes) = self.storage.get_checkpoint(key)? { + let checkpoint = IndexCheckpoint::from_bytes(&bytes)?; + info!( + index = %updater.name(), + last_sequence = checkpoint.last_sequence, + "Loaded checkpoint" + ); + self.checkpoints.insert(index_type, checkpoint); + } else { + debug!(index = %updater.name(), "No existing checkpoint, starting from 0"); + self.checkpoints + .insert(index_type, IndexCheckpoint::new(index_type)); + } + } + Ok(()) + } + + /// Save checkpoints to storage. + pub fn save_checkpoints(&self) -> Result<(), IndexingError> { + for (index_type, checkpoint) in &self.checkpoints { + let key = index_type.checkpoint_key(); + let bytes = checkpoint.to_bytes()?; + self.storage.put_checkpoint(key, &bytes)?; + debug!( + index_type = %index_type, + last_sequence = checkpoint.last_sequence, + "Saved checkpoint" + ); + } + Ok(()) + } + + /// Get the minimum last_sequence across all checkpoints. + /// + /// This is the starting point for the next batch - we need + /// to process from here to ensure all indexes are caught up. + fn min_checkpoint_sequence(&self) -> u64 { + self.checkpoints + .values() + .map(|c| c.last_sequence) + .min() + .unwrap_or(0) + } + + /// Process a batch of outbox entries from storage. + /// + /// Returns the processing result including per-index stats. + pub fn process_batch(&mut self, batch_size: usize) -> Result { + let start_sequence = self.min_checkpoint_sequence(); + let limit = batch_size.max(1); + + info!( + start_sequence = start_sequence, + limit = limit, + "Fetching outbox entries" + ); + + let entries = self.storage.get_outbox_entries(start_sequence, limit)?; + + if entries.is_empty() { + debug!("No outbox entries to process"); + return Ok(ProcessResult::new()); + } + + info!(count = entries.len(), "Processing outbox entries"); + + let mut result = ProcessResult::new(); + + // Process each updater + for updater in &self.updaters { + let index_type = updater.index_type(); + let checkpoint = self + .checkpoints + .get(&index_type) + .cloned() + .unwrap_or_else(|| IndexCheckpoint::new(index_type)); + + // Filter entries this updater hasn't seen yet + // When processed_count is 0, this is a fresh checkpoint and we should + // process from the beginning (including sequence 0) + let new_entries: Vec<_> = entries + .iter() + .filter(|(seq, _)| { + if checkpoint.processed_count == 0 { + true // Fresh checkpoint - process all available entries + } else { + *seq > checkpoint.last_sequence + } + }) + .cloned() + .collect(); + + if new_entries.is_empty() { + debug!(index = %updater.name(), "No new entries for this index"); + result.add_result(index_type, UpdateResult::new()); + continue; + } + + debug!( + index = %updater.name(), + count = new_entries.len(), + "Processing entries" + ); + + let mut update_result = UpdateResult::new(); + + for (sequence, entry) in &new_entries { + match updater.index_document(entry) { + Ok(()) => { + update_result.record_success(); + } + Err(e) => { + warn!( + index = %updater.name(), + sequence = sequence, + event_id = %entry.event_id, + error = %e, + "Failed to index document" + ); + if self.config.continue_on_error { + update_result.record_error(); + } else { + return Err(e); + } + } + } + update_result.set_sequence(*sequence); + } + + result.add_result(index_type, update_result); + } + + // Commit if configured + if self.config.commit_after_batch && result.has_updates() { + self.commit()?; + result.committed = true; + + // Update checkpoints after successful commit + if let Some(last_seq) = result.last_sequence { + for (index_type, checkpoint) in &mut self.checkpoints { + if let Some(idx_result) = result.by_index.get(index_type) { + if idx_result.last_sequence > checkpoint.last_sequence { + checkpoint + .update(idx_result.last_sequence, idx_result.processed as u64); + } + } + } + self.save_checkpoints()?; + + info!( + last_sequence = last_seq, + total_processed = result.total_processed, + "Batch processing complete" + ); + } + } + + Ok(result) + } + + /// Commit all indexes. + pub fn commit(&self) -> Result<(), IndexingError> { + for updater in &self.updaters { + updater.commit()?; + debug!(index = %updater.name(), "Committed"); + } + Ok(()) + } + + /// Process entries until caught up or max iterations reached. + /// + /// Returns total processing stats across all batches. + pub fn process_until_caught_up( + &mut self, + max_iterations: usize, + ) -> Result { + let mut total_result = ProcessResult::new(); + let mut iterations = 0; + + loop { + if iterations >= max_iterations { + info!(iterations = iterations, "Reached max iterations"); + break; + } + + let batch_result = self.process_batch(self.config.batch_size)?; + + if !batch_result.has_updates() && batch_result.last_sequence.is_none() { + debug!("No more entries to process"); + break; + } + + // Merge results + total_result.total_processed += batch_result.total_processed; + if let Some(seq) = batch_result.last_sequence { + total_result.last_sequence = Some(seq); + } + for (index_type, result) in batch_result.by_index { + total_result + .by_index + .entry(index_type) + .or_default() + .merge(&result); + } + + iterations += 1; + } + + total_result.committed = true; + Ok(total_result) + } + + /// Clean up processed outbox entries. + /// + /// Deletes entries up to the minimum checkpoint sequence. + /// Only call after confirming all indexes are caught up. + pub fn cleanup_outbox(&self) -> Result { + let min_seq = self.min_checkpoint_sequence(); + if min_seq == 0 { + return Ok(0); + } + + // Delete entries that all indexes have processed + // Use min_seq - 1 to keep the last processed entry for safety + let up_to = min_seq.saturating_sub(1); + let deleted = self.storage.delete_outbox_entries(up_to)?; + + info!( + min_sequence = min_seq, + deleted = deleted, + "Cleaned up outbox entries" + ); + + Ok(deleted) + } + + /// Get the current checkpoint for an index type. + pub fn get_checkpoint(&self, index_type: IndexType) -> Option<&IndexCheckpoint> { + self.checkpoints.get(&index_type) + } + + /// Get all registered updater names. + pub fn updater_names(&self) -> Vec<&str> { + self.updaters.iter().map(|u| u.name()).collect() + } + + /// Get the number of registered updaters. + pub fn updater_count(&self) -> usize { + self.updaters.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use memory_types::OutboxEntry; + use tempfile::TempDir; + + // Mock updater for testing + struct MockUpdater { + index_type: IndexType, + name: &'static str, + should_fail: bool, + } + + impl MockUpdater { + fn new(index_type: IndexType, name: &'static str) -> Self { + Self { + index_type, + name, + should_fail: false, + } + } + + #[allow(dead_code)] + fn failing(index_type: IndexType, name: &'static str) -> Self { + Self { + index_type, + name, + should_fail: true, + } + } + } + + impl IndexUpdater for MockUpdater { + fn index_document(&self, _entry: &OutboxEntry) -> Result<(), IndexingError> { + if self.should_fail { + Err(IndexingError::Index("Mock failure".to_string())) + } else { + Ok(()) + } + } + + fn remove_document(&self, _doc_id: &str) -> Result<(), IndexingError> { + Ok(()) + } + + fn commit(&self) -> Result<(), IndexingError> { + Ok(()) + } + + fn index_type(&self) -> IndexType { + self.index_type + } + + fn name(&self) -> &str { + self.name + } + } + + fn create_test_storage() -> (Arc, TempDir) { + let temp_dir = TempDir::new().unwrap(); + let storage = Storage::open(temp_dir.path()).unwrap(); + (Arc::new(storage), temp_dir) + } + + #[test] + fn test_pipeline_creation() { + let (storage, _temp) = create_test_storage(); + let pipeline = IndexingPipeline::new(storage, PipelineConfig::default()); + + assert_eq!(pipeline.updater_count(), 0); + } + + #[test] + fn test_add_updater() { + let (storage, _temp) = create_test_storage(); + let mut pipeline = IndexingPipeline::new(storage, PipelineConfig::default()); + + pipeline.add_updater(Box::new(MockUpdater::new(IndexType::Bm25, "bm25"))); + pipeline.add_updater(Box::new(MockUpdater::new(IndexType::Vector, "vector"))); + + assert_eq!(pipeline.updater_count(), 2); + assert_eq!(pipeline.updater_names(), vec!["bm25", "vector"]); + } + + #[test] + fn test_load_save_checkpoints() { + let (storage, _temp) = create_test_storage(); + let mut pipeline = IndexingPipeline::new(storage.clone(), PipelineConfig::default()); + + pipeline.add_updater(Box::new(MockUpdater::new(IndexType::Bm25, "bm25"))); + pipeline.load_checkpoints().unwrap(); + + // Should have a checkpoint now + let cp = pipeline.get_checkpoint(IndexType::Bm25); + assert!(cp.is_some()); + assert_eq!(cp.unwrap().last_sequence, 0); + + // Modify and save + pipeline + .checkpoints + .get_mut(&IndexType::Bm25) + .unwrap() + .update(42, 10); + pipeline.save_checkpoints().unwrap(); + + // Create new pipeline and load + let mut pipeline2 = IndexingPipeline::new(storage, PipelineConfig::default()); + pipeline2.add_updater(Box::new(MockUpdater::new(IndexType::Bm25, "bm25"))); + pipeline2.load_checkpoints().unwrap(); + + let cp2 = pipeline2.get_checkpoint(IndexType::Bm25); + assert!(cp2.is_some()); + assert_eq!(cp2.unwrap().last_sequence, 42); + } + + #[test] + fn test_process_batch_empty() { + let (storage, _temp) = create_test_storage(); + let mut pipeline = IndexingPipeline::new(storage, PipelineConfig::default()); + + pipeline.add_updater(Box::new(MockUpdater::new(IndexType::Bm25, "bm25"))); + pipeline.load_checkpoints().unwrap(); + + let result = pipeline.process_batch(100).unwrap(); + assert!(!result.has_updates()); + assert!(result.last_sequence.is_none()); + } + + #[test] + fn test_process_batch_with_entries() { + let (storage, _temp_dir) = create_test_storage(); + + // Add some outbox entries via events + for i in 0..5 { + let event_id = format!("event-{}", i); + let outbox_entry = OutboxEntry::for_index(event_id.clone(), i * 1000); + let outbox_bytes = outbox_entry.to_bytes().unwrap(); + storage + .put_event(&ulid::Ulid::new().to_string(), b"test", &outbox_bytes) + .unwrap(); + } + + let mut pipeline = IndexingPipeline::new(storage, PipelineConfig::default()); + pipeline.add_updater(Box::new(MockUpdater::new(IndexType::Bm25, "bm25"))); + pipeline.load_checkpoints().unwrap(); + + let result = pipeline.process_batch(100).unwrap(); + assert!(result.has_updates()); + assert_eq!(result.total_processed, 5); + assert!(result.last_sequence.is_some()); + assert!(result.committed); + } + + #[test] + fn test_process_until_caught_up() { + let (storage, _temp_dir) = create_test_storage(); + + // Add outbox entries + for i in 0..10 { + let event_id = format!("event-{}", i); + let outbox_entry = OutboxEntry::for_index(event_id.clone(), i * 1000); + let outbox_bytes = outbox_entry.to_bytes().unwrap(); + storage + .put_event(&ulid::Ulid::new().to_string(), b"test", &outbox_bytes) + .unwrap(); + } + + let config = PipelineConfig::default().with_batch_size(3); + let mut pipeline = IndexingPipeline::new(storage, config); + pipeline.add_updater(Box::new(MockUpdater::new(IndexType::Bm25, "bm25"))); + pipeline.load_checkpoints().unwrap(); + + let result = pipeline.process_until_caught_up(100).unwrap(); + assert_eq!(result.total_processed, 10); + } + + #[test] + fn test_cleanup_outbox() { + let (storage, _temp_dir) = create_test_storage(); + + // Add outbox entries + for i in 0..5 { + let event_id = format!("event-{}", i); + let outbox_entry = OutboxEntry::for_index(event_id.clone(), i * 1000); + let outbox_bytes = outbox_entry.to_bytes().unwrap(); + storage + .put_event(&ulid::Ulid::new().to_string(), b"test", &outbox_bytes) + .unwrap(); + } + + let mut pipeline = IndexingPipeline::new(storage.clone(), PipelineConfig::default()); + pipeline.add_updater(Box::new(MockUpdater::new(IndexType::Bm25, "bm25"))); + pipeline.load_checkpoints().unwrap(); + + // Process all entries + pipeline.process_until_caught_up(100).unwrap(); + + // Cleanup should delete processed entries + let deleted = pipeline.cleanup_outbox().unwrap(); + // Deletes up to min_seq - 1, so 4 entries (0, 1, 2, 3) + assert!(deleted >= 3); + + // Verify remaining entries + let remaining = storage.get_outbox_entries(0, 100).unwrap(); + assert!(remaining.len() <= 2); // At most entries 4 and maybe 5 + } + + #[test] + fn test_min_checkpoint_sequence() { + let (storage, _temp) = create_test_storage(); + let mut pipeline = IndexingPipeline::new(storage, PipelineConfig::default()); + + pipeline.add_updater(Box::new(MockUpdater::new(IndexType::Bm25, "bm25"))); + pipeline.add_updater(Box::new(MockUpdater::new(IndexType::Vector, "vector"))); + pipeline.load_checkpoints().unwrap(); + + // Both start at 0 + assert_eq!(pipeline.min_checkpoint_sequence(), 0); + + // Update one + pipeline + .checkpoints + .get_mut(&IndexType::Bm25) + .unwrap() + .update(10, 5); + + // Min should still be 0 (vector hasn't caught up) + assert_eq!(pipeline.min_checkpoint_sequence(), 0); + + // Update both + pipeline + .checkpoints + .get_mut(&IndexType::Vector) + .unwrap() + .update(8, 4); + + // Min should now be 8 + assert_eq!(pipeline.min_checkpoint_sequence(), 8); + } + + #[test] + fn test_pipeline_config() { + let config = PipelineConfig::default() + .with_batch_size(50) + .with_continue_on_error(false) + .with_commit_after_batch(false); + + assert_eq!(config.batch_size, 50); + assert!(!config.continue_on_error); + assert!(!config.commit_after_batch); + } + + #[test] + fn test_process_result() { + let mut result = ProcessResult::new(); + assert!(!result.has_updates()); + + let update1 = UpdateResult { + processed: 5, + skipped: 2, + errors: 1, + last_sequence: 10, + }; + + result.add_result(IndexType::Bm25, update1); + assert!(result.has_updates()); + assert_eq!(result.total_processed, 5); + assert_eq!(result.last_sequence, Some(10)); + + let update2 = UpdateResult { + processed: 3, + skipped: 0, + errors: 0, + last_sequence: 15, + }; + + result.add_result(IndexType::Vector, update2); + assert_eq!(result.total_processed, 8); + assert_eq!(result.last_sequence, Some(15)); + } +} diff --git a/crates/memory-indexing/src/rebuild.rs b/crates/memory-indexing/src/rebuild.rs new file mode 100644 index 0000000..b7fafeb --- /dev/null +++ b/crates/memory-indexing/src/rebuild.rs @@ -0,0 +1,477 @@ +//! Index rebuild functionality for reconstructing search indexes from storage. +//! +//! Provides utilities for rebuilding BM25 and vector indexes from scratch +//! by iterating through all TOC nodes and grips in storage. + +use std::sync::Arc; + +use tracing::{debug, info, warn}; + +use memory_storage::Storage; +use memory_types::{Grip, TocLevel, TocNode}; + +use crate::bm25_updater::Bm25IndexUpdater; +use crate::checkpoint::{IndexCheckpoint, IndexType}; +use crate::error::IndexingError; +use crate::updater::IndexUpdater; +use crate::vector_updater::VectorIndexUpdater; + +/// Configuration for index rebuild operations. +#[derive(Debug, Clone)] +pub struct RebuildConfig { + /// Number of documents to process before reporting progress. + pub batch_size: usize, + /// Which index types to rebuild. + pub index_types: Vec, + /// Whether to clear existing indexes before rebuilding. + pub clear_first: bool, + /// Whether to continue on individual document errors. + pub continue_on_error: bool, +} + +impl Default for RebuildConfig { + fn default() -> Self { + Self { + batch_size: 100, + index_types: vec![IndexType::Bm25, IndexType::Vector], + clear_first: true, + continue_on_error: true, + } + } +} + +impl RebuildConfig { + /// Create a config for BM25 only. + pub fn bm25_only() -> Self { + Self { + index_types: vec![IndexType::Bm25], + ..Default::default() + } + } + + /// Create a config for vector only. + pub fn vector_only() -> Self { + Self { + index_types: vec![IndexType::Vector], + ..Default::default() + } + } + + /// Set the batch size. + pub fn with_batch_size(mut self, size: usize) -> Self { + self.batch_size = size; + self + } + + /// Set whether to clear indexes first. + pub fn with_clear_first(mut self, clear: bool) -> Self { + self.clear_first = clear; + self + } + + /// Set whether to continue on errors. + pub fn with_continue_on_error(mut self, continue_on_error: bool) -> Self { + self.continue_on_error = continue_on_error; + self + } +} + +/// Progress tracking for rebuild operations. +#[derive(Debug, Clone, Default)] +pub struct RebuildProgress { + /// Total documents processed. + pub total_processed: u64, + /// Number of TOC nodes indexed. + pub toc_nodes_indexed: u64, + /// Number of grips indexed. + pub grips_indexed: u64, + /// Number of errors encountered. + pub errors: u64, + /// Number of documents skipped (already indexed or empty). + pub skipped: u64, + /// Whether the rebuild completed successfully. + pub completed: bool, +} + +impl RebuildProgress { + /// Create a new progress tracker. + pub fn new() -> Self { + Self::default() + } + + /// Record a successful TOC node index. + pub fn record_toc_node(&mut self) { + self.toc_nodes_indexed += 1; + self.total_processed += 1; + } + + /// Record a successful grip index. + pub fn record_grip(&mut self) { + self.grips_indexed += 1; + self.total_processed += 1; + } + + /// Record an error. + pub fn record_error(&mut self) { + self.errors += 1; + self.total_processed += 1; + } + + /// Record a skipped document. + pub fn record_skip(&mut self) { + self.skipped += 1; + self.total_processed += 1; + } + + /// Mark as completed. + pub fn mark_completed(&mut self) { + self.completed = true; + } +} + +/// Result of a rebuild operation. +#[derive(Debug)] +pub struct RebuildResult { + /// Progress statistics. + pub progress: RebuildProgress, + /// Time taken in milliseconds. + pub elapsed_ms: u64, + /// Per-index results. + pub index_results: Vec<(IndexType, IndexCheckpoint)>, +} + +/// Trait for receiving rebuild progress updates. +pub trait ProgressCallback: Send { + /// Called after each batch of documents is processed. + fn on_progress(&self, progress: &RebuildProgress); +} + +/// A no-op progress callback for when progress reporting isn't needed. +pub struct NoOpProgressCallback; + +impl ProgressCallback for NoOpProgressCallback { + fn on_progress(&self, _progress: &RebuildProgress) {} +} + +/// A callback that logs progress at info level. +pub struct LoggingProgressCallback { + batch_size: usize, +} + +impl LoggingProgressCallback { + /// Create a new logging progress callback. + pub fn new(batch_size: usize) -> Self { + Self { batch_size } + } +} + +impl ProgressCallback for LoggingProgressCallback { + fn on_progress(&self, progress: &RebuildProgress) { + if progress + .total_processed + .is_multiple_of(self.batch_size as u64) + { + info!( + total = progress.total_processed, + toc_nodes = progress.toc_nodes_indexed, + grips = progress.grips_indexed, + errors = progress.errors, + "Rebuild progress" + ); + } + } +} + +/// Iterate through all TOC nodes in storage. +/// +/// Returns nodes from all levels, ordered by level (Year -> Month -> Week -> Day -> Segment). +pub fn iter_all_toc_nodes(storage: &Storage) -> Result, IndexingError> { + let mut all_nodes = Vec::new(); + + // Iterate through all TOC levels + for level in &[ + TocLevel::Year, + TocLevel::Month, + TocLevel::Week, + TocLevel::Day, + TocLevel::Segment, + ] { + let nodes = storage + .get_toc_nodes_by_level(*level, None, None) + .map_err(IndexingError::Storage)?; + all_nodes.extend(nodes); + } + + debug!(count = all_nodes.len(), "Found TOC nodes in storage"); + Ok(all_nodes) +} + +/// Iterate through all grips in storage. +/// +/// Uses prefix iteration on the grips column family. +pub fn iter_all_grips(storage: &Storage) -> Result, IndexingError> { + let mut grips = Vec::new(); + + // Use prefix_iterator with empty prefix to get all grips + // Grips are stored with their grip_id as key + // We need to filter out index entries (which start with "node:") + let entries = storage + .prefix_iterator("grips", b"grip:") + .map_err(IndexingError::Storage)?; + + for (key, value) in entries { + let key_str = String::from_utf8_lossy(&key); + // Only process grip entries, not index entries + if key_str.starts_with("grip:") { + match Grip::from_bytes(&value) { + Ok(grip) => grips.push(grip), + Err(e) => { + warn!(key = %key_str, error = %e, "Failed to deserialize grip"); + } + } + } + } + + debug!(count = grips.len(), "Found grips in storage"); + Ok(grips) +} + +/// Rebuild BM25 index from storage. +pub fn rebuild_bm25_index( + storage: Arc, + updater: &Bm25IndexUpdater, + config: &RebuildConfig, + progress_callback: &P, +) -> Result { + let mut progress = RebuildProgress::new(); + + info!("Starting BM25 index rebuild..."); + + // Iterate through all TOC nodes + let nodes = iter_all_toc_nodes(&storage)?; + info!(count = nodes.len(), "Found TOC nodes to index"); + + for node in nodes { + match updater.index_node(&node) { + Ok(()) => { + progress.record_toc_node(); + } + Err(e) => { + if config.continue_on_error { + warn!(node_id = %node.node_id, error = %e, "Failed to index TOC node"); + progress.record_error(); + } else { + return Err(e); + } + } + } + + if progress + .total_processed + .is_multiple_of(config.batch_size as u64) + { + progress_callback.on_progress(&progress); + } + } + + // Iterate through all grips + let grips = iter_all_grips(&storage)?; + info!(count = grips.len(), "Found grips to index"); + + for grip in grips { + match updater.index_grip_direct(&grip) { + Ok(()) => { + progress.record_grip(); + } + Err(e) => { + if config.continue_on_error { + warn!(grip_id = %grip.grip_id, error = %e, "Failed to index grip"); + progress.record_error(); + } else { + return Err(e); + } + } + } + + if progress + .total_processed + .is_multiple_of(config.batch_size as u64) + { + progress_callback.on_progress(&progress); + } + } + + // Commit the index + updater.commit()?; + progress.mark_completed(); + progress_callback.on_progress(&progress); + + info!( + toc_nodes = progress.toc_nodes_indexed, + grips = progress.grips_indexed, + errors = progress.errors, + "BM25 index rebuild complete" + ); + + Ok(progress) +} + +/// Rebuild vector index from storage. +pub fn rebuild_vector_index( + storage: Arc, + updater: &VectorIndexUpdater, + config: &RebuildConfig, + progress_callback: &P, +) -> Result { + let mut progress = RebuildProgress::new(); + + info!("Starting vector index rebuild..."); + + // Iterate through all TOC nodes + let nodes = iter_all_toc_nodes(&storage)?; + info!(count = nodes.len(), "Found TOC nodes to index"); + + for node in nodes { + match updater.index_node(&node) { + Ok(true) => { + progress.record_toc_node(); + } + Ok(false) => { + progress.record_skip(); + } + Err(e) => { + if config.continue_on_error { + warn!(node_id = %node.node_id, error = %e, "Failed to index TOC node"); + progress.record_error(); + } else { + return Err(e); + } + } + } + + if progress + .total_processed + .is_multiple_of(config.batch_size as u64) + { + progress_callback.on_progress(&progress); + } + } + + // Iterate through all grips + let grips = iter_all_grips(&storage)?; + info!(count = grips.len(), "Found grips to index"); + + for grip in grips { + match updater.index_grip_direct(&grip) { + Ok(true) => { + progress.record_grip(); + } + Ok(false) => { + progress.record_skip(); + } + Err(e) => { + if config.continue_on_error { + warn!(grip_id = %grip.grip_id, error = %e, "Failed to index grip"); + progress.record_error(); + } else { + return Err(e); + } + } + } + + if progress + .total_processed + .is_multiple_of(config.batch_size as u64) + { + progress_callback.on_progress(&progress); + } + } + + // Commit the index + updater.commit()?; + progress.mark_completed(); + progress_callback.on_progress(&progress); + + info!( + toc_nodes = progress.toc_nodes_indexed, + grips = progress.grips_indexed, + skipped = progress.skipped, + errors = progress.errors, + "Vector index rebuild complete" + ); + + Ok(progress) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_rebuild_config_default() { + let config = RebuildConfig::default(); + assert_eq!(config.batch_size, 100); + assert_eq!(config.index_types.len(), 2); + assert!(config.clear_first); + assert!(config.continue_on_error); + } + + #[test] + fn test_rebuild_config_bm25_only() { + let config = RebuildConfig::bm25_only(); + assert_eq!(config.index_types.len(), 1); + assert_eq!(config.index_types[0], IndexType::Bm25); + } + + #[test] + fn test_rebuild_config_vector_only() { + let config = RebuildConfig::vector_only(); + assert_eq!(config.index_types.len(), 1); + assert_eq!(config.index_types[0], IndexType::Vector); + } + + #[test] + fn test_rebuild_config_builder() { + let config = RebuildConfig::default() + .with_batch_size(50) + .with_clear_first(false) + .with_continue_on_error(false); + + assert_eq!(config.batch_size, 50); + assert!(!config.clear_first); + assert!(!config.continue_on_error); + } + + #[test] + fn test_rebuild_progress() { + let mut progress = RebuildProgress::new(); + assert_eq!(progress.total_processed, 0); + + progress.record_toc_node(); + assert_eq!(progress.toc_nodes_indexed, 1); + assert_eq!(progress.total_processed, 1); + + progress.record_grip(); + assert_eq!(progress.grips_indexed, 1); + assert_eq!(progress.total_processed, 2); + + progress.record_error(); + assert_eq!(progress.errors, 1); + assert_eq!(progress.total_processed, 3); + + progress.record_skip(); + assert_eq!(progress.skipped, 1); + assert_eq!(progress.total_processed, 4); + + assert!(!progress.completed); + progress.mark_completed(); + assert!(progress.completed); + } + + #[test] + fn test_no_op_progress_callback() { + let callback = NoOpProgressCallback; + let progress = RebuildProgress::new(); + callback.on_progress(&progress); // Should not panic + } +} diff --git a/crates/memory-indexing/src/updater.rs b/crates/memory-indexing/src/updater.rs new file mode 100644 index 0000000..c6c8d71 --- /dev/null +++ b/crates/memory-indexing/src/updater.rs @@ -0,0 +1,161 @@ +//! Index updater trait for incremental index updates. +//! +//! Defines the interface for index-specific update operations. +//! Each index type (BM25, vector) implements this trait. + +use crate::checkpoint::IndexType; +use crate::error::IndexingError; +use memory_types::OutboxEntry; + +/// Trait for index-specific update operations. +/// +/// Implementations handle the details of converting outbox entries +/// to index-specific documents and managing commits. +pub trait IndexUpdater: Send + Sync { + /// Index a new or updated document from an outbox entry. + /// + /// The entry contains an event_id that can be used to fetch + /// the full event data from storage if needed. + fn index_document(&self, entry: &OutboxEntry) -> Result<(), IndexingError>; + + /// Remove a document from the index by its ID. + /// + /// The doc_id could be an event_id, node_id, or grip_id depending + /// on what was indexed. + fn remove_document(&self, doc_id: &str) -> Result<(), IndexingError>; + + /// Commit pending changes to make them visible. + /// + /// This may be expensive - batch updates before calling. + fn commit(&self) -> Result<(), IndexingError>; + + /// Get the index type this updater manages. + fn index_type(&self) -> IndexType; + + /// Get the name of this updater for logging. + fn name(&self) -> &str; +} + +/// Result of processing a batch of outbox entries. +#[derive(Debug, Default, Clone)] +pub struct UpdateResult { + /// Number of entries successfully processed + pub processed: usize, + /// Number of entries skipped (already indexed, invalid, etc.) + pub skipped: usize, + /// Number of errors encountered + pub errors: usize, + /// The highest sequence number processed + pub last_sequence: u64, +} + +impl UpdateResult { + /// Create a new empty result. + pub fn new() -> Self { + Self::default() + } + + /// Record a successful processing. + pub fn record_success(&mut self) { + self.processed += 1; + } + + /// Record a skipped entry. + pub fn record_skip(&mut self) { + self.skipped += 1; + } + + /// Record an error. + pub fn record_error(&mut self) { + self.errors += 1; + } + + /// Set the last sequence number processed. + pub fn set_sequence(&mut self, seq: u64) { + self.last_sequence = seq; + } + + /// Merge another result into this one. + pub fn merge(&mut self, other: &UpdateResult) { + self.processed += other.processed; + self.skipped += other.skipped; + self.errors += other.errors; + if other.last_sequence > self.last_sequence { + self.last_sequence = other.last_sequence; + } + } + + /// Check if any entries were processed successfully. + pub fn has_updates(&self) -> bool { + self.processed > 0 + } + + /// Total number of entries handled (success + skip + error). + pub fn total(&self) -> usize { + self.processed + self.skipped + self.errors + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_update_result_new() { + let result = UpdateResult::new(); + assert_eq!(result.processed, 0); + assert_eq!(result.skipped, 0); + assert_eq!(result.errors, 0); + assert_eq!(result.last_sequence, 0); + } + + #[test] + fn test_update_result_record() { + let mut result = UpdateResult::new(); + result.record_success(); + result.record_success(); + result.record_skip(); + result.record_error(); + result.set_sequence(42); + + assert_eq!(result.processed, 2); + assert_eq!(result.skipped, 1); + assert_eq!(result.errors, 1); + assert_eq!(result.last_sequence, 42); + assert_eq!(result.total(), 4); + assert!(result.has_updates()); + } + + #[test] + fn test_update_result_merge() { + let mut result1 = UpdateResult { + processed: 5, + skipped: 2, + errors: 1, + last_sequence: 10, + }; + + let result2 = UpdateResult { + processed: 3, + skipped: 1, + errors: 0, + last_sequence: 15, + }; + + result1.merge(&result2); + + assert_eq!(result1.processed, 8); + assert_eq!(result1.skipped, 3); + assert_eq!(result1.errors, 1); + assert_eq!(result1.last_sequence, 15); + } + + #[test] + fn test_update_result_no_updates() { + let mut result = UpdateResult::new(); + result.record_skip(); + result.record_error(); + + assert!(!result.has_updates()); + } +} diff --git a/crates/memory-indexing/src/vector_updater.rs b/crates/memory-indexing/src/vector_updater.rs new file mode 100644 index 0000000..92c7ee0 --- /dev/null +++ b/crates/memory-indexing/src/vector_updater.rs @@ -0,0 +1,596 @@ +//! Vector index updater for HNSW-based semantic search. +//! +//! Wraps HnswIndex and CandleEmbedder to handle outbox-driven vector indexing. +//! Generates embeddings from text content and stores vectors with metadata. + +use std::sync::{Arc, RwLock}; + +use tracing::{debug, warn}; + +use memory_embeddings::{CandleEmbedder, EmbeddingModel}; +use memory_storage::Storage; +use memory_types::{Grip, OutboxAction, OutboxEntry, TocNode}; +use memory_vector::{DocType, HnswIndex, VectorEntry, VectorIndex, VectorMetadata}; + +use crate::checkpoint::IndexType; +use crate::error::IndexingError; +use crate::updater::{IndexUpdater, UpdateResult}; + +/// Vector index updater using HNSW and Candle embeddings. +/// +/// Generates embeddings for TOC nodes and grips, then stores +/// them in the HNSW index with metadata for retrieval. +pub struct VectorIndexUpdater { + index: Arc>, + embedder: Arc, + metadata: Arc, + storage: Arc, +} + +impl VectorIndexUpdater { + /// Create a new vector updater. + pub fn new( + index: Arc>, + embedder: Arc, + metadata: Arc, + storage: Arc, + ) -> Self { + Self { + index, + embedder, + metadata, + storage, + } + } + + /// Extract text content from a TOC node for embedding. + fn extract_toc_text(node: &TocNode) -> String { + let mut parts = vec![node.title.clone()]; + + for bullet in &node.bullets { + parts.push(bullet.text.clone()); + } + + if !node.keywords.is_empty() { + parts.push(node.keywords.join(" ")); + } + + parts.join(". ") + } + + /// Index a TOC node. + fn index_toc_node(&self, node: &TocNode) -> Result { + let doc_id = &node.node_id; + + // Check if already indexed + if self + .metadata + .find_by_doc_id(doc_id) + .map_err(|e| IndexingError::Index(format!("Metadata lookup error: {}", e)))? + .is_some() + { + debug!(doc_id = %doc_id, "TOC node already indexed, skipping"); + return Ok(false); + } + + let text = Self::extract_toc_text(node); + if text.trim().is_empty() { + debug!(doc_id = %doc_id, "Empty text, skipping"); + return Ok(false); + } + + // Generate embedding + let embedding = self + .embedder + .embed(&text) + .map_err(|e| IndexingError::Index(format!("Embedding error: {}", e)))?; + + // Get next vector ID + let vector_id = self + .metadata + .next_vector_id() + .map_err(|e| IndexingError::Index(format!("Metadata error: {}", e)))?; + + // Add to HNSW index + { + let mut index = self + .index + .write() + .map_err(|e| IndexingError::Index(format!("Index lock error: {}", e)))?; + index + .add(vector_id, &embedding) + .map_err(|e| IndexingError::Index(format!("HNSW add error: {}", e)))?; + } + + // Store metadata + let entry = VectorEntry::new( + vector_id, + DocType::TocNode, + doc_id.to_string(), + node.created_at.timestamp_millis(), + &text, + ); + self.metadata + .put(&entry) + .map_err(|e| IndexingError::Index(format!("Metadata put error: {}", e)))?; + + debug!(vector_id = vector_id, doc_id = %doc_id, "Indexed TOC node vector"); + Ok(true) + } + + /// Index a grip. + fn index_grip(&self, grip: &Grip) -> Result { + let doc_id = &grip.grip_id; + + // Check if already indexed + if self + .metadata + .find_by_doc_id(doc_id) + .map_err(|e| IndexingError::Index(format!("Metadata lookup error: {}", e)))? + .is_some() + { + debug!(doc_id = %doc_id, "Grip already indexed, skipping"); + return Ok(false); + } + + let text = &grip.excerpt; + if text.trim().is_empty() { + debug!(doc_id = %doc_id, "Empty excerpt, skipping"); + return Ok(false); + } + + // Generate embedding + let embedding = self + .embedder + .embed(text) + .map_err(|e| IndexingError::Index(format!("Embedding error: {}", e)))?; + + // Get next vector ID + let vector_id = self + .metadata + .next_vector_id() + .map_err(|e| IndexingError::Index(format!("Metadata error: {}", e)))?; + + // Add to HNSW index + { + let mut index = self + .index + .write() + .map_err(|e| IndexingError::Index(format!("Index lock error: {}", e)))?; + index + .add(vector_id, &embedding) + .map_err(|e| IndexingError::Index(format!("HNSW add error: {}", e)))?; + } + + // Store metadata + let entry = VectorEntry::new( + vector_id, + DocType::Grip, + doc_id.to_string(), + grip.timestamp.timestamp_millis(), + text, + ); + self.metadata + .put(&entry) + .map_err(|e| IndexingError::Index(format!("Metadata put error: {}", e)))?; + + debug!(vector_id = vector_id, doc_id = %doc_id, "Indexed grip vector"); + Ok(true) + } + + /// Process an outbox entry. + fn process_entry(&self, entry: &OutboxEntry) -> Result { + match entry.action { + OutboxAction::IndexEvent => { + debug!(event_id = %entry.event_id, "Processing index event for vector"); + + // Try to find a grip for this event + if let Some(grip) = self.find_grip_for_event(&entry.event_id)? { + return self.index_grip(&grip); + } + + debug!(event_id = %entry.event_id, "No grip found for event, skipping"); + Ok(false) + } + OutboxAction::UpdateToc => { + debug!(event_id = %entry.event_id, "Skipping TOC update action"); + Ok(false) + } + } + } + + /// Find a grip that references this event. + fn find_grip_for_event(&self, event_id: &str) -> Result, IndexingError> { + // Simplified lookup - return None for now + // In a full implementation, this would query storage + debug!(event_id = %event_id, "Looking up grip for event"); + Ok(None) + } + + /// Process a batch of outbox entries. + pub fn process_batch( + &self, + entries: &[(u64, OutboxEntry)], + ) -> Result { + let mut result = UpdateResult::new(); + + for (sequence, entry) in entries { + match self.process_entry(entry) { + Ok(true) => { + result.record_success(); + } + Ok(false) => { + result.record_skip(); + } + Err(e) => { + warn!( + sequence = sequence, + event_id = %entry.event_id, + error = %e, + "Failed to process entry for vector index" + ); + result.record_error(); + } + } + result.set_sequence(*sequence); + } + + Ok(result) + } + + /// Index a TOC node directly (for bulk indexing). + pub fn index_node(&self, node: &TocNode) -> Result { + self.index_toc_node(node) + } + + /// Index a grip directly (for bulk indexing). + pub fn index_grip_direct(&self, grip: &Grip) -> Result { + self.index_grip(grip) + } + + /// Remove a vector by document ID. + pub fn remove_by_doc_id(&self, doc_id: &str) -> Result { + // Find the vector ID for this document + let entry = self + .metadata + .find_by_doc_id(doc_id) + .map_err(|e| IndexingError::Index(format!("Metadata lookup error: {}", e)))?; + + if let Some(entry) = entry { + // Remove from HNSW + { + let mut index = self + .index + .write() + .map_err(|e| IndexingError::Index(format!("Index lock error: {}", e)))?; + index + .remove(entry.vector_id) + .map_err(|e| IndexingError::Index(format!("HNSW remove error: {}", e)))?; + } + + // Remove metadata + self.metadata + .delete(entry.vector_id) + .map_err(|e| IndexingError::Index(format!("Metadata delete error: {}", e)))?; + + debug!(vector_id = entry.vector_id, doc_id = %doc_id, "Removed vector"); + Ok(true) + } else { + debug!(doc_id = %doc_id, "Vector not found for removal"); + Ok(false) + } + } + + /// Get the underlying storage reference. + pub fn storage(&self) -> &Arc { + &self.storage + } + + /// Get the embedding dimension. + pub fn dimension(&self) -> usize { + self.embedder.info().dimension + } +} + +impl IndexUpdater for VectorIndexUpdater { + fn index_document(&self, entry: &OutboxEntry) -> Result<(), IndexingError> { + let _ = self.process_entry(entry)?; + Ok(()) + } + + fn remove_document(&self, doc_id: &str) -> Result<(), IndexingError> { + let _ = self.remove_by_doc_id(doc_id)?; + Ok(()) + } + + fn commit(&self) -> Result<(), IndexingError> { + // Save the HNSW index to disk + let index = self + .index + .read() + .map_err(|e| IndexingError::Index(format!("Index lock error: {}", e)))?; + index + .save() + .map_err(|e| IndexingError::Index(format!("HNSW save error: {}", e)))?; + Ok(()) + } + + fn index_type(&self) -> IndexType { + IndexType::Vector + } + + fn name(&self) -> &str { + "vector" + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use memory_embeddings::{Embedding, EmbeddingError, ModelInfo}; + use memory_types::TocLevel; + use memory_vector::HnswConfig; + use tempfile::TempDir; + + // Mock embedder for testing + struct MockEmbedder { + dimension: usize, + info: ModelInfo, + } + + impl MockEmbedder { + fn new(dimension: usize) -> Self { + Self { + dimension, + info: ModelInfo { + name: "mock".to_string(), + dimension, + max_sequence_length: 512, + }, + } + } + } + + impl EmbeddingModel for MockEmbedder { + fn info(&self) -> &ModelInfo { + &self.info + } + + fn embed(&self, _text: &str) -> Result { + // Return a simple embedding of the correct dimension + let values: Vec = (0..self.dimension) + .map(|i| (i as f32) / (self.dimension as f32)) + .collect(); + Ok(Embedding::new(values)) + } + } + + fn create_test_components( + temp_dir: &TempDir, + ) -> ( + Arc>, + Arc, + Arc, + Arc, + ) { + let storage_path = temp_dir.path().join("storage"); + std::fs::create_dir_all(&storage_path).unwrap(); + let storage = Arc::new(Storage::open(&storage_path).unwrap()); + + let vector_path = temp_dir.path().join("vector"); + std::fs::create_dir_all(&vector_path).unwrap(); + + let config = HnswConfig::new(64, &vector_path).with_capacity(1000); + let index = Arc::new(RwLock::new(HnswIndex::open_or_create(config).unwrap())); + + let embedder = Arc::new(MockEmbedder::new(64)); + + let metadata_path = temp_dir.path().join("metadata"); + std::fs::create_dir_all(&metadata_path).unwrap(); + let metadata = Arc::new(VectorMetadata::open(&metadata_path).unwrap()); + + (index, embedder, metadata, storage) + } + + #[test] + fn test_vector_updater_creation() { + let temp_dir = TempDir::new().unwrap(); + let (index, embedder, metadata, storage) = create_test_components(&temp_dir); + + let updater = VectorIndexUpdater::new(index, embedder, metadata, storage); + assert_eq!(updater.index_type(), IndexType::Vector); + assert_eq!(updater.name(), "vector"); + assert_eq!(updater.dimension(), 64); + } + + #[test] + fn test_extract_toc_text() { + use memory_types::TocBullet; + + let mut node = TocNode::new( + "test-node".to_string(), + TocLevel::Day, + "Test Title".to_string(), + Utc::now(), + Utc::now(), + ); + node.bullets.push(TocBullet::new("First bullet")); + node.bullets.push(TocBullet::new("Second bullet")); + node.keywords = vec!["key1".to_string(), "key2".to_string()]; + + let text = VectorIndexUpdater::::extract_toc_text(&node); + + assert!(text.contains("Test Title")); + assert!(text.contains("First bullet")); + assert!(text.contains("Second bullet")); + assert!(text.contains("key1")); + assert!(text.contains("key2")); + } + + #[test] + fn test_index_toc_node() { + let temp_dir = TempDir::new().unwrap(); + let (index, embedder, metadata, storage) = create_test_components(&temp_dir); + + let updater = VectorIndexUpdater::new(index.clone(), embedder, metadata.clone(), storage); + + let node = TocNode::new( + "toc:day:2024-01-15".to_string(), + TocLevel::Day, + "Monday, January 15".to_string(), + Utc::now(), + Utc::now(), + ); + + let indexed = updater.index_node(&node).unwrap(); + assert!(indexed); + + // Should find in metadata + let found = metadata.find_by_doc_id("toc:day:2024-01-15").unwrap(); + assert!(found.is_some()); + + // Index should have one vector + let idx = index.read().unwrap(); + assert_eq!(idx.len(), 1); + } + + #[test] + fn test_index_toc_node_duplicate() { + let temp_dir = TempDir::new().unwrap(); + let (index, embedder, metadata, storage) = create_test_components(&temp_dir); + + let updater = VectorIndexUpdater::new(index, embedder, metadata, storage); + + let node = TocNode::new( + "toc:day:2024-01-15".to_string(), + TocLevel::Day, + "Monday, January 15".to_string(), + Utc::now(), + Utc::now(), + ); + + // First index + let indexed1 = updater.index_node(&node).unwrap(); + assert!(indexed1); + + // Second index should skip + let indexed2 = updater.index_node(&node).unwrap(); + assert!(!indexed2); + } + + #[test] + fn test_index_grip() { + let temp_dir = TempDir::new().unwrap(); + let (index, embedder, metadata, storage) = create_test_components(&temp_dir); + + let updater = VectorIndexUpdater::new(index.clone(), embedder, metadata.clone(), storage); + + let grip = Grip::new( + "grip:12345".to_string(), + "User asked about authentication".to_string(), + "event-001".to_string(), + "event-003".to_string(), + Utc::now(), + "test".to_string(), + ); + + let indexed = updater.index_grip_direct(&grip).unwrap(); + assert!(indexed); + + // Should find in metadata + let found = metadata.find_by_doc_id("grip:12345").unwrap(); + assert!(found.is_some()); + } + + #[test] + fn test_remove_by_doc_id() { + let temp_dir = TempDir::new().unwrap(); + let (index, embedder, metadata, storage) = create_test_components(&temp_dir); + + let updater = VectorIndexUpdater::new(index.clone(), embedder, metadata.clone(), storage); + + let node = TocNode::new( + "toc:day:2024-01-15".to_string(), + TocLevel::Day, + "Monday, January 15".to_string(), + Utc::now(), + Utc::now(), + ); + + // Index first + updater.index_node(&node).unwrap(); + assert_eq!(index.read().unwrap().len(), 1); + + // Remove + let removed = updater.remove_by_doc_id("toc:day:2024-01-15").unwrap(); + assert!(removed); + + // Should be gone from metadata + let found = metadata.find_by_doc_id("toc:day:2024-01-15").unwrap(); + assert!(found.is_none()); + } + + #[test] + fn test_remove_nonexistent() { + let temp_dir = TempDir::new().unwrap(); + let (index, embedder, metadata, storage) = create_test_components(&temp_dir); + + let updater = VectorIndexUpdater::new(index, embedder, metadata, storage); + + let removed = updater.remove_by_doc_id("nonexistent").unwrap(); + assert!(!removed); + } + + #[test] + fn test_process_batch_empty() { + let temp_dir = TempDir::new().unwrap(); + let (index, embedder, metadata, storage) = create_test_components(&temp_dir); + + let updater = VectorIndexUpdater::new(index, embedder, metadata, storage); + + let entries: Vec<(u64, OutboxEntry)> = vec![]; + let result = updater.process_batch(&entries).unwrap(); + + assert_eq!(result.total(), 0); + } + + #[test] + fn test_process_batch_with_entries() { + let temp_dir = TempDir::new().unwrap(); + let (index, embedder, metadata, storage) = create_test_components(&temp_dir); + + let updater = VectorIndexUpdater::new(index, embedder, metadata, storage); + + let entries = vec![ + (0, OutboxEntry::for_index("event-1".to_string(), 1000)), + (1, OutboxEntry::for_index("event-2".to_string(), 2000)), + (2, OutboxEntry::for_toc("event-3".to_string(), 3000)), + ]; + + let result = updater.process_batch(&entries).unwrap(); + + // All should be skipped (no grips found) + assert_eq!(result.skipped, 3); + assert_eq!(result.last_sequence, 2); + } + + #[test] + fn test_commit() { + let temp_dir = TempDir::new().unwrap(); + let (index, embedder, metadata, storage) = create_test_components(&temp_dir); + + let updater = VectorIndexUpdater::new(index, embedder, metadata, storage); + + let node = TocNode::new( + "toc:day:2024-01-15".to_string(), + TocLevel::Day, + "Monday, January 15".to_string(), + Utc::now(), + Utc::now(), + ); + + updater.index_node(&node).unwrap(); + updater.commit().unwrap(); + } +} diff --git a/crates/memory-ingest/src/main.rs b/crates/memory-ingest/src/main.rs index 813f2df..d639d3d 100644 --- a/crates/memory-ingest/src/main.rs +++ b/crates/memory-ingest/src/main.rs @@ -127,7 +127,7 @@ fn main() { } }; - let _ = rt.block_on(async { + rt.block_on(async { if let Ok(mut client) = MemoryClient::connect_default().await { // Ignore result - fail-open let _ = client.ingest(event).await; @@ -190,17 +190,44 @@ mod tests { #[test] fn test_map_cch_event_type_all_types() { - assert!(matches!(map_cch_event_type("SessionStart"), HookEventType::SessionStart)); - assert!(matches!(map_cch_event_type("UserPromptSubmit"), HookEventType::UserPromptSubmit)); - assert!(matches!(map_cch_event_type("AssistantResponse"), HookEventType::AssistantResponse)); - assert!(matches!(map_cch_event_type("PreToolUse"), HookEventType::ToolUse)); - assert!(matches!(map_cch_event_type("PostToolUse"), HookEventType::ToolResult)); + assert!(matches!( + map_cch_event_type("SessionStart"), + HookEventType::SessionStart + )); + assert!(matches!( + map_cch_event_type("UserPromptSubmit"), + HookEventType::UserPromptSubmit + )); + assert!(matches!( + map_cch_event_type("AssistantResponse"), + HookEventType::AssistantResponse + )); + assert!(matches!( + map_cch_event_type("PreToolUse"), + HookEventType::ToolUse + )); + assert!(matches!( + map_cch_event_type("PostToolUse"), + HookEventType::ToolResult + )); assert!(matches!(map_cch_event_type("Stop"), HookEventType::Stop)); - assert!(matches!(map_cch_event_type("SessionEnd"), HookEventType::Stop)); - assert!(matches!(map_cch_event_type("SubagentStart"), HookEventType::SubagentStart)); - assert!(matches!(map_cch_event_type("SubagentStop"), HookEventType::SubagentStop)); + assert!(matches!( + map_cch_event_type("SessionEnd"), + HookEventType::Stop + )); + assert!(matches!( + map_cch_event_type("SubagentStart"), + HookEventType::SubagentStart + )); + assert!(matches!( + map_cch_event_type("SubagentStop"), + HookEventType::SubagentStop + )); // Unknown defaults to UserPromptSubmit - assert!(matches!(map_cch_event_type("UnknownType"), HookEventType::UserPromptSubmit)); + assert!(matches!( + map_cch_event_type("UnknownType"), + HookEventType::UserPromptSubmit + )); } #[test] diff --git a/crates/memory-scheduler/Cargo.toml b/crates/memory-scheduler/Cargo.toml index 06a00c1..79107ec 100644 --- a/crates/memory-scheduler/Cargo.toml +++ b/crates/memory-scheduler/Cargo.toml @@ -8,7 +8,7 @@ description = "Background job scheduler for agent-memory daemon" [features] default = ["jobs"] -jobs = ["memory-toc", "memory-storage", "memory-types"] +jobs = ["memory-toc", "memory-storage", "memory-types", "memory-search", "memory-indexing"] [dependencies] # Cron scheduling @@ -41,6 +41,8 @@ rand = "0.8" memory-toc = { path = "../memory-toc", optional = true } memory-storage = { path = "../memory-storage", optional = true } memory-types = { path = "../memory-types", optional = true } +memory-search = { path = "../memory-search", optional = true } +memory-indexing = { path = "../memory-indexing", optional = true } [dev-dependencies] tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/memory-scheduler/src/error.rs b/crates/memory-scheduler/src/error.rs index 91735d2..07eb355 100644 --- a/crates/memory-scheduler/src/error.rs +++ b/crates/memory-scheduler/src/error.rs @@ -32,6 +32,10 @@ pub enum SchedulerError { /// Scheduler is not running #[error("Scheduler is not running")] NotRunning, + + /// Job execution timed out + #[error("Job timed out after {0} seconds")] + Timeout(u64), } impl From for SchedulerError { @@ -60,5 +64,9 @@ mod tests { let err = SchedulerError::NotRunning; assert!(err.to_string().contains("not running")); + + let err = SchedulerError::Timeout(30); + assert!(err.to_string().contains("timed out")); + assert!(err.to_string().contains("30")); } } diff --git a/crates/memory-scheduler/src/jitter.rs b/crates/memory-scheduler/src/jitter.rs index fea8325..07e12d4 100644 --- a/crates/memory-scheduler/src/jitter.rs +++ b/crates/memory-scheduler/src/jitter.rs @@ -1,7 +1,9 @@ -//! Jitter utilities for distributed scheduling. +//! Jitter and timeout utilities for distributed scheduling. //! //! Jitter adds a random delay before job execution to prevent thundering herd //! problems when many instances are scheduled at the same time. +//! +//! Timeout wraps job execution to prevent runaway jobs from blocking the scheduler. use std::time::Duration; @@ -64,6 +66,60 @@ impl JitterConfig { } } +/// Configuration for job execution timeout. +/// +/// Timeout limits how long a job can run before being cancelled. This prevents +/// runaway jobs from blocking the scheduler indefinitely. +/// +/// # Example +/// +/// ``` +/// use memory_scheduler::TimeoutConfig; +/// +/// // Create timeout config with 5 minute limit +/// let config = TimeoutConfig::new(300); +/// assert!(config.is_enabled()); +/// +/// // No timeout +/// let no_timeout = TimeoutConfig::none(); +/// assert!(!no_timeout.is_enabled()); +/// ``` +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct TimeoutConfig { + /// Maximum job execution time in seconds (0 = no timeout). + pub timeout_secs: u64, +} + +impl TimeoutConfig { + /// Create a new timeout configuration with the given maximum duration. + /// + /// # Arguments + /// + /// * `timeout_secs` - Maximum execution time in seconds. Set to 0 for no timeout. + pub fn new(timeout_secs: u64) -> Self { + Self { timeout_secs } + } + + /// Create a timeout configuration with no limit. + pub fn none() -> Self { + Self { timeout_secs: 0 } + } + + /// Get the timeout as a Duration, or None if no timeout is configured. + pub fn as_duration(&self) -> Option { + if self.timeout_secs == 0 { + None + } else { + Some(Duration::from_secs(self.timeout_secs)) + } + } + + /// Check if timeout is enabled. + pub fn is_enabled(&self) -> bool { + self.timeout_secs > 0 + } +} + /// Execute a future with jitter delay. /// /// Applies a random delay (up to `max_jitter_secs`) before executing the @@ -186,4 +242,42 @@ mod tests { let unique: std::collections::HashSet<_> = samples.iter().map(|d| d.as_millis()).collect(); assert!(unique.len() > 1, "Jitter should produce varied values"); } + + // TimeoutConfig tests + + #[test] + fn test_timeout_zero_is_disabled() { + let config = TimeoutConfig::new(0); + assert!(!config.is_enabled()); + assert!(config.as_duration().is_none()); + } + + #[test] + fn test_timeout_enabled() { + let config = TimeoutConfig::new(300); + assert!(config.is_enabled()); + assert_eq!(config.as_duration(), Some(Duration::from_secs(300))); + } + + #[test] + fn test_timeout_default() { + let config = TimeoutConfig::default(); + assert_eq!(config.timeout_secs, 0); + assert!(!config.is_enabled()); + } + + #[test] + fn test_timeout_none() { + let config = TimeoutConfig::none(); + assert_eq!(config.timeout_secs, 0); + assert!(!config.is_enabled()); + } + + #[test] + fn test_timeout_serialization() { + let config = TimeoutConfig::new(60); + let json = serde_json::to_string(&config).unwrap(); + let config_back: TimeoutConfig = serde_json::from_str(&json).unwrap(); + assert_eq!(config, config_back); + } } diff --git a/crates/memory-scheduler/src/jobs/compaction.rs b/crates/memory-scheduler/src/jobs/compaction.rs index 524f72b..3a2b679 100644 --- a/crates/memory-scheduler/src/jobs/compaction.rs +++ b/crates/memory-scheduler/src/jobs/compaction.rs @@ -15,7 +15,7 @@ use tracing::info; use memory_storage::Storage; -use crate::{JitterConfig, OverlapPolicy, SchedulerError, SchedulerService}; +use crate::{JitterConfig, OverlapPolicy, SchedulerError, SchedulerService, TimeoutConfig}; /// Configuration for the compaction job. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -28,6 +28,9 @@ pub struct CompactionJobConfig { /// Max jitter in seconds (default: 600 = 10 min) pub jitter_secs: u64, + + /// Timeout in seconds (default: 3600 = 1 hour) + pub timeout_secs: u64, } impl Default for CompactionJobConfig { @@ -36,6 +39,7 @@ impl Default for CompactionJobConfig { cron: "0 0 4 * * 0".to_string(), timezone: "UTC".to_string(), jitter_secs: 600, + timeout_secs: 3600, // 1 hour } } } @@ -66,6 +70,7 @@ pub async fn create_compaction_job( Some(&config.timezone), OverlapPolicy::Skip, JitterConfig::new(config.jitter_secs), + TimeoutConfig::new(config.timeout_secs), move || { let storage = storage.clone(); async move { @@ -94,6 +99,7 @@ mod tests { assert_eq!(config.cron, "0 0 4 * * 0"); assert_eq!(config.timezone, "UTC"); assert_eq!(config.jitter_secs, 600); + assert_eq!(config.timeout_secs, 3600); } #[test] diff --git a/crates/memory-scheduler/src/jobs/indexing.rs b/crates/memory-scheduler/src/jobs/indexing.rs new file mode 100644 index 0000000..35e785c --- /dev/null +++ b/crates/memory-scheduler/src/jobs/indexing.rs @@ -0,0 +1,331 @@ +//! Outbox indexing scheduled job. +//! +//! Processes outbox entries in batches and updates search indexes +//! (BM25 and vector) with checkpoint tracking for crash recovery. +//! +//! # Architecture +//! +//! The indexing job follows the outbox pattern: +//! 1. Events are written with outbox entries atomically +//! 2. This job periodically consumes outbox entries +//! 3. Each index updater (BM25, vector) processes entries +//! 4. Checkpoints track progress for crash recovery +//! 5. Processed entries are cleaned up after all indexes catch up +//! +//! # Default Schedule +//! +//! By default, the job runs every minute to minimize latency between +//! writes and searchability, while keeping checkpoint overhead low. + +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use tokio::sync::Mutex; +use tracing::{debug, info, warn}; + +use memory_indexing::{IndexingPipeline, PipelineConfig}; + +use crate::{JitterConfig, OverlapPolicy, SchedulerError, SchedulerService, TimeoutConfig}; + +/// Configuration for the indexing job. +/// +/// Controls the job schedule, batch processing, and checkpoint behavior. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IndexingJobConfig { + /// Cron expression (default: "0 * * * * *" = every minute) + /// + /// The default one-minute interval balances latency with efficiency. + /// For higher throughput systems, consider "*/30 * * * * *" (every 30s). + pub cron: String, + + /// Timezone for scheduling (default: "UTC") + pub timezone: String, + + /// Max jitter in seconds (default: 10) + /// + /// Jitter spreads job execution to avoid thundering herd when + /// multiple daemon instances run on the same schedule. + pub jitter_secs: u64, + + /// Maximum entries to process per batch (default: 100) + /// + /// Larger batches are more efficient but increase memory usage + /// and latency between checkpoint commits. + pub batch_size: usize, + + /// Maximum iterations per job run (default: 10) + /// + /// Limits how many batches are processed in a single job execution. + /// Prevents long-running jobs from blocking other scheduled work. + pub max_iterations: usize, + + /// Whether to cleanup processed outbox entries (default: true) + /// + /// When enabled, entries that all indexes have processed are + /// deleted to reclaim storage space. + pub cleanup_after_processing: bool, + + /// Whether to continue processing on individual entry errors (default: true) + /// + /// When enabled, errors on individual entries are logged but don't + /// stop the batch. When disabled, any error fails the entire batch. + pub continue_on_error: bool, + + /// Whether to commit after each batch (default: true) + /// + /// When enabled, checkpoints are saved after each batch for crash + /// recovery. When disabled, commits only happen at job completion. + pub commit_after_batch: bool, + + /// Timeout in seconds for job execution (default: 300 = 5 minutes) + /// + /// Prevents runaway jobs from blocking the scheduler. Set to 0 for + /// no timeout (not recommended in production). + pub timeout_secs: u64, +} + +impl Default for IndexingJobConfig { + fn default() -> Self { + Self { + cron: "0 * * * * *".to_string(), // Every minute + timezone: "UTC".to_string(), + jitter_secs: 10, + batch_size: 100, + max_iterations: 10, + cleanup_after_processing: true, + continue_on_error: true, + commit_after_batch: true, + timeout_secs: 300, // 5 minutes + } + } +} + +impl IndexingJobConfig { + /// Create a new config with the given cron expression. + pub fn with_cron(mut self, cron: impl Into) -> Self { + self.cron = cron.into(); + self + } + + /// Set the timezone. + pub fn with_timezone(mut self, timezone: impl Into) -> Self { + self.timezone = timezone.into(); + self + } + + /// Set the batch size. + pub fn with_batch_size(mut self, batch_size: usize) -> Self { + self.batch_size = batch_size; + self + } + + /// Set the max iterations. + pub fn with_max_iterations(mut self, max_iterations: usize) -> Self { + self.max_iterations = max_iterations; + self + } + + /// Set whether to cleanup after processing. + pub fn with_cleanup(mut self, cleanup: bool) -> Self { + self.cleanup_after_processing = cleanup; + self + } + + /// Set the timeout in seconds. + pub fn with_timeout(mut self, timeout_secs: u64) -> Self { + self.timeout_secs = timeout_secs; + self + } + + /// Convert to PipelineConfig for the indexing pipeline. + pub fn to_pipeline_config(&self) -> PipelineConfig { + PipelineConfig::default() + .with_batch_size(self.batch_size) + .with_continue_on_error(self.continue_on_error) + .with_commit_after_batch(self.commit_after_batch) + } +} + +/// Register the indexing job with the scheduler. +/// +/// Creates a job that processes outbox entries and updates search indexes. +/// The pipeline must already have index updaters registered. +/// +/// # Arguments +/// +/// * `scheduler` - The scheduler service to register the job with +/// * `pipeline` - Pre-configured indexing pipeline with updaters +/// * `config` - Configuration for job schedule and batch processing +/// +/// # Errors +/// +/// Returns error if job registration fails (invalid cron, invalid timezone). +/// +/// # Example +/// +/// ```ignore +/// use memory_indexing::{IndexingPipeline, PipelineConfig, Bm25IndexUpdater}; +/// use memory_scheduler::{SchedulerService, create_indexing_job, IndexingJobConfig}; +/// +/// let mut pipeline = IndexingPipeline::new(storage.clone(), PipelineConfig::default()); +/// pipeline.add_updater(Box::new(bm25_updater)); +/// pipeline.load_checkpoints()?; +/// +/// let pipeline = Arc::new(Mutex::new(pipeline)); +/// +/// create_indexing_job(&scheduler, pipeline, IndexingJobConfig::default()).await?; +/// ``` +pub async fn create_indexing_job( + scheduler: &SchedulerService, + pipeline: Arc>, + config: IndexingJobConfig, +) -> Result<(), SchedulerError> { + let max_iterations = config.max_iterations; + let cleanup_after = config.cleanup_after_processing; + + scheduler + .register_job( + "outbox_indexing", + &config.cron, + Some(&config.timezone), + OverlapPolicy::Skip, + JitterConfig::new(config.jitter_secs), + TimeoutConfig::new(config.timeout_secs), + move || { + let pipeline = pipeline.clone(); + async move { run_indexing_job(pipeline, max_iterations, cleanup_after).await } + }, + ) + .await?; + + info!("Registered outbox indexing job"); + Ok(()) +} + +/// Execute the indexing job. +/// +/// Processes outbox entries in batches until caught up or max iterations reached. +async fn run_indexing_job( + pipeline: Arc>, + max_iterations: usize, + cleanup_after: bool, +) -> Result<(), String> { + // Acquire pipeline lock (tokio::sync::Mutex for async-friendly locking) + let mut pipeline = pipeline.lock().await; + + debug!(max_iterations = max_iterations, "Starting indexing job run"); + + // Process until caught up or max iterations + let result = pipeline + .process_until_caught_up(max_iterations) + .map_err(|e| format!("Indexing failed: {}", e))?; + + if result.has_updates() { + info!( + total_processed = result.total_processed, + last_sequence = ?result.last_sequence, + "Indexing job processed entries" + ); + } else { + debug!("Indexing job: no entries to process"); + } + + // Cleanup processed entries if enabled + if cleanup_after && result.has_updates() { + match pipeline.cleanup_outbox() { + Ok(deleted) => { + if deleted > 0 { + info!(deleted = deleted, "Cleaned up processed outbox entries"); + } + } + Err(e) => { + // Log but don't fail the job - cleanup can be retried + warn!(error = %e, "Failed to cleanup outbox entries"); + } + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_default_config() { + let config = IndexingJobConfig::default(); + + assert_eq!(config.cron, "0 * * * * *"); + assert_eq!(config.timezone, "UTC"); + assert_eq!(config.jitter_secs, 10); + assert_eq!(config.batch_size, 100); + assert_eq!(config.max_iterations, 10); + assert!(config.cleanup_after_processing); + assert!(config.continue_on_error); + assert!(config.commit_after_batch); + assert_eq!(config.timeout_secs, 300); + } + + #[test] + fn test_config_builder() { + let config = IndexingJobConfig::default() + .with_cron("*/30 * * * * *") + .with_timezone("America/New_York") + .with_batch_size(50) + .with_max_iterations(5) + .with_cleanup(false); + + assert_eq!(config.cron, "*/30 * * * * *"); + assert_eq!(config.timezone, "America/New_York"); + assert_eq!(config.batch_size, 50); + assert_eq!(config.max_iterations, 5); + assert!(!config.cleanup_after_processing); + } + + #[test] + fn test_config_serialization() { + let config = IndexingJobConfig::default(); + let json = serde_json::to_string(&config).unwrap(); + let decoded: IndexingJobConfig = serde_json::from_str(&json).unwrap(); + + assert_eq!(config.cron, decoded.cron); + assert_eq!(config.timezone, decoded.timezone); + assert_eq!(config.jitter_secs, decoded.jitter_secs); + assert_eq!(config.batch_size, decoded.batch_size); + assert_eq!(config.max_iterations, decoded.max_iterations); + assert_eq!( + config.cleanup_after_processing, + decoded.cleanup_after_processing + ); + assert_eq!(config.continue_on_error, decoded.continue_on_error); + assert_eq!(config.commit_after_batch, decoded.commit_after_batch); + } + + #[test] + fn test_to_pipeline_config() { + let config = IndexingJobConfig::default().with_batch_size(50); + + let pipeline_config = config.to_pipeline_config(); + + assert_eq!(pipeline_config.batch_size, 50); + assert!(pipeline_config.continue_on_error); + assert!(pipeline_config.commit_after_batch); + } + + #[test] + fn test_config_json_format() { + let config = IndexingJobConfig::default(); + let json = serde_json::to_string_pretty(&config).unwrap(); + + // Verify expected fields are present in JSON + assert!(json.contains("\"cron\"")); + assert!(json.contains("\"timezone\"")); + assert!(json.contains("\"jitter_secs\"")); + assert!(json.contains("\"batch_size\"")); + assert!(json.contains("\"max_iterations\"")); + assert!(json.contains("\"cleanup_after_processing\"")); + assert!(json.contains("\"continue_on_error\"")); + assert!(json.contains("\"commit_after_batch\"")); + } +} diff --git a/crates/memory-scheduler/src/jobs/mod.rs b/crates/memory-scheduler/src/jobs/mod.rs index cafa7c6..fa76306 100644 --- a/crates/memory-scheduler/src/jobs/mod.rs +++ b/crates/memory-scheduler/src/jobs/mod.rs @@ -7,9 +7,21 @@ //! //! - **rollup**: TOC rollup jobs for day/week/month aggregation //! - **compaction**: RocksDB compaction for storage optimization +//! - **search**: Search index commit job for making documents searchable +//! - **indexing**: Outbox indexing job for processing new entries into indexes pub mod compaction; pub mod rollup; +#[cfg(feature = "jobs")] +pub mod indexing; +#[cfg(feature = "jobs")] +pub mod search; + pub use compaction::{create_compaction_job, CompactionJobConfig}; pub use rollup::{create_rollup_jobs, RollupJobConfig}; + +#[cfg(feature = "jobs")] +pub use indexing::{create_indexing_job, IndexingJobConfig}; +#[cfg(feature = "jobs")] +pub use search::{create_index_commit_job, IndexCommitJobConfig}; diff --git a/crates/memory-scheduler/src/jobs/rollup.rs b/crates/memory-scheduler/src/jobs/rollup.rs index 5343177..2fe3940 100644 --- a/crates/memory-scheduler/src/jobs/rollup.rs +++ b/crates/memory-scheduler/src/jobs/rollup.rs @@ -24,7 +24,7 @@ use memory_toc::rollup::RollupJob; use memory_toc::summarizer::Summarizer; use memory_types::TocLevel; -use crate::{JitterConfig, OverlapPolicy, SchedulerError, SchedulerService}; +use crate::{JitterConfig, OverlapPolicy, SchedulerError, SchedulerService, TimeoutConfig}; /// Configuration for TOC rollup jobs. /// @@ -45,6 +45,9 @@ pub struct RollupJobConfig { /// Max jitter in seconds (default: 300 = 5 min) pub jitter_secs: u64, + + /// Timeout in seconds for each rollup job (default: 1800 = 30 minutes) + pub timeout_secs: u64, } impl Default for RollupJobConfig { @@ -55,6 +58,7 @@ impl Default for RollupJobConfig { month_cron: "0 0 3 1 * *".to_string(), timezone: "UTC".to_string(), jitter_secs: 300, + timeout_secs: 1800, // 30 minutes } } } @@ -83,6 +87,8 @@ pub async fn create_rollup_jobs( summarizer: Arc, config: RollupJobConfig, ) -> Result<(), SchedulerError> { + let timeout = TimeoutConfig::new(config.timeout_secs); + // Day rollup job let storage_day = storage.clone(); let summarizer_day = summarizer.clone(); @@ -93,6 +99,7 @@ pub async fn create_rollup_jobs( Some(&config.timezone), OverlapPolicy::Skip, JitterConfig::new(config.jitter_secs), + timeout.clone(), move || { let storage = storage_day.clone(); let summarizer = summarizer_day.clone(); @@ -111,6 +118,7 @@ pub async fn create_rollup_jobs( Some(&config.timezone), OverlapPolicy::Skip, JitterConfig::new(config.jitter_secs), + timeout.clone(), move || { let storage = storage_week.clone(); let summarizer = summarizer_week.clone(); @@ -129,6 +137,7 @@ pub async fn create_rollup_jobs( Some(&config.timezone), OverlapPolicy::Skip, JitterConfig::new(config.jitter_secs), + timeout, move || { let storage = storage_month.clone(); let summarizer = summarizer_month.clone(); @@ -199,6 +208,7 @@ mod tests { assert_eq!(config.month_cron, "0 0 3 1 * *"); assert_eq!(config.timezone, "UTC"); assert_eq!(config.jitter_secs, 300); + assert_eq!(config.timeout_secs, 1800); } #[test] diff --git a/crates/memory-scheduler/src/jobs/search.rs b/crates/memory-scheduler/src/jobs/search.rs new file mode 100644 index 0000000..3a2f224 --- /dev/null +++ b/crates/memory-scheduler/src/jobs/search.rs @@ -0,0 +1,99 @@ +//! Search index scheduled jobs. +//! +//! Periodic commit job to make indexed documents searchable. + +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; + +use memory_search::SearchIndexer; + +use crate::{JitterConfig, OverlapPolicy, SchedulerError, SchedulerService, TimeoutConfig}; + +/// Configuration for index commit job. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IndexCommitJobConfig { + /// Cron expression (default: "0 * * * * *" = every minute) + pub cron: String, + /// Timezone (default: "UTC") + pub timezone: String, + /// Max jitter in seconds (default: 10) + pub jitter_secs: u64, + /// Timeout in seconds (default: 60 = 1 minute) + pub timeout_secs: u64, +} + +impl Default for IndexCommitJobConfig { + fn default() -> Self { + Self { + cron: "0 * * * * *".to_string(), // Every minute + timezone: "UTC".to_string(), + jitter_secs: 10, + timeout_secs: 60, // 1 minute + } + } +} + +/// Register the index commit job with the scheduler. +/// +/// This job periodically commits the search index to make +/// newly indexed documents visible to search queries. +pub async fn create_index_commit_job( + scheduler: &SchedulerService, + indexer: Arc, + config: IndexCommitJobConfig, +) -> Result<(), SchedulerError> { + scheduler + .register_job( + "search_index_commit", + &config.cron, + Some(&config.timezone), + OverlapPolicy::Skip, + JitterConfig::new(config.jitter_secs), + TimeoutConfig::new(config.timeout_secs), + move || { + let indexer = indexer.clone(); + async move { + match indexer.commit() { + Ok(opstamp) => { + tracing::debug!(opstamp, "Search index committed"); + Ok(()) + } + Err(e) => { + tracing::warn!(error = %e, "Search index commit failed"); + Err(e.to_string()) + } + } + } + }, + ) + .await?; + + tracing::info!("Registered search index commit job"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_default_config() { + let config = IndexCommitJobConfig::default(); + assert_eq!(config.cron, "0 * * * * *"); + assert_eq!(config.timezone, "UTC"); + assert_eq!(config.jitter_secs, 10); + assert_eq!(config.timeout_secs, 60); + } + + #[test] + fn test_config_serialization() { + let config = IndexCommitJobConfig::default(); + let json = serde_json::to_string(&config).unwrap(); + let decoded: IndexCommitJobConfig = serde_json::from_str(&json).unwrap(); + + assert_eq!(config.cron, decoded.cron); + assert_eq!(config.timezone, decoded.timezone); + assert_eq!(config.jitter_secs, decoded.jitter_secs); + } +} diff --git a/crates/memory-scheduler/src/lib.rs b/crates/memory-scheduler/src/lib.rs index ec09a4a..6068688 100644 --- a/crates/memory-scheduler/src/lib.rs +++ b/crates/memory-scheduler/src/lib.rs @@ -50,7 +50,7 @@ pub mod jobs; pub use config::SchedulerConfig; pub use error::SchedulerError; -pub use jitter::{with_jitter, JitterConfig}; +pub use jitter::{with_jitter, JitterConfig, TimeoutConfig}; pub use overlap::{OverlapGuard, OverlapPolicy, RunGuard}; pub use registry::{JobRegistry, JobResult, JobStatus}; pub use scheduler::{validate_cron_expression, SchedulerService}; @@ -58,4 +58,8 @@ pub use scheduler::{validate_cron_expression, SchedulerService}; #[cfg(feature = "jobs")] pub use jobs::compaction::{create_compaction_job, CompactionJobConfig}; #[cfg(feature = "jobs")] +pub use jobs::indexing::{create_indexing_job, IndexingJobConfig}; +#[cfg(feature = "jobs")] pub use jobs::rollup::{create_rollup_jobs, RollupJobConfig}; +#[cfg(feature = "jobs")] +pub use jobs::search::{create_index_commit_job, IndexCommitJobConfig}; diff --git a/crates/memory-scheduler/src/scheduler.rs b/crates/memory-scheduler/src/scheduler.rs index 1fd7404..3099ebd 100644 --- a/crates/memory-scheduler/src/scheduler.rs +++ b/crates/memory-scheduler/src/scheduler.rs @@ -13,7 +13,7 @@ use tokio_cron_scheduler::{Job, JobScheduler}; use tokio_util::sync::CancellationToken; use tracing::{debug, info, warn}; -use crate::jitter::JitterConfig; +use crate::jitter::{JitterConfig, TimeoutConfig}; use crate::overlap::{OverlapGuard, OverlapPolicy}; use crate::registry::{JobRegistry, JobResult}; use crate::{SchedulerConfig, SchedulerError}; @@ -45,11 +45,7 @@ pub fn validate_cron_expression(expr: &str) -> Result<(), SchedulerError> { // tokio-cron-scheduler uses croner internally for parsing match Job::new_async(expr, |_uuid, _lock| Box::pin(async {})) { Ok(_) => Ok(()), - Err(e) => Err(SchedulerError::InvalidCron(format!( - "'{}': {}", - expr, - e - ))), + Err(e) => Err(SchedulerError::InvalidCron(format!("'{}': {}", expr, e))), } } @@ -267,6 +263,7 @@ impl SchedulerService { /// - Job status tracking via the registry /// - Overlap policy to prevent concurrent execution /// - Jitter for distributed scheduling + /// - Timeout to prevent runaway jobs /// /// # Arguments /// @@ -275,12 +272,13 @@ impl SchedulerService { /// * `timezone` - IANA timezone string, or None to use config default /// * `overlap_policy` - How to handle overlapping executions /// * `jitter` - Random delay configuration before execution + /// * `timeout` - Maximum execution time configuration /// * `job_fn` - Async function returning `Result<(), String>` /// /// # Example /// /// ```ignore - /// use memory_scheduler::{OverlapPolicy, JitterConfig}; + /// use memory_scheduler::{OverlapPolicy, JitterConfig, TimeoutConfig}; /// /// scheduler.register_job( /// "hourly-rollup", @@ -288,12 +286,14 @@ impl SchedulerService { /// None, /// OverlapPolicy::Skip, /// JitterConfig::new(30), + /// TimeoutConfig::new(300), // 5 minute timeout /// || async { do_rollup().await }, /// ).await?; /// /// // Check job status /// let status = scheduler.registry().get_status("hourly-rollup"); /// ``` + #[allow(clippy::too_many_arguments)] pub async fn register_job( &self, name: &str, @@ -301,6 +301,7 @@ impl SchedulerService { timezone: Option<&str>, overlap_policy: OverlapPolicy, jitter: JitterConfig, + timeout: TimeoutConfig, job_fn: F, ) -> Result where @@ -325,13 +326,15 @@ impl SchedulerService { let registry = self.registry.clone(); let overlap_guard = Arc::new(OverlapGuard::new(overlap_policy)); let max_jitter_secs = jitter.max_jitter_secs; + let timeout_duration = timeout.as_duration(); - // Create timezone-aware job with overlap and jitter support + // Create timezone-aware job with overlap, jitter, and timeout support let job = Job::new_async_tz(cron_expr, tz, move |_uuid, _lock| { let name = job_name.clone(); let registry = registry.clone(); let guard = overlap_guard.clone(); let job_fn = job_fn.clone(); + let timeout_dur = timeout_duration; Box::pin(async move { // Check if job is paused @@ -366,12 +369,30 @@ impl SchedulerService { } } - // Execute the job function - let result = match job_fn().await { - Ok(()) => JobResult::Success, - Err(e) => { - warn!(job = %name, error = %e, "Job failed"); - JobResult::Failed(e) + // Execute the job function with optional timeout + let result = match timeout_dur { + Some(duration) => { + debug!(job = %name, timeout_secs = duration.as_secs(), "Executing with timeout"); + match tokio::time::timeout(duration, job_fn()).await { + Ok(Ok(())) => JobResult::Success, + Ok(Err(e)) => { + warn!(job = %name, error = %e, "Job failed"); + JobResult::Failed(e) + } + Err(_) => { + warn!(job = %name, timeout_secs = duration.as_secs(), "Job timed out"); + JobResult::Failed(format!("Job timed out after {} seconds", duration.as_secs())) + } + } + } + None => { + match job_fn().await { + Ok(()) => JobResult::Success, + Err(e) => { + warn!(job = %name, error = %e, "Job failed"); + JobResult::Failed(e) + } + } } }; @@ -396,6 +417,7 @@ impl SchedulerService { timezone = %tz.name(), overlap = ?overlap_policy, jitter_secs = max_jitter_secs, + timeout_secs = timeout.timeout_secs, "Job registered" ); @@ -539,8 +561,8 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn test_add_cron_job_valid_expression() { - use std::sync::Arc; use std::sync::atomic::AtomicU32; + use std::sync::Arc; let config = SchedulerConfig { shutdown_timeout_secs: 1, @@ -637,7 +659,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn test_register_job_adds_to_registry() { - use crate::{JitterConfig, OverlapPolicy}; + use crate::{JitterConfig, OverlapPolicy, TimeoutConfig}; let config = SchedulerConfig::default(); let scheduler = SchedulerService::new(config).await.unwrap(); @@ -649,6 +671,7 @@ mod tests { None, OverlapPolicy::Skip, JitterConfig::none(), + TimeoutConfig::none(), || async { Ok(()) }, ) .await @@ -668,7 +691,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn test_pause_resume_job() { - use crate::{JitterConfig, OverlapPolicy}; + use crate::{JitterConfig, OverlapPolicy, TimeoutConfig}; let config = SchedulerConfig::default(); let scheduler = SchedulerService::new(config).await.unwrap(); @@ -680,6 +703,7 @@ mod tests { None, OverlapPolicy::Skip, JitterConfig::none(), + TimeoutConfig::none(), || async { Ok(()) }, ) .await @@ -717,7 +741,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn test_register_job_with_overlap_policy() { - use crate::{JitterConfig, OverlapPolicy}; + use crate::{JitterConfig, OverlapPolicy, TimeoutConfig}; let config = SchedulerConfig::default(); let scheduler = SchedulerService::new(config).await.unwrap(); @@ -730,6 +754,7 @@ mod tests { None, OverlapPolicy::Skip, JitterConfig::none(), + TimeoutConfig::none(), || async { Ok(()) }, ) .await @@ -743,6 +768,7 @@ mod tests { None, OverlapPolicy::Concurrent, JitterConfig::none(), + TimeoutConfig::none(), || async { Ok(()) }, ) .await @@ -755,7 +781,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn test_register_job_with_jitter() { - use crate::{JitterConfig, OverlapPolicy}; + use crate::{JitterConfig, OverlapPolicy, TimeoutConfig}; let config = SchedulerConfig::default(); let scheduler = SchedulerService::new(config).await.unwrap(); @@ -768,6 +794,7 @@ mod tests { None, OverlapPolicy::Skip, JitterConfig::new(30), + TimeoutConfig::none(), || async { Ok(()) }, ) .await @@ -779,7 +806,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn test_register_job_invalid_cron() { - use crate::{JitterConfig, OverlapPolicy}; + use crate::{JitterConfig, OverlapPolicy, TimeoutConfig}; let config = SchedulerConfig::default(); let scheduler = SchedulerService::new(config).await.unwrap(); @@ -791,6 +818,7 @@ mod tests { None, OverlapPolicy::Skip, JitterConfig::none(), + TimeoutConfig::none(), || async { Ok(()) }, ) .await; @@ -800,7 +828,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn test_register_job_invalid_timezone() { - use crate::{JitterConfig, OverlapPolicy}; + use crate::{JitterConfig, OverlapPolicy, TimeoutConfig}; let config = SchedulerConfig::default(); let scheduler = SchedulerService::new(config).await.unwrap(); @@ -812,6 +840,7 @@ mod tests { Some("Invalid/Timezone"), OverlapPolicy::Skip, JitterConfig::none(), + TimeoutConfig::none(), || async { Ok(()) }, ) .await; @@ -821,9 +850,9 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn test_register_job_execution_tracking() { - use crate::{JitterConfig, OverlapPolicy}; - use std::sync::Arc; + use crate::{JitterConfig, OverlapPolicy, TimeoutConfig}; use std::sync::atomic::AtomicU32; + use std::sync::Arc; let config = SchedulerConfig { shutdown_timeout_secs: 1, @@ -842,6 +871,7 @@ mod tests { None, OverlapPolicy::Skip, JitterConfig::none(), + TimeoutConfig::none(), move || { let c = counter_clone.clone(); async move { @@ -866,4 +896,85 @@ mod tests { // The job may or may not have run depending on timing // The key test is that if it ran, the registry would be updated } + + #[tokio::test(flavor = "multi_thread")] + async fn test_register_job_with_timeout() { + use crate::{JitterConfig, OverlapPolicy, TimeoutConfig}; + + let config = SchedulerConfig::default(); + let scheduler = SchedulerService::new(config).await.unwrap(); + + // Register with timeout + let uuid = scheduler + .register_job( + "timeout-job", + "0 0 * * * *", + None, + OverlapPolicy::Skip, + JitterConfig::none(), + TimeoutConfig::new(300), // 5 minute timeout + || async { Ok(()) }, + ) + .await + .unwrap(); + + assert!(!uuid.is_nil()); + assert!(scheduler.registry().is_registered("timeout-job")); + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_register_job_timeout_triggers() { + use crate::{JitterConfig, JobResult, OverlapPolicy, TimeoutConfig}; + use std::sync::atomic::AtomicBool; + use std::sync::Arc; + + let config = SchedulerConfig { + shutdown_timeout_secs: 1, + ..Default::default() + }; + let mut scheduler = SchedulerService::new(config).await.unwrap(); + + let job_started = Arc::new(AtomicBool::new(false)); + let job_started_clone = job_started.clone(); + + // Register a job that takes longer than the timeout + scheduler + .register_job( + "slow-job", + "*/1 * * * * *", // Every second + None, + OverlapPolicy::Skip, + JitterConfig::none(), + TimeoutConfig::new(1), // 1 second timeout + move || { + let started = job_started_clone.clone(); + async move { + started.store(true, Ordering::SeqCst); + // Sleep for longer than timeout + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + Ok(()) + } + }, + ) + .await + .unwrap(); + + // Start scheduler and let it run + scheduler.start().await.unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(2500)).await; + scheduler.shutdown().await.unwrap(); + + // If the job ran, it should have been marked as failed due to timeout + let status = scheduler.registry().get_status("slow-job"); + if let Some(s) = status { + if s.run_count > 0 { + // Job ran and should have timed out + assert!( + matches!(s.last_result, Some(JobResult::Failed(ref msg)) if msg.contains("timed out")), + "Expected timeout failure, got {:?}", + s.last_result + ); + } + } + } } diff --git a/crates/memory-search/Cargo.toml b/crates/memory-search/Cargo.toml new file mode 100644 index 0000000..4e80007 --- /dev/null +++ b/crates/memory-search/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "memory-search" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Full-text search for Agent Memory using Tantivy" + +[dependencies] +tantivy = { workspace = true } +memory-types = { workspace = true } +tokio = { workspace = true } +chrono = { workspace = true } +tracing = { workspace = true } +thiserror = { workspace = true } +serde = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +tokio = { workspace = true, features = ["test-util", "macros", "rt-multi-thread"] } diff --git a/crates/memory-search/src/document.rs b/crates/memory-search/src/document.rs new file mode 100644 index 0000000..766c688 --- /dev/null +++ b/crates/memory-search/src/document.rs @@ -0,0 +1,194 @@ +//! Document mapping from domain types to Tantivy documents. +//! +//! Converts TocNode and Grip into indexable Tantivy documents. + +use tantivy::doc; +use tantivy::TantivyDocument; + +use memory_types::{Grip, TocNode}; + +use crate::schema::{DocType, SearchSchema}; + +/// Convert a TocNode to a Tantivy document. +/// +/// Text field contains: title + all bullet texts +/// Keywords field contains: joined keywords +pub fn toc_node_to_doc(schema: &SearchSchema, node: &TocNode) -> TantivyDocument { + // Combine title and bullets for searchable text + let mut text_parts = vec![node.title.clone()]; + for bullet in &node.bullets { + text_parts.push(bullet.text.clone()); + } + let text = text_parts.join(" "); + + // Join keywords with space + let keywords = node.keywords.join(" "); + + // Timestamp in milliseconds + let timestamp = node.start_time.timestamp_millis().to_string(); + + doc!( + schema.doc_type => DocType::TocNode.as_str(), + schema.doc_id => node.node_id.clone(), + schema.level => node.level.to_string(), + schema.text => text, + schema.keywords => keywords, + schema.timestamp_ms => timestamp + ) +} + +/// Convert a Grip to a Tantivy document. +/// +/// Text field contains: excerpt +/// Level field is empty (not applicable to grips) +pub fn grip_to_doc(schema: &SearchSchema, grip: &Grip) -> TantivyDocument { + let timestamp = grip.timestamp.timestamp_millis().to_string(); + + doc!( + schema.doc_type => DocType::Grip.as_str(), + schema.doc_id => grip.grip_id.clone(), + schema.level => "", // Not applicable for grips + schema.text => grip.excerpt.clone(), + schema.keywords => "", // Grips don't have keywords + schema.timestamp_ms => timestamp + ) +} + +/// Extract text content from a TocNode for indexing. +/// +/// Returns combined title and bullet text. +pub fn extract_toc_text(node: &TocNode) -> String { + let mut parts = vec![node.title.clone()]; + for bullet in &node.bullets { + parts.push(bullet.text.clone()); + } + parts.join(" ") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::schema::build_teleport_schema; + use chrono::Utc; + use memory_types::{TocBullet, TocLevel}; + use tantivy::schema::Value; + + fn sample_toc_node() -> TocNode { + let mut node = TocNode::new( + "toc:day:2024-01-15".to_string(), + TocLevel::Day, + "Monday, January 15, 2024".to_string(), + Utc::now(), + Utc::now(), + ); + node.bullets = vec![ + TocBullet::new("Discussed Rust memory safety"), + TocBullet::new("Implemented authentication flow"), + ]; + node.keywords = vec!["rust".to_string(), "memory".to_string(), "auth".to_string()]; + node + } + + fn sample_grip() -> Grip { + Grip::new( + "grip-12345".to_string(), + "User asked about borrow checker semantics".to_string(), + "event-001".to_string(), + "event-003".to_string(), + Utc::now(), + "segment_summarizer".to_string(), + ) + } + + #[test] + fn test_toc_node_to_doc() { + let schema = build_teleport_schema(); + let node = sample_toc_node(); + + let doc = toc_node_to_doc(&schema, &node); + + // Verify doc_type + let doc_type = doc.get_first(schema.doc_type).unwrap(); + assert_eq!(doc_type.as_str(), Some("toc_node")); + + // Verify doc_id + let doc_id = doc.get_first(schema.doc_id).unwrap(); + assert_eq!(doc_id.as_str(), Some("toc:day:2024-01-15")); + + // Verify text contains title and bullets + let text = doc.get_first(schema.text).unwrap(); + let text_str = text.as_str().unwrap(); + assert!(text_str.contains("Monday, January 15, 2024")); + assert!(text_str.contains("Rust memory safety")); + } + + #[test] + fn test_grip_to_doc() { + let schema = build_teleport_schema(); + let grip = sample_grip(); + + let doc = grip_to_doc(&schema, &grip); + + let doc_type = doc.get_first(schema.doc_type).unwrap(); + assert_eq!(doc_type.as_str(), Some("grip")); + + let text = doc.get_first(schema.text).unwrap(); + assert!(text.as_str().unwrap().contains("borrow checker")); + } + + #[test] + fn test_extract_toc_text() { + let node = sample_toc_node(); + let text = extract_toc_text(&node); + + assert!(text.contains("Monday, January 15, 2024")); + assert!(text.contains("Discussed Rust memory safety")); + assert!(text.contains("Implemented authentication flow")); + } + + #[test] + fn test_toc_node_with_empty_bullets() { + let schema = build_teleport_schema(); + let node = TocNode::new( + "toc:day:2024-01-15".to_string(), + TocLevel::Day, + "A simple title".to_string(), + Utc::now(), + Utc::now(), + ); + + let doc = toc_node_to_doc(&schema, &node); + + let text = doc.get_first(schema.text).unwrap(); + assert_eq!(text.as_str(), Some("A simple title")); + } + + #[test] + fn test_toc_node_with_empty_keywords() { + let schema = build_teleport_schema(); + let mut node = TocNode::new( + "toc:day:2024-01-15".to_string(), + TocLevel::Day, + "A simple title".to_string(), + Utc::now(), + Utc::now(), + ); + node.keywords = vec![]; + + let doc = toc_node_to_doc(&schema, &node); + + let keywords = doc.get_first(schema.keywords).unwrap(); + assert_eq!(keywords.as_str(), Some("")); + } + + #[test] + fn test_grip_doc_level_is_empty() { + let schema = build_teleport_schema(); + let grip = sample_grip(); + + let doc = grip_to_doc(&schema, &grip); + + let level = doc.get_first(schema.level).unwrap(); + assert_eq!(level.as_str(), Some("")); + } +} diff --git a/crates/memory-search/src/error.rs b/crates/memory-search/src/error.rs new file mode 100644 index 0000000..38e3739 --- /dev/null +++ b/crates/memory-search/src/error.rs @@ -0,0 +1,35 @@ +//! Search error types. + +use thiserror::Error; + +/// Errors that can occur during search operations. +#[derive(Debug, Error)] +pub enum SearchError { + /// Tantivy index error + #[error("Tantivy error: {0}")] + Tantivy(#[from] tantivy::TantivyError), + + /// Query parse error + #[error("Query parse error: {0}")] + QueryParse(#[from] tantivy::query::QueryParserError), + + /// IO error + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + + /// Index not found + #[error("Index not found at path: {0}")] + IndexNotFound(String), + + /// Document not found + #[error("Document not found: {0}")] + DocumentNotFound(String), + + /// Schema mismatch + #[error("Schema mismatch: {0}")] + SchemaMismatch(String), + + /// Index is locked (another process has it open) + #[error("Index is locked: {0}")] + IndexLocked(String), +} diff --git a/crates/memory-search/src/index.rs b/crates/memory-search/src/index.rs new file mode 100644 index 0000000..bf4ad9e --- /dev/null +++ b/crates/memory-search/src/index.rs @@ -0,0 +1,179 @@ +//! Tantivy index management. +//! +//! Handles index creation, opening, and lifecycle. + +use std::path::{Path, PathBuf}; + +use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy}; +use tracing::{debug, info}; + +use crate::error::SearchError; +use crate::schema::{build_teleport_schema, SearchSchema}; + +/// Default memory budget for IndexWriter (50MB) +const DEFAULT_WRITER_MEMORY_MB: usize = 50; + +/// Search index configuration +#[derive(Debug, Clone)] +pub struct SearchIndexConfig { + /// Path to index directory + pub index_path: PathBuf, + /// Memory budget for writer in MB + pub writer_memory_mb: usize, +} + +impl Default for SearchIndexConfig { + fn default() -> Self { + Self { + index_path: PathBuf::from("./bm25-index"), + writer_memory_mb: DEFAULT_WRITER_MEMORY_MB, + } + } +} + +impl SearchIndexConfig { + pub fn new(index_path: impl Into) -> Self { + Self { + index_path: index_path.into(), + writer_memory_mb: DEFAULT_WRITER_MEMORY_MB, + } + } + + pub fn with_memory_mb(mut self, mb: usize) -> Self { + self.writer_memory_mb = mb; + self + } +} + +/// Wrapper for Tantivy index with schema access. +pub struct SearchIndex { + index: Index, + schema: SearchSchema, + config: SearchIndexConfig, +} + +impl SearchIndex { + /// Open existing index or create new one. + pub fn open_or_create(config: SearchIndexConfig) -> Result { + let index = open_or_create_index(&config.index_path)?; + let schema = SearchSchema::from_schema(index.schema())?; + + info!(path = ?config.index_path, "Opened search index"); + + Ok(Self { + index, + schema, + config, + }) + } + + /// Get the search schema + pub fn schema(&self) -> &SearchSchema { + &self.schema + } + + /// Get the underlying Tantivy index + pub fn index(&self) -> &Index { + &self.index + } + + /// Create an IndexWriter with configured memory budget + pub fn writer(&self) -> Result { + let memory_budget = self.config.writer_memory_mb * 1024 * 1024; + let writer = self.index.writer(memory_budget)?; + debug!( + memory_mb = self.config.writer_memory_mb, + "Created index writer" + ); + Ok(writer) + } + + /// Create an IndexReader with OnCommit reload policy + pub fn reader(&self) -> Result { + let reader = self + .index + .reader_builder() + .reload_policy(ReloadPolicy::OnCommitWithDelay) + .try_into()?; + debug!("Created index reader"); + Ok(reader) + } + + /// Get the index path + pub fn path(&self) -> &Path { + &self.config.index_path + } + + /// Check if index exists at the configured path + pub fn exists(&self) -> bool { + self.config.index_path.join("meta.json").exists() + } +} + +/// Open an existing index or create a new one. +/// +/// Uses MmapDirectory for persistence. +pub fn open_or_create_index(path: &Path) -> Result { + if path.join("meta.json").exists() { + debug!(path = ?path, "Opening existing index"); + let index = Index::open_in_dir(path)?; + Ok(index) + } else { + info!(path = ?path, "Creating new index"); + std::fs::create_dir_all(path)?; + let schema = build_teleport_schema(); + let index = Index::create_in_dir(path, schema.schema().clone())?; + Ok(index) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_create_new_index() { + let temp_dir = TempDir::new().unwrap(); + let config = SearchIndexConfig::new(temp_dir.path()); + + let index = SearchIndex::open_or_create(config).unwrap(); + assert!(index.exists()); + } + + #[test] + fn test_reopen_existing_index() { + let temp_dir = TempDir::new().unwrap(); + let config = SearchIndexConfig::new(temp_dir.path()); + + // Create index + let _index1 = SearchIndex::open_or_create(config.clone()).unwrap(); + + // Reopen + let index2 = SearchIndex::open_or_create(config).unwrap(); + assert!(index2.exists()); + } + + #[test] + fn test_create_writer_and_reader() { + let temp_dir = TempDir::new().unwrap(); + let config = SearchIndexConfig::new(temp_dir.path()); + let index = SearchIndex::open_or_create(config).unwrap(); + + let _writer = index.writer().unwrap(); + let _reader = index.reader().unwrap(); + } + + #[test] + fn test_config_default() { + let config = SearchIndexConfig::default(); + assert_eq!(config.index_path, PathBuf::from("./bm25-index")); + assert_eq!(config.writer_memory_mb, DEFAULT_WRITER_MEMORY_MB); + } + + #[test] + fn test_config_with_memory() { + let config = SearchIndexConfig::new("/tmp/test").with_memory_mb(100); + assert_eq!(config.writer_memory_mb, 100); + } +} diff --git a/crates/memory-search/src/indexer.rs b/crates/memory-search/src/indexer.rs new file mode 100644 index 0000000..67b67f5 --- /dev/null +++ b/crates/memory-search/src/indexer.rs @@ -0,0 +1,405 @@ +//! Search indexer for adding documents to the Tantivy index. +//! +//! The indexer wraps IndexWriter with shared access via `Arc`. +//! Documents are not visible until commit() is called. + +use std::sync::{Arc, Mutex}; + +use tantivy::{IndexWriter, Term}; +use tracing::{debug, info, warn}; + +use memory_types::{Grip, TocNode}; + +use crate::document::{grip_to_doc, toc_node_to_doc}; +use crate::error::SearchError; +use crate::index::SearchIndex; +use crate::schema::SearchSchema; + +/// Manages document indexing operations. +/// +/// Wraps IndexWriter for shared access across components. +/// Commit batches documents for visibility. +pub struct SearchIndexer { + writer: Arc>, + schema: SearchSchema, +} + +impl SearchIndexer { + /// Create a new indexer from a SearchIndex. + pub fn new(index: &SearchIndex) -> Result { + let writer = index.writer()?; + let schema = index.schema().clone(); + + Ok(Self { + writer: Arc::new(Mutex::new(writer)), + schema, + }) + } + + /// Create from an existing writer (for testing or shared use). + pub fn from_writer(writer: IndexWriter, schema: SearchSchema) -> Self { + Self { + writer: Arc::new(Mutex::new(writer)), + schema, + } + } + + /// Get a clone of the writer Arc for sharing. + pub fn writer_handle(&self) -> Arc> { + self.writer.clone() + } + + /// Index a TOC node. + /// + /// If a document with the same node_id exists, it will be replaced. + pub fn index_toc_node(&self, node: &TocNode) -> Result<(), SearchError> { + let doc = toc_node_to_doc(&self.schema, node); + + let writer = self + .writer + .lock() + .map_err(|e| SearchError::IndexLocked(e.to_string()))?; + + // Delete existing document with same ID (for update) + let term = Term::from_field_text(self.schema.doc_id, &node.node_id); + writer.delete_term(term); + + // Add new document + writer.add_document(doc)?; + + debug!(node_id = %node.node_id, level = %node.level, "Indexed TOC node"); + Ok(()) + } + + /// Index a grip. + /// + /// If a document with the same grip_id exists, it will be replaced. + pub fn index_grip(&self, grip: &Grip) -> Result<(), SearchError> { + let doc = grip_to_doc(&self.schema, grip); + + let writer = self + .writer + .lock() + .map_err(|e| SearchError::IndexLocked(e.to_string()))?; + + // Delete existing document with same ID (for update) + let term = Term::from_field_text(self.schema.doc_id, &grip.grip_id); + writer.delete_term(term); + + // Add new document + writer.add_document(doc)?; + + debug!(grip_id = %grip.grip_id, "Indexed grip"); + Ok(()) + } + + /// Index multiple TOC nodes in batch. + pub fn index_toc_nodes(&self, nodes: &[TocNode]) -> Result { + let writer = self + .writer + .lock() + .map_err(|e| SearchError::IndexLocked(e.to_string()))?; + + let mut count = 0; + for node in nodes { + let doc = toc_node_to_doc(&self.schema, node); + + // Delete existing + let term = Term::from_field_text(self.schema.doc_id, &node.node_id); + writer.delete_term(term); + + // Add new + writer.add_document(doc)?; + count += 1; + } + + debug!(count, "Indexed TOC nodes batch"); + Ok(count) + } + + /// Index multiple grips in batch. + pub fn index_grips(&self, grips: &[Grip]) -> Result { + let writer = self + .writer + .lock() + .map_err(|e| SearchError::IndexLocked(e.to_string()))?; + + let mut count = 0; + for grip in grips { + let doc = grip_to_doc(&self.schema, grip); + + // Delete existing + let term = Term::from_field_text(self.schema.doc_id, &grip.grip_id); + writer.delete_term(term); + + // Add new + writer.add_document(doc)?; + count += 1; + } + + debug!(count, "Indexed grips batch"); + Ok(count) + } + + /// Delete a document by ID. + pub fn delete_document(&self, doc_id: &str) -> Result<(), SearchError> { + let writer = self + .writer + .lock() + .map_err(|e| SearchError::IndexLocked(e.to_string()))?; + + let term = Term::from_field_text(self.schema.doc_id, doc_id); + writer.delete_term(term); + + debug!(doc_id, "Deleted document"); + Ok(()) + } + + /// Commit pending changes to make them searchable. + /// + /// This is expensive - batch document adds and commit periodically. + pub fn commit(&self) -> Result { + let mut writer = self + .writer + .lock() + .map_err(|e| SearchError::IndexLocked(e.to_string()))?; + + let opstamp = writer.commit()?; + info!(opstamp, "Committed index changes"); + Ok(opstamp) + } + + /// Rollback uncommitted changes. + pub fn rollback(&self) -> Result { + let mut writer = self + .writer + .lock() + .map_err(|e| SearchError::IndexLocked(e.to_string()))?; + + let opstamp = writer.rollback()?; + warn!(opstamp, "Rolled back index changes"); + Ok(opstamp) + } + + /// Get the current commit opstamp. + pub fn pending_ops(&self) -> Result { + let writer = self + .writer + .lock() + .map_err(|e| SearchError::IndexLocked(e.to_string()))?; + + Ok(writer.commit_opstamp()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::index::{SearchIndex, SearchIndexConfig}; + use chrono::Utc; + use memory_types::{TocBullet, TocLevel}; + use tempfile::TempDir; + + fn sample_toc_node(id: &str) -> TocNode { + let mut node = TocNode::new( + id.to_string(), + TocLevel::Day, + format!("Test Node {}", id), + Utc::now(), + Utc::now(), + ); + node.bullets = vec![TocBullet::new("Test bullet content")]; + node.keywords = vec!["test".to_string()]; + node + } + + fn sample_grip(id: &str) -> Grip { + Grip::new( + id.to_string(), + "Test excerpt content".to_string(), + "event-001".to_string(), + "event-002".to_string(), + Utc::now(), + "test".to_string(), + ) + } + + #[test] + fn test_index_toc_node() { + let temp_dir = TempDir::new().unwrap(); + let config = SearchIndexConfig::new(temp_dir.path()); + let index = SearchIndex::open_or_create(config).unwrap(); + let indexer = SearchIndexer::new(&index).unwrap(); + + let node = sample_toc_node("node-1"); + indexer.index_toc_node(&node).unwrap(); + indexer.commit().unwrap(); + } + + #[test] + fn test_index_grip() { + let temp_dir = TempDir::new().unwrap(); + let config = SearchIndexConfig::new(temp_dir.path()); + let index = SearchIndex::open_or_create(config).unwrap(); + let indexer = SearchIndexer::new(&index).unwrap(); + + let grip = sample_grip("grip-1"); + indexer.index_grip(&grip).unwrap(); + indexer.commit().unwrap(); + } + + #[test] + fn test_index_batch() { + let temp_dir = TempDir::new().unwrap(); + let config = SearchIndexConfig::new(temp_dir.path()); + let index = SearchIndex::open_or_create(config).unwrap(); + let indexer = SearchIndexer::new(&index).unwrap(); + + let nodes: Vec = (0..5) + .map(|i| sample_toc_node(&format!("node-{}", i))) + .collect(); + + let count = indexer.index_toc_nodes(&nodes).unwrap(); + assert_eq!(count, 5); + indexer.commit().unwrap(); + } + + #[test] + fn test_index_grips_batch() { + let temp_dir = TempDir::new().unwrap(); + let config = SearchIndexConfig::new(temp_dir.path()); + let index = SearchIndex::open_or_create(config).unwrap(); + let indexer = SearchIndexer::new(&index).unwrap(); + + let grips: Vec = (0..3) + .map(|i| sample_grip(&format!("grip-{}", i))) + .collect(); + + let count = indexer.index_grips(&grips).unwrap(); + assert_eq!(count, 3); + indexer.commit().unwrap(); + } + + #[test] + fn test_update_existing_document() { + let temp_dir = TempDir::new().unwrap(); + let config = SearchIndexConfig::new(temp_dir.path()); + let index = SearchIndex::open_or_create(config).unwrap(); + let indexer = SearchIndexer::new(&index).unwrap(); + + // Index initial version + let mut node = sample_toc_node("node-1"); + node.title = "Version 1".to_string(); + indexer.index_toc_node(&node).unwrap(); + indexer.commit().unwrap(); + + // Index updated version (same ID) + node.title = "Version 2".to_string(); + node.version = 2; + indexer.index_toc_node(&node).unwrap(); + indexer.commit().unwrap(); + + // Should only have one document + let reader = index.reader().unwrap(); + let searcher = reader.searcher(); + let num_docs: u64 = searcher + .segment_readers() + .iter() + .map(|r| r.num_docs() as u64) + .sum(); + assert_eq!(num_docs, 1); + } + + #[test] + fn test_delete_document() { + let temp_dir = TempDir::new().unwrap(); + let config = SearchIndexConfig::new(temp_dir.path()); + let index = SearchIndex::open_or_create(config).unwrap(); + let indexer = SearchIndexer::new(&index).unwrap(); + + // Index a node + let node = sample_toc_node("node-to-delete"); + indexer.index_toc_node(&node).unwrap(); + indexer.commit().unwrap(); + + // Verify it exists + let reader = index.reader().unwrap(); + let searcher = reader.searcher(); + let num_docs_before: u64 = searcher + .segment_readers() + .iter() + .map(|r| r.num_docs() as u64) + .sum(); + assert_eq!(num_docs_before, 1); + + // Delete it + indexer.delete_document("node-to-delete").unwrap(); + indexer.commit().unwrap(); + + // Verify it's gone (need new reader after commit) + let reader = index.reader().unwrap(); + let searcher = reader.searcher(); + let num_docs_after: u64 = searcher + .segment_readers() + .iter() + .map(|r| r.num_docs() as u64) + .sum(); + assert_eq!(num_docs_after, 0); + } + + #[test] + fn test_writer_handle() { + let temp_dir = TempDir::new().unwrap(); + let config = SearchIndexConfig::new(temp_dir.path()); + let index = SearchIndex::open_or_create(config).unwrap(); + let indexer = SearchIndexer::new(&index).unwrap(); + + // Get handle and verify it's shareable + let handle1 = indexer.writer_handle(); + let handle2 = indexer.writer_handle(); + + // Both handles should point to the same Arc + assert!(Arc::ptr_eq(&handle1, &handle2)); + } + + #[test] + fn test_rollback() { + let temp_dir = TempDir::new().unwrap(); + let config = SearchIndexConfig::new(temp_dir.path()); + let index = SearchIndex::open_or_create(config).unwrap(); + let indexer = SearchIndexer::new(&index).unwrap(); + + // Index a node but don't commit + let node = sample_toc_node("node-to-rollback"); + indexer.index_toc_node(&node).unwrap(); + + // Rollback + indexer.rollback().unwrap(); + + // Commit (after rollback, writer is reset) + indexer.commit().unwrap(); + + // Verify no documents exist + let reader = index.reader().unwrap(); + let searcher = reader.searcher(); + let num_docs: u64 = searcher + .segment_readers() + .iter() + .map(|r| r.num_docs() as u64) + .sum(); + assert_eq!(num_docs, 0); + } + + #[test] + fn test_pending_ops() { + let temp_dir = TempDir::new().unwrap(); + let config = SearchIndexConfig::new(temp_dir.path()); + let index = SearchIndex::open_or_create(config).unwrap(); + let indexer = SearchIndexer::new(&index).unwrap(); + + // Check we can get the opstamp + let opstamp = indexer.pending_ops().unwrap(); + // Initial opstamp should be 0 + assert_eq!(opstamp, 0); + } +} diff --git a/crates/memory-search/src/lib.rs b/crates/memory-search/src/lib.rs new file mode 100644 index 0000000..a79a84a --- /dev/null +++ b/crates/memory-search/src/lib.rs @@ -0,0 +1,32 @@ +//! # memory-search +//! +//! Full-text search for Agent Memory using Tantivy. +//! +//! This crate provides BM25 keyword search for "teleporting" directly to +//! relevant TOC nodes or grips without traversing the hierarchy. +//! +//! ## Features +//! - Embedded Tantivy index with MmapDirectory for persistence +//! - Schema for indexing TOC node summaries and grip excerpts +//! - BM25 scoring for relevance ranking +//! - Document type filtering (toc_node vs grip) +//! +//! ## Requirements +//! - TEL-01: Tantivy embedded index +//! - TEL-02: BM25 search returns ranked results +//! - TEL-03: Relevance scores for agent decision-making +//! - TEL-04: Incremental index updates + +pub mod document; +pub mod error; +pub mod index; +pub mod indexer; +pub mod schema; +pub mod searcher; + +pub use document::{extract_toc_text, grip_to_doc, toc_node_to_doc}; +pub use error::SearchError; +pub use index::{open_or_create_index, SearchIndex, SearchIndexConfig}; +pub use indexer::SearchIndexer; +pub use schema::{build_teleport_schema, DocType, SearchSchema}; +pub use searcher::{SearchOptions, TeleportResult, TeleportSearcher}; diff --git a/crates/memory-search/src/schema.rs b/crates/memory-search/src/schema.rs new file mode 100644 index 0000000..236408c --- /dev/null +++ b/crates/memory-search/src/schema.rs @@ -0,0 +1,174 @@ +//! Tantivy schema definition for teleport search. +//! +//! Indexes two document types: +//! - TOC nodes: title + bullets + keywords +//! - Grips: excerpt text + +use tantivy::schema::{Field, Schema, STORED, STRING, TEXT}; + +use crate::SearchError; + +/// Document types stored in the index +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DocType { + TocNode, + Grip, +} + +impl DocType { + pub fn as_str(&self) -> &'static str { + match self { + DocType::TocNode => "toc_node", + DocType::Grip => "grip", + } + } + + /// Parse from string, returning None for unknown types. + pub fn parse(s: &str) -> Option { + match s { + "toc_node" => Some(DocType::TocNode), + "grip" => Some(DocType::Grip), + _ => None, + } + } +} + +impl std::str::FromStr for DocType { + type Err = String; + + fn from_str(s: &str) -> Result { + Self::parse(s).ok_or_else(|| format!("unknown doc type: {}", s)) + } +} + +/// Schema field handles for efficient access +#[derive(Debug, Clone)] +pub struct SearchSchema { + schema: Schema, + /// Document type: "toc_node" or "grip" (STRING | STORED) + pub doc_type: Field, + /// Primary key: node_id or grip_id (STRING | STORED) + pub doc_id: Field, + /// TOC level for toc_node: "year", "month", etc. (STRING) + pub level: Field, + /// Searchable text: title+bullets for TOC, excerpt for grip (TEXT) + pub text: Field, + /// Keywords/tags (TEXT | STORED) + pub keywords: Field, + /// Timestamp in milliseconds (STRING | STORED for recency) + pub timestamp_ms: Field, +} + +impl SearchSchema { + /// Get the underlying Tantivy schema + pub fn schema(&self) -> &Schema { + &self.schema + } + + /// Create a SearchSchema from an existing Tantivy Schema + pub fn from_schema(schema: Schema) -> Result { + let doc_type = schema + .get_field("doc_type") + .map_err(|_| SearchError::SchemaMismatch("missing doc_type field".into()))?; + let doc_id = schema + .get_field("doc_id") + .map_err(|_| SearchError::SchemaMismatch("missing doc_id field".into()))?; + let level = schema + .get_field("level") + .map_err(|_| SearchError::SchemaMismatch("missing level field".into()))?; + let text = schema + .get_field("text") + .map_err(|_| SearchError::SchemaMismatch("missing text field".into()))?; + let keywords = schema + .get_field("keywords") + .map_err(|_| SearchError::SchemaMismatch("missing keywords field".into()))?; + let timestamp_ms = schema + .get_field("timestamp_ms") + .map_err(|_| SearchError::SchemaMismatch("missing timestamp_ms field".into()))?; + + Ok(Self { + schema, + doc_type, + doc_id, + level, + text, + keywords, + timestamp_ms, + }) + } +} + +/// Build the teleport search schema. +/// +/// Schema fields: +/// - doc_type: STRING | STORED - "toc_node" or "grip" +/// - doc_id: STRING | STORED - node_id or grip_id +/// - level: STRING - TOC level (for filtering) +/// - text: TEXT - searchable content +/// - keywords: TEXT | STORED - keywords/tags +/// - timestamp_ms: STRING | STORED - for recency info +pub fn build_teleport_schema() -> SearchSchema { + let mut schema_builder = Schema::builder(); + + // Document type for filtering: "toc_node" or "grip" + let doc_type = schema_builder.add_text_field("doc_type", STRING | STORED); + + // Primary key - node_id or grip_id + let doc_id = schema_builder.add_text_field("doc_id", STRING | STORED); + + // TOC level (for toc_node only): "year", "month", "week", "day", "segment" + let level = schema_builder.add_text_field("level", STRING); + + // Searchable text content (title + bullets for TOC, excerpt for grip) + let text = schema_builder.add_text_field("text", TEXT); + + // Keywords (indexed and stored for retrieval) + let keywords = schema_builder.add_text_field("keywords", TEXT | STORED); + + // Timestamp for recency (stored as string for simplicity) + let timestamp_ms = schema_builder.add_text_field("timestamp_ms", STRING | STORED); + + let schema = schema_builder.build(); + + SearchSchema { + schema, + doc_type, + doc_id, + level, + text, + keywords, + timestamp_ms, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_schema() { + let schema = build_teleport_schema(); + assert!(schema.schema.get_field("doc_type").is_ok()); + assert!(schema.schema.get_field("doc_id").is_ok()); + assert!(schema.schema.get_field("text").is_ok()); + } + + #[test] + fn test_doc_type_conversion() { + assert_eq!(DocType::TocNode.as_str(), "toc_node"); + assert_eq!(DocType::parse("grip"), Some(DocType::Grip)); + assert_eq!(DocType::parse("invalid"), None); + // Test FromStr trait + assert_eq!("toc_node".parse::().unwrap(), DocType::TocNode); + assert!("invalid".parse::().is_err()); + } + + #[test] + fn test_from_schema() { + let original = build_teleport_schema(); + let rebuilt = SearchSchema::from_schema(original.schema().clone()).unwrap(); + assert_eq!(rebuilt.doc_type, original.doc_type); + assert_eq!(rebuilt.doc_id, original.doc_id); + assert_eq!(rebuilt.text, original.text); + } +} diff --git a/crates/memory-search/src/searcher.rs b/crates/memory-search/src/searcher.rs new file mode 100644 index 0000000..fe0bedb --- /dev/null +++ b/crates/memory-search/src/searcher.rs @@ -0,0 +1,451 @@ +//! Search implementation using BM25 scoring. +//! +//! Provides keyword search over TOC nodes and grips. + +use tantivy::collector::TopDocs; +use tantivy::query::{BooleanQuery, Occur, QueryParser, TermQuery}; +use tantivy::schema::{IndexRecordOption, Value}; +use tantivy::{IndexReader, Term}; +use tracing::{debug, info}; + +use crate::error::SearchError; +use crate::index::SearchIndex; +use crate::schema::{DocType, SearchSchema}; + +/// A search result with relevance score. +#[derive(Debug, Clone)] +pub struct TeleportResult { + /// Document ID (node_id or grip_id) + pub doc_id: String, + /// Document type + pub doc_type: DocType, + /// BM25 relevance score + pub score: f32, + /// Keywords from the document (if stored) + pub keywords: Option, + /// Timestamp in milliseconds + pub timestamp_ms: Option, +} + +/// Search options for filtering and limiting results. +#[derive(Debug, Clone, Default)] +pub struct SearchOptions { + /// Filter by document type (None = all types) + pub doc_type: Option, + /// Maximum results to return + pub limit: usize, +} + +impl SearchOptions { + pub fn new() -> Self { + Self { + doc_type: None, + limit: 10, + } + } + + pub fn with_limit(mut self, limit: usize) -> Self { + self.limit = limit; + self + } + + pub fn with_doc_type(mut self, doc_type: DocType) -> Self { + self.doc_type = Some(doc_type); + self + } + + pub fn toc_only() -> Self { + Self::new().with_doc_type(DocType::TocNode) + } + + pub fn grips_only() -> Self { + Self::new().with_doc_type(DocType::Grip) + } +} + +/// Searcher for teleport queries using BM25 ranking. +pub struct TeleportSearcher { + reader: IndexReader, + schema: SearchSchema, + query_parser: QueryParser, +} + +impl TeleportSearcher { + /// Create a new searcher from a SearchIndex. + pub fn new(index: &SearchIndex) -> Result { + let reader = index.reader()?; + let schema = index.schema().clone(); + + // Create query parser targeting text and keywords fields + let query_parser = + QueryParser::for_index(index.index(), vec![schema.text, schema.keywords]); + + Ok(Self { + reader, + schema, + query_parser, + }) + } + + /// Reload the reader to see recent commits. + pub fn reload(&self) -> Result<(), SearchError> { + self.reader.reload()?; + debug!("Reloaded search reader"); + Ok(()) + } + + /// Search with a query string. + /// + /// Uses BM25 scoring over text and keywords fields. + pub fn search( + &self, + query_str: &str, + options: SearchOptions, + ) -> Result, SearchError> { + if query_str.trim().is_empty() { + return Ok(Vec::new()); + } + + let searcher = self.reader.searcher(); + + // Parse the text query + let text_query = self.query_parser.parse_query(query_str)?; + + // Apply document type filter if specified + let final_query = if let Some(doc_type) = options.doc_type { + let type_term = Term::from_field_text(self.schema.doc_type, doc_type.as_str()); + let type_query = TermQuery::new(type_term, IndexRecordOption::Basic); + + Box::new(BooleanQuery::new(vec![ + (Occur::Must, text_query), + (Occur::Must, Box::new(type_query)), + ])) + } else { + text_query + }; + + // Execute search + let top_docs = searcher.search(&final_query, &TopDocs::with_limit(options.limit))?; + + // Map results + let mut results = Vec::with_capacity(top_docs.len()); + for (score, doc_address) in top_docs { + let doc: tantivy::TantivyDocument = searcher.doc(doc_address)?; + + // Extract fields + let doc_type_str = doc + .get_first(self.schema.doc_type) + .and_then(|v| v.as_str()) + .unwrap_or(""); + + let doc_id = doc + .get_first(self.schema.doc_id) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + + let keywords = doc + .get_first(self.schema.keywords) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .filter(|s| !s.is_empty()); + + let timestamp_ms = doc + .get_first(self.schema.timestamp_ms) + .and_then(|v| v.as_str()) + .and_then(|s| s.parse::().ok()); + + let doc_type = doc_type_str.parse::().unwrap_or(DocType::TocNode); + + results.push(TeleportResult { + doc_id, + doc_type, + score, + keywords, + timestamp_ms, + }); + } + + info!( + query = query_str, + results = results.len(), + "Teleport search complete" + ); + + Ok(results) + } + + /// Search TOC nodes only. + pub fn search_toc( + &self, + query_str: &str, + limit: usize, + ) -> Result, SearchError> { + self.search(query_str, SearchOptions::toc_only().with_limit(limit)) + } + + /// Search grips only. + pub fn search_grips( + &self, + query_str: &str, + limit: usize, + ) -> Result, SearchError> { + self.search(query_str, SearchOptions::grips_only().with_limit(limit)) + } + + /// Get the number of indexed documents. + pub fn num_docs(&self) -> u64 { + let searcher = self.reader.searcher(); + searcher + .segment_readers() + .iter() + .map(|r| r.num_docs() as u64) + .sum() + } +} + +// Implement Send + Sync for TeleportSearcher to allow use with Arc +// TeleportSearcher is safe to share between threads as: +// - IndexReader is thread-safe +// - SearchSchema is Clone and contains only Field handles +// - QueryParser is used within methods that hold the appropriate locks +unsafe impl Send for TeleportSearcher {} +unsafe impl Sync for TeleportSearcher {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::index::{SearchIndex, SearchIndexConfig}; + use crate::indexer::SearchIndexer; + use chrono::Utc; + use memory_types::{Grip, TocBullet, TocLevel, TocNode}; + use tempfile::TempDir; + + fn sample_toc_node(id: &str, title: &str, bullet: &str) -> TocNode { + let mut node = TocNode::new( + id.to_string(), + TocLevel::Day, + title.to_string(), + Utc::now(), + Utc::now(), + ); + node.bullets = vec![TocBullet::new(bullet)]; + node.keywords = vec!["test".to_string()]; + node + } + + fn sample_grip(id: &str, excerpt: &str) -> Grip { + Grip::new( + id.to_string(), + excerpt.to_string(), + "event-001".to_string(), + "event-002".to_string(), + Utc::now(), + "test".to_string(), + ) + } + + fn setup_index() -> (TempDir, SearchIndex) { + let temp_dir = TempDir::new().unwrap(); + let config = SearchIndexConfig::new(temp_dir.path()); + let index = SearchIndex::open_or_create(config).unwrap(); + (temp_dir, index) + } + + #[test] + fn test_search_toc_nodes() { + let (_temp_dir, index) = setup_index(); + let indexer = SearchIndexer::new(&index).unwrap(); + + // Index some nodes + let node1 = sample_toc_node("node-1", "Rust Memory Safety", "Discussed borrow checker"); + let node2 = sample_toc_node("node-2", "Python Performance", "Talked about async/await"); + + indexer.index_toc_node(&node1).unwrap(); + indexer.index_toc_node(&node2).unwrap(); + indexer.commit().unwrap(); + + let searcher = TeleportSearcher::new(&index).unwrap(); + + // Search for "rust" + let results = searcher.search_toc("rust", 10).unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].doc_id, "node-1"); + assert!(results[0].score > 0.0); + } + + #[test] + fn test_search_grips() { + let (_temp_dir, index) = setup_index(); + let indexer = SearchIndexer::new(&index).unwrap(); + + let grip1 = sample_grip("grip-1", "User asked about memory allocation"); + let grip2 = sample_grip("grip-2", "Discussed database performance"); + + indexer.index_grip(&grip1).unwrap(); + indexer.index_grip(&grip2).unwrap(); + indexer.commit().unwrap(); + + let searcher = TeleportSearcher::new(&index).unwrap(); + + let results = searcher.search_grips("memory", 10).unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].doc_id, "grip-1"); + } + + #[test] + fn test_search_all_types() { + let (_temp_dir, index) = setup_index(); + let indexer = SearchIndexer::new(&index).unwrap(); + + let node = sample_toc_node("node-1", "Memory Discussion", "Talked about allocation"); + let grip = sample_grip("grip-1", "Memory allocation in Rust"); + + indexer.index_toc_node(&node).unwrap(); + indexer.index_grip(&grip).unwrap(); + indexer.commit().unwrap(); + + let searcher = TeleportSearcher::new(&index).unwrap(); + + // Search all types + let results = searcher + .search("memory", SearchOptions::new().with_limit(10)) + .unwrap(); + assert_eq!(results.len(), 2); + } + + #[test] + fn test_bm25_ranking() { + let (_temp_dir, index) = setup_index(); + let indexer = SearchIndexer::new(&index).unwrap(); + + // Node with "rust" once + let node1 = sample_toc_node("node-1", "Rust basics", "Introduction"); + // Node with "rust" multiple times + let node2 = sample_toc_node( + "node-2", + "Advanced Rust", + "Deep dive into Rust ownership and Rust lifetimes", + ); + + indexer.index_toc_node(&node1).unwrap(); + indexer.index_toc_node(&node2).unwrap(); + indexer.commit().unwrap(); + + let searcher = TeleportSearcher::new(&index).unwrap(); + + let results = searcher.search_toc("rust", 10).unwrap(); + assert_eq!(results.len(), 2); + // Node2 should rank higher (more occurrences of "rust") + assert_eq!(results[0].doc_id, "node-2"); + assert!(results[0].score > results[1].score); + } + + #[test] + fn test_empty_query() { + let (_temp_dir, index) = setup_index(); + let searcher = TeleportSearcher::new(&index).unwrap(); + + let results = searcher.search("", SearchOptions::new()).unwrap(); + assert!(results.is_empty()); + } + + #[test] + fn test_whitespace_only_query() { + let (_temp_dir, index) = setup_index(); + let searcher = TeleportSearcher::new(&index).unwrap(); + + let results = searcher.search(" ", SearchOptions::new()).unwrap(); + assert!(results.is_empty()); + } + + #[test] + fn test_num_docs() { + let (_temp_dir, index) = setup_index(); + let indexer = SearchIndexer::new(&index).unwrap(); + + indexer + .index_toc_node(&sample_toc_node("node-1", "Test", "Content")) + .unwrap(); + indexer + .index_grip(&sample_grip("grip-1", "Excerpt")) + .unwrap(); + indexer.commit().unwrap(); + + let searcher = TeleportSearcher::new(&index).unwrap(); + assert_eq!(searcher.num_docs(), 2); + } + + #[test] + fn test_search_options_builder() { + let options = SearchOptions::new() + .with_limit(20) + .with_doc_type(DocType::TocNode); + + assert_eq!(options.limit, 20); + assert_eq!(options.doc_type, Some(DocType::TocNode)); + } + + #[test] + fn test_search_options_toc_only() { + let options = SearchOptions::toc_only(); + assert_eq!(options.doc_type, Some(DocType::TocNode)); + assert_eq!(options.limit, 10); + } + + #[test] + fn test_search_options_grips_only() { + let options = SearchOptions::grips_only(); + assert_eq!(options.doc_type, Some(DocType::Grip)); + assert_eq!(options.limit, 10); + } + + #[test] + fn test_search_with_keywords() { + let (_temp_dir, index) = setup_index(); + let indexer = SearchIndexer::new(&index).unwrap(); + + let mut node = TocNode::new( + "node-1".to_string(), + TocLevel::Day, + "Generic Title".to_string(), + Utc::now(), + Utc::now(), + ); + node.keywords = vec!["rust".to_string(), "memory".to_string()]; + + indexer.index_toc_node(&node).unwrap(); + indexer.commit().unwrap(); + + let searcher = TeleportSearcher::new(&index).unwrap(); + + // Search by keyword + let results = searcher.search_toc("rust", 10).unwrap(); + assert_eq!(results.len(), 1); + assert!(results[0].keywords.is_some()); + } + + #[test] + fn test_reload() { + let (_temp_dir, index) = setup_index(); + let searcher = TeleportSearcher::new(&index).unwrap(); + + // Reload should succeed + searcher.reload().unwrap(); + } + + #[test] + fn test_no_results_for_nonexistent_term() { + let (_temp_dir, index) = setup_index(); + let indexer = SearchIndexer::new(&index).unwrap(); + + let node = sample_toc_node("node-1", "Rust Discussion", "Talked about ownership"); + indexer.index_toc_node(&node).unwrap(); + indexer.commit().unwrap(); + + let searcher = TeleportSearcher::new(&index).unwrap(); + + let results = searcher.search_toc("nonexistentterm12345", 10).unwrap(); + assert!(results.is_empty()); + } +} diff --git a/crates/memory-service/Cargo.toml b/crates/memory-service/Cargo.toml index dfe0cbb..057a16d 100644 --- a/crates/memory-service/Cargo.toml +++ b/crates/memory-service/Cargo.toml @@ -8,6 +8,11 @@ license.workspace = true memory-types = { workspace = true } memory-storage = { workspace = true } memory-scheduler = { workspace = true } +memory-search = { workspace = true } +memory-toc = { workspace = true } +memory-embeddings = { workspace = true } +memory-vector = { workspace = true } +memory-topics = { workspace = true } tokio = { workspace = true } tonic = { workspace = true } tonic-health = { workspace = true } diff --git a/crates/memory-service/src/hybrid.rs b/crates/memory-service/src/hybrid.rs new file mode 100644 index 0000000..8fbb598 --- /dev/null +++ b/crates/memory-service/src/hybrid.rs @@ -0,0 +1,237 @@ +//! HybridSearch RPC implementation. +//! +//! Combines BM25 and vector search using Reciprocal Rank Fusion (RRF). +//! RRF_score(doc) = sum(weight_i / (k + rank_i(doc))) +//! where k=60 is the standard constant. + +use std::collections::HashMap; +use std::sync::Arc; + +use tonic::{Request, Response, Status}; +use tracing::{debug, info}; + +use crate::pb::{ + HybridMode, HybridSearchRequest, HybridSearchResponse, VectorMatch, VectorTeleportRequest, +}; +use crate::vector::VectorTeleportHandler; + +/// Standard RRF constant (from original RRF paper) +const RRF_K: f32 = 60.0; + +/// Handler for hybrid search operations. +pub struct HybridSearchHandler { + vector_handler: Arc, + // BM25 integration will be added when Phase 11 is complete +} + +impl HybridSearchHandler { + /// Create a new hybrid search handler. + pub fn new(vector_handler: Arc) -> Self { + Self { vector_handler } + } + + /// Check if BM25 search is available. + pub fn bm25_available(&self) -> bool { + // TODO: Will be true when Phase 11 is integrated + false + } + + /// Check if vector search is available. + pub fn vector_available(&self) -> bool { + self.vector_handler.is_available() + } + + /// Handle HybridSearch RPC request. + pub async fn hybrid_search( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let query = &req.query; + let top_k = if req.top_k > 0 { + req.top_k as usize + } else { + 10 + }; + let mode = HybridMode::try_from(req.mode).unwrap_or(HybridMode::Hybrid); + let bm25_weight = if req.bm25_weight > 0.0 { + req.bm25_weight + } else { + 0.5 + }; + let vector_weight = if req.vector_weight > 0.0 { + req.vector_weight + } else { + 0.5 + }; + + debug!(query = %query, mode = ?mode, "HybridSearch request"); + + // Determine actual mode based on availability + let (actual_mode, matches) = match mode { + HybridMode::VectorOnly => ( + HybridMode::VectorOnly, + self.vector_search(query, top_k, &req).await?, + ), + HybridMode::Bm25Only => (HybridMode::Bm25Only, self.bm25_search(query, top_k).await?), + HybridMode::Hybrid | HybridMode::Unspecified => { + if self.vector_available() && self.bm25_available() { + let fused = self + .fuse_rrf(query, top_k, bm25_weight, vector_weight, &req) + .await?; + (HybridMode::Hybrid, fused) + } else if self.vector_available() { + ( + HybridMode::VectorOnly, + self.vector_search(query, top_k, &req).await?, + ) + } else if self.bm25_available() { + (HybridMode::Bm25Only, self.bm25_search(query, top_k).await?) + } else { + (HybridMode::Unspecified, vec![]) + } + } + }; + + info!(query = %query, mode = ?actual_mode, results = matches.len(), "HybridSearch complete"); + + Ok(Response::new(HybridSearchResponse { + matches, + mode_used: actual_mode as i32, + bm25_available: self.bm25_available(), + vector_available: self.vector_available(), + })) + } + + /// Perform vector-only search. + async fn vector_search( + &self, + query: &str, + top_k: usize, + req: &HybridSearchRequest, + ) -> Result, Status> { + let vector_req = VectorTeleportRequest { + query: query.to_string(), + top_k: top_k as i32, + min_score: 0.0, + time_filter: req.time_filter, + target: req.target, + }; + let response = self + .vector_handler + .vector_teleport(Request::new(vector_req)) + .await?; + Ok(response.into_inner().matches) + } + + /// Perform BM25-only search. + async fn bm25_search(&self, _query: &str, _top_k: usize) -> Result, Status> { + // TODO: Integrate with Phase 11 BM25 when complete + Ok(vec![]) + } + + /// Fuse results using Reciprocal Rank Fusion. + async fn fuse_rrf( + &self, + query: &str, + top_k: usize, + bm25_weight: f32, + vector_weight: f32, + req: &HybridSearchRequest, + ) -> Result, Status> { + // Fetch more results for fusion + let fetch_k = top_k * 2; + + let vector_results = self.vector_search(query, fetch_k, req).await?; + let bm25_results = self.bm25_search(query, fetch_k).await?; + + let mut rrf: HashMap = HashMap::new(); + + // Accumulate vector RRF scores + for (rank, m) in vector_results.into_iter().enumerate() { + let score = vector_weight / (RRF_K + rank as f32 + 1.0); + let entry = rrf + .entry(m.doc_id.clone()) + .or_insert_with(|| RrfEntry::from(&m)); + entry.rrf_score += score; + } + + // Accumulate BM25 RRF scores + for (rank, m) in bm25_results.into_iter().enumerate() { + let score = bm25_weight / (RRF_K + rank as f32 + 1.0); + let entry = rrf + .entry(m.doc_id.clone()) + .or_insert_with(|| RrfEntry::from(&m)); + entry.rrf_score += score; + } + + // Sort by RRF score and truncate + let mut entries: Vec<_> = rrf.into_values().collect(); + entries.sort_by(|a, b| { + b.rrf_score + .partial_cmp(&a.rrf_score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + entries.truncate(top_k); + + // Convert to VectorMatch + Ok(entries + .into_iter() + .map(|e| VectorMatch { + doc_id: e.doc_id, + doc_type: e.doc_type, + score: e.rrf_score, + text_preview: e.text_preview, + timestamp_ms: e.timestamp_ms, + }) + .collect()) + } +} + +/// Entry for RRF accumulation. +struct RrfEntry { + doc_id: String, + doc_type: String, + text_preview: String, + timestamp_ms: i64, + rrf_score: f32, +} + +impl From<&VectorMatch> for RrfEntry { + fn from(m: &VectorMatch) -> Self { + Self { + doc_id: m.doc_id.clone(), + doc_type: m.doc_type.clone(), + text_preview: m.text_preview.clone(), + timestamp_ms: m.timestamp_ms, + rrf_score: 0.0, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_rrf_k_constant() { + // Verify RRF_K is the standard value from the paper + assert_eq!(RRF_K, 60.0); + } + + #[test] + fn test_rrf_entry_from_vector_match() { + let m = VectorMatch { + doc_id: "test-123".to_string(), + doc_type: "toc_node".to_string(), + score: 0.95, + text_preview: "Test preview".to_string(), + timestamp_ms: 1234567890, + }; + + let entry = RrfEntry::from(&m); + assert_eq!(entry.doc_id, "test-123"); + assert_eq!(entry.doc_type, "toc_node"); + assert_eq!(entry.rrf_score, 0.0); // Should start at 0 + } +} diff --git a/crates/memory-service/src/ingest.rs b/crates/memory-service/src/ingest.rs index b095c56..22056bd 100644 --- a/crates/memory-service/src/ingest.rs +++ b/crates/memory-service/src/ingest.rs @@ -12,32 +12,39 @@ use tonic::{Request, Response, Status}; use tracing::{debug, error, info}; use memory_scheduler::SchedulerService; +use memory_search::TeleportSearcher; use memory_storage::Storage; use memory_types::{Event, EventRole, EventType, OutboxEntry}; +use crate::hybrid::HybridSearchHandler; use crate::pb::{ - memory_service_server::MemoryService, - BrowseTocRequest, BrowseTocResponse, - Event as ProtoEvent, - EventRole as ProtoEventRole, - EventType as ProtoEventType, - ExpandGripRequest, ExpandGripResponse, - GetEventsRequest, GetEventsResponse, - GetNodeRequest, GetNodeResponse, - GetSchedulerStatusRequest, GetSchedulerStatusResponse, - GetTocRootRequest, GetTocRootResponse, - IngestEventRequest, - IngestEventResponse, - PauseJobRequest, PauseJobResponse, - ResumeJobRequest, ResumeJobResponse, + memory_service_server::MemoryService, BrowseTocRequest, BrowseTocResponse, Event as ProtoEvent, + EventRole as ProtoEventRole, EventType as ProtoEventType, ExpandGripRequest, + ExpandGripResponse, GetEventsRequest, GetEventsResponse, GetNodeRequest, GetNodeResponse, + GetRelatedTopicsRequest, GetRelatedTopicsResponse, GetSchedulerStatusRequest, + GetSchedulerStatusResponse, GetTocRootRequest, GetTocRootResponse, GetTopTopicsRequest, + GetTopTopicsResponse, GetTopicGraphStatusRequest, GetTopicGraphStatusResponse, + GetTopicsByQueryRequest, GetTopicsByQueryResponse, GetVectorIndexStatusRequest, + HybridSearchRequest, HybridSearchResponse, IngestEventRequest, IngestEventResponse, + PauseJobRequest, PauseJobResponse, ResumeJobRequest, ResumeJobResponse, SearchChildrenRequest, + SearchChildrenResponse, SearchNodeRequest, SearchNodeResponse, TeleportSearchRequest, + TeleportSearchResponse, VectorIndexStatus, VectorTeleportRequest, VectorTeleportResponse, }; use crate::query; use crate::scheduler_service::SchedulerGrpcService; +use crate::search_service; +use crate::teleport_service; +use crate::topics::TopicGraphHandler; +use crate::vector::VectorTeleportHandler; /// Implementation of the MemoryService gRPC service. pub struct MemoryServiceImpl { storage: Arc, scheduler_service: Option, + teleport_searcher: Option>, + vector_service: Option>, + hybrid_service: Option>, + topic_service: Option>, } impl MemoryServiceImpl { @@ -46,6 +53,10 @@ impl MemoryServiceImpl { Self { storage, scheduler_service: None, + teleport_searcher: None, + vector_service: None, + hybrid_service: None, + topic_service: None, } } @@ -57,6 +68,106 @@ impl MemoryServiceImpl { Self { storage, scheduler_service: Some(SchedulerGrpcService::new(scheduler)), + teleport_searcher: None, + vector_service: None, + hybrid_service: None, + topic_service: None, + } + } + + /// Create a new MemoryServiceImpl with storage, scheduler, and teleport searcher. + /// + /// When teleport searcher is provided, the TeleportSearch RPC will be functional. + pub fn with_scheduler_and_search( + storage: Arc, + scheduler: Arc, + searcher: Arc, + ) -> Self { + Self { + storage, + scheduler_service: Some(SchedulerGrpcService::new(scheduler)), + teleport_searcher: Some(searcher), + vector_service: None, + hybrid_service: None, + topic_service: None, + } + } + + /// Create a new MemoryServiceImpl with storage and teleport searcher (no scheduler). + pub fn with_search(storage: Arc, searcher: Arc) -> Self { + Self { + storage, + scheduler_service: None, + teleport_searcher: Some(searcher), + vector_service: None, + hybrid_service: None, + topic_service: None, + } + } + + /// Create a new MemoryServiceImpl with storage and vector search. + /// + /// When vector service is provided, VectorTeleport and HybridSearch RPCs will be functional. + pub fn with_vector(storage: Arc, vector_handler: Arc) -> Self { + let hybrid_handler = Arc::new(HybridSearchHandler::new(vector_handler.clone())); + Self { + storage, + scheduler_service: None, + teleport_searcher: None, + vector_service: Some(vector_handler), + hybrid_service: Some(hybrid_handler), + topic_service: None, + } + } + + /// Create a new MemoryServiceImpl with storage and topic graph. + /// + /// When topic service is provided, the topic graph RPCs will be functional. + pub fn with_topics(storage: Arc, topic_handler: Arc) -> Self { + Self { + storage, + scheduler_service: None, + teleport_searcher: None, + vector_service: None, + hybrid_service: None, + topic_service: Some(topic_handler), + } + } + + /// Create a new MemoryServiceImpl with all services. + pub fn with_all_services( + storage: Arc, + scheduler: Arc, + searcher: Arc, + vector_handler: Arc, + ) -> Self { + let hybrid_handler = Arc::new(HybridSearchHandler::new(vector_handler.clone())); + Self { + storage, + scheduler_service: Some(SchedulerGrpcService::new(scheduler)), + teleport_searcher: Some(searcher), + vector_service: Some(vector_handler), + hybrid_service: Some(hybrid_handler), + topic_service: None, + } + } + + /// Create a new MemoryServiceImpl with all services including topics. + pub fn with_all_services_and_topics( + storage: Arc, + scheduler: Arc, + searcher: Arc, + vector_handler: Arc, + topic_handler: Arc, + ) -> Self { + let hybrid_handler = Arc::new(HybridSearchHandler::new(vector_handler.clone())); + Self { + storage, + scheduler_service: Some(SchedulerGrpcService::new(scheduler)), + teleport_searcher: Some(searcher), + vector_service: Some(vector_handler), + hybrid_service: Some(hybrid_handler), + topic_service: Some(topic_handler), } } @@ -87,6 +198,7 @@ impl MemoryServiceImpl { } /// Convert proto Event to domain Event + #[allow(clippy::result_large_err)] fn convert_event(proto: ProtoEvent) -> Result { let timestamp = Utc .timestamp_millis_opt(proto.timestamp_ms) @@ -94,10 +206,10 @@ impl MemoryServiceImpl { .ok_or_else(|| Status::invalid_argument("Invalid timestamp"))?; let role = Self::convert_role( - ProtoEventRole::try_from(proto.role).unwrap_or(ProtoEventRole::Unspecified) + ProtoEventRole::try_from(proto.role).unwrap_or(ProtoEventRole::Unspecified), ); let event_type = Self::convert_event_type( - ProtoEventType::try_from(proto.event_type).unwrap_or(ProtoEventType::Unspecified) + ProtoEventType::try_from(proto.event_type).unwrap_or(ProtoEventType::Unspecified), ); let mut event = Event::new( @@ -130,9 +242,9 @@ impl MemoryService for MemoryServiceImpl { ) -> Result, Status> { let req = request.into_inner(); - let proto_event = req.event.ok_or_else(|| { - Status::invalid_argument("Event is required") - })?; + let proto_event = req + .event + .ok_or_else(|| Status::invalid_argument("Event is required"))?; // Validate event_id if proto_event.event_id.is_empty() { @@ -165,7 +277,9 @@ impl MemoryService for MemoryServiceImpl { })?; // Store event with atomic outbox write - let (_, created) = self.storage.put_event(&event_id, &event_bytes, &outbox_bytes) + let (_, created) = self + .storage + .put_event(&event_id, &event_bytes, &outbox_bytes) .map_err(|e| { error!("Failed to store event: {}", e); Status::internal(format!("Storage error: {}", e)) @@ -177,10 +291,7 @@ impl MemoryService for MemoryServiceImpl { debug!("Event already exists (idempotent): {}", event_id); } - Ok(Response::new(IngestEventResponse { - event_id, - created, - })) + Ok(Response::new(IngestEventResponse { event_id, created })) } /// Get root TOC nodes (year level). @@ -266,6 +377,144 @@ impl MemoryService for MemoryServiceImpl { })), } } + + /// Search within a single TOC node. + /// + /// Per SEARCH-01: SearchNode searches node's fields for query terms. + async fn search_node( + &self, + request: Request, + ) -> Result, Status> { + search_service::search_node(Arc::clone(&self.storage), request).await + } + + /// Search across children of a parent node. + /// + /// Per SEARCH-02: SearchChildren searches all children of parent. + async fn search_children( + &self, + request: Request, + ) -> Result, Status> { + search_service::search_children(Arc::clone(&self.storage), request).await + } + + /// Teleport search for TOC nodes or grips using BM25 ranking. + /// + /// Per TEL-01 through TEL-04: BM25 search with relevance scores. + async fn teleport_search( + &self, + request: Request, + ) -> Result, Status> { + match &self.teleport_searcher { + Some(searcher) => { + teleport_service::handle_teleport_search(searcher.clone(), request).await + } + None => Err(Status::unavailable("Search index not configured")), + } + } + + /// Vector semantic search using HNSW index. + /// + /// Per VEC-01: Semantic similarity search over TOC nodes and grips. + async fn vector_teleport( + &self, + request: Request, + ) -> Result, Status> { + match &self.vector_service { + Some(svc) => svc.vector_teleport(request).await, + None => Err(Status::unavailable("Vector index not enabled")), + } + } + + /// Hybrid BM25 + vector search using RRF fusion. + /// + /// Per VEC-02: Combines BM25 and vector scores using RRF. + async fn hybrid_search( + &self, + request: Request, + ) -> Result, Status> { + match &self.hybrid_service { + Some(svc) => svc.hybrid_search(request).await, + None => Err(Status::unavailable("Vector index not enabled")), + } + } + + /// Get vector index status and statistics. + /// + /// Per VEC-03: Returns index availability and stats. + async fn get_vector_index_status( + &self, + request: Request, + ) -> Result, Status> { + match &self.vector_service { + Some(svc) => svc.get_vector_index_status(request).await, + None => Ok(Response::new(VectorIndexStatus { + available: false, + vector_count: 0, + dimension: 0, + last_indexed: String::new(), + index_path: String::new(), + size_bytes: 0, + })), + } + } + + /// Get topic graph status and statistics. + /// + /// Per TOPIC-08: Returns topic graph availability and stats. + async fn get_topic_graph_status( + &self, + request: Request, + ) -> Result, Status> { + match &self.topic_service { + Some(svc) => svc.get_topic_graph_status(request).await, + None => Ok(Response::new(GetTopicGraphStatusResponse { + topic_count: 0, + relationship_count: 0, + last_updated: String::new(), + available: false, + })), + } + } + + /// Get topics matching a query. + /// + /// Per TOPIC-08: Search topics by keywords. + async fn get_topics_by_query( + &self, + request: Request, + ) -> Result, Status> { + match &self.topic_service { + Some(svc) => svc.get_topics_by_query(request).await, + None => Err(Status::unavailable("Topic graph not enabled")), + } + } + + /// Get topics related to a specific topic. + /// + /// Per TOPIC-08: Navigate topic relationships. + async fn get_related_topics( + &self, + request: Request, + ) -> Result, Status> { + match &self.topic_service { + Some(svc) => svc.get_related_topics(request).await, + None => Err(Status::unavailable("Topic graph not enabled")), + } + } + + /// Get top topics by importance score. + /// + /// Per TOPIC-08: Get most important topics. + async fn get_top_topics( + &self, + request: Request, + ) -> Result, Status> { + match &self.topic_service { + Some(svc) => svc.get_top_topics(request).await, + None => Err(Status::unavailable("Topic graph not enabled")), + } + } } #[cfg(test)] @@ -320,14 +569,18 @@ mod tests { }; // First ingestion - let response1 = service.ingest_event(Request::new(IngestEventRequest { - event: Some(event.clone()), - })).await.unwrap(); + let response1 = service + .ingest_event(Request::new(IngestEventRequest { + event: Some(event.clone()), + })) + .await + .unwrap(); // Second ingestion (same event_id) - let response2 = service.ingest_event(Request::new(IngestEventRequest { - event: Some(event), - })).await.unwrap(); + let response2 = service + .ingest_event(Request::new(IngestEventRequest { event: Some(event) })) + .await + .unwrap(); assert!(response1.into_inner().created); assert!(!response2.into_inner().created); // Idempotent diff --git a/crates/memory-service/src/lib.rs b/crates/memory-service/src/lib.rs index 443d079..dd398dd 100644 --- a/crates/memory-service/src/lib.rs +++ b/crates/memory-service/src/lib.rs @@ -4,21 +4,31 @@ //! - IngestEvent RPC for event ingestion (ING-01) //! - Query RPCs for TOC navigation (QRY-01 through QRY-05) //! - Scheduler RPCs for job status and control (SCHED-05) +//! - Teleport search RPC for BM25 keyword search (TEL-01 through TEL-04) +//! - Vector search RPCs for semantic search (VEC-01 through VEC-03) +//! - Topic graph RPCs for topic navigation (TOPIC-08) //! - Health check endpoint (GRPC-03) //! - Reflection endpoint for debugging (GRPC-04) +pub mod hybrid; pub mod ingest; pub mod query; pub mod scheduler_service; +pub mod search_service; pub mod server; +pub mod teleport_service; +pub mod topics; +pub mod vector; pub mod pb { tonic::include_proto!("memory"); - pub const FILE_DESCRIPTOR_SET: &[u8] = - tonic::include_file_descriptor_set!("memory_descriptor"); + pub const FILE_DESCRIPTOR_SET: &[u8] = tonic::include_file_descriptor_set!("memory_descriptor"); } +pub use hybrid::HybridSearchHandler; pub use ingest::MemoryServiceImpl; pub use scheduler_service::SchedulerGrpcService; pub use server::{run_server, run_server_with_scheduler, run_server_with_shutdown}; +pub use topics::TopicGraphHandler; +pub use vector::VectorTeleportHandler; diff --git a/crates/memory-service/src/query.rs b/crates/memory-service/src/query.rs index 467aa7e..29c56a0 100644 --- a/crates/memory-service/src/query.rs +++ b/crates/memory-service/src/query.rs @@ -8,20 +8,15 @@ use tonic::{Request, Response, Status}; use tracing::{debug, warn}; use memory_storage::Storage; -use memory_types::{Event, EventRole, EventType, TocLevel as DomainTocLevel, TocNode as DomainTocNode}; +use memory_types::{ + Event, EventRole, EventType, TocLevel as DomainTocLevel, TocNode as DomainTocNode, +}; use crate::pb::{ - BrowseTocRequest, BrowseTocResponse, - Event as ProtoEvent, - EventRole as ProtoEventRole, - EventType as ProtoEventType, - ExpandGripRequest, ExpandGripResponse, - GetEventsRequest, GetEventsResponse, - GetNodeRequest, GetNodeResponse, - GetTocRootRequest, GetTocRootResponse, - Grip as ProtoGrip, - TocBullet as ProtoTocBullet, - TocLevel as ProtoTocLevel, + BrowseTocRequest, BrowseTocResponse, Event as ProtoEvent, EventRole as ProtoEventRole, + EventType as ProtoEventType, ExpandGripRequest, ExpandGripResponse, GetEventsRequest, + GetEventsResponse, GetNodeRequest, GetNodeResponse, GetTocRootRequest, GetTocRootResponse, + Grip as ProtoGrip, TocBullet as ProtoTocBullet, TocLevel as ProtoTocLevel, TocNode as ProtoTocNode, }; @@ -34,14 +29,12 @@ pub async fn get_toc_root( ) -> Result, Status> { debug!("GetTocRoot request"); - let year_nodes = storage.get_toc_nodes_by_level(DomainTocLevel::Year, None, None) + let year_nodes = storage + .get_toc_nodes_by_level(DomainTocLevel::Year, None, None) .map_err(|e| Status::internal(format!("Storage error: {}", e)))?; // Sort by time descending (most recent first) - let mut nodes: Vec = year_nodes - .into_iter() - .map(domain_to_proto_node) - .collect(); + let mut nodes: Vec = year_nodes.into_iter().map(domain_to_proto_node).collect(); nodes.reverse(); Ok(Response::new(GetTocRootResponse { nodes })) @@ -61,7 +54,8 @@ pub async fn get_node( return Err(Status::invalid_argument("node_id is required")); } - let node = storage.get_toc_node(&req.node_id) + let node = storage + .get_toc_node(&req.node_id) .map_err(|e| Status::internal(format!("Storage error: {}", e)))?; let proto_node = node.map(domain_to_proto_node); @@ -77,20 +71,29 @@ pub async fn browse_toc( request: Request, ) -> Result, Status> { let req = request.into_inner(); - debug!("BrowseToc request: parent={}, limit={}", req.parent_id, req.limit); + debug!( + "BrowseToc request: parent={}, limit={}", + req.parent_id, req.limit + ); if req.parent_id.is_empty() { return Err(Status::invalid_argument("parent_id is required")); } - let limit = if req.limit <= 0 { 20 } else { req.limit as usize }; - let offset: usize = req.continuation_token + let limit = if req.limit <= 0 { + 20 + } else { + req.limit as usize + }; + let offset: usize = req + .continuation_token .as_ref() .and_then(|t| t.parse().ok()) .unwrap_or(0); // Get all child nodes - let all_children = storage.get_child_nodes(&req.parent_id) + let all_children = storage + .get_child_nodes(&req.parent_id) .map_err(|e| Status::internal(format!("Storage error: {}", e)))?; // Apply pagination @@ -125,11 +128,19 @@ pub async fn get_events( request: Request, ) -> Result, Status> { let req = request.into_inner(); - debug!("GetEvents request: from={} to={} limit={}", req.from_timestamp_ms, req.to_timestamp_ms, req.limit); + debug!( + "GetEvents request: from={} to={} limit={}", + req.from_timestamp_ms, req.to_timestamp_ms, req.limit + ); - let limit = if req.limit <= 0 { 50 } else { req.limit as usize }; + let limit = if req.limit <= 0 { + 50 + } else { + req.limit as usize + }; - let raw_events = storage.get_events_in_range(req.from_timestamp_ms, req.to_timestamp_ms) + let raw_events = storage + .get_events_in_range(req.from_timestamp_ms, req.to_timestamp_ms) .map_err(|e| Status::internal(format!("Storage error: {}", e)))?; let has_more = raw_events.len() > limit; @@ -189,7 +200,8 @@ pub async fn expand_grip( let start_time = grip_time.saturating_sub(time_window_ms); let end_time = grip_time.saturating_add(time_window_ms); - let raw_events = storage.get_events_in_range(start_time, end_time) + let raw_events = storage + .get_events_in_range(start_time, end_time) .map_err(|e| Status::internal(format!("Storage error: {}", e)))?; // Deserialize events @@ -272,7 +284,8 @@ fn domain_to_proto_node(node: DomainTocNode) -> ProtoTocNode { DomainTocLevel::Segment => ProtoTocLevel::Segment, }; - let bullets: Vec = node.bullets + let bullets: Vec = node + .bullets .into_iter() .map(|b| ProtoTocBullet { text: b.text, @@ -282,7 +295,13 @@ fn domain_to_proto_node(node: DomainTocNode) -> ProtoTocNode { // Generate summary from first bullet text if available let summary = if !bullets.is_empty() { - Some(bullets.iter().map(|b| b.text.clone()).collect::>().join(" ")) + Some( + bullets + .iter() + .map(|b| b.text.clone()) + .collect::>() + .join(" "), + ) } else { None }; @@ -335,7 +354,6 @@ fn domain_to_proto_event(event: Event) -> ProtoEvent { mod tests { use super::*; use chrono::{TimeZone, Utc}; - use memory_types::TocBullet; use tempfile::TempDir; fn create_test_storage() -> (Arc, TempDir) { diff --git a/crates/memory-service/src/scheduler_service.rs b/crates/memory-service/src/scheduler_service.rs index af76e3a..09364d4 100644 --- a/crates/memory-service/src/scheduler_service.rs +++ b/crates/memory-service/src/scheduler_service.rs @@ -146,7 +146,7 @@ impl SchedulerGrpcService { #[cfg(test)] mod tests { use super::*; - use memory_scheduler::{JitterConfig, OverlapPolicy, SchedulerConfig}; + use memory_scheduler::{JitterConfig, OverlapPolicy, SchedulerConfig, TimeoutConfig}; async fn create_test_scheduler() -> Arc { let config = SchedulerConfig::default(); @@ -178,6 +178,7 @@ mod tests { None, OverlapPolicy::Skip, JitterConfig::none(), + TimeoutConfig::none(), || async { Ok(()) }, ) .await @@ -206,6 +207,7 @@ mod tests { None, OverlapPolicy::Skip, JitterConfig::none(), + TimeoutConfig::none(), || async { Ok(()) }, ) .await @@ -250,6 +252,7 @@ mod tests { None, OverlapPolicy::Skip, JitterConfig::none(), + TimeoutConfig::none(), || async { Ok(()) }, ) .await diff --git a/crates/memory-service/src/search_service.rs b/crates/memory-service/src/search_service.rs new file mode 100644 index 0000000..ed85194 --- /dev/null +++ b/crates/memory-service/src/search_service.rs @@ -0,0 +1,374 @@ +//! Search RPC implementations. +//! +//! Per SEARCH-01, SEARCH-02: TOC node search via term matching. + +use std::cmp::Ordering; +use std::sync::Arc; + +use tonic::{Request, Response, Status}; +use tracing::debug; + +use memory_storage::Storage; +use memory_toc::search::{ + search_node as core_search_node, SearchField as DomainSearchField, + SearchMatch as DomainSearchMatch, +}; +use memory_types::TocLevel as DomainTocLevel; + +use crate::pb::{ + SearchChildrenRequest, SearchChildrenResponse, SearchField as ProtoSearchField, + SearchMatch as ProtoSearchMatch, SearchNodeRequest, SearchNodeResponse, + SearchNodeResult as ProtoSearchNodeResult, TocLevel as ProtoTocLevel, +}; + +/// Convert proto SearchField to domain SearchField. +fn proto_to_domain_field(proto: i32) -> Option { + match ProtoSearchField::try_from(proto) { + Ok(ProtoSearchField::Title) => Some(DomainSearchField::Title), + Ok(ProtoSearchField::Summary) => Some(DomainSearchField::Summary), + Ok(ProtoSearchField::Bullets) => Some(DomainSearchField::Bullets), + Ok(ProtoSearchField::Keywords) => Some(DomainSearchField::Keywords), + _ => None, // Unspecified or invalid + } +} + +/// Convert domain SearchField to proto SearchField. +fn domain_to_proto_field(domain: DomainSearchField) -> i32 { + match domain { + DomainSearchField::Title => ProtoSearchField::Title as i32, + DomainSearchField::Summary => ProtoSearchField::Summary as i32, + DomainSearchField::Bullets => ProtoSearchField::Bullets as i32, + DomainSearchField::Keywords => ProtoSearchField::Keywords as i32, + } +} + +/// Convert domain SearchMatch to proto SearchMatch. +fn domain_to_proto_match(m: DomainSearchMatch) -> ProtoSearchMatch { + ProtoSearchMatch { + field: domain_to_proto_field(m.field), + text: m.text, + grip_ids: m.grip_ids, + score: m.score, + } +} + +/// Convert domain TocLevel to proto TocLevel. +fn domain_to_proto_level(level: DomainTocLevel) -> i32 { + match level { + DomainTocLevel::Year => ProtoTocLevel::Year as i32, + DomainTocLevel::Month => ProtoTocLevel::Month as i32, + DomainTocLevel::Week => ProtoTocLevel::Week as i32, + DomainTocLevel::Day => ProtoTocLevel::Day as i32, + DomainTocLevel::Segment => ProtoTocLevel::Segment as i32, + } +} + +/// Search within a single TOC node. +/// +/// Per SEARCH-01: SearchNode searches node's fields for query terms. +pub async fn search_node( + storage: Arc, + request: Request, +) -> Result, Status> { + let req = request.into_inner(); + debug!( + "SearchNode request: node_id={}, query={}", + req.node_id, req.query + ); + + if req.node_id.is_empty() { + return Err(Status::invalid_argument("node_id is required")); + } + + if req.query.trim().is_empty() { + return Err(Status::invalid_argument("query is required")); + } + + // Load the node + let node = storage + .get_toc_node(&req.node_id) + .map_err(|e| Status::internal(format!("Storage error: {}", e)))? + .ok_or_else(|| Status::not_found("Node not found"))?; + + // Convert proto fields to domain fields + let fields: Vec = req + .fields + .iter() + .filter_map(|f| proto_to_domain_field(*f)) + .collect(); + + // Execute search + let matches = core_search_node(&node, &req.query, &fields); + + // Apply limit + let limit = if req.limit > 0 { + req.limit as usize + } else { + 10 + }; + let matches: Vec = matches + .into_iter() + .take(limit) + .map(domain_to_proto_match) + .collect(); + + Ok(Response::new(SearchNodeResponse { + matched: !matches.is_empty(), + matches, + node_id: req.node_id, + level: domain_to_proto_level(node.level), + })) +} + +/// Search across children of a parent node. +/// +/// Per SEARCH-02: SearchChildren searches all children of parent. +pub async fn search_children( + storage: Arc, + request: Request, +) -> Result, Status> { + let req = request.into_inner(); + debug!( + "SearchChildren request: parent_id={}, query={}", + req.parent_id, req.query + ); + + if req.query.trim().is_empty() { + return Err(Status::invalid_argument("query is required")); + } + + // Get children of parent (empty parent_id = root level years) + let children = if req.parent_id.is_empty() { + storage + .get_toc_nodes_by_level(DomainTocLevel::Year, None, None) + .map_err(|e| Status::internal(format!("Storage error: {}", e)))? + } else { + storage + .get_child_nodes(&req.parent_id) + .map_err(|e| Status::internal(format!("Storage error: {}", e)))? + }; + + // Convert proto fields to domain fields + let fields: Vec = req + .fields + .iter() + .filter_map(|f| proto_to_domain_field(*f)) + .collect(); + + // Search each child and collect results + let mut results: Vec = Vec::new(); + for child in children { + let matches = core_search_node(&child, &req.query, &fields); + if !matches.is_empty() { + // Calculate aggregate score (average of match scores) + let relevance = matches.iter().map(|m| m.score).sum::() / matches.len() as f32; + + results.push(ProtoSearchNodeResult { + node_id: child.node_id.clone(), + title: child.title.clone(), + level: domain_to_proto_level(child.level), + matches: matches.into_iter().map(domain_to_proto_match).collect(), + relevance_score: relevance, + }); + } + } + + // Sort by relevance score descending + results.sort_by(|a, b| { + b.relevance_score + .partial_cmp(&a.relevance_score) + .unwrap_or(Ordering::Equal) + }); + + // Apply limit + let limit = if req.limit > 0 { + req.limit as usize + } else { + 10 + }; + let has_more = results.len() > limit; + let results: Vec = results.into_iter().take(limit).collect(); + + Ok(Response::new(SearchChildrenResponse { results, has_more })) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn create_test_storage() -> (Arc, TempDir) { + let temp_dir = TempDir::new().unwrap(); + let storage = Arc::new(Storage::open(temp_dir.path()).unwrap()); + (storage, temp_dir) + } + + #[tokio::test] + async fn test_search_node_not_found() { + let (storage, _temp) = create_test_storage(); + let request = Request::new(SearchNodeRequest { + node_id: "nonexistent".to_string(), + query: "test".to_string(), + fields: vec![], + limit: 10, + token_budget: 0, + }); + let result = search_node(storage, request).await; + assert!(result.is_err()); + assert_eq!(result.unwrap_err().code(), tonic::Code::NotFound); + } + + #[tokio::test] + async fn test_search_node_empty_node_id() { + let (storage, _temp) = create_test_storage(); + let request = Request::new(SearchNodeRequest { + node_id: "".to_string(), + query: "test".to_string(), + fields: vec![], + limit: 10, + token_budget: 0, + }); + let result = search_node(storage, request).await; + assert!(result.is_err()); + assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument); + } + + #[tokio::test] + async fn test_search_node_empty_query() { + let (storage, _temp) = create_test_storage(); + let request = Request::new(SearchNodeRequest { + node_id: "toc:year:2026".to_string(), + query: "".to_string(), + fields: vec![], + limit: 10, + token_budget: 0, + }); + let result = search_node(storage, request).await; + assert!(result.is_err()); + assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument); + } + + #[tokio::test] + async fn test_search_node_whitespace_query() { + let (storage, _temp) = create_test_storage(); + let request = Request::new(SearchNodeRequest { + node_id: "toc:year:2026".to_string(), + query: " ".to_string(), + fields: vec![], + limit: 10, + token_budget: 0, + }); + let result = search_node(storage, request).await; + assert!(result.is_err()); + assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument); + } + + #[tokio::test] + async fn test_search_children_empty_query() { + let (storage, _temp) = create_test_storage(); + let request = Request::new(SearchChildrenRequest { + parent_id: "".to_string(), + query: " ".to_string(), + child_level: 0, + fields: vec![], + limit: 10, + token_budget: 0, + }); + let result = search_children(storage, request).await; + assert!(result.is_err()); + assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument); + } + + #[tokio::test] + async fn test_search_children_empty_results() { + let (storage, _temp) = create_test_storage(); + let request = Request::new(SearchChildrenRequest { + parent_id: "".to_string(), + query: "jwt token".to_string(), + child_level: 0, + fields: vec![], + limit: 10, + token_budget: 0, + }); + // Should succeed with empty results (no nodes in storage) + let result = search_children(storage, request).await; + assert!(result.is_ok()); + let response = result.unwrap().into_inner(); + assert!(response.results.is_empty()); + assert!(!response.has_more); + } + + #[test] + fn test_proto_to_domain_field_title() { + let result = proto_to_domain_field(ProtoSearchField::Title as i32); + assert_eq!(result, Some(DomainSearchField::Title)); + } + + #[test] + fn test_proto_to_domain_field_summary() { + let result = proto_to_domain_field(ProtoSearchField::Summary as i32); + assert_eq!(result, Some(DomainSearchField::Summary)); + } + + #[test] + fn test_proto_to_domain_field_bullets() { + let result = proto_to_domain_field(ProtoSearchField::Bullets as i32); + assert_eq!(result, Some(DomainSearchField::Bullets)); + } + + #[test] + fn test_proto_to_domain_field_keywords() { + let result = proto_to_domain_field(ProtoSearchField::Keywords as i32); + assert_eq!(result, Some(DomainSearchField::Keywords)); + } + + #[test] + fn test_proto_to_domain_field_unspecified() { + let result = proto_to_domain_field(ProtoSearchField::Unspecified as i32); + assert_eq!(result, None); + } + + #[test] + fn test_proto_to_domain_field_invalid() { + let result = proto_to_domain_field(999); + assert_eq!(result, None); + } + + #[test] + fn test_domain_to_proto_field_roundtrip() { + for domain in [ + DomainSearchField::Title, + DomainSearchField::Summary, + DomainSearchField::Bullets, + DomainSearchField::Keywords, + ] { + let proto = domain_to_proto_field(domain); + let back = proto_to_domain_field(proto); + assert_eq!(back, Some(domain)); + } + } + + #[test] + fn test_domain_to_proto_level() { + assert_eq!( + domain_to_proto_level(DomainTocLevel::Year), + ProtoTocLevel::Year as i32 + ); + assert_eq!( + domain_to_proto_level(DomainTocLevel::Month), + ProtoTocLevel::Month as i32 + ); + assert_eq!( + domain_to_proto_level(DomainTocLevel::Week), + ProtoTocLevel::Week as i32 + ); + assert_eq!( + domain_to_proto_level(DomainTocLevel::Day), + ProtoTocLevel::Day as i32 + ); + assert_eq!( + domain_to_proto_level(DomainTocLevel::Segment), + ProtoTocLevel::Segment as i32 + ); + } +} diff --git a/crates/memory-service/src/server.rs b/crates/memory-service/src/server.rs index dc95319..bb40a00 100644 --- a/crates/memory-service/src/server.rs +++ b/crates/memory-service/src/server.rs @@ -191,7 +191,8 @@ mod tests { let server_handle = tokio::spawn(async move { run_server_with_shutdown(addr, storage, async { rx.await.ok(); - }).await + }) + .await }); // Give server time to start diff --git a/crates/memory-service/src/teleport_service.rs b/crates/memory-service/src/teleport_service.rs new file mode 100644 index 0000000..8417653 --- /dev/null +++ b/crates/memory-service/src/teleport_service.rs @@ -0,0 +1,249 @@ +//! Teleport search handler. +//! +//! Provides BM25 keyword search over TOC nodes and grips. + +use std::sync::Arc; + +use memory_search::{DocType, SearchOptions, TeleportSearcher}; +use tonic::{Request, Response, Status}; +use tracing::debug; + +use crate::pb::{ + TeleportDocType, TeleportSearchRequest, TeleportSearchResponse, TeleportSearchResult, +}; + +/// Handle TeleportSearch RPC. +pub async fn handle_teleport_search( + searcher: Arc, + request: Request, +) -> Result, Status> { + let req = request.into_inner(); + + debug!(query = %req.query, "Processing teleport search"); + + // Build search options + let mut options = SearchOptions::new(); + + // Set limit (default 10, max 100) + let limit = if req.limit > 0 { + (req.limit as usize).min(100) + } else { + 10 + }; + options = options.with_limit(limit); + + // Set doc type filter + if req.doc_type == TeleportDocType::TocNode as i32 { + options = options.with_doc_type(DocType::TocNode); + } else if req.doc_type == TeleportDocType::Grip as i32 { + options = options.with_doc_type(DocType::Grip); + } + + // Execute search (blocking operation, use spawn_blocking) + let query = req.query.clone(); + let searcher_clone = searcher.clone(); + let results = tokio::task::spawn_blocking(move || searcher_clone.search(&query, options)) + .await + .map_err(|e| Status::internal(format!("Search task failed: {}", e)))? + .map_err(|e| Status::internal(format!("Search failed: {}", e)))?; + + // Get total docs + let total_docs = searcher.num_docs(); + + // Map to proto results + let proto_results: Vec = results + .into_iter() + .map(|r| TeleportSearchResult { + doc_id: r.doc_id, + doc_type: match r.doc_type { + DocType::TocNode => TeleportDocType::TocNode as i32, + DocType::Grip => TeleportDocType::Grip as i32, + }, + score: r.score, + keywords: r.keywords, + timestamp_ms: r.timestamp_ms, + }) + .collect(); + + Ok(Response::new(TeleportSearchResponse { + results: proto_results, + total_docs, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use memory_search::{SearchIndex, SearchIndexConfig, SearchIndexer}; + use memory_types::{Grip, TocBullet, TocLevel, TocNode}; + use tempfile::TempDir; + + fn sample_toc_node(id: &str, title: &str, bullet: &str) -> TocNode { + let mut node = TocNode::new( + id.to_string(), + TocLevel::Day, + title.to_string(), + Utc::now(), + Utc::now(), + ); + node.bullets = vec![TocBullet::new(bullet)]; + node.keywords = vec!["test".to_string()]; + node + } + + fn sample_grip(id: &str, excerpt: &str) -> Grip { + Grip::new( + id.to_string(), + excerpt.to_string(), + "event-001".to_string(), + "event-002".to_string(), + Utc::now(), + "test".to_string(), + ) + } + + fn setup_searcher() -> (TempDir, Arc) { + let temp_dir = TempDir::new().unwrap(); + let config = SearchIndexConfig::new(temp_dir.path()); + let index = SearchIndex::open_or_create(config).unwrap(); + + // Index some test data + let indexer = SearchIndexer::new(&index).unwrap(); + indexer + .index_toc_node(&sample_toc_node( + "node-1", + "Rust Memory Safety", + "Discussed borrow checker", + )) + .unwrap(); + indexer + .index_grip(&sample_grip("grip-1", "User asked about memory allocation")) + .unwrap(); + indexer.commit().unwrap(); + + let searcher = Arc::new(TeleportSearcher::new(&index).unwrap()); + (temp_dir, searcher) + } + + #[tokio::test] + async fn test_handle_teleport_search_all_types() { + let (_temp_dir, searcher) = setup_searcher(); + + let request = Request::new(TeleportSearchRequest { + query: "memory".to_string(), + doc_type: TeleportDocType::Unspecified as i32, + limit: 10, + }); + + let response = handle_teleport_search(searcher, request).await.unwrap(); + let resp = response.into_inner(); + + // Should find both node and grip + assert_eq!(resp.results.len(), 2); + assert!(resp.total_docs >= 2); + } + + #[tokio::test] + async fn test_handle_teleport_search_toc_only() { + let (_temp_dir, searcher) = setup_searcher(); + + let request = Request::new(TeleportSearchRequest { + query: "memory".to_string(), + doc_type: TeleportDocType::TocNode as i32, + limit: 10, + }); + + let response = handle_teleport_search(searcher, request).await.unwrap(); + let resp = response.into_inner(); + + // Should find only the node + assert_eq!(resp.results.len(), 1); + assert_eq!(resp.results[0].doc_type, TeleportDocType::TocNode as i32); + } + + #[tokio::test] + async fn test_handle_teleport_search_grip_only() { + let (_temp_dir, searcher) = setup_searcher(); + + let request = Request::new(TeleportSearchRequest { + query: "memory".to_string(), + doc_type: TeleportDocType::Grip as i32, + limit: 10, + }); + + let response = handle_teleport_search(searcher, request).await.unwrap(); + let resp = response.into_inner(); + + // Should find only the grip + assert_eq!(resp.results.len(), 1); + assert_eq!(resp.results[0].doc_type, TeleportDocType::Grip as i32); + } + + #[tokio::test] + async fn test_handle_teleport_search_limit() { + let (_temp_dir, searcher) = setup_searcher(); + + let request = Request::new(TeleportSearchRequest { + query: "memory".to_string(), + doc_type: TeleportDocType::Unspecified as i32, + limit: 1, + }); + + let response = handle_teleport_search(searcher, request).await.unwrap(); + let resp = response.into_inner(); + + // Should respect limit + assert_eq!(resp.results.len(), 1); + } + + #[tokio::test] + async fn test_handle_teleport_search_empty_query() { + let (_temp_dir, searcher) = setup_searcher(); + + let request = Request::new(TeleportSearchRequest { + query: "".to_string(), + doc_type: TeleportDocType::Unspecified as i32, + limit: 10, + }); + + let response = handle_teleport_search(searcher, request).await.unwrap(); + let resp = response.into_inner(); + + // Empty query returns empty results + assert!(resp.results.is_empty()); + } + + #[tokio::test] + async fn test_handle_teleport_search_no_matches() { + let (_temp_dir, searcher) = setup_searcher(); + + let request = Request::new(TeleportSearchRequest { + query: "nonexistentterm12345".to_string(), + doc_type: TeleportDocType::Unspecified as i32, + limit: 10, + }); + + let response = handle_teleport_search(searcher, request).await.unwrap(); + let resp = response.into_inner(); + + assert!(resp.results.is_empty()); + } + + #[tokio::test] + async fn test_handle_teleport_search_default_limit() { + let (_temp_dir, searcher) = setup_searcher(); + + let request = Request::new(TeleportSearchRequest { + query: "memory".to_string(), + doc_type: TeleportDocType::Unspecified as i32, + limit: 0, // Should default to 10 + }); + + let response = handle_teleport_search(searcher, request).await.unwrap(); + let resp = response.into_inner(); + + // Should still return results (limit defaults to 10) + assert!(!resp.results.is_empty()); + } +} diff --git a/crates/memory-service/src/topics.rs b/crates/memory-service/src/topics.rs new file mode 100644 index 0000000..fe39e9c --- /dev/null +++ b/crates/memory-service/src/topics.rs @@ -0,0 +1,352 @@ +//! Topic Graph RPC implementations. +//! +//! Provides gRPC handlers for topic navigation: +//! - GetTopicGraphStatus: Check if topic graph is available +//! - GetTopicsByQuery: Search topics by keywords +//! - GetRelatedTopics: Get topics related to a given topic +//! - GetTopTopics: Get top topics by importance score + +use std::sync::Arc; + +use chrono::Utc; +use tonic::{Request, Response, Status}; +use tracing::{debug, info}; + +use memory_topics::{RelationshipType, TopicStorage}; + +use crate::pb::{ + GetRelatedTopicsRequest, GetRelatedTopicsResponse, GetTopTopicsRequest, GetTopTopicsResponse, + GetTopicGraphStatusRequest, GetTopicGraphStatusResponse, GetTopicsByQueryRequest, + GetTopicsByQueryResponse, Topic as ProtoTopic, TopicRelationship as ProtoTopicRelationship, +}; + +/// Handler for topic graph operations. +pub struct TopicGraphHandler { + storage: Arc, +} + +impl TopicGraphHandler { + /// Create a new topic graph handler. + pub fn new(storage: Arc) -> Self { + Self { storage } + } + + /// Check if the topic graph is available. + pub fn is_available(&self) -> bool { + self.storage + .get_stats() + .map(|s| s.topic_count > 0) + .unwrap_or(false) + } + + /// Handle GetTopicGraphStatus RPC request. + pub async fn get_topic_graph_status( + &self, + _request: Request, + ) -> Result, Status> { + debug!("GetTopicGraphStatus request"); + + let stats = self.storage.get_stats().map_err(|e| { + tracing::error!("Failed to get topic stats: {}", e); + Status::internal(format!("Failed to get topic stats: {}", e)) + })?; + + let last_updated = if stats.last_extraction_ms > 0 { + chrono::DateTime::from_timestamp_millis(stats.last_extraction_ms) + .map(|dt| dt.to_rfc3339()) + .unwrap_or_default() + } else { + String::new() + }; + + Ok(Response::new(GetTopicGraphStatusResponse { + topic_count: stats.topic_count, + relationship_count: stats.relationship_count, + last_updated, + available: stats.topic_count > 0, + })) + } + + /// Handle GetTopicsByQuery RPC request. + pub async fn get_topics_by_query( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let query = req.query.to_lowercase(); + let limit = if req.limit > 0 { + req.limit as usize + } else { + 10 + }; + + debug!(query = %query, limit = limit, "GetTopicsByQuery request"); + + let all_topics = self.storage.list_topics().map_err(|e| { + tracing::error!("Failed to list topics: {}", e); + Status::internal(format!("Failed to list topics: {}", e)) + })?; + + // Filter topics by query matching label or keywords + let query_terms: Vec<&str> = query.split_whitespace().collect(); + let mut matching_topics: Vec<_> = all_topics + .into_iter() + .filter(|topic| { + let label_lower = topic.label.to_lowercase(); + let keywords_lower: Vec = + topic.keywords.iter().map(|k| k.to_lowercase()).collect(); + + query_terms.iter().any(|term| { + label_lower.contains(term) || keywords_lower.iter().any(|k| k.contains(term)) + }) + }) + .collect(); + + // Sort by importance score descending + matching_topics.sort_by(|a, b| { + b.importance_score + .partial_cmp(&a.importance_score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + // Limit results + matching_topics.truncate(limit); + + let proto_topics: Vec = + matching_topics.into_iter().map(topic_to_proto).collect(); + + info!(query = %query, results = proto_topics.len(), "GetTopicsByQuery complete"); + + Ok(Response::new(GetTopicsByQueryResponse { + topics: proto_topics, + })) + } + + /// Handle GetRelatedTopics RPC request. + pub async fn get_related_topics( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let topic_id = &req.topic_id; + let limit = if req.limit > 0 { + req.limit as usize + } else { + 10 + }; + + // Parse optional relationship type filter + let rel_type_filter = if req.relationship_type.is_empty() { + None + } else { + parse_relationship_type(&req.relationship_type) + }; + + debug!( + topic_id = %topic_id, + relationship_type = ?rel_type_filter, + limit = limit, + "GetRelatedTopics request" + ); + + // Verify source topic exists + let _source_topic = self + .storage + .get_topic(topic_id) + .map_err(|e| { + tracing::error!("Failed to get topic: {}", e); + Status::internal(format!("Failed to get topic: {}", e)) + })? + .ok_or_else(|| Status::not_found(format!("Topic not found: {}", topic_id)))?; + + // Get relationships for the topic + let relationships = self + .storage + .get_relationships_filtered(topic_id, rel_type_filter) + .map_err(|e| { + tracing::error!("Failed to get relationships: {}", e); + Status::internal(format!("Failed to get relationships: {}", e)) + })?; + + // Limit relationships + let limited_rels: Vec<_> = relationships.into_iter().take(limit).collect(); + + // Fetch related topics + let mut related_topics = Vec::new(); + let mut proto_relationships = Vec::new(); + + for rel in &limited_rels { + if let Ok(Some(topic)) = self.storage.get_topic(&rel.target_id) { + related_topics.push(topic_to_proto(topic)); + proto_relationships.push(relationship_to_proto(rel)); + } + } + + info!( + topic_id = %topic_id, + results = related_topics.len(), + "GetRelatedTopics complete" + ); + + Ok(Response::new(GetRelatedTopicsResponse { + related_topics, + relationships: proto_relationships, + })) + } + + /// Handle GetTopTopics RPC request. + pub async fn get_top_topics( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let limit = if req.limit > 0 { + req.limit as usize + } else { + 10 + }; + let _days = if req.days > 0 { req.days } else { 30 }; + + debug!(limit = limit, days = _days, "GetTopTopics request"); + + // Get top topics sorted by importance + let topics = self.storage.get_top_topics(limit).map_err(|e| { + tracing::error!("Failed to get top topics: {}", e); + Status::internal(format!("Failed to get top topics: {}", e)) + })?; + + // Note: The days parameter could be used to filter topics by last_mentioned_at + // within the lookback window. For now, we rely on the importance scorer's + // time-decay which already factors in recency. Future enhancement could + // filter out topics not mentioned within the days window. + let now = Utc::now(); + let cutoff = now - chrono::Duration::days(_days as i64); + + let filtered_topics: Vec<_> = topics + .into_iter() + .filter(|t| t.last_mentioned_at >= cutoff) + .collect(); + + let proto_topics: Vec = + filtered_topics.into_iter().map(topic_to_proto).collect(); + + info!( + limit = limit, + results = proto_topics.len(), + "GetTopTopics complete" + ); + + Ok(Response::new(GetTopTopicsResponse { + topics: proto_topics, + })) + } +} + +/// Convert a domain Topic to a proto Topic. +fn topic_to_proto(topic: memory_topics::Topic) -> ProtoTopic { + ProtoTopic { + id: topic.topic_id, + label: topic.label, + importance_score: topic.importance_score as f32, + keywords: topic.keywords, + created_at: topic.created_at.to_rfc3339(), + last_mention: topic.last_mentioned_at.to_rfc3339(), + } +} + +/// Convert a domain TopicRelationship to a proto TopicRelationship. +fn relationship_to_proto(rel: &memory_topics::TopicRelationship) -> ProtoTopicRelationship { + ProtoTopicRelationship { + source_id: rel.source_id.clone(), + target_id: rel.target_id.clone(), + relationship_type: rel.relationship_type.to_string(), + strength: rel.strength, + } +} + +/// Parse a relationship type string to RelationshipType. +fn parse_relationship_type(s: &str) -> Option { + match s.to_lowercase().as_str() { + "co-occurrence" | "cooccurrence" | "coo" => Some(RelationshipType::CoOccurrence), + "semantic" | "sem" => Some(RelationshipType::Semantic), + "hierarchical" | "hie" => Some(RelationshipType::Hierarchical), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use memory_topics::{Topic, TopicRelationship}; + + #[test] + fn test_topic_to_proto() { + let now = Utc::now(); + let topic = Topic { + topic_id: "topic-123".to_string(), + label: "Machine Learning".to_string(), + embedding: vec![0.1, 0.2, 0.3], + importance_score: 0.85, + node_count: 10, + created_at: now, + last_mentioned_at: now, + status: memory_topics::TopicStatus::Active, + keywords: vec!["ml".to_string(), "ai".to_string()], + }; + + let proto = topic_to_proto(topic); + + assert_eq!(proto.id, "topic-123"); + assert_eq!(proto.label, "Machine Learning"); + assert!((proto.importance_score - 0.85).abs() < f32::EPSILON); + assert_eq!(proto.keywords, vec!["ml", "ai"]); + } + + #[test] + fn test_relationship_to_proto() { + let rel = TopicRelationship::new( + "topic-a".to_string(), + "topic-b".to_string(), + RelationshipType::Semantic, + 0.75, + ); + + let proto = relationship_to_proto(&rel); + + assert_eq!(proto.source_id, "topic-a"); + assert_eq!(proto.target_id, "topic-b"); + assert_eq!(proto.relationship_type, "semantic"); + assert!((proto.strength - 0.75).abs() < f32::EPSILON); + } + + #[test] + fn test_parse_relationship_type() { + assert_eq!( + parse_relationship_type("co-occurrence"), + Some(RelationshipType::CoOccurrence) + ); + assert_eq!( + parse_relationship_type("semantic"), + Some(RelationshipType::Semantic) + ); + assert_eq!( + parse_relationship_type("hierarchical"), + Some(RelationshipType::Hierarchical) + ); + assert_eq!( + parse_relationship_type("coo"), + Some(RelationshipType::CoOccurrence) + ); + assert_eq!( + parse_relationship_type("sem"), + Some(RelationshipType::Semantic) + ); + assert_eq!( + parse_relationship_type("hie"), + Some(RelationshipType::Hierarchical) + ); + assert_eq!(parse_relationship_type("unknown"), None); + assert_eq!(parse_relationship_type(""), None); + } +} diff --git a/crates/memory-service/src/vector.rs b/crates/memory-service/src/vector.rs new file mode 100644 index 0000000..99bc70e --- /dev/null +++ b/crates/memory-service/src/vector.rs @@ -0,0 +1,161 @@ +//! VectorTeleport RPC implementation. +//! +//! Provides semantic similarity search over TOC nodes and grips +//! using HNSW vector index. + +use std::sync::Arc; + +use tonic::{Request, Response, Status}; +use tracing::{debug, info}; + +use memory_embeddings::{CandleEmbedder, EmbeddingModel}; +use memory_vector::{DocType, HnswIndex, VectorIndex, VectorMetadata}; + +use crate::pb::{ + GetVectorIndexStatusRequest, VectorIndexStatus, VectorMatch, VectorTargetType, + VectorTeleportRequest, VectorTeleportResponse, +}; + +/// Handler for vector search operations. +pub struct VectorTeleportHandler { + embedder: Arc, + index: Arc>, + metadata: Arc, +} + +impl VectorTeleportHandler { + /// Create a new vector teleport handler. + pub fn new( + embedder: Arc, + index: Arc>, + metadata: Arc, + ) -> Self { + Self { + embedder, + index, + metadata, + } + } + + /// Check if the vector index is available for search. + pub fn is_available(&self) -> bool { + let index = self.index.read().unwrap(); + index.len() > 0 + } + + /// Get the current vector index status. + pub fn get_status(&self) -> VectorIndexStatus { + let index = self.index.read().unwrap(); + let stats = index.stats(); + VectorIndexStatus { + available: stats.available && stats.vector_count > 0, + vector_count: stats.vector_count as i64, + dimension: stats.dimension as i32, + last_indexed: String::new(), + index_path: index.index_file().to_string_lossy().to_string(), + size_bytes: stats.size_bytes as i64, + } + } + + /// Handle VectorTeleport RPC request. + pub async fn vector_teleport( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let query = &req.query; + let top_k = if req.top_k > 0 { + req.top_k as usize + } else { + 10 + }; + let min_score = req.min_score; + + debug!(query = %query, top_k = top_k, "VectorTeleport request"); + + let status = self.get_status(); + if !status.available { + return Ok(Response::new(VectorTeleportResponse { + matches: vec![], + index_status: Some(status), + })); + } + + // Embed query using spawn_blocking for CPU-bound work + let embedder = self.embedder.clone(); + let query_owned = query.to_string(); + let embedding = tokio::task::spawn_blocking(move || embedder.embed(&query_owned)) + .await + .map_err(|e| Status::internal(format!("Task error: {}", e)))? + .map_err(|e| Status::internal(format!("Embedding failed: {}", e)))?; + + // Search index + let results = { + let index = self.index.read().unwrap(); + index + .search(&embedding, top_k) + .map_err(|e| Status::internal(format!("Search failed: {}", e)))? + }; + + // Convert to matches with metadata lookup + let mut matches = Vec::new(); + for result in results { + if result.score < min_score { + continue; + } + + if let Ok(Some(entry)) = self.metadata.get(result.vector_id) { + // Target type filter + if !self.matches_target(req.target, entry.doc_type) { + continue; + } + + // Time filter + if let Some(ref tf) = req.time_filter { + if entry.created_at < tf.start_ms || entry.created_at >= tf.end_ms { + continue; + } + } + + matches.push(VectorMatch { + doc_id: entry.doc_id, + doc_type: entry.doc_type.as_str().to_string(), + score: result.score, + text_preview: entry.text_preview, + timestamp_ms: entry.created_at, + }); + } + } + + info!(query = %query, results = matches.len(), "VectorTeleport complete"); + + Ok(Response::new(VectorTeleportResponse { + matches, + index_status: Some(status), + })) + } + + /// Handle GetVectorIndexStatus RPC request. + pub async fn get_vector_index_status( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(self.get_status())) + } + + /// Check if a document type matches the target filter. + fn matches_target(&self, target: i32, doc_type: DocType) -> bool { + match VectorTargetType::try_from(target) { + Ok(VectorTargetType::Unspecified) | Ok(VectorTargetType::All) => true, + Ok(VectorTargetType::TocNode) => doc_type == DocType::TocNode, + Ok(VectorTargetType::Grip) => doc_type == DocType::Grip, + Err(_) => true, + } + } +} + +#[cfg(test)] +mod tests { + // Integration tests require embedding model download + // Run with: cargo test -p memory-service --features integration -- --ignored +} diff --git a/crates/memory-storage/src/column_families.rs b/crates/memory-storage/src/column_families.rs index 2d12b4f..1490571 100644 --- a/crates/memory-storage/src/column_families.rs +++ b/crates/memory-storage/src/column_families.rs @@ -28,6 +28,15 @@ pub const CF_OUTBOX: &str = "outbox"; /// Column family name for background job checkpoints pub const CF_CHECKPOINTS: &str = "checkpoints"; +/// Column family for topic records +pub const CF_TOPICS: &str = "topics"; + +/// Column family for topic-node links +pub const CF_TOPIC_LINKS: &str = "topic_links"; + +/// Column family for topic relationships +pub const CF_TOPIC_RELS: &str = "topic_rels"; + /// All column family names pub const ALL_CF_NAMES: &[&str] = &[ CF_EVENTS, @@ -36,6 +45,9 @@ pub const ALL_CF_NAMES: &[&str] = &[ CF_GRIPS, CF_OUTBOX, CF_CHECKPOINTS, + CF_TOPICS, + CF_TOPIC_LINKS, + CF_TOPIC_RELS, ]; /// Create column family options for events (append-only, compressed) @@ -65,5 +77,8 @@ pub fn build_cf_descriptors() -> Vec { ColumnFamilyDescriptor::new(CF_GRIPS, Options::default()), ColumnFamilyDescriptor::new(CF_OUTBOX, outbox_options()), ColumnFamilyDescriptor::new(CF_CHECKPOINTS, Options::default()), + ColumnFamilyDescriptor::new(CF_TOPICS, Options::default()), + ColumnFamilyDescriptor::new(CF_TOPIC_LINKS, Options::default()), + ColumnFamilyDescriptor::new(CF_TOPIC_RELS, Options::default()), ] } diff --git a/crates/memory-storage/src/db.rs b/crates/memory-storage/src/db.rs index b6fc021..ea582ee 100644 --- a/crates/memory-storage/src/db.rs +++ b/crates/memory-storage/src/db.rs @@ -6,14 +6,18 @@ //! - Single-key and range reads //! - Idempotent writes (ING-03) -use rocksdb::{DB, Options, WriteBatch, IteratorMode, Direction}; +use rocksdb::{Direction, IteratorMode, Options, WriteBatch, DB}; use std::path::Path; use std::sync::atomic::{AtomicU64, Ordering}; use tracing::{debug, info}; -use crate::column_families::{build_cf_descriptors, ALL_CF_NAMES, CF_EVENTS, CF_OUTBOX, CF_CHECKPOINTS, CF_TOC_NODES, CF_TOC_LATEST, CF_GRIPS}; +use crate::column_families::{ + build_cf_descriptors, ALL_CF_NAMES, CF_CHECKPOINTS, CF_EVENTS, CF_GRIPS, CF_OUTBOX, + CF_TOC_LATEST, CF_TOC_NODES, +}; use crate::error::StorageError; -use crate::keys::{EventKey, OutboxKey, CheckpointKey}; +use crate::keys::{CheckpointKey, EventKey, OutboxKey}; +use memory_types::OutboxEntry; // Re-export TocLevel for use in this crate pub use memory_types::TocLevel; @@ -55,7 +59,8 @@ impl Storage { /// Load the highest outbox sequence number from storage fn load_outbox_sequence(db: &DB) -> Result { - let cf = db.cf_handle(CF_OUTBOX) + let cf = db + .cf_handle(CF_OUTBOX) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_OUTBOX.to_string()))?; // Iterate in reverse to find highest key @@ -82,9 +87,13 @@ impl Storage { event_bytes: &[u8], outbox_bytes: &[u8], ) -> Result<(EventKey, bool), StorageError> { - let events_cf = self.db.cf_handle(CF_EVENTS) + let events_cf = self + .db + .cf_handle(CF_EVENTS) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_EVENTS.to_string()))?; - let outbox_cf = self.db.cf_handle(CF_OUTBOX) + let outbox_cf = self + .db + .cf_handle(CF_OUTBOX) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_OUTBOX.to_string()))?; // Parse event_id to get key (ING-03: idempotent using event_id) @@ -104,14 +113,19 @@ impl Storage { batch.put_cf(&outbox_cf, outbox_key.to_bytes(), outbox_bytes); self.db.write(batch)?; - debug!("Stored event {} with outbox seq {}", event_id, outbox_key.sequence); + debug!( + "Stored event {} with outbox seq {}", + event_id, outbox_key.sequence + ); Ok((event_key, true)) } /// Get an event by its event_id pub fn get_event(&self, event_id: &str) -> Result>, StorageError> { - let events_cf = self.db.cf_handle(CF_EVENTS) + let events_cf = self + .db + .cf_handle(CF_EVENTS) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_EVENTS.to_string()))?; let event_key = EventKey::from_event_id(event_id)?; @@ -127,7 +141,9 @@ impl Storage { start_ms: i64, end_ms: i64, ) -> Result)>, StorageError> { - let events_cf = self.db.cf_handle(CF_EVENTS) + let events_cf = self + .db + .cf_handle(CF_EVENTS) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_EVENTS.to_string()))?; let start_prefix = EventKey::prefix_start(start_ms); @@ -153,8 +169,14 @@ impl Storage { } /// Store a checkpoint for crash recovery (STOR-03) - pub fn put_checkpoint(&self, job_name: &str, checkpoint_bytes: &[u8]) -> Result<(), StorageError> { - let cf = self.db.cf_handle(CF_CHECKPOINTS) + pub fn put_checkpoint( + &self, + job_name: &str, + checkpoint_bytes: &[u8], + ) -> Result<(), StorageError> { + let cf = self + .db + .cf_handle(CF_CHECKPOINTS) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_CHECKPOINTS.to_string()))?; let key = CheckpointKey::new(job_name); @@ -164,7 +186,9 @@ impl Storage { /// Get a checkpoint for crash recovery (STOR-03) pub fn get_checkpoint(&self, job_name: &str) -> Result>, StorageError> { - let cf = self.db.cf_handle(CF_CHECKPOINTS) + let cf = self + .db + .cf_handle(CF_CHECKPOINTS) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_CHECKPOINTS.to_string()))?; let key = CheckpointKey::new(job_name); @@ -172,6 +196,78 @@ impl Storage { Ok(result) } + // ==================== Outbox Methods ==================== + + /// Get outbox entries starting from a sequence number. + /// + /// Returns Vec of (sequence, entry) tuples in sequence order. + /// Used by indexing pipelines to consume outbox entries. + pub fn get_outbox_entries( + &self, + start_sequence: u64, + limit: usize, + ) -> Result, StorageError> { + let cf = self + .db + .cf_handle(CF_OUTBOX) + .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_OUTBOX.to_string()))?; + + let start_key = OutboxKey::new(start_sequence); + let iter = self.db.iterator_cf( + &cf, + IteratorMode::From(&start_key.to_bytes(), Direction::Forward), + ); + + let mut results = Vec::new(); + for item in iter.take(limit) { + let (key, value) = item?; + let outbox_key = OutboxKey::from_bytes(&key)?; + let entry = OutboxEntry::from_bytes(&value) + .map_err(|e| StorageError::Serialization(e.to_string()))?; + results.push((outbox_key.sequence, entry)); + } + + Ok(results) + } + + /// Delete outbox entries up to and including a sequence number. + /// + /// Used to clean up processed outbox entries after all indexes + /// have been updated. Returns count of deleted entries. + pub fn delete_outbox_entries(&self, up_to_sequence: u64) -> Result { + let cf = self + .db + .cf_handle(CF_OUTBOX) + .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_OUTBOX.to_string()))?; + + // Collect keys to delete + let iter = self.db.iterator_cf(&cf, IteratorMode::Start); + let mut batch = WriteBatch::default(); + let mut count = 0; + + for item in iter { + let (key, _) = item?; + let outbox_key = OutboxKey::from_bytes(&key)?; + + if outbox_key.sequence > up_to_sequence { + break; + } + + batch.delete_cf(&cf, &key); + count += 1; + } + + if count > 0 { + self.db.write(batch)?; + debug!( + "Deleted {} outbox entries up to sequence {}", + count, up_to_sequence + ); + } + + Ok(count) + } + /// Flush all column families to disk pub fn flush(&self) -> Result<(), StorageError> { for cf_name in ALL_CF_NAMES { @@ -189,14 +285,20 @@ impl Storage { /// Appends a new version rather than mutating. /// Updates toc_latest to point to new version. pub fn put_toc_node(&self, node: &memory_types::TocNode) -> Result<(), StorageError> { - let nodes_cf = self.db.cf_handle(CF_TOC_NODES) + let nodes_cf = self + .db + .cf_handle(CF_TOC_NODES) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_TOC_NODES.to_string()))?; - let latest_cf = self.db.cf_handle(CF_TOC_LATEST) + let latest_cf = self + .db + .cf_handle(CF_TOC_LATEST) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_TOC_LATEST.to_string()))?; // Get current version let latest_key = format!("latest:{}", node.node_id); - let current_version = self.db.get_cf(&latest_cf, &latest_key)? + let current_version = self + .db + .get_cf(&latest_cf, &latest_key)? .map(|b| { if b.len() >= 4 { u32::from_be_bytes([b[0], b[1], b[2], b[3]]) @@ -213,13 +315,14 @@ impl Storage { let mut versioned_node = node.clone(); versioned_node.version = new_version; - let node_bytes = versioned_node.to_bytes() + let node_bytes = versioned_node + .to_bytes() .map_err(|e| StorageError::Serialization(e.to_string()))?; // Atomic write: node + latest pointer let mut batch = WriteBatch::default(); batch.put_cf(&nodes_cf, versioned_key.as_bytes(), &node_bytes); - batch.put_cf(&latest_cf, latest_key.as_bytes(), &new_version.to_be_bytes()); + batch.put_cf(&latest_cf, latest_key.as_bytes(), new_version.to_be_bytes()); self.db.write(batch)?; @@ -228,10 +331,17 @@ impl Storage { } /// Get the latest version of a TOC node. - pub fn get_toc_node(&self, node_id: &str) -> Result, StorageError> { - let nodes_cf = self.db.cf_handle(CF_TOC_NODES) + pub fn get_toc_node( + &self, + node_id: &str, + ) -> Result, StorageError> { + let nodes_cf = self + .db + .cf_handle(CF_TOC_NODES) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_TOC_NODES.to_string()))?; - let latest_cf = self.db.cf_handle(CF_TOC_LATEST) + let latest_cf = self + .db + .cf_handle(CF_TOC_LATEST) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_TOC_LATEST.to_string()))?; // Get latest version number @@ -260,9 +370,13 @@ impl Storage { start_time: Option>, end_time: Option>, ) -> Result, StorageError> { - let nodes_cf = self.db.cf_handle(CF_TOC_NODES) + let nodes_cf = self + .db + .cf_handle(CF_TOC_NODES) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_TOC_NODES.to_string()))?; - let latest_cf = self.db.cf_handle(CF_TOC_LATEST) + let latest_cf = self + .db + .cf_handle(CF_TOC_LATEST) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_TOC_LATEST.to_string()))?; let level_prefix = format!("latest:toc:{}:", level); @@ -295,7 +409,9 @@ impl Storage { // Filter by time range if specified let include = match (start_time, end_time) { - (Some(start), Some(end)) => node.end_time >= start && node.start_time <= end, + (Some(start), Some(end)) => { + node.end_time >= start && node.start_time <= end + } (Some(start), None) => node.end_time >= start, (None, Some(end)) => node.start_time <= end, (None, None) => true, @@ -315,7 +431,10 @@ impl Storage { } /// Get child nodes of a parent node. - pub fn get_child_nodes(&self, parent_node_id: &str) -> Result, StorageError> { + pub fn get_child_nodes( + &self, + parent_node_id: &str, + ) -> Result, StorageError> { let parent = self.get_toc_node(parent_node_id)?; match parent { Some(node) => { @@ -336,18 +455,22 @@ impl Storage { /// Store a grip. pub fn put_grip(&self, grip: &memory_types::Grip) -> Result<(), StorageError> { - let grips_cf = self.db.cf_handle(CF_GRIPS) + let grips_cf = self + .db + .cf_handle(CF_GRIPS) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_GRIPS.to_string()))?; - let grip_bytes = grip.to_bytes() + let grip_bytes = grip + .to_bytes() .map_err(|e| StorageError::Serialization(e.to_string()))?; - self.db.put_cf(&grips_cf, grip.grip_id.as_bytes(), &grip_bytes)?; + self.db + .put_cf(&grips_cf, grip.grip_id.as_bytes(), &grip_bytes)?; // If linked to a TOC node, create index entry if let Some(ref node_id) = grip.toc_node_id { let index_key = format!("node:{}:{}", node_id, grip.grip_id); - self.db.put_cf(&grips_cf, index_key.as_bytes(), &[])?; + self.db.put_cf(&grips_cf, index_key.as_bytes(), [])?; } debug!(grip_id = %grip.grip_id, "Stored grip"); @@ -356,7 +479,9 @@ impl Storage { /// Get a grip by ID. pub fn get_grip(&self, grip_id: &str) -> Result, StorageError> { - let grips_cf = self.db.cf_handle(CF_GRIPS) + let grips_cf = self + .db + .cf_handle(CF_GRIPS) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_GRIPS.to_string()))?; match self.db.get_cf(&grips_cf, grip_id.as_bytes())? { @@ -370,8 +495,13 @@ impl Storage { } /// Get all grips linked to a TOC node. - pub fn get_grips_for_node(&self, node_id: &str) -> Result, StorageError> { - let grips_cf = self.db.cf_handle(CF_GRIPS) + pub fn get_grips_for_node( + &self, + node_id: &str, + ) -> Result, StorageError> { + let grips_cf = self + .db + .cf_handle(CF_GRIPS) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_GRIPS.to_string()))?; let prefix = format!("node:{}:", node_id); @@ -403,7 +533,9 @@ impl Storage { /// Delete a grip and its index entry. pub fn delete_grip(&self, grip_id: &str) -> Result<(), StorageError> { - let grips_cf = self.db.cf_handle(CF_GRIPS) + let grips_cf = self + .db + .cf_handle(CF_GRIPS) .ok_or_else(|| StorageError::ColumnFamilyNotFound(CF_GRIPS.to_string()))?; // Get grip first to find index entry @@ -422,6 +554,80 @@ impl Storage { Ok(()) } + // ===== Generic Column Family Operations ===== + + /// Put a value into a specific column family. + /// + /// This is a low-level method for use by other crates that manage their own + /// column families (e.g., memory-topics). + pub fn put(&self, cf_name: &str, key: &[u8], value: &[u8]) -> Result<(), StorageError> { + let cf = self + .db + .cf_handle(cf_name) + .ok_or_else(|| StorageError::ColumnFamilyNotFound(cf_name.to_string()))?; + self.db.put_cf(&cf, key, value)?; + Ok(()) + } + + /// Get a value from a specific column family. + /// + /// This is a low-level method for use by other crates that manage their own + /// column families (e.g., memory-topics). + pub fn get(&self, cf_name: &str, key: &[u8]) -> Result>, StorageError> { + let cf = self + .db + .cf_handle(cf_name) + .ok_or_else(|| StorageError::ColumnFamilyNotFound(cf_name.to_string()))?; + let result = self.db.get_cf(&cf, key)?; + Ok(result) + } + + /// Delete a value from a specific column family. + /// + /// This is a low-level method for use by other crates that manage their own + /// column families (e.g., memory-topics). + pub fn delete(&self, cf_name: &str, key: &[u8]) -> Result<(), StorageError> { + let cf = self + .db + .cf_handle(cf_name) + .ok_or_else(|| StorageError::ColumnFamilyNotFound(cf_name.to_string()))?; + self.db.delete_cf(&cf, key)?; + Ok(()) + } + + /// Iterate over entries with a given prefix in a column family. + /// + /// Returns an iterator of (key, value) pairs. + /// This is a low-level method for use by other crates that manage their own + /// column families (e.g., memory-topics). + #[allow(clippy::type_complexity)] + pub fn prefix_iterator( + &self, + cf_name: &str, + prefix: &[u8], + ) -> Result, Vec)>, StorageError> { + let cf = self + .db + .cf_handle(cf_name) + .ok_or_else(|| StorageError::ColumnFamilyNotFound(cf_name.to_string()))?; + + let mut results = Vec::new(); + let iter = self + .db + .iterator_cf(&cf, IteratorMode::From(prefix, Direction::Forward)); + + for item in iter { + let (key, value) = item?; + // Stop if we've passed the prefix + if !key.starts_with(prefix) { + break; + } + results.push((key.to_vec(), value.to_vec())); + } + + Ok(results) + } + // ===== Admin Operations ===== /// Trigger manual compaction on all column families. @@ -431,7 +637,14 @@ impl Storage { info!("Starting full compaction..."); self.db.compact_range::<&[u8], &[u8]>(None, None); - for cf_name in &[CF_EVENTS, CF_TOC_NODES, CF_TOC_LATEST, CF_GRIPS, CF_OUTBOX, CF_CHECKPOINTS] { + for cf_name in &[ + CF_EVENTS, + CF_TOC_NODES, + CF_TOC_LATEST, + CF_GRIPS, + CF_OUTBOX, + CF_CHECKPOINTS, + ] { if let Some(cf) = self.db.cf_handle(cf_name) { self.db.compact_range_cf::<&[u8], &[u8]>(&cf, None, None); } @@ -442,7 +655,9 @@ impl Storage { /// Trigger compaction on a specific column family. pub fn compact_cf(&self, cf_name: &str) -> Result<(), StorageError> { - let cf = self.db.cf_handle(cf_name) + let cf = self + .db + .cf_handle(cf_name) .ok_or_else(|| StorageError::ColumnFamilyNotFound(cf_name.to_string()))?; info!(cf = %cf_name, "Starting compaction..."); self.db.compact_range_cf::<&[u8], &[u8]>(&cf, None, None); @@ -458,22 +673,22 @@ impl Storage { // Count events if let Some(cf) = self.db.cf_handle(CF_EVENTS) { - stats.event_count = self.count_cf_entries(&cf)?; + stats.event_count = self.count_cf_entries(cf)?; } // Count TOC nodes if let Some(cf) = self.db.cf_handle(CF_TOC_NODES) { - stats.toc_node_count = self.count_cf_entries(&cf)?; + stats.toc_node_count = self.count_cf_entries(cf)?; } // Count grips if let Some(cf) = self.db.cf_handle(CF_GRIPS) { - stats.grip_count = self.count_cf_entries(&cf)?; + stats.grip_count = self.count_cf_entries(cf)?; } // Count outbox entries if let Some(cf) = self.db.cf_handle(CF_OUTBOX) { - stats.outbox_count = self.count_cf_entries(&cf)?; + stats.outbox_count = self.count_cf_entries(cf)?; } // Get disk usage @@ -539,7 +754,11 @@ mod tests { let (storage, _temp) = create_test_storage(); // Verify all CFs exist by trying to get handles for cf_name in ALL_CF_NAMES { - assert!(storage.db.cf_handle(cf_name).is_some(), "CF {} should exist", cf_name); + assert!( + storage.db.cf_handle(cf_name).is_some(), + "CF {} should exist", + cf_name + ); } } @@ -551,7 +770,9 @@ mod tests { let event_bytes = b"test event data"; let outbox_bytes = b"outbox entry"; - let (key, created) = storage.put_event(&event_id, event_bytes, outbox_bytes).unwrap(); + let (key, created) = storage + .put_event(&event_id, event_bytes, outbox_bytes) + .unwrap(); assert!(created); assert_eq!(key.event_id(), event_id); @@ -567,8 +788,12 @@ mod tests { let event_bytes = b"test event data"; let outbox_bytes = b"outbox entry"; - let (_, created1) = storage.put_event(&event_id, event_bytes, outbox_bytes).unwrap(); - let (_, created2) = storage.put_event(&event_id, event_bytes, outbox_bytes).unwrap(); + let (_, created1) = storage + .put_event(&event_id, event_bytes, outbox_bytes) + .unwrap(); + let (_, created2) = storage + .put_event(&event_id, event_bytes, outbox_bytes) + .unwrap(); assert!(created1); assert!(!created2); // Second write should be idempotent @@ -587,9 +812,15 @@ mod tests { let ulid2 = ulid::Ulid::from_parts(ts2 as u64, rand::random()); let ulid3 = ulid::Ulid::from_parts(ts3 as u64, rand::random()); - storage.put_event(&ulid1.to_string(), b"event1", b"outbox1").unwrap(); - storage.put_event(&ulid2.to_string(), b"event2", b"outbox2").unwrap(); - storage.put_event(&ulid3.to_string(), b"event3", b"outbox3").unwrap(); + storage + .put_event(&ulid1.to_string(), b"event1", b"outbox1") + .unwrap(); + storage + .put_event(&ulid2.to_string(), b"event2", b"outbox2") + .unwrap(); + storage + .put_event(&ulid3.to_string(), b"event3", b"outbox3") + .unwrap(); // Query range [1500, 2500) should only get event2 let results = storage.get_events_in_range(1500, 2500).unwrap(); @@ -695,7 +926,9 @@ mod tests { chrono::Utc::now(), chrono::Utc::now(), ); - parent.child_node_ids.push("toc:segment:2024-01-15:abc123".to_string()); + parent + .child_node_ids + .push("toc:segment:2024-01-15:abc123".to_string()); storage.put_toc_node(&parent).unwrap(); // Get children @@ -736,7 +969,8 @@ mod tests { "event-015".to_string(), chrono::Utc::now(), "segment_summarizer".to_string(), - ).with_toc_node("toc:day:2024-01-29".to_string()); + ) + .with_toc_node("toc:day:2024-01-29".to_string()); storage.put_grip(&grip).unwrap(); @@ -764,16 +998,157 @@ mod tests { "event-002".to_string(), chrono::Utc::now(), "test".to_string(), - ).with_toc_node("toc:day:2024-01-30".to_string()); + ) + .with_toc_node("toc:day:2024-01-30".to_string()); storage.put_grip(&grip).unwrap(); - assert!(storage.get_grip("grip:1706540400000:del123").unwrap().is_some()); + assert!(storage + .get_grip("grip:1706540400000:del123") + .unwrap() + .is_some()); storage.delete_grip("grip:1706540400000:del123").unwrap(); - assert!(storage.get_grip("grip:1706540400000:del123").unwrap().is_none()); + assert!(storage + .get_grip("grip:1706540400000:del123") + .unwrap() + .is_none()); // Index should also be deleted let grips = storage.get_grips_for_node("toc:day:2024-01-30").unwrap(); assert!(grips.is_empty()); } + + // ==================== Outbox Tests ==================== + + #[test] + fn test_get_outbox_entries_empty() { + let (storage, _temp) = create_test_storage(); + + let entries = storage.get_outbox_entries(0, 10).unwrap(); + assert!(entries.is_empty()); + } + + #[test] + fn test_get_outbox_entries_after_event() { + let (storage, _temp) = create_test_storage(); + + // Create an event which also creates an outbox entry + let event_id = ulid::Ulid::new().to_string(); + let outbox_entry = memory_types::OutboxEntry::for_index(event_id.clone(), 1000); + let outbox_bytes = outbox_entry.to_bytes().unwrap(); + + storage + .put_event(&event_id, b"test event", &outbox_bytes) + .unwrap(); + + // Read outbox entries + let entries = storage.get_outbox_entries(0, 10).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].0, 0); // First sequence is 0 + assert_eq!(entries[0].1.event_id, event_id); + } + + #[test] + fn test_get_outbox_entries_with_limit() { + let (storage, _temp) = create_test_storage(); + + // Create multiple events + for i in 0..5 { + let event_id = ulid::Ulid::new().to_string(); + let outbox_entry = memory_types::OutboxEntry::for_index(event_id.clone(), i * 1000); + let outbox_bytes = outbox_entry.to_bytes().unwrap(); + storage + .put_event(&event_id, b"test", &outbox_bytes) + .unwrap(); + } + + // Read with limit + let entries = storage.get_outbox_entries(0, 3).unwrap(); + assert_eq!(entries.len(), 3); + assert_eq!(entries[0].0, 0); + assert_eq!(entries[1].0, 1); + assert_eq!(entries[2].0, 2); + } + + #[test] + fn test_get_outbox_entries_from_offset() { + let (storage, _temp) = create_test_storage(); + + // Create multiple events + let mut event_ids = Vec::new(); + for i in 0..5 { + let event_id = ulid::Ulid::new().to_string(); + event_ids.push(event_id.clone()); + let outbox_entry = memory_types::OutboxEntry::for_index(event_id.clone(), i * 1000); + let outbox_bytes = outbox_entry.to_bytes().unwrap(); + storage + .put_event(&event_id, b"test", &outbox_bytes) + .unwrap(); + } + + // Read starting from sequence 2 + let entries = storage.get_outbox_entries(2, 10).unwrap(); + assert_eq!(entries.len(), 3); + assert_eq!(entries[0].0, 2); + assert_eq!(entries[0].1.event_id, event_ids[2]); + assert_eq!(entries[1].0, 3); + assert_eq!(entries[2].0, 4); + } + + #[test] + fn test_delete_outbox_entries() { + let (storage, _temp) = create_test_storage(); + + // Create multiple events + for i in 0..5 { + let event_id = ulid::Ulid::new().to_string(); + let outbox_entry = memory_types::OutboxEntry::for_index(event_id.clone(), i * 1000); + let outbox_bytes = outbox_entry.to_bytes().unwrap(); + storage + .put_event(&event_id, b"test", &outbox_bytes) + .unwrap(); + } + + // Delete entries up to sequence 2 (inclusive) + let deleted = storage.delete_outbox_entries(2).unwrap(); + assert_eq!(deleted, 3); // Sequences 0, 1, 2 + + // Verify remaining entries + let entries = storage.get_outbox_entries(0, 10).unwrap(); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].0, 3); + assert_eq!(entries[1].0, 4); + } + + #[test] + fn test_delete_outbox_entries_none() { + let (storage, _temp) = create_test_storage(); + + // Delete from empty outbox + let deleted = storage.delete_outbox_entries(10).unwrap(); + assert_eq!(deleted, 0); + } + + #[test] + fn test_delete_outbox_entries_all() { + let (storage, _temp) = create_test_storage(); + + // Create multiple events + for i in 0..3 { + let event_id = ulid::Ulid::new().to_string(); + let outbox_entry = memory_types::OutboxEntry::for_index(event_id.clone(), i * 1000); + let outbox_bytes = outbox_entry.to_bytes().unwrap(); + storage + .put_event(&event_id, b"test", &outbox_bytes) + .unwrap(); + } + + // Delete all entries + let deleted = storage.delete_outbox_entries(100).unwrap(); + assert_eq!(deleted, 3); + + // Verify all gone + let entries = storage.get_outbox_entries(0, 10).unwrap(); + assert!(entries.is_empty()); + } } diff --git a/crates/memory-storage/src/keys.rs b/crates/memory-storage/src/keys.rs index 10f75fd..e24119f 100644 --- a/crates/memory-storage/src/keys.rs +++ b/crates/memory-storage/src/keys.rs @@ -7,8 +7,8 @@ //! //! This format enables efficient time-range scans via RocksDB prefix iteration. -use ulid::Ulid; use crate::error::StorageError; +use ulid::Ulid; /// Key for event storage /// Format: evt:{timestamp_ms:013}:{ulid} @@ -37,7 +37,8 @@ impl EventKey { /// Create an event key from an event_id string (the ULID portion) /// Uses the ULID's embedded timestamp pub fn from_event_id(event_id: &str) -> Result { - let ulid: Ulid = event_id.parse() + let ulid: Ulid = event_id + .parse() .map_err(|e| StorageError::Key(format!("Invalid event_id ULID: {}", e)))?; // ULID contains timestamp - extract it let timestamp_ms = ulid.timestamp_ms() as i64; @@ -55,19 +56,24 @@ impl EventKey { pub fn from_bytes(bytes: &[u8]) -> Result { let s = std::str::from_utf8(bytes) .map_err(|e| StorageError::Key(format!("Invalid UTF-8: {}", e)))?; - Self::from_str(s) + Self::parse(s) } /// Parse from string format - pub fn from_str(s: &str) -> Result { + pub fn parse(s: &str) -> Result { let parts: Vec<&str> = s.split(':').collect(); if parts.len() != 3 || parts[0] != "evt" { - return Err(StorageError::Key(format!("Invalid event key format: {}", s))); + return Err(StorageError::Key(format!( + "Invalid event key format: {}", + s + ))); } - let timestamp_ms: i64 = parts[1].parse() + let timestamp_ms: i64 = parts[1] + .parse() .map_err(|e| StorageError::Key(format!("Invalid timestamp: {}", e)))?; - let ulid: Ulid = parts[2].parse() + let ulid: Ulid = parts[2] + .parse() .map_err(|e| StorageError::Key(format!("Invalid ULID: {}", e)))?; Ok(Self { timestamp_ms, ulid }) @@ -115,10 +121,14 @@ impl OutboxKey { let parts: Vec<&str> = s.split(':').collect(); if parts.len() != 2 || parts[0] != "outbox" { - return Err(StorageError::Key(format!("Invalid outbox key format: {}", s))); + return Err(StorageError::Key(format!( + "Invalid outbox key format: {}", + s + ))); } - let sequence: u64 = parts[1].parse() + let sequence: u64 = parts[1] + .parse() .map_err(|e| StorageError::Key(format!("Invalid sequence: {}", e)))?; Ok(Self { sequence }) @@ -135,7 +145,9 @@ pub struct CheckpointKey { impl CheckpointKey { pub fn new(job_name: impl Into) -> Self { - Self { job_name: job_name.into() } + Self { + job_name: job_name.into(), + } } pub fn to_bytes(&self) -> Vec { diff --git a/crates/memory-storage/src/lib.rs b/crates/memory-storage/src/lib.rs index ff30c13..f202303 100644 --- a/crates/memory-storage/src/lib.rs +++ b/crates/memory-storage/src/lib.rs @@ -12,6 +12,10 @@ pub mod db; pub mod error; pub mod keys; +pub use column_families::{ + CF_CHECKPOINTS, CF_EVENTS, CF_GRIPS, CF_OUTBOX, CF_TOC_LATEST, CF_TOC_NODES, CF_TOPICS, + CF_TOPIC_LINKS, CF_TOPIC_RELS, +}; pub use db::{Storage, StorageStats}; pub use error::StorageError; -pub use keys::{EventKey, OutboxKey, CheckpointKey}; +pub use keys::{CheckpointKey, EventKey, OutboxKey}; diff --git a/crates/memory-toc/src/builder.rs b/crates/memory-toc/src/builder.rs index ecae24a..c411c32 100644 --- a/crates/memory-toc/src/builder.rs +++ b/crates/memory-toc/src/builder.rs @@ -2,15 +2,15 @@ //! //! Builds TOC nodes from segments and ensures parent nodes exist. -use std::sync::Arc; use chrono::{DateTime, Utc}; +use std::sync::Arc; use tracing::{debug, info}; use memory_storage::Storage; use memory_types::{Segment, TocBullet, TocLevel, TocNode}; use crate::node_id::{generate_node_id, generate_title, get_parent_node_id, get_time_boundaries}; -use crate::summarizer::{extract_grips, Summary, Summarizer, SummarizerError}; +use crate::summarizer::{extract_grips, Summarizer, SummarizerError, Summary}; /// Error type for TOC building. #[derive(Debug, thiserror::Error)] @@ -36,7 +36,10 @@ pub struct TocBuilder { impl TocBuilder { /// Create a new TocBuilder. pub fn new(storage: Arc, summarizer: Arc) -> Self { - Self { storage, summarizer } + Self { + storage, + summarizer, + } } /// Process a segment and create/update TOC nodes. @@ -47,7 +50,9 @@ impl TocBuilder { /// 3. Extracts grips from events based on bullets (SUMM-03) pub async fn process_segment(&self, segment: &Segment) -> Result { if segment.events.is_empty() { - return Err(BuilderError::InvalidSegment("Segment has no events".to_string())); + return Err(BuilderError::InvalidSegment( + "Segment has no events".to_string(), + )); } info!( @@ -75,7 +80,9 @@ impl TocBuilder { // Link bullet to grip if we know which bullet it supports if let Some(bullet_idx) = extracted.bullet_index { if bullet_idx < segment_node.bullets.len() { - segment_node.bullets[bullet_idx].grip_ids.push(grip.grip_id.clone()); + segment_node.bullets[bullet_idx] + .grip_ids + .push(grip.grip_id.clone()); } } @@ -97,16 +104,18 @@ impl TocBuilder { } /// Create a segment-level TOC node. - fn create_segment_node(&self, segment: &Segment, summary: &Summary) -> Result { - let node_id = format!("toc:segment:{}:{}", + fn create_segment_node( + &self, + segment: &Segment, + summary: &Summary, + ) -> Result { + let node_id = format!( + "toc:segment:{}:{}", segment.start_time.format("%Y-%m-%d"), segment.segment_id.trim_start_matches("seg:") ); - let bullets: Vec = summary.bullets - .iter() - .map(|s| TocBullet::new(s)) - .collect(); + let bullets: Vec = summary.bullets.iter().map(TocBullet::new).collect(); let mut node = TocNode::new( node_id, @@ -144,7 +153,8 @@ impl TocBuilder { } } else { // Create parent node with placeholder summary - let parent_node = self.create_parent_node(&parent_id, parent_level, child_node, ¤t_id)?; + let parent_node = + self.create_parent_node(&parent_id, parent_level, child_node, ¤t_id)?; self.storage.put_toc_node(&parent_node)?; debug!( parent = %parent_id, @@ -174,13 +184,7 @@ impl TocBuilder { let (start_time, end_time) = get_time_boundaries(level, child.start_time); let title = generate_title(level, child.start_time); - let mut node = TocNode::new( - parent_id.to_string(), - level, - title, - start_time, - end_time, - ); + let mut node = TocNode::new(parent_id.to_string(), level, title, start_time, end_time); node.child_node_ids.push(child_id.to_string()); // Placeholder bullet - will be replaced by rollup job @@ -192,17 +196,19 @@ impl TocBuilder { /// Get all segment nodes for a day. pub fn get_segments_for_day(&self, date: DateTime) -> Result, BuilderError> { let day_id = generate_node_id(TocLevel::Day, date); - self.storage.get_child_nodes(&day_id).map_err(BuilderError::from) + self.storage + .get_child_nodes(&day_id) + .map_err(BuilderError::from) } } #[cfg(test)] mod tests { use super::*; + use crate::summarizer::MockSummarizer; use chrono::TimeZone; use memory_types::{Event, EventRole, EventType}; use tempfile::TempDir; - use crate::summarizer::MockSummarizer; fn create_test_storage() -> (Arc, TempDir) { let temp_dir = TempDir::new().unwrap(); diff --git a/crates/memory-toc/src/config.rs b/crates/memory-toc/src/config.rs index 9389f5e..55fecd8 100644 --- a/crates/memory-toc/src/config.rs +++ b/crates/memory-toc/src/config.rs @@ -77,6 +77,9 @@ mod tests { let config = TocConfig::default(); let json = serde_json::to_string(&config).unwrap(); let decoded: TocConfig = serde_json::from_str(&json).unwrap(); - assert_eq!(config.segmentation.token_threshold, decoded.segmentation.token_threshold); + assert_eq!( + config.segmentation.token_threshold, + decoded.segmentation.token_threshold + ); } } diff --git a/crates/memory-toc/src/expand.rs b/crates/memory-toc/src/expand.rs index ce97f90..356dbae 100644 --- a/crates/memory-toc/src/expand.rs +++ b/crates/memory-toc/src/expand.rs @@ -2,8 +2,8 @@ //! //! Per GRIP-04: ExpandGrip returns context events around excerpt. -use std::sync::Arc; use chrono::Duration; +use std::sync::Arc; use tracing::debug; use memory_storage::Storage; @@ -101,7 +101,9 @@ impl GripExpander { /// Expand a grip by ID, retrieving context events. pub fn expand(&self, grip_id: &str) -> Result { // Get the grip - let grip = self.storage.get_grip(grip_id)? + let grip = self + .storage + .get_grip(grip_id)? .ok_or_else(|| ExpandError::GripNotFound(grip_id.to_string()))?; self.expand_grip(&grip) @@ -184,19 +186,14 @@ impl GripExpander { /// Parse timestamp from ULID event ID. fn parse_ulid_timestamp(event_id: &str) -> Option> { - ulid::Ulid::from_string(event_id) - .ok() - .and_then(|u| { - let ms = u.timestamp_ms(); - chrono::DateTime::from_timestamp_millis(ms as i64) - }) + ulid::Ulid::from_string(event_id).ok().and_then(|u| { + let ms = u.timestamp_ms(); + chrono::DateTime::from_timestamp_millis(ms as i64) + }) } /// Convenience function to expand a grip. -pub fn expand_grip( - storage: Arc, - grip_id: &str, -) -> Result { +pub fn expand_grip(storage: Arc, grip_id: &str) -> Result { GripExpander::new(storage).expand(grip_id) } @@ -225,7 +222,9 @@ mod tests { let event_bytes = serde_json::to_vec(&event).unwrap(); let outbox_bytes = b"outbox"; - storage.put_event(&event.event_id, &event_bytes, outbox_bytes).unwrap(); + storage + .put_event(&event.event_id, &event_bytes, outbox_bytes) + .unwrap(); event } @@ -256,8 +255,8 @@ mod tests { let expanded = expander.expand(&grip.grip_id).unwrap(); assert_eq!(expanded.excerpt_events.len(), 2); - assert!(expanded.events_before.len() >= 1); - assert!(expanded.events_after.len() >= 1); + assert!(!expanded.events_before.is_empty()); + assert!(!expanded.events_after.is_empty()); } #[test] @@ -292,7 +291,7 @@ mod tests { let expanded = expander.expand(&grip.grip_id).unwrap(); let all = expanded.all_events(); - assert!(all.len() >= 1); // At least the excerpt event + assert!(!all.is_empty()); // At least the excerpt event } #[test] @@ -301,7 +300,11 @@ mod tests { // Create many events for i in 0..10 { - create_and_store_event(&storage, &format!("Event {}", i), 1706540000000 + i * 100000); + create_and_store_event( + &storage, + &format!("Event {}", i), + 1706540000000 + i * 100000, + ); } let event = create_and_store_event(&storage, "Target", 1706540500000); diff --git a/crates/memory-toc/src/grip_id.rs b/crates/memory-toc/src/grip_id.rs index 05c31db..63c97b6 100644 --- a/crates/memory-toc/src/grip_id.rs +++ b/crates/memory-toc/src/grip_id.rs @@ -22,8 +22,10 @@ pub fn parse_grip_timestamp(grip_id: &str) -> Option> { return None; } - parts[1].parse::().ok() - .and_then(|ms| chrono::DateTime::from_timestamp_millis(ms)) + parts[1] + .parse::() + .ok() + .and_then(chrono::DateTime::from_timestamp_millis) } /// Check if a string is a valid grip ID format. diff --git a/crates/memory-toc/src/lib.rs b/crates/memory-toc/src/lib.rs index 6cd004f..739bb57 100644 --- a/crates/memory-toc/src/lib.rs +++ b/crates/memory-toc/src/lib.rs @@ -7,6 +7,7 @@ //! - Node ID generation //! - Grip ID generation and provenance //! - Grip expansion for context retrieval (GRIP-04) +//! - TOC node search with term-overlap scoring (Phase 10.5) pub mod builder; pub mod config; @@ -14,14 +15,18 @@ pub mod expand; pub mod grip_id; pub mod node_id; pub mod rollup; +pub mod search; pub mod segmenter; pub mod summarizer; pub use builder::{BuilderError, TocBuilder}; pub use config::{SegmentationConfig, TocConfig}; -pub use expand::{expand_grip, ExpandConfig, ExpandedGrip, ExpandError, GripExpander}; +pub use expand::{expand_grip, ExpandConfig, ExpandError, ExpandedGrip, GripExpander}; pub use grip_id::{generate_grip_id, is_valid_grip_id, parse_grip_timestamp}; pub use node_id::{generate_node_id, generate_title, get_parent_node_id, parse_level}; -pub use rollup::{RollupCheckpoint, RollupError, RollupJob, run_all_rollups}; +pub use rollup::{run_all_rollups, RollupCheckpoint, RollupError, RollupJob}; +pub use search::{search_node, term_overlap_score, SearchField, SearchMatch}; pub use segmenter::{segment_events, SegmentBuilder, TokenCounter}; -pub use summarizer::{ApiSummarizer, ApiSummarizerConfig, MockSummarizer, Summary, Summarizer, SummarizerError}; +pub use summarizer::{ + ApiSummarizer, ApiSummarizerConfig, MockSummarizer, Summarizer, SummarizerError, Summary, +}; diff --git a/crates/memory-toc/src/node_id.rs b/crates/memory-toc/src/node_id.rs index 320aa50..6db13d2 100644 --- a/crates/memory-toc/src/node_id.rs +++ b/crates/memory-toc/src/node_id.rs @@ -59,7 +59,11 @@ pub fn get_parent_node_id(node_id: &str) -> Option { if parts.len() >= 3 { if let Ok(date) = chrono::NaiveDate::parse_from_str(parts[2], "%Y-%m-%d") { let iso_week = date.iso_week(); - return Some(format!("toc:week:{}:W{:02}", iso_week.year(), iso_week.week())); + return Some(format!( + "toc:week:{}:W{:02}", + iso_week.year(), + iso_week.week() + )); } } None @@ -73,7 +77,8 @@ pub fn get_parent_node_id(node_id: &str) -> Option { parts[3].trim_start_matches('W').parse::(), ) { // Get the Thursday of the week to determine the month - if let Some(date) = chrono::NaiveDate::from_isoywd_opt(year, week, Weekday::Thu) { + if let Some(date) = chrono::NaiveDate::from_isoywd_opt(year, week, Weekday::Thu) + { return Some(format!("toc:month:{}:{:02}", date.year(), date.month())); } } @@ -131,22 +136,31 @@ pub fn get_time_boundaries(level: TocLevel, time: DateTime) -> (DateTime { let start = Utc.with_ymd_and_hms(time.year(), 1, 1, 0, 0, 0).unwrap(); - let end = Utc.with_ymd_and_hms(time.year() + 1, 1, 1, 0, 0, 0).unwrap() - Duration::milliseconds(1); + let end = Utc + .with_ymd_and_hms(time.year() + 1, 1, 1, 0, 0, 0) + .unwrap() + - Duration::milliseconds(1); (start, end) } TocLevel::Month => { - let start = Utc.with_ymd_and_hms(time.year(), time.month(), 1, 0, 0, 0).unwrap(); + let start = Utc + .with_ymd_and_hms(time.year(), time.month(), 1, 0, 0, 0) + .unwrap(); let next_month = if time.month() == 12 { - Utc.with_ymd_and_hms(time.year() + 1, 1, 1, 0, 0, 0).unwrap() + Utc.with_ymd_and_hms(time.year() + 1, 1, 1, 0, 0, 0) + .unwrap() } else { - Utc.with_ymd_and_hms(time.year(), time.month() + 1, 1, 0, 0, 0).unwrap() + Utc.with_ymd_and_hms(time.year(), time.month() + 1, 1, 0, 0, 0) + .unwrap() }; let end = next_month - Duration::milliseconds(1); (start, end) } TocLevel::Week => { let iso_week = time.iso_week(); - let monday = chrono::NaiveDate::from_isoywd_opt(iso_week.year(), iso_week.week(), Weekday::Mon).unwrap(); + let monday = + chrono::NaiveDate::from_isoywd_opt(iso_week.year(), iso_week.week(), Weekday::Mon) + .unwrap(); let start = Utc.from_utc_datetime(&monday.and_time(NaiveTime::MIN)); let end = start + Duration::days(7) - Duration::milliseconds(1); (start, end) diff --git a/crates/memory-toc/src/rollup.rs b/crates/memory-toc/src/rollup.rs index bdb1ef2..1e13170 100644 --- a/crates/memory-toc/src/rollup.rs +++ b/crates/memory-toc/src/rollup.rs @@ -3,15 +3,15 @@ //! Per TOC-05: Day/Week/Month rollup jobs with checkpointing. //! Per SUMM-04: Rollup summarizer aggregates child node summaries. -use std::sync::Arc; use chrono::{DateTime, Duration, Utc}; use serde::{Deserialize, Serialize}; +use std::sync::Arc; use tracing::{debug, info}; use memory_storage::Storage; use memory_types::{TocBullet, TocLevel, TocNode}; -use crate::summarizer::{Summary, Summarizer, SummarizerError}; +use crate::summarizer::{Summarizer, SummarizerError, Summary}; /// Checkpoint for rollup job crash recovery. /// @@ -101,15 +101,32 @@ impl RollupJob { } /// Create rollup jobs for all levels. - pub fn create_all( - storage: Arc, - summarizer: Arc, - ) -> Vec { + pub fn create_all(storage: Arc, summarizer: Arc) -> Vec { vec![ - Self::new(storage.clone(), summarizer.clone(), TocLevel::Day, Duration::hours(1)), - Self::new(storage.clone(), summarizer.clone(), TocLevel::Week, Duration::hours(24)), - Self::new(storage.clone(), summarizer.clone(), TocLevel::Month, Duration::hours(24)), - Self::new(storage.clone(), summarizer.clone(), TocLevel::Year, Duration::days(7)), + Self::new( + storage.clone(), + summarizer.clone(), + TocLevel::Day, + Duration::hours(1), + ), + Self::new( + storage.clone(), + summarizer.clone(), + TocLevel::Week, + Duration::hours(24), + ), + Self::new( + storage.clone(), + summarizer.clone(), + TocLevel::Month, + Duration::hours(24), + ), + Self::new( + storage.clone(), + summarizer.clone(), + TocLevel::Year, + Duration::days(7), + ), ] } @@ -122,15 +139,15 @@ impl RollupJob { // Load checkpoint let checkpoint = self.load_checkpoint(&job_name)?; - let start_time = checkpoint.map(|c| c.last_processed_time).unwrap_or(DateTime::::MIN_UTC); + let start_time = checkpoint + .map(|c| c.last_processed_time) + .unwrap_or(DateTime::::MIN_UTC); // Get nodes at this level that need rollup let cutoff_time = Utc::now() - self.min_age; - let nodes = self.storage.get_toc_nodes_by_level( - self.level, - Some(start_time), - Some(cutoff_time), - )?; + let nodes = + self.storage + .get_toc_nodes_by_level(self.level, Some(start_time), Some(cutoff_time))?; let mut processed = 0; @@ -154,11 +171,13 @@ impl RollupJob { // Convert children to summaries let summaries: Vec = children .iter() - .map(|c| Summary::new( - c.title.clone(), - c.bullets.iter().map(|b| b.text.clone()).collect(), - c.keywords.clone(), - )) + .map(|c| { + Summary::new( + c.title.clone(), + c.bullets.iter().map(|b| b.text.clone()).collect(), + c.keywords.clone(), + ) + }) .collect(); // Generate rollup summary @@ -167,7 +186,8 @@ impl RollupJob { // Update node with rollup summary let mut updated_node = node.clone(); updated_node.title = rollup_summary.title; - updated_node.bullets = rollup_summary.bullets + updated_node.bullets = rollup_summary + .bullets .into_iter() .map(TocBullet::new) .collect(); @@ -220,7 +240,8 @@ impl RollupJob { created_at: Utc::now(), }; - let bytes = checkpoint.to_bytes() + let bytes = checkpoint + .to_bytes() .map_err(|e| RollupError::Checkpoint(e.to_string()))?; self.storage.put_checkpoint(job_name, &bytes)?; @@ -246,11 +267,11 @@ pub async fn run_all_rollups( #[cfg(test)] mod tests { use super::*; + use crate::builder::TocBuilder; + use crate::summarizer::MockSummarizer; + use chrono::TimeZone; use memory_types::{Event, EventRole, EventType, Segment}; use tempfile::TempDir; - use chrono::TimeZone; - use crate::summarizer::MockSummarizer; - use crate::builder::TocBuilder; fn create_test_storage() -> (Arc, TempDir) { let temp_dir = TempDir::new().unwrap(); @@ -307,16 +328,14 @@ mod tests { // Create segment in the past let past_time = Utc::now() - Duration::days(2); - let events = vec![ - Event::new( - ulid::Ulid::new().to_string(), - "session".to_string(), - past_time, - EventType::UserMessage, - EventRole::User, - "Test event".to_string(), - ), - ]; + let events = vec![Event::new( + ulid::Ulid::new().to_string(), + "session".to_string(), + past_time, + EventType::UserMessage, + EventRole::User, + "Test event".to_string(), + )]; let segment = Segment::new( "seg:test".to_string(), events.clone(), @@ -338,6 +357,7 @@ mod tests { let result = job.run().await.unwrap(); // Result depends on whether the day node has children // This tests the basic flow works without errors - assert!(result >= 0); + // result is a count of nodes processed + let _ = result; } } diff --git a/crates/memory-toc/src/search.rs b/crates/memory-toc/src/search.rs new file mode 100644 index 0000000..dd281a4 --- /dev/null +++ b/crates/memory-toc/src/search.rs @@ -0,0 +1,491 @@ +//! Search functionality for TOC nodes. +//! +//! Provides term-overlap scoring to search within TOC node content +//! without external index dependencies. +//! +//! Per Phase 10.5: This is the "always works" foundation that +//! later phases (BM25, vector) build upon. + +use memory_types::TocNode; + +/// Represents searchable fields in a TOC node. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SearchField { + /// The node title + Title, + /// Summary derived from bullets + Summary, + /// Individual bullet points + Bullets, + /// Keywords associated with the node + Keywords, +} + +/// Represents a match result from searching a TOC node. +#[derive(Debug, Clone)] +pub struct SearchMatch { + /// Which field matched + pub field: SearchField, + /// The text that matched + pub text: String, + /// Grip IDs associated with this match (for provenance) + pub grip_ids: Vec, + /// Relevance score (0.0 to 1.0) + pub score: f32, +} + +impl SearchMatch { + /// Create a new search match. + pub fn new(field: SearchField, text: String, grip_ids: Vec, score: f32) -> Self { + Self { + field, + text, + grip_ids, + score, + } + } +} + +/// Calculate term overlap score (0.0-1.0). +/// +/// Returns the ratio of matched terms to total terms. +/// Returns None if no terms match or if terms list is empty. +/// +/// # Arguments +/// * `text` - The text to search within +/// * `terms` - Search terms (should be lowercase, >= 3 chars) +/// +/// # Example +/// ``` +/// use memory_toc::search::term_overlap_score; +/// +/// let terms = vec!["jwt".to_string(), "token".to_string()]; +/// let score = term_overlap_score("JWT authentication with token refresh", &terms); +/// assert_eq!(score, Some(1.0)); // Both terms match +/// ``` +pub fn term_overlap_score(text: &str, terms: &[String]) -> Option { + if terms.is_empty() { + return None; + } + + let text_lower = text.to_lowercase(); + let matched_count = terms + .iter() + .filter(|term| text_lower.contains(term.as_str())) + .count(); + + if matched_count == 0 { + None + } else { + Some(matched_count as f32 / terms.len() as f32) + } +} + +/// Parse query string into normalized search terms. +/// +/// - Splits on whitespace +/// - Filters terms shorter than 3 characters +/// - Converts to lowercase +fn parse_query(query: &str) -> Vec { + query + .split_whitespace() + .filter(|w| w.len() >= 3) + .map(|w| w.to_lowercase()) + .collect() +} + +/// Check if a search field is enabled. +fn field_enabled(fields: &[SearchField], target: SearchField) -> bool { + fields.is_empty() || fields.contains(&target) +} + +/// Search within a single node's fields for matching terms. +/// +/// # Arguments +/// * `node` - The TOC node to search +/// * `query` - Space-separated search terms +/// * `fields` - Which fields to search (empty = all fields) +/// +/// # Returns +/// Vector of SearchMatch sorted by score descending +/// +/// # Example +/// ``` +/// use memory_toc::search::{search_node, SearchField}; +/// use memory_types::{TocNode, TocLevel, TocBullet}; +/// use chrono::Utc; +/// +/// let mut node = TocNode::new( +/// "node:1".to_string(), +/// TocLevel::Day, +/// "JWT Debugging Session".to_string(), +/// Utc::now(), +/// Utc::now(), +/// ); +/// node.bullets = vec![TocBullet::new("Fixed token expiration bug")]; +/// +/// let matches = search_node(&node, "jwt token", &[]); +/// assert!(!matches.is_empty()); +/// ``` +pub fn search_node(node: &TocNode, query: &str, fields: &[SearchField]) -> Vec { + let terms = parse_query(query); + if terms.is_empty() { + return Vec::new(); + } + + let mut matches = Vec::new(); + + // Search title + if field_enabled(fields, SearchField::Title) { + if let Some(score) = term_overlap_score(&node.title, &terms) { + matches.push(SearchMatch::new( + SearchField::Title, + node.title.clone(), + Vec::new(), + score, + )); + } + } + + // Search summary (derived from bullets) + if field_enabled(fields, SearchField::Summary) { + let summary: String = node + .bullets + .iter() + .map(|b| b.text.as_str()) + .collect::>() + .join(" "); + + if !summary.is_empty() { + if let Some(score) = term_overlap_score(&summary, &terms) { + // Collect all grip IDs from bullets for the summary match + let grip_ids: Vec = node + .bullets + .iter() + .flat_map(|b| b.grip_ids.clone()) + .collect(); + + matches.push(SearchMatch::new( + SearchField::Summary, + summary, + grip_ids, + score, + )); + } + } + } + + // Search individual bullets + if field_enabled(fields, SearchField::Bullets) { + for bullet in &node.bullets { + if let Some(score) = term_overlap_score(&bullet.text, &terms) { + matches.push(SearchMatch::new( + SearchField::Bullets, + bullet.text.clone(), + bullet.grip_ids.clone(), + score, + )); + } + } + } + + // Search keywords + if field_enabled(fields, SearchField::Keywords) { + for keyword in &node.keywords { + // Keyword matching: if any term matches the keyword exactly + // (case-insensitive), score is 1.0 + let keyword_lower = keyword.to_lowercase(); + if terms.iter().any(|term| term == &keyword_lower) { + matches.push(SearchMatch::new( + SearchField::Keywords, + keyword.clone(), + Vec::new(), + 1.0, + )); + } + } + } + + // Sort by score descending + matches.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + matches +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::{TimeZone, Utc}; + use memory_types::{TocBullet, TocLevel}; + + fn make_test_node( + title: &str, + bullets: Vec<(&str, Vec<&str>)>, + keywords: Vec<&str>, + ) -> TocNode { + let mut node = TocNode::new( + "test:node:1".to_string(), + TocLevel::Segment, + title.to_string(), + Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(), + Utc.with_ymd_and_hms(2026, 1, 1, 23, 59, 59).unwrap(), + ); + node.bullets = bullets + .into_iter() + .map(|(text, grips)| { + TocBullet::new(text).with_grips(grips.into_iter().map(|s| s.to_string()).collect()) + }) + .collect(); + node.keywords = keywords.into_iter().map(|s| s.to_string()).collect(); + node + } + + #[test] + fn test_term_overlap_single_match() { + let terms = vec!["jwt".to_string()]; + let score = term_overlap_score("JWT authentication system", &terms); + assert!(score.is_some()); + assert_eq!(score.unwrap(), 1.0); + } + + #[test] + fn test_term_overlap_partial_match() { + let terms = vec!["jwt".to_string(), "debugging".to_string()]; + let score = term_overlap_score("JWT authentication", &terms); + assert!(score.is_some()); + assert_eq!(score.unwrap(), 0.5); + } + + #[test] + fn test_term_overlap_no_match() { + let terms = vec!["vector".to_string(), "embedding".to_string()]; + let score = term_overlap_score("JWT authentication", &terms); + assert!(score.is_none()); + } + + #[test] + fn test_term_overlap_empty_terms() { + let terms: Vec = vec![]; + let score = term_overlap_score("JWT authentication", &terms); + assert!(score.is_none()); + } + + #[test] + fn test_term_overlap_case_insensitive() { + let terms = vec!["jwt".to_string(), "token".to_string()]; + let score = term_overlap_score("JWT TOKEN Authentication", &terms); + assert!(score.is_some()); + assert_eq!(score.unwrap(), 1.0); + } + + #[test] + fn test_parse_query_filters_short_terms() { + let terms = parse_query("to jwt is the token"); + // "to" (2) and "is" (2) should be filtered, "jwt" (3) and "the" (3) and "token" (5) kept + assert!(terms.contains(&"jwt".to_string())); + assert!(terms.contains(&"the".to_string())); + assert!(terms.contains(&"token".to_string())); + assert!(!terms.contains(&"to".to_string())); + assert!(!terms.contains(&"is".to_string())); + } + + #[test] + fn test_parse_query_lowercase() { + let terms = parse_query("JWT Token"); + assert!(terms.contains(&"jwt".to_string())); + assert!(terms.contains(&"token".to_string())); + } + + #[test] + fn test_search_node_title_match() { + let node = make_test_node("JWT Token Debugging Session", vec![], vec![]); + let matches = search_node(&node, "jwt debugging", &[SearchField::Title]); + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].field, SearchField::Title); + assert_eq!(matches[0].score, 1.0); // Both terms match + } + + #[test] + fn test_search_node_title_partial_match() { + let node = make_test_node("JWT Token Session", vec![], vec![]); + let matches = search_node(&node, "jwt debugging", &[SearchField::Title]); + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].field, SearchField::Title); + assert_eq!(matches[0].score, 0.5); // Only "jwt" matches + } + + #[test] + fn test_search_node_title_no_match() { + let node = make_test_node("Database Migration", vec![], vec![]); + let matches = search_node(&node, "jwt authentication", &[SearchField::Title]); + assert!(matches.is_empty()); + } + + #[test] + fn test_search_node_bullet_with_grips() { + let node = make_test_node( + "Session", + vec![("Fixed JWT expiration bug", vec!["grip:123"])], + vec![], + ); + let matches = search_node(&node, "jwt bug", &[SearchField::Bullets]); + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].field, SearchField::Bullets); + assert_eq!(matches[0].grip_ids, vec!["grip:123"]); + assert_eq!(matches[0].score, 1.0); + } + + #[test] + fn test_search_node_multiple_bullets() { + let node = make_test_node( + "Session", + vec![ + ("Fixed JWT expiration bug", vec!["grip:1"]), + ("Added token refresh", vec!["grip:2"]), + ("Updated documentation", vec!["grip:3"]), + ], + vec![], + ); + let matches = search_node(&node, "jwt token", &[SearchField::Bullets]); + // First bullet matches "jwt", second matches "token" + assert_eq!(matches.len(), 2); + } + + #[test] + fn test_search_node_keyword_match() { + let node = make_test_node("Session", vec![], vec!["authentication", "JWT"]); + let matches = search_node(&node, "jwt", &[SearchField::Keywords]); + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].field, SearchField::Keywords); + assert_eq!(matches[0].score, 1.0); + assert_eq!(matches[0].text, "JWT"); + } + + #[test] + fn test_search_node_keyword_no_partial_match() { + let node = make_test_node("Session", vec![], vec!["authentication"]); + // "auth" should not match "authentication" for keywords (exact match required) + let matches = search_node(&node, "auth", &[SearchField::Keywords]); + assert!(matches.is_empty()); + } + + #[test] + fn test_search_node_summary_match() { + let node = make_test_node( + "Session", + vec![ + ("Fixed JWT bug", vec!["grip:1"]), + ("Added token refresh", vec!["grip:2"]), + ], + vec![], + ); + let matches = search_node(&node, "jwt token", &[SearchField::Summary]); + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].field, SearchField::Summary); + // Summary should include both grips + assert!(matches[0].grip_ids.contains(&"grip:1".to_string())); + assert!(matches[0].grip_ids.contains(&"grip:2".to_string())); + assert_eq!(matches[0].score, 1.0); + } + + #[test] + fn test_search_node_short_terms_filtered() { + let node = make_test_node("The JWT Token", vec![], vec![]); + // "to" (2 chars) is filtered, "jwt" (3 chars) kept + let matches = search_node(&node, "to jwt", &[SearchField::Title]); + assert_eq!(matches.len(), 1); + // Only "jwt" should be used, and it matches + assert_eq!(matches[0].score, 1.0); + } + + #[test] + fn test_search_node_all_terms_filtered() { + let node = make_test_node("JWT Token", vec![], vec![]); + // All terms < 3 chars + let matches = search_node(&node, "to is a", &[SearchField::Title]); + assert!(matches.is_empty()); + } + + #[test] + fn test_search_node_all_fields() { + let node = make_test_node( + "JWT Session", + vec![("Implemented authentication", vec!["grip:1"])], + vec!["token"], + ); + // Empty fields means search all + let matches = search_node(&node, "jwt token authentication", &[]); + // Should find matches in: + // - Title: "jwt" matches + // - Summary: "authentication" matches + // - Bullets: "authentication" matches + // - Keywords: "token" matches exactly + assert!(!matches.is_empty()); + + let field_types: Vec = matches.iter().map(|m| m.field).collect(); + assert!(field_types.contains(&SearchField::Title)); + assert!(field_types.contains(&SearchField::Summary)); + assert!(field_types.contains(&SearchField::Bullets)); + assert!(field_types.contains(&SearchField::Keywords)); + } + + #[test] + fn test_search_node_sorted_by_score() { + let node = make_test_node( + "Session", // No match + vec![ + ("JWT debugging today", vec!["grip:1"]), // 1 of 2 = 0.5 + ("JWT authentication and token refresh", vec!["grip:2"]), // 2 of 2 = 1.0 + ], + vec![], + ); + let matches = search_node(&node, "jwt authentication", &[SearchField::Bullets]); + assert_eq!(matches.len(), 2); + // Higher score should be first + assert!(matches[0].score >= matches[1].score); + assert_eq!(matches[0].score, 1.0); + assert_eq!(matches[1].score, 0.5); + } + + #[test] + fn test_search_node_empty_query() { + let node = make_test_node("JWT Session", vec![], vec![]); + let matches = search_node(&node, "", &[]); + assert!(matches.is_empty()); + } + + #[test] + fn test_search_node_empty_node() { + let node = make_test_node("", vec![], vec![]); + let matches = search_node(&node, "jwt token", &[]); + assert!(matches.is_empty()); + } + + #[test] + fn test_search_node_grips_propagated_from_bullets() { + let node = make_test_node( + "Session", + vec![ + ("Fixed JWT bug", vec!["grip:aaa", "grip:bbb"]), + ("Added token refresh", vec!["grip:ccc"]), + ], + vec![], + ); + + // Test bullet match includes its grips + let bullet_matches = search_node(&node, "jwt bug", &[SearchField::Bullets]); + assert_eq!(bullet_matches.len(), 1); + assert_eq!(bullet_matches[0].grip_ids.len(), 2); + assert!(bullet_matches[0].grip_ids.contains(&"grip:aaa".to_string())); + assert!(bullet_matches[0].grip_ids.contains(&"grip:bbb".to_string())); + + // Test summary match includes all grips + let summary_matches = search_node(&node, "fixed added", &[SearchField::Summary]); + assert_eq!(summary_matches.len(), 1); + assert_eq!(summary_matches[0].grip_ids.len(), 3); + } +} diff --git a/crates/memory-toc/src/segmenter.rs b/crates/memory-toc/src/segmenter.rs index 33f62b8..a242e6a 100644 --- a/crates/memory-toc/src/segmenter.rs +++ b/crates/memory-toc/src/segmenter.rs @@ -18,7 +18,9 @@ pub struct TokenCounter { impl TokenCounter { pub fn new(max_tool_result_chars: usize) -> Self { - Self { max_tool_result_chars } + Self { + max_tool_result_chars, + } } /// Count tokens in event text. @@ -322,7 +324,7 @@ mod tests { fn test_segment_builder_token_boundary() { let config = SegmentationConfig { time_threshold_ms: 1000000, // Very high to not trigger - token_threshold: 10, // Very low to trigger + token_threshold: 10, // Very low to trigger overlap_time_ms: 500, overlap_tokens: 5, max_tool_result_chars: 1000, @@ -359,7 +361,9 @@ mod tests { builder.add_event(create_event_at("Late", 1400)); // Trigger boundary - let segment1 = builder.add_event(create_event_at("After gap", 5000)).unwrap(); + let segment1 = builder + .add_event(create_event_at("After gap", 5000)) + .unwrap(); assert_eq!(segment1.events.len(), 3); // Add more events and flush diff --git a/crates/memory-toc/src/summarizer/api.rs b/crates/memory-toc/src/summarizer/api.rs index cbf9bd6..331105d 100644 --- a/crates/memory-toc/src/summarizer/api.rs +++ b/crates/memory-toc/src/summarizer/api.rs @@ -10,12 +10,12 @@ use tracing::{debug, error, warn}; use memory_types::Event; -use super::{Summary, Summarizer, SummarizerError}; +use super::{Summarizer, SummarizerError, Summary}; /// Configuration for API-based summarizer. #[derive(Debug, Clone)] pub struct ApiSummarizerConfig { - /// API base URL (e.g., "https://api.openai.com/v1") + /// API base URL (e.g., `https://api.openai.com/v1`) pub base_url: String, /// Model to use (e.g., "gpt-4o-mini", "claude-3-haiku-20240307") @@ -249,7 +249,10 @@ Guidelines: let response = self .client .post(&url) - .header("Authorization", format!("Bearer {}", self.config.api_key.expose_secret())) + .header( + "Authorization", + format!("Bearer {}", self.config.api_key.expose_secret()), + ) .header("Content-Type", "application/json") .json(&request) .send() diff --git a/crates/memory-toc/src/summarizer/grip_extractor.rs b/crates/memory-toc/src/summarizer/grip_extractor.rs index 00d1edb..b644032 100644 --- a/crates/memory-toc/src/summarizer/grip_extractor.rs +++ b/crates/memory-toc/src/summarizer/grip_extractor.rs @@ -81,10 +81,7 @@ impl GripExtractor { /// Find the best matching events for a bullet point. fn find_best_match(&self, events: &[Event], bullet: &str, source: &str) -> Option { // Extract key terms from bullet - let key_terms: Vec<&str> = bullet - .split_whitespace() - .filter(|w| w.len() > 3) - .collect(); + let key_terms: Vec<&str> = bullet.split_whitespace().filter(|w| w.len() > 3).collect(); if key_terms.is_empty() { return None; @@ -164,11 +161,7 @@ impl Default for GripExtractor { } /// Convenience function to extract grips from events. -pub fn extract_grips( - events: &[Event], - bullets: &[String], - source: &str, -) -> Vec { +pub fn extract_grips(events: &[Event], bullets: &[String], source: &str) -> Vec { GripExtractor::new().extract_grips(events, bullets, source) } @@ -193,13 +186,14 @@ mod tests { fn test_extract_grips_basic() { let events = vec![ create_test_event("How do I implement authentication?", 1706540400000), - create_test_event("You can use JWT tokens for stateless authentication", 1706540500000), + create_test_event( + "You can use JWT tokens for stateless authentication", + 1706540500000, + ), create_test_event("That sounds good, let me try it", 1706540600000), ]; - let bullets = vec![ - "Discussed JWT authentication implementation".to_string(), - ]; + let bullets = vec!["Discussed JWT authentication implementation".to_string()]; let grips = extract_grips(&events, &bullets, "test"); diff --git a/crates/memory-toc/src/summarizer/mock.rs b/crates/memory-toc/src/summarizer/mock.rs index 6463b88..798765e 100644 --- a/crates/memory-toc/src/summarizer/mock.rs +++ b/crates/memory-toc/src/summarizer/mock.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use memory_types::Event; -use super::{Summary, Summarizer, SummarizerError}; +use super::{Summarizer, SummarizerError, Summary}; /// Mock summarizer that generates deterministic summaries. /// @@ -47,11 +47,7 @@ impl Summarizer for MockSummarizer { let first_event = &events[0]; let last_event = &events[events.len() - 1]; - let title = format!( - "{} {} events", - self.title_prefix, - events.len() - ); + let title = format!("{} {} events", self.title_prefix, events.len()); let bullets = vec![ format!("First message: {}", truncate(&first_event.text, 50)), @@ -70,11 +66,7 @@ impl Summarizer for MockSummarizer { return Err(SummarizerError::NoEvents); } - let title = format!( - "{} {} child summaries", - self.title_prefix, - summaries.len() - ); + let title = format!("{} {} child summaries", self.title_prefix, summaries.len()); // Collect bullets from children (first bullet from each) let bullets: Vec = summaries @@ -84,10 +76,8 @@ impl Summarizer for MockSummarizer { .collect(); // Merge keywords from all children - let mut all_keywords: Vec = summaries - .iter() - .flat_map(|s| s.keywords.clone()) - .collect(); + let mut all_keywords: Vec = + summaries.iter().flat_map(|s| s.keywords.clone()).collect(); all_keywords.sort(); all_keywords.dedup(); let keywords = all_keywords.into_iter().take(7).collect(); @@ -107,7 +97,11 @@ fn truncate(text: &str, max_len: usize) -> String { /// Extract mock keywords from events (simple word extraction). fn extract_mock_keywords(events: &[Event]) -> Vec { - let all_text: String = events.iter().map(|e| e.text.as_str()).collect::>().join(" "); + let all_text: String = events + .iter() + .map(|e| e.text.as_str()) + .collect::>() + .join(" "); // Simple keyword extraction: split by whitespace, filter short words let words: Vec = all_text @@ -118,7 +112,8 @@ fn extract_mock_keywords(events: &[Event]) -> Vec { .collect(); // Count and sort by frequency - let mut word_counts: std::collections::HashMap = std::collections::HashMap::new(); + let mut word_counts: std::collections::HashMap = + std::collections::HashMap::new(); for word in words { *word_counts.entry(word).or_insert(0) += 1; } @@ -132,9 +127,9 @@ fn extract_mock_keywords(events: &[Event]) -> Vec { /// Check if word is a common stopword. fn is_stopword(word: &str) -> bool { const STOPWORDS: &[&str] = &[ - "the", "and", "for", "that", "this", "with", "from", "have", "has", - "been", "were", "will", "would", "could", "should", "there", "their", - "what", "when", "where", "which", "about", "into", "through", + "the", "and", "for", "that", "this", "with", "from", "have", "has", "been", "were", "will", + "would", "could", "should", "there", "their", "what", "when", "where", "which", "about", + "into", "through", ]; STOPWORDS.contains(&word) } diff --git a/crates/memory-toc/src/summarizer/mod.rs b/crates/memory-toc/src/summarizer/mod.rs index fd8198e..23c0bd3 100644 --- a/crates/memory-toc/src/summarizer/mod.rs +++ b/crates/memory-toc/src/summarizer/mod.rs @@ -100,7 +100,10 @@ mod tests { fn test_summary_creation() { let summary = Summary::new( "Discussed authentication".to_string(), - vec!["Implemented JWT".to_string(), "Fixed token refresh".to_string()], + vec![ + "Implemented JWT".to_string(), + "Fixed token refresh".to_string(), + ], vec!["auth".to_string(), "jwt".to_string()], ); diff --git a/crates/memory-topics/Cargo.toml b/crates/memory-topics/Cargo.toml new file mode 100644 index 0000000..71c3ebd --- /dev/null +++ b/crates/memory-topics/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "memory-topics" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +memory-types = { workspace = true } +memory-storage = { workspace = true } +hdbscan = { workspace = true } +tokio = { workspace = true, features = ["rt"] } +tracing = { workspace = true } +thiserror = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +chrono = { workspace = true, features = ["serde"] } +ulid = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +tokio = { workspace = true, features = ["test-util", "macros", "rt-multi-thread"] } diff --git a/crates/memory-topics/src/config.rs b/crates/memory-topics/src/config.rs new file mode 100644 index 0000000..ecdb24f --- /dev/null +++ b/crates/memory-topics/src/config.rs @@ -0,0 +1,286 @@ +//! Topic configuration. + +use serde::{Deserialize, Serialize}; + +/// Master configuration for topic functionality. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TopicsConfig { + /// Master switch - topics disabled by default + #[serde(default)] + pub enabled: bool, + + /// Extraction settings + #[serde(default)] + pub extraction: ExtractionConfig, + + /// Labeling settings + #[serde(default)] + pub labeling: LabelingConfig, + + /// Importance scoring settings + #[serde(default)] + pub importance: ImportanceConfig, + + /// Relationship detection settings + #[serde(default)] + pub relationships: RelationshipsConfig, + + /// Lifecycle management settings + #[serde(default)] + pub lifecycle: LifecycleConfig, +} + +#[allow(clippy::derivable_impls)] +impl Default for TopicsConfig { + fn default() -> Self { + Self { + enabled: false, // Disabled by default per TOPIC-07 + extraction: ExtractionConfig::default(), + labeling: LabelingConfig::default(), + importance: ImportanceConfig::default(), + relationships: RelationshipsConfig::default(), + lifecycle: LifecycleConfig::default(), + } + } +} + +/// Topic extraction configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ExtractionConfig { + /// Minimum cluster size for HDBSCAN + #[serde(default = "default_min_cluster_size")] + pub min_cluster_size: usize, + + /// Minimum similarity threshold for cluster membership + #[serde(default = "default_similarity_threshold")] + pub similarity_threshold: f32, + + /// Cron schedule for extraction job + #[serde(default = "default_extraction_schedule")] + pub schedule: String, + + /// Maximum nodes to process per batch + #[serde(default = "default_batch_size")] + pub batch_size: usize, +} + +impl Default for ExtractionConfig { + fn default() -> Self { + Self { + min_cluster_size: default_min_cluster_size(), + similarity_threshold: default_similarity_threshold(), + schedule: default_extraction_schedule(), + batch_size: default_batch_size(), + } + } +} + +fn default_min_cluster_size() -> usize { + 3 +} +fn default_similarity_threshold() -> f32 { + 0.75 +} +fn default_extraction_schedule() -> String { + "0 4 * * *".to_string() // 4 AM daily +} +fn default_batch_size() -> usize { + 500 +} + +/// Topic labeling configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LabelingConfig { + /// Whether to use LLM for labeling + #[serde(default = "default_true")] + pub use_llm: bool, + + /// Fall back to keyword extraction if LLM fails + #[serde(default = "default_true")] + pub fallback_to_keywords: bool, + + /// Maximum label length + #[serde(default = "default_max_label_length")] + pub max_label_length: usize, + + /// Number of top keywords to extract + #[serde(default = "default_top_keywords")] + pub top_keywords: usize, +} + +impl Default for LabelingConfig { + fn default() -> Self { + Self { + use_llm: default_true(), + fallback_to_keywords: default_true(), + max_label_length: default_max_label_length(), + top_keywords: default_top_keywords(), + } + } +} + +fn default_true() -> bool { + true +} +fn default_max_label_length() -> usize { + 50 +} +fn default_top_keywords() -> usize { + 5 +} + +/// Importance scoring configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ImportanceConfig { + /// Half-life in days for decay + #[serde(default = "default_half_life_days")] + pub half_life_days: u32, + + /// Boost multiplier for mentions within 7 days + #[serde(default = "default_recency_boost")] + pub recency_boost: f64, +} + +impl Default for ImportanceConfig { + fn default() -> Self { + Self { + half_life_days: default_half_life_days(), + recency_boost: default_recency_boost(), + } + } +} + +fn default_half_life_days() -> u32 { + 30 +} +fn default_recency_boost() -> f64 { + 2.0 +} + +/// Relationship detection configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RelationshipsConfig { + /// Minimum similarity for "similar" relationship + #[serde(default = "default_similar_threshold")] + pub similar_threshold: f32, + + /// Maximum hierarchy depth + #[serde(default = "default_max_hierarchy_depth")] + pub max_hierarchy_depth: usize, + + /// Enable parent/child detection + #[serde(default = "default_true")] + pub enable_hierarchy: bool, +} + +impl Default for RelationshipsConfig { + fn default() -> Self { + Self { + similar_threshold: default_similar_threshold(), + max_hierarchy_depth: default_max_hierarchy_depth(), + enable_hierarchy: default_true(), + } + } +} + +fn default_similar_threshold() -> f32 { + 0.8 +} +fn default_max_hierarchy_depth() -> usize { + 3 +} + +/// Lifecycle management configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LifecycleConfig { + /// Days of inactivity before pruning + #[serde(default = "default_prune_after_days")] + pub prune_after_days: u32, + + /// Cron schedule for pruning job + #[serde(default = "default_prune_schedule")] + pub prune_schedule: String, + + /// Enable automatic resurrection on re-mention + #[serde(default = "default_true")] + pub auto_resurrect: bool, +} + +impl Default for LifecycleConfig { + fn default() -> Self { + Self { + prune_after_days: default_prune_after_days(), + prune_schedule: default_prune_schedule(), + auto_resurrect: default_true(), + } + } +} + +fn default_prune_after_days() -> u32 { + 90 +} +fn default_prune_schedule() -> String { + "0 5 * * 0".to_string() // 5 AM Sunday +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_default_config_disabled() { + let config = TopicsConfig::default(); + assert!(!config.enabled); + } + + #[test] + fn test_extraction_defaults() { + let config = ExtractionConfig::default(); + assert_eq!(config.min_cluster_size, 3); + assert!((config.similarity_threshold - 0.75).abs() < f32::EPSILON); + } + + #[test] + fn test_labeling_defaults() { + let config = LabelingConfig::default(); + assert!(config.use_llm); + assert!(config.fallback_to_keywords); + assert_eq!(config.max_label_length, 50); + assert_eq!(config.top_keywords, 5); + } + + #[test] + fn test_importance_defaults() { + let config = ImportanceConfig::default(); + assert_eq!(config.half_life_days, 30); + assert!((config.recency_boost - 2.0).abs() < f64::EPSILON); + } + + #[test] + fn test_relationships_defaults() { + let config = RelationshipsConfig::default(); + assert!((config.similar_threshold - 0.8).abs() < f32::EPSILON); + assert_eq!(config.max_hierarchy_depth, 3); + assert!(config.enable_hierarchy); + } + + #[test] + fn test_lifecycle_defaults() { + let config = LifecycleConfig::default(); + assert_eq!(config.prune_after_days, 90); + assert_eq!(config.prune_schedule, "0 5 * * 0"); + assert!(config.auto_resurrect); + } + + #[test] + fn test_config_serialization() { + let config = TopicsConfig::default(); + let json = serde_json::to_string(&config).unwrap(); + let parsed: TopicsConfig = serde_json::from_str(&json).unwrap(); + assert_eq!(config.enabled, parsed.enabled); + assert_eq!( + config.extraction.min_cluster_size, + parsed.extraction.min_cluster_size + ); + } +} diff --git a/crates/memory-topics/src/error.rs b/crates/memory-topics/src/error.rs new file mode 100644 index 0000000..c83e32d --- /dev/null +++ b/crates/memory-topics/src/error.rs @@ -0,0 +1,43 @@ +//! Topic error types. + +use thiserror::Error; + +/// Errors that can occur during topic operations. +#[derive(Debug, Error)] +pub enum TopicsError { + /// Storage error + #[error("Storage error: {0}")] + Storage(#[from] memory_storage::StorageError), + + /// Clustering error + #[error("Clustering error: {0}")] + Clustering(String), + + /// Serialization error + #[error("Serialization error: {0}")] + Serialization(#[from] serde_json::Error), + + /// Topic not found + #[error("Topic not found: {0}")] + NotFound(String), + + /// Invalid configuration + #[error("Invalid configuration: {0}")] + InvalidConfig(String), + + /// Feature disabled + #[error("Topic graph is disabled")] + Disabled, + + /// Embedding error + #[error("Embedding error: {0}")] + Embedding(String), + + /// Invalid input + #[error("Invalid input: {0}")] + InvalidInput(String), + + /// Cycle detected in relationships + #[error("Cycle detected in topic relationships")] + CycleDetected, +} diff --git a/crates/memory-topics/src/extraction.rs b/crates/memory-topics/src/extraction.rs new file mode 100644 index 0000000..49bc3a1 --- /dev/null +++ b/crates/memory-topics/src/extraction.rs @@ -0,0 +1,422 @@ +//! Topic extraction using HDBSCAN clustering. + +use hdbscan::{Hdbscan, HdbscanHyperParams}; +use tracing::{debug, info}; +use ulid::Ulid; + +use crate::config::ExtractionConfig; +use crate::error::TopicsError; +use crate::labeling::{ClusterDocument, TopicLabel, TopicLabeler}; +use crate::similarity::calculate_centroid; +use crate::types::Topic; + +/// Input for topic extraction: node ID with its embedding. +#[derive(Debug, Clone)] +pub struct NodeEmbedding { + /// TOC node identifier + pub node_id: String, + /// Embedding vector + pub embedding: Vec, + /// Summary text (for keyword extraction) + pub summary: String, +} + +/// Result of clustering: cluster ID to node IDs. +#[derive(Debug)] +pub struct ClusterResult { + /// Cluster label (-1 = noise) + pub label: i32, + /// Node IDs in this cluster + pub node_ids: Vec, + /// Node embeddings in this cluster + pub embeddings: Vec>, + /// Summaries for keyword extraction + pub summaries: Vec, +} + +/// Topic extractor using HDBSCAN clustering. +pub struct TopicExtractor { + config: ExtractionConfig, +} + +impl TopicExtractor { + /// Create a new topic extractor. + pub fn new(config: ExtractionConfig) -> Self { + Self { config } + } + + /// Cluster embeddings using HDBSCAN. + /// + /// Returns clusters grouped by label. Label -1 indicates noise points. + pub fn cluster(&self, nodes: &[NodeEmbedding]) -> Result, TopicsError> { + if nodes.len() < self.config.min_cluster_size { + debug!( + count = nodes.len(), + min = self.config.min_cluster_size, + "Not enough nodes for clustering" + ); + return Ok(vec![]); + } + + info!(count = nodes.len(), "Starting HDBSCAN clustering"); + + // Convert to 2D array format expected by hdbscan (f64) + let data: Vec> = nodes + .iter() + .map(|n| n.embedding.iter().map(|&x| x as f64).collect()) + .collect(); + + // Create clusterer with custom params + let params = HdbscanHyperParams::builder() + .min_cluster_size(self.config.min_cluster_size) + .build(); + + let clusterer = Hdbscan::new(&data, params); + + // Run clustering (blocking - should be called from spawn_blocking) + let labels = clusterer + .cluster() + .map_err(|e| TopicsError::Clustering(e.to_string()))?; + + info!( + labels = labels.len(), + unique = count_unique_clusters(&labels), + "Clustering complete" + ); + + // Group nodes by cluster label + let clusters = group_by_cluster(nodes, &labels); + + Ok(clusters) + } + + /// Create topics from cluster results. + /// + /// Each cluster becomes a topic with a centroid embedding. + pub fn create_topics(&self, clusters: &[ClusterResult]) -> Vec { + clusters + .iter() + .filter(|c| c.label >= 0) // Skip noise + .map(|cluster| { + let topic_id = Ulid::new().to_string(); + + // Calculate centroid + let embedding_refs: Vec<&[f32]> = + cluster.embeddings.iter().map(|e| e.as_slice()).collect(); + let centroid = calculate_centroid(&embedding_refs); + + // Create topic with placeholder label (labeling is Plan 14-02) + let mut topic = Topic::new( + topic_id, + format!("Topic {}", cluster.label), // Placeholder + centroid, + ); + topic.node_count = cluster.node_ids.len() as u32; + + topic + }) + .collect() + } + + /// Get configuration. + pub fn config(&self) -> &ExtractionConfig { + &self.config + } + + /// Create topics from cluster results with labeling. + /// + /// Each cluster becomes a topic with a centroid embedding and a label + /// generated by the provided labeler. + pub fn create_labeled_topics( + &self, + clusters: &[ClusterResult], + labeler: &dyn TopicLabeler, + ) -> Result, TopicsError> { + let mut topics = Vec::new(); + + for cluster in clusters.iter().filter(|c| c.label >= 0) { + let topic_id = Ulid::new().to_string(); + + // Calculate centroid + let embedding_refs: Vec<&[f32]> = + cluster.embeddings.iter().map(|e| e.as_slice()).collect(); + let centroid = calculate_centroid(&embedding_refs); + + // Create cluster documents for labeling + let documents: Vec = cluster + .node_ids + .iter() + .zip(cluster.summaries.iter()) + .map(|(id, summary)| ClusterDocument::new(id.clone(), summary.clone())) + .collect(); + + // Generate label using the labeler + let TopicLabel { + label, + keywords, + confidence: _, + } = labeler.label_cluster(&documents)?; + + // Create topic with generated label + let mut topic = Topic::new(topic_id, label, centroid); + topic.node_count = cluster.node_ids.len() as u32; + topic.keywords = keywords; + + topics.push(topic); + } + + Ok(topics) + } +} + +/// Count unique non-noise clusters. +fn count_unique_clusters(labels: &[i32]) -> usize { + let mut unique: std::collections::HashSet = labels.iter().copied().collect(); + unique.remove(&-1); // Remove noise label + unique.len() +} + +/// Group nodes by cluster label. +fn group_by_cluster(nodes: &[NodeEmbedding], labels: &[i32]) -> Vec { + use std::collections::HashMap; + + let mut groups: HashMap = HashMap::new(); + + for (node, &label) in nodes.iter().zip(labels.iter()) { + let cluster = groups.entry(label).or_insert_with(|| ClusterResult { + label, + node_ids: Vec::new(), + embeddings: Vec::new(), + summaries: Vec::new(), + }); + cluster.node_ids.push(node.node_id.clone()); + cluster.embeddings.push(node.embedding.clone()); + cluster.summaries.push(node.summary.clone()); + } + + groups.into_values().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_node(id: &str, embedding: Vec) -> NodeEmbedding { + NodeEmbedding { + node_id: id.to_string(), + embedding, + summary: format!("Summary for {}", id), + } + } + + #[test] + fn test_cluster_insufficient_nodes() { + let config = ExtractionConfig { + min_cluster_size: 5, + ..Default::default() + }; + let extractor = TopicExtractor::new(config); + + let nodes = vec![ + make_node("n1", vec![1.0, 0.0, 0.0]), + make_node("n2", vec![0.0, 1.0, 0.0]), + ]; + + let result = extractor.cluster(&nodes).unwrap(); + assert!(result.is_empty()); + } + + #[test] + fn test_count_unique_clusters() { + let labels = vec![0, 0, 1, 1, -1, 2, -1]; + assert_eq!(count_unique_clusters(&labels), 3); // 0, 1, 2 + } + + #[test] + fn test_count_unique_clusters_all_noise() { + let labels = vec![-1, -1, -1]; + assert_eq!(count_unique_clusters(&labels), 0); + } + + #[test] + fn test_count_unique_clusters_empty() { + let labels: Vec = vec![]; + assert_eq!(count_unique_clusters(&labels), 0); + } + + #[test] + fn test_group_by_cluster() { + let nodes = vec![ + make_node("n1", vec![1.0]), + make_node("n2", vec![2.0]), + make_node("n3", vec![3.0]), + ]; + let labels = vec![0, 0, 1]; + + let groups = group_by_cluster(&nodes, &labels); + assert_eq!(groups.len(), 2); + + let cluster_0 = groups.iter().find(|c| c.label == 0).unwrap(); + assert_eq!(cluster_0.node_ids.len(), 2); + + let cluster_1 = groups.iter().find(|c| c.label == 1).unwrap(); + assert_eq!(cluster_1.node_ids.len(), 1); + } + + #[test] + fn test_group_by_cluster_with_noise() { + let nodes = vec![ + make_node("n1", vec![1.0]), + make_node("n2", vec![2.0]), + make_node("n3", vec![3.0]), + ]; + let labels = vec![0, -1, 0]; + + let groups = group_by_cluster(&nodes, &labels); + assert_eq!(groups.len(), 2); // cluster 0 and noise (-1) + + let noise = groups.iter().find(|c| c.label == -1).unwrap(); + assert_eq!(noise.node_ids.len(), 1); + assert_eq!(noise.node_ids[0], "n2"); + } + + #[test] + fn test_create_topics_skips_noise() { + let config = ExtractionConfig::default(); + let extractor = TopicExtractor::new(config); + + let clusters = vec![ + ClusterResult { + label: -1, // Noise + node_ids: vec!["n1".to_string()], + embeddings: vec![vec![1.0, 0.0]], + summaries: vec!["Summary".to_string()], + }, + ClusterResult { + label: 0, + node_ids: vec!["n2".to_string(), "n3".to_string()], + embeddings: vec![vec![1.0, 0.0], vec![0.0, 1.0]], + summaries: vec!["Sum1".to_string(), "Sum2".to_string()], + }, + ]; + + let topics = extractor.create_topics(&clusters); + assert_eq!(topics.len(), 1); // Only non-noise cluster + assert_eq!(topics[0].node_count, 2); + } + + #[test] + fn test_create_topics_generates_ulid() { + let config = ExtractionConfig::default(); + let extractor = TopicExtractor::new(config); + + let clusters = vec![ClusterResult { + label: 0, + node_ids: vec!["n1".to_string()], + embeddings: vec![vec![1.0, 0.0]], + summaries: vec!["Summary".to_string()], + }]; + + let topics = extractor.create_topics(&clusters); + assert_eq!(topics.len(), 1); + // ULID is 26 characters + assert_eq!(topics[0].topic_id.len(), 26); + } + + #[test] + fn test_create_topics_empty_clusters() { + let config = ExtractionConfig::default(); + let extractor = TopicExtractor::new(config); + + let clusters: Vec = vec![]; + let topics = extractor.create_topics(&clusters); + assert!(topics.is_empty()); + } + + #[test] + fn test_extractor_config() { + let config = ExtractionConfig { + min_cluster_size: 10, + similarity_threshold: 0.9, + ..Default::default() + }; + let extractor = TopicExtractor::new(config); + + assert_eq!(extractor.config().min_cluster_size, 10); + assert!((extractor.config().similarity_threshold - 0.9).abs() < f32::EPSILON); + } + + #[test] + fn test_create_labeled_topics() { + use crate::config::LabelingConfig; + use crate::labeling::KeywordLabeler; + + let config = ExtractionConfig::default(); + let extractor = TopicExtractor::new(config); + let labeler = KeywordLabeler::new(LabelingConfig::default()); + + let clusters = vec![ClusterResult { + label: 0, + node_ids: vec!["n1".to_string(), "n2".to_string()], + embeddings: vec![vec![1.0, 0.0], vec![0.0, 1.0]], + summaries: vec![ + "rust programming systems memory".to_string(), + "rust ownership borrowing safety".to_string(), + ], + }]; + + let topics = extractor + .create_labeled_topics(&clusters, &labeler) + .unwrap(); + assert_eq!(topics.len(), 1); + assert!(!topics[0].label.is_empty()); + assert!(!topics[0].keywords.is_empty()); + assert_eq!(topics[0].node_count, 2); + } + + #[test] + fn test_create_labeled_topics_skips_noise() { + use crate::config::LabelingConfig; + use crate::labeling::KeywordLabeler; + + let config = ExtractionConfig::default(); + let extractor = TopicExtractor::new(config); + let labeler = KeywordLabeler::new(LabelingConfig::default()); + + let clusters = vec![ + ClusterResult { + label: -1, // Noise - should be skipped + node_ids: vec!["noise".to_string()], + embeddings: vec![vec![1.0, 0.0]], + summaries: vec!["noise summary".to_string()], + }, + ClusterResult { + label: 0, + node_ids: vec!["valid".to_string()], + embeddings: vec![vec![0.0, 1.0]], + summaries: vec!["valid summary content".to_string()], + }, + ]; + + let topics = extractor + .create_labeled_topics(&clusters, &labeler) + .unwrap(); + assert_eq!(topics.len(), 1); // Only non-noise cluster + } + + #[test] + fn test_create_labeled_topics_empty() { + use crate::config::LabelingConfig; + use crate::labeling::KeywordLabeler; + + let config = ExtractionConfig::default(); + let extractor = TopicExtractor::new(config); + let labeler = KeywordLabeler::new(LabelingConfig::default()); + + let clusters: Vec = vec![]; + let topics = extractor + .create_labeled_topics(&clusters, &labeler) + .unwrap(); + assert!(topics.is_empty()); + } +} diff --git a/crates/memory-topics/src/importance.rs b/crates/memory-topics/src/importance.rs new file mode 100644 index 0000000..d042da6 --- /dev/null +++ b/crates/memory-topics/src/importance.rs @@ -0,0 +1,493 @@ +//! Time-decayed importance scoring for topics. +//! +//! Uses an exponential decay model with half-life to calculate topic importance. +//! Recent mentions receive a boost, and scores never decay below a minimum threshold. + +use chrono::{DateTime, Utc}; + +use crate::config::ImportanceConfig; +use crate::types::Topic; + +/// Minimum importance score to prevent decay to zero. +const DEFAULT_MIN_SCORE: f64 = 0.01; + +/// Days threshold for maximum recency boost. +const RECENCY_BOOST_THRESHOLD_DAYS: f64 = 1.0; + +/// Days over which recency boost linearly decays. +const RECENCY_BOOST_DECAY_DAYS: f64 = 7.0; + +/// Calculates time-decayed importance scores for topics. +/// +/// The scoring formula combines: +/// - Base score from node count (logarithmic scaling) +/// - Exponential decay based on time since last mention +/// - Recency boost for very recent mentions (< 7 days) +/// +/// # Example +/// ``` +/// use memory_topics::importance::ImportanceScorer; +/// use memory_topics::config::ImportanceConfig; +/// use chrono::Utc; +/// +/// let config = ImportanceConfig::default(); +/// let scorer = ImportanceScorer::new(config); +/// +/// let score = scorer.calculate_score(10, Utc::now(), Utc::now()); +/// assert!(score > 0.0); +/// ``` +pub struct ImportanceScorer { + config: ImportanceConfig, + min_score: f64, +} + +impl ImportanceScorer { + /// Create a new importance scorer with the given configuration. + pub fn new(config: ImportanceConfig) -> Self { + Self { + config, + min_score: DEFAULT_MIN_SCORE, + } + } + + /// Create a scorer with a custom minimum score. + pub fn with_min_score(config: ImportanceConfig, min_score: f64) -> Self { + Self { config, min_score } + } + + /// Get the configured half-life in days. + pub fn half_life_days(&self) -> u32 { + self.config.half_life_days + } + + /// Get the configured recency boost factor. + pub fn recency_boost_factor(&self) -> f64 { + self.config.recency_boost + } + + /// Get the minimum score threshold. + pub fn min_score(&self) -> f64 { + self.min_score + } + + /// Calculate importance score for a topic. + /// + /// Formula: `score = base_score * decay_factor * recency_boost` + /// + /// - `base_score = ln(1 + node_count)` (logarithmic to prevent large clusters from dominating) + /// - `decay_factor = 2^(-days_since_mention / half_life)` (exponential decay) + /// - `recency_boost` = boost factor for mentions within 7 days + /// + /// # Arguments + /// * `node_count` - Number of nodes linked to the topic + /// * `last_mentioned_at` - Timestamp of last topic mention + /// * `now` - Current timestamp for decay calculation + /// + /// # Returns + /// Importance score (always >= min_score) + pub fn calculate_score( + &self, + node_count: u32, + last_mentioned_at: DateTime, + now: DateTime, + ) -> f64 { + let days_since = self.days_between(last_mentioned_at, now); + let base = self.base_score(node_count); + let decay = self.decay_factor(days_since); + let boost = self.recency_boost(days_since); + + (base * decay * boost).max(self.min_score) + } + + /// Update topic importance based on new mention. + /// + /// Updates the topic's `last_mentioned_at` timestamp and recalculates + /// the importance score. Also increments `node_count` by 1. + /// + /// # Arguments + /// * `topic` - Topic to update (mutated in place) + /// * `now` - Current timestamp + pub fn on_topic_mentioned(&self, topic: &mut Topic, now: DateTime) { + topic.last_mentioned_at = now; + topic.node_count += 1; + topic.importance_score = + self.calculate_score(topic.node_count, topic.last_mentioned_at, now); + } + + /// Touch a topic without incrementing node count. + /// + /// Updates the `last_mentioned_at` timestamp and recalculates importance + /// without changing the node count. Useful for re-references to existing links. + /// + /// # Arguments + /// * `topic` - Topic to update (mutated in place) + /// * `now` - Current timestamp + pub fn touch_topic(&self, topic: &mut Topic, now: DateTime) { + topic.last_mentioned_at = now; + topic.importance_score = + self.calculate_score(topic.node_count, topic.last_mentioned_at, now); + } + + /// Batch recalculate all topic scores. + /// + /// Useful for periodic refresh jobs to update decayed scores. + /// + /// # Arguments + /// * `topics` - Slice of topics to update (mutated in place) + /// * `now` - Current timestamp for decay calculation + /// + /// # Returns + /// Number of topics updated + pub fn recalculate_all(&self, topics: &mut [Topic], now: DateTime) -> u32 { + let mut count = 0; + for topic in topics.iter_mut() { + let new_score = self.calculate_score(topic.node_count, topic.last_mentioned_at, now); + if (new_score - topic.importance_score).abs() > f64::EPSILON { + topic.importance_score = new_score; + count += 1; + } + } + count + } + + /// Calculate exponential decay factor. + /// + /// Uses the half-life formula: `decay = 2^(-days / half_life)` + /// + /// # Returns + /// Value between min_score and 1.0 + fn decay_factor(&self, days_since: f64) -> f64 { + let half_life = f64::from(self.config.half_life_days); + let factor = 2.0_f64.powf(-days_since / half_life); + factor.max(self.min_score) + } + + /// Calculate base score from node count using logarithmic scaling. + /// + /// Uses `ln(1 + node_count)` to prevent topics with many nodes + /// from dominating while still rewarding larger clusters. + fn base_score(&self, node_count: u32) -> f64 { + (1.0 + f64::from(node_count)).ln() + } + + /// Calculate recency boost for very recent mentions. + /// + /// - Mentions < 1 day ago: full boost + /// - Mentions 1-7 days ago: linear decay from boost to 1.0 + /// - Mentions > 7 days ago: no boost (1.0) + fn recency_boost(&self, days_since: f64) -> f64 { + if days_since < RECENCY_BOOST_THRESHOLD_DAYS { + // Full boost for very recent mentions + self.config.recency_boost + } else if days_since < RECENCY_BOOST_DECAY_DAYS { + // Linear decay from boost to 1.0 over the decay period + let remaining_boost = self.config.recency_boost - 1.0; + let decay_progress = (days_since - RECENCY_BOOST_THRESHOLD_DAYS) + / (RECENCY_BOOST_DECAY_DAYS - RECENCY_BOOST_THRESHOLD_DAYS); + 1.0 + remaining_boost * (1.0 - decay_progress) + } else { + // No boost for older mentions + 1.0 + } + } + + /// Calculate days between two timestamps. + fn days_between(&self, from: DateTime, to: DateTime) -> f64 { + let duration = to.signed_duration_since(from); + duration.num_seconds() as f64 / 86400.0 + } +} + +impl Default for ImportanceScorer { + fn default() -> Self { + Self::new(ImportanceConfig::default()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Duration; + + fn create_test_topic() -> Topic { + Topic::new( + "test-topic-id".to_string(), + "Test Topic".to_string(), + vec![0.1, 0.2, 0.3], + ) + } + + #[test] + fn test_scorer_defaults() { + let scorer = ImportanceScorer::default(); + assert_eq!(scorer.half_life_days(), 30); + assert!((scorer.recency_boost_factor() - 2.0).abs() < f64::EPSILON); + assert!((scorer.min_score() - DEFAULT_MIN_SCORE).abs() < f64::EPSILON); + } + + #[test] + fn test_scorer_with_custom_min_score() { + let config = ImportanceConfig::default(); + let scorer = ImportanceScorer::with_min_score(config, 0.05); + assert!((scorer.min_score() - 0.05).abs() < f64::EPSILON); + } + + #[test] + fn test_base_score_logarithmic() { + let scorer = ImportanceScorer::default(); + + // Base score uses ln(1 + node_count) + let score_0 = scorer.base_score(0); + let score_1 = scorer.base_score(1); + let score_10 = scorer.base_score(10); + let score_100 = scorer.base_score(100); + + // Verify logarithmic scaling + assert!((score_0 - 1.0_f64.ln()).abs() < f64::EPSILON); // ln(1) = 0 + assert!((score_1 - 2.0_f64.ln()).abs() < f64::EPSILON); // ln(2) + assert!((score_10 - 11.0_f64.ln()).abs() < f64::EPSILON); // ln(11) + assert!((score_100 - 101.0_f64.ln()).abs() < f64::EPSILON); // ln(101) + + // Verify diminishing returns + assert!(score_10 < score_100); + assert!((score_100 - score_10) < (score_10 - score_0)); + } + + #[test] + fn test_decay_factor_at_half_life() { + let scorer = ImportanceScorer::default(); + + // At exactly one half-life, decay should be 0.5 + let decay = scorer.decay_factor(30.0); + assert!((decay - 0.5).abs() < 0.001); + } + + #[test] + fn test_decay_factor_at_zero() { + let scorer = ImportanceScorer::default(); + + // At time zero, no decay + let decay = scorer.decay_factor(0.0); + assert!((decay - 1.0).abs() < f64::EPSILON); + } + + #[test] + fn test_decay_factor_at_two_half_lives() { + let scorer = ImportanceScorer::default(); + + // At two half-lives, decay should be 0.25 + let decay = scorer.decay_factor(60.0); + assert!((decay - 0.25).abs() < 0.001); + } + + #[test] + fn test_decay_factor_never_below_min() { + let scorer = ImportanceScorer::default(); + + // Even after many half-lives, score stays above minimum + let decay = scorer.decay_factor(365.0); // ~12 half-lives + assert!(decay >= scorer.min_score()); + } + + #[test] + fn test_recency_boost_very_recent() { + let scorer = ImportanceScorer::default(); + + // Less than 1 day: full boost + let boost = scorer.recency_boost(0.5); + assert!((boost - 2.0).abs() < f64::EPSILON); + } + + #[test] + fn test_recency_boost_within_week() { + let scorer = ImportanceScorer::default(); + + // At 4 days (midpoint of 1-7 day range): partial boost + let boost = scorer.recency_boost(4.0); + // Should be halfway between 2.0 and 1.0 + assert!(boost > 1.0); + assert!(boost < 2.0); + assert!((boost - 1.5).abs() < 0.01); + } + + #[test] + fn test_recency_boost_after_week() { + let scorer = ImportanceScorer::default(); + + // After 7 days: no boost + let boost = scorer.recency_boost(10.0); + assert!((boost - 1.0).abs() < f64::EPSILON); + } + + #[test] + fn test_calculate_score_combines_factors() { + let scorer = ImportanceScorer::default(); + let now = Utc::now(); + let mentioned_now = now; + let mentioned_30_days_ago = now - Duration::days(30); + + // Score for recent mention with 10 nodes + let recent_score = scorer.calculate_score(10, mentioned_now, now); + + // Score for old mention with same nodes + let old_score = scorer.calculate_score(10, mentioned_30_days_ago, now); + + // Recent should be higher due to recency boost and no decay + assert!(recent_score > old_score); + + // Old score should be approximately half (one half-life) * no recency boost + // recent = ln(11) * 1.0 * 2.0 = ~4.79 + // old = ln(11) * 0.5 * 1.0 = ~1.20 + assert!(recent_score > old_score * 2.0); + } + + #[test] + fn test_calculate_score_minimum() { + let scorer = ImportanceScorer::default(); + let now = Utc::now(); + let ancient = now - Duration::days(365 * 10); // 10 years ago + + // Even very old topics have minimum score + let score = scorer.calculate_score(0, ancient, now); + assert!(score >= scorer.min_score()); + } + + #[test] + fn test_on_topic_mentioned() { + let scorer = ImportanceScorer::default(); + let mut topic = create_test_topic(); + let initial_node_count = topic.node_count; + let initial_score = topic.importance_score; + + let now = Utc::now(); + scorer.on_topic_mentioned(&mut topic, now); + + // Node count should increase + assert_eq!(topic.node_count, initial_node_count + 1); + // Timestamp should update + assert_eq!(topic.last_mentioned_at, now); + // Score should be recalculated (and likely increase due to recency boost) + assert!(topic.importance_score > 0.0); + assert_ne!(topic.importance_score, initial_score); + } + + #[test] + fn test_touch_topic_no_node_increment() { + let scorer = ImportanceScorer::default(); + let mut topic = create_test_topic(); + topic.node_count = 5; + let initial_node_count = topic.node_count; + + let now = Utc::now(); + scorer.touch_topic(&mut topic, now); + + // Node count should NOT increase + assert_eq!(topic.node_count, initial_node_count); + // Timestamp should update + assert_eq!(topic.last_mentioned_at, now); + } + + #[test] + fn test_recalculate_all() { + let scorer = ImportanceScorer::default(); + let base_time = Utc::now() - Duration::days(15); + + let mut topics = vec![ + Topic::new("t1".to_string(), "Topic 1".to_string(), vec![0.1]), + Topic::new("t2".to_string(), "Topic 2".to_string(), vec![0.2]), + ]; + + // Set different ages + topics[0].node_count = 5; + topics[0].last_mentioned_at = base_time; + topics[0].importance_score = 0.0; + + topics[1].node_count = 10; + topics[1].last_mentioned_at = base_time - Duration::days(30); + topics[1].importance_score = 0.0; + + let now = Utc::now(); + let updated = scorer.recalculate_all(&mut topics, now); + + assert_eq!(updated, 2); + assert!(topics[0].importance_score > 0.0); + assert!(topics[1].importance_score > 0.0); + // Topic 0 (more recent) should have higher score despite fewer nodes + assert!(topics[0].importance_score > topics[1].importance_score); + } + + #[test] + fn test_recalculate_all_skips_unchanged() { + let scorer = ImportanceScorer::default(); + let now = Utc::now(); + + let mut topics = vec![Topic::new( + "t1".to_string(), + "Topic 1".to_string(), + vec![0.1], + )]; + + // Pre-calculate correct score + topics[0].node_count = 5; + topics[0].last_mentioned_at = now; + topics[0].importance_score = + scorer.calculate_score(topics[0].node_count, topics[0].last_mentioned_at, now); + + // Should skip since score is already correct + let updated = scorer.recalculate_all(&mut topics, now); + assert_eq!(updated, 0); + } + + #[test] + fn test_days_between() { + let scorer = ImportanceScorer::default(); + let now = Utc::now(); + let yesterday = now - Duration::days(1); + let week_ago = now - Duration::days(7); + + assert!((scorer.days_between(yesterday, now) - 1.0).abs() < 0.001); + assert!((scorer.days_between(week_ago, now) - 7.0).abs() < 0.001); + } + + #[test] + fn test_days_between_fractional() { + let scorer = ImportanceScorer::default(); + let now = Utc::now(); + let twelve_hours_ago = now - Duration::hours(12); + + let days = scorer.days_between(twelve_hours_ago, now); + assert!((days - 0.5).abs() < 0.001); + } + + #[test] + fn test_importance_ordering() { + let scorer = ImportanceScorer::default(); + let now = Utc::now(); + + // Create scenarios with different combinations + let scenarios = [ + (10, now, "recent-many"), // Recent, many nodes + (10, now - Duration::days(30), "old-many"), // Old, many nodes + (1, now, "recent-few"), // Recent, few nodes + (1, now - Duration::days(30), "old-few"), // Old, few nodes + ]; + + let scores: Vec<(f64, &str)> = scenarios + .iter() + .map(|(nodes, time, label)| (scorer.calculate_score(*nodes, *time, now), *label)) + .collect(); + + // Verify expected ordering: recent-many > old-many >= recent-few > old-few + assert!( + scores.iter().find(|(_, l)| *l == "recent-many").unwrap().0 + > scores.iter().find(|(_, l)| *l == "old-many").unwrap().0, + "Recent with many nodes should beat old with many nodes" + ); + + assert!( + scores.iter().find(|(_, l)| *l == "recent-few").unwrap().0 + > scores.iter().find(|(_, l)| *l == "old-few").unwrap().0, + "Recent with few nodes should beat old with few nodes" + ); + } +} diff --git a/crates/memory-topics/src/labeling.rs b/crates/memory-topics/src/labeling.rs new file mode 100644 index 0000000..482442c --- /dev/null +++ b/crates/memory-topics/src/labeling.rs @@ -0,0 +1,350 @@ +//! Topic labeling using keyword extraction. +//! +//! Provides label generation for topic clusters using TF-IDF keyword extraction. + +use crate::config::LabelingConfig; +use crate::error::TopicsError; +use crate::tfidf::TfIdf; + +/// Document within a cluster for labeling. +#[derive(Debug, Clone)] +pub struct ClusterDocument { + /// Document identifier + pub doc_id: String, + /// Full text content + pub text: String, + /// Pre-extracted keywords (optional) + pub keywords: Vec, +} + +impl ClusterDocument { + /// Create a new cluster document. + pub fn new(doc_id: String, text: String) -> Self { + Self { + doc_id, + text, + keywords: Vec::new(), + } + } + + /// Create a cluster document with pre-extracted keywords. + pub fn with_keywords(doc_id: String, text: String, keywords: Vec) -> Self { + Self { + doc_id, + text, + keywords, + } + } +} + +/// Generated topic label with metadata. +#[derive(Debug, Clone)] +pub struct TopicLabel { + /// Human-readable label (2-5 words) + pub label: String, + /// Top keywords for this topic + pub keywords: Vec, + /// Confidence score (0.0 - 1.0) + pub confidence: f32, +} + +impl TopicLabel { + /// Create a new topic label. + pub fn new(label: String, keywords: Vec, confidence: f32) -> Self { + Self { + label, + keywords, + confidence, + } + } +} + +/// Trait for generating topic labels from cluster documents. +pub trait TopicLabeler: Send + Sync { + /// Generate a label for a cluster of documents. + fn label_cluster(&self, documents: &[ClusterDocument]) -> Result; +} + +/// Keyword-based topic labeler using TF-IDF. +/// +/// This is the default labeler that requires no external dependencies. +/// It extracts top keywords using TF-IDF scoring and generates a concise label. +pub struct KeywordLabeler { + config: LabelingConfig, +} + +impl KeywordLabeler { + /// Create a new keyword labeler. + pub fn new(config: LabelingConfig) -> Self { + Self { config } + } + + /// Extract keywords using TF-IDF scoring. + /// + /// Returns keywords sorted by TF-IDF score (highest first). + fn extract_keywords(&self, documents: &[ClusterDocument]) -> Vec<(String, f32)> { + if documents.is_empty() { + return Vec::new(); + } + + // Collect all texts + let texts: Vec<&str> = documents.iter().map(|d| d.text.as_str()).collect(); + + // Use TF-IDF to extract keywords + let tfidf = TfIdf::new(&texts); + + // Get aggregated scores across all documents in the cluster + tfidf.top_terms(self.config.top_keywords * 2) // Get extra for filtering + } + + /// Generate a label from top keywords. + /// + /// Creates a concise 2-5 word label from the most important keywords. + fn generate_label(&self, keywords: &[(String, f32)]) -> String { + if keywords.is_empty() { + return "Unknown Topic".to_string(); + } + + // Take top keywords for label (max 5) + let label_words: Vec<&str> = keywords + .iter() + .take(5) + .map(|(word, _)| word.as_str()) + .collect(); + + // Join with spaces and truncate if needed + let label = label_words.join(" "); + self.truncate_label(&label) + } + + /// Truncate label to max length, breaking at word boundary. + fn truncate_label(&self, label: &str) -> String { + if label.len() <= self.config.max_label_length { + return label.to_string(); + } + + // Find last space before max length + let truncated = &label[..self.config.max_label_length]; + if let Some(last_space) = truncated.rfind(' ') { + truncated[..last_space].to_string() + } else { + truncated.to_string() + } + } + + /// Calculate confidence based on keyword distribution. + /// + /// Higher confidence when keywords have distinct high scores. + fn calculate_confidence(&self, keywords: &[(String, f32)]) -> f32 { + if keywords.is_empty() { + return 0.0; + } + if keywords.len() == 1 { + return keywords[0].1.min(1.0); + } + + // Confidence based on top keyword score relative to sum + let top_score = keywords[0].1; + let total_score: f32 = keywords.iter().map(|(_, s)| s).sum(); + + if total_score == 0.0 { + return 0.0; + } + + // Normalize: if top keyword dominates, confidence is higher + let ratio = top_score / total_score; + + // Scale to 0.5-1.0 range (having any keywords gives at least 0.5) + 0.5 + (ratio * 0.5) + } +} + +impl TopicLabeler for KeywordLabeler { + fn label_cluster(&self, documents: &[ClusterDocument]) -> Result { + if documents.is_empty() { + return Err(TopicsError::InvalidInput( + "Cannot label empty cluster".to_string(), + )); + } + + // Extract keywords using TF-IDF + let keywords = self.extract_keywords(documents); + + // Generate label from top keywords + let label = self.generate_label(&keywords); + + // Calculate confidence + let confidence = self.calculate_confidence(&keywords); + + // Return top N keywords as configured + let top_keywords: Vec = keywords + .into_iter() + .take(self.config.top_keywords) + .map(|(word, _)| word) + .collect(); + + Ok(TopicLabel::new(label, top_keywords, confidence)) + } +} + +impl Default for KeywordLabeler { + fn default() -> Self { + Self::new(LabelingConfig::default()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_doc(id: &str, text: &str) -> ClusterDocument { + ClusterDocument::new(id.to_string(), text.to_string()) + } + + #[test] + fn test_cluster_document_new() { + let doc = ClusterDocument::new("doc1".to_string(), "Some text".to_string()); + assert_eq!(doc.doc_id, "doc1"); + assert_eq!(doc.text, "Some text"); + assert!(doc.keywords.is_empty()); + } + + #[test] + fn test_cluster_document_with_keywords() { + let doc = ClusterDocument::with_keywords( + "doc1".to_string(), + "Some text".to_string(), + vec!["keyword1".to_string(), "keyword2".to_string()], + ); + assert_eq!(doc.keywords.len(), 2); + } + + #[test] + fn test_topic_label_new() { + let label = TopicLabel::new("Test Label".to_string(), vec!["keyword1".to_string()], 0.85); + assert_eq!(label.label, "Test Label"); + assert_eq!(label.keywords.len(), 1); + assert!((label.confidence - 0.85).abs() < f32::EPSILON); + } + + #[test] + fn test_keyword_labeler_empty_documents() { + let labeler = KeywordLabeler::default(); + let result = labeler.label_cluster(&[]); + assert!(result.is_err()); + } + + #[test] + fn test_keyword_labeler_single_document() { + let labeler = KeywordLabeler::default(); + let docs = vec![make_doc( + "d1", + "machine learning algorithms neural networks deep learning", + )]; + + let result = labeler.label_cluster(&docs).unwrap(); + assert!(!result.label.is_empty()); + assert!(result.confidence > 0.0); + } + + #[test] + fn test_keyword_labeler_multiple_documents() { + let labeler = KeywordLabeler::default(); + let docs = vec![ + make_doc("d1", "machine learning algorithms for classification"), + make_doc("d2", "deep learning neural networks training"), + make_doc("d3", "machine learning model optimization techniques"), + ]; + + let result = labeler.label_cluster(&docs).unwrap(); + assert!(!result.label.is_empty()); + assert!(!result.keywords.is_empty()); + assert!(result.confidence >= 0.5); + } + + #[test] + fn test_truncate_label_short() { + let config = LabelingConfig { + max_label_length: 50, + ..Default::default() + }; + let labeler = KeywordLabeler::new(config); + + let label = labeler.truncate_label("short label"); + assert_eq!(label, "short label"); + } + + #[test] + fn test_truncate_label_long() { + let config = LabelingConfig { + max_label_length: 20, + ..Default::default() + }; + let labeler = KeywordLabeler::new(config); + + let label = labeler.truncate_label("this is a very long label that needs truncation"); + assert!(label.len() <= 20); + assert!(!label.ends_with(' ')); // Should break at word boundary + } + + #[test] + fn test_generate_label_empty_keywords() { + let labeler = KeywordLabeler::default(); + let label = labeler.generate_label(&[]); + assert_eq!(label, "Unknown Topic"); + } + + #[test] + fn test_calculate_confidence_empty() { + let labeler = KeywordLabeler::default(); + let confidence = labeler.calculate_confidence(&[]); + assert!((confidence - 0.0).abs() < f32::EPSILON); + } + + #[test] + fn test_calculate_confidence_single() { + let labeler = KeywordLabeler::default(); + let keywords = vec![("word".to_string(), 0.8)]; + let confidence = labeler.calculate_confidence(&keywords); + assert!((confidence - 0.8).abs() < f32::EPSILON); + } + + #[test] + fn test_calculate_confidence_dominant_keyword() { + let labeler = KeywordLabeler::default(); + let keywords = vec![("dominant".to_string(), 0.9), ("minor".to_string(), 0.1)]; + let confidence = labeler.calculate_confidence(&keywords); + // 0.9 / 1.0 = 0.9, scaled to 0.5 + 0.45 = 0.95 + assert!(confidence > 0.9); + } + + #[test] + fn test_calculate_confidence_even_distribution() { + let labeler = KeywordLabeler::default(); + let keywords = vec![ + ("word1".to_string(), 0.25), + ("word2".to_string(), 0.25), + ("word3".to_string(), 0.25), + ("word4".to_string(), 0.25), + ]; + let confidence = labeler.calculate_confidence(&keywords); + // 0.25 / 1.0 = 0.25, scaled to 0.5 + 0.125 = 0.625 + assert!((confidence - 0.625).abs() < 0.01); + } + + #[test] + fn test_extract_keywords_returns_scored_terms() { + let labeler = KeywordLabeler::default(); + let docs = vec![ + make_doc("d1", "rust programming language systems"), + make_doc("d2", "rust memory safety ownership"), + ]; + + let keywords = labeler.extract_keywords(&docs); + assert!(!keywords.is_empty()); + // Keywords should be sorted by score (descending) + for i in 1..keywords.len() { + assert!(keywords[i - 1].1 >= keywords[i].1); + } + } +} diff --git a/crates/memory-topics/src/lib.rs b/crates/memory-topics/src/lib.rs new file mode 100644 index 0000000..6bf19a1 --- /dev/null +++ b/crates/memory-topics/src/lib.rs @@ -0,0 +1,49 @@ +//! # memory-topics +//! +//! Semantic topic extraction and management for Agent Memory. +//! +//! This crate enables conceptual discovery through topics extracted from +//! TOC summaries using embedding clustering. Topics have time-decayed +//! importance scores and can form relationships (similar, parent/child). +//! +//! ## Features +//! - HDBSCAN clustering for automatic topic detection +//! - TF-IDF keyword extraction for topic labeling +//! - Optional LLM-enhanced labeling with keyword fallback +//! - Time-decayed importance scoring +//! - Topic relationships (similar, parent, child) +//! - Optional feature - disabled by default +//! +//! ## Requirements +//! - TOPIC-01: Topic extraction from TOC summaries +//! - TOPIC-02: Topics stored in CF_TOPICS +//! - TOPIC-07: Optional via configuration +//! - TOPIC-08: GetTopicGraphStatus RPC for discovery + +pub mod config; +pub mod error; +pub mod extraction; +pub mod importance; +pub mod labeling; +pub mod lifecycle; +pub mod llm_labeler; +pub mod relationships; +pub mod similarity; +pub mod storage; +pub mod tfidf; +pub mod types; + +pub use config::{ImportanceConfig, LabelingConfig, TopicsConfig}; +pub use error::TopicsError; +pub use extraction::TopicExtractor; +pub use importance::ImportanceScorer; +pub use labeling::{ClusterDocument, KeywordLabeler, TopicLabel, TopicLabeler}; +pub use lifecycle::{LifecycleStats, TopicLifecycleManager}; +pub use llm_labeler::{LlmClient, LlmLabeler, NoOpLlmClient}; +pub use relationships::{RelationshipBuilder, TopicGraphBuilder}; +pub use similarity::{calculate_centroid, cosine_similarity}; +pub use storage::TopicStorage; +pub use tfidf::TfIdf; +pub use types::{ + Embedding, RelationshipType, Topic, TopicId, TopicLink, TopicRelationship, TopicStatus, +}; diff --git a/crates/memory-topics/src/lifecycle.rs b/crates/memory-topics/src/lifecycle.rs new file mode 100644 index 0000000..4866fa2 --- /dev/null +++ b/crates/memory-topics/src/lifecycle.rs @@ -0,0 +1,680 @@ +//! Topic lifecycle management. +//! +//! This module provides tools for managing the lifecycle of topics: +//! - Running extraction cycles to discover new topics +//! - Refreshing importance scores +//! - Pruning stale topics +//! - Merging similar topics +//! +//! ## Usage +//! +//! ```rust,ignore +//! use memory_topics::lifecycle::{TopicLifecycleManager, LifecycleStats}; +//! use memory_topics::TopicStorage; +//! use std::sync::Arc; +//! +//! let storage = Arc::new(/* ... */); +//! let topic_storage = TopicStorage::new(storage); +//! let manager = TopicLifecycleManager::new(topic_storage); +//! +//! // Get lifecycle statistics +//! let stats = manager.get_lifecycle_stats()?; +//! println!("Active topics: {}", stats.active_topics); +//! +//! // Prune stale topics (not mentioned in 90 days) +//! let pruned = manager.prune_stale_topics(90)?; +//! println!("Pruned {} topics", pruned); +//! ``` + +use chrono::{DateTime, Duration, Utc}; +use serde::{Deserialize, Serialize}; +use tracing::{debug, info, instrument, warn}; + +use crate::config::ImportanceConfig; +use crate::error::TopicsError; +use crate::importance::ImportanceScorer; +use crate::similarity::cosine_similarity; +use crate::storage::TopicStorage; +use crate::types::{Topic, TopicStatus}; + +/// Statistics about the topic lifecycle state. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct LifecycleStats { + /// Number of active topics + pub active_topics: usize, + /// Number of archived/pruned topics + pub archived_topics: usize, + /// Total number of relationships + pub total_relationships: usize, + /// Timestamp of last extraction cycle + pub last_extraction: Option>, + /// Timestamp of last prune operation + pub last_prune: Option>, +} + +impl LifecycleStats { + /// Create a new lifecycle stats instance. + pub fn new() -> Self { + Self::default() + } + + /// Get total topic count (active + archived). + pub fn total_topics(&self) -> usize { + self.active_topics + self.archived_topics + } +} + +/// Manages the lifecycle of topics. +/// +/// The `TopicLifecycleManager` provides operations for maintaining the health +/// and relevance of the topic graph over time. +pub struct TopicLifecycleManager<'a> { + storage: &'a TopicStorage, + importance_config: ImportanceConfig, + /// Timestamp of last extraction cycle (in-memory tracking) + last_extraction: Option>, + /// Timestamp of last prune operation (in-memory tracking) + last_prune: Option>, +} + +impl<'a> TopicLifecycleManager<'a> { + /// Create a new lifecycle manager with default importance configuration. + pub fn new(storage: &'a TopicStorage) -> Self { + Self { + storage, + importance_config: ImportanceConfig::default(), + last_extraction: None, + last_prune: None, + } + } + + /// Create a lifecycle manager with custom importance configuration. + pub fn with_importance_config(storage: &'a TopicStorage, config: ImportanceConfig) -> Self { + Self { + storage, + importance_config: config, + last_extraction: None, + last_prune: None, + } + } + + /// Run a topic extraction cycle. + /// + /// This is a placeholder that marks extraction as having run. + /// The actual extraction logic is handled by the scheduler job. + /// + /// # Returns + /// + /// Number of new topics extracted (placeholder returns 0). + #[instrument(skip(self))] + pub fn run_extraction_cycle(&mut self) -> Result { + info!("Running topic extraction cycle"); + self.last_extraction = Some(Utc::now()); + + // Note: Actual extraction is performed by the scheduler job using TopicExtractor. + // This method primarily updates the last_extraction timestamp for tracking. + // A real implementation would: + // 1. Query recent TOC nodes since last extraction + // 2. Get embeddings for those nodes + // 3. Run HDBSCAN clustering + // 4. Label and store new topics + + debug!("Extraction cycle complete (placeholder)"); + Ok(0) + } + + /// Refresh importance scores for all topics. + /// + /// Recalculates time-decayed importance scores based on current time. + /// Topics are updated in storage only if their scores have changed. + /// + /// # Returns + /// + /// Number of topics whose scores were updated. + #[instrument(skip(self))] + pub fn refresh_importance_scores(&mut self) -> Result { + let scorer = ImportanceScorer::new(self.importance_config.clone()); + let updated = self.storage.refresh_importance_scores(&scorer)?; + info!(updated_count = updated, "Refreshed importance scores"); + Ok(updated) + } + + /// Prune topics that haven't been mentioned in the specified number of days. + /// + /// Topics are marked as `Pruned` status rather than deleted, allowing + /// potential resurrection if they are mentioned again. + /// + /// # Arguments + /// + /// * `days` - Number of days of inactivity before pruning + /// + /// # Returns + /// + /// Number of topics pruned. + #[instrument(skip(self))] + pub fn prune_stale_topics(&mut self, days: u32) -> Result { + let now = Utc::now(); + let threshold = now - Duration::days(i64::from(days)); + self.last_prune = Some(now); + + let topics = self.storage.list_topics()?; + let mut pruned_count = 0; + + for topic in topics { + if topic.last_mentioned_at < threshold { + let mut pruned_topic = topic.clone(); + pruned_topic.status = TopicStatus::Pruned; + self.storage.save_topic(&pruned_topic)?; + pruned_count += 1; + debug!( + topic_id = %topic.topic_id, + last_mentioned = %topic.last_mentioned_at, + "Pruned stale topic" + ); + } + } + + info!( + days = days, + pruned_count = pruned_count, + "Pruned stale topics" + ); + Ok(pruned_count) + } + + /// Merge topics that are highly similar. + /// + /// Topics with embedding similarity above the threshold are merged. + /// The topic with higher importance is kept, and relationships from + /// the merged topic are transferred. + /// + /// # Arguments + /// + /// * `threshold` - Minimum cosine similarity for merging (0.0 - 1.0) + /// + /// # Returns + /// + /// Number of topic pairs merged. + #[instrument(skip(self))] + pub fn merge_similar_topics(&mut self, threshold: f32) -> Result { + if !(0.0..=1.0).contains(&threshold) { + return Err(TopicsError::InvalidInput(format!( + "Threshold must be between 0.0 and 1.0, got {}", + threshold + ))); + } + + let topics = self.storage.list_topics()?; + let n = topics.len(); + let mut merged_count = 0; + let mut merged_ids: std::collections::HashSet = std::collections::HashSet::new(); + + // Find similar topic pairs + for i in 0..n { + if merged_ids.contains(&topics[i].topic_id) { + continue; + } + + for j in (i + 1)..n { + if merged_ids.contains(&topics[j].topic_id) { + continue; + } + + let similarity = cosine_similarity(&topics[i].embedding, &topics[j].embedding); + + if similarity >= threshold { + // Determine which topic to keep (higher importance) + let (keeper, merged) = + if topics[i].importance_score >= topics[j].importance_score { + (&topics[i], &topics[j]) + } else { + (&topics[j], &topics[i]) + }; + + // Mark merged topic as pruned + let mut pruned = merged.clone(); + pruned.status = TopicStatus::Pruned; + self.storage.save_topic(&pruned)?; + + // Update keeper with combined node count and keywords + let mut updated_keeper = keeper.clone(); + updated_keeper.node_count += merged.node_count; + + // Merge keywords (deduplicate) + for keyword in &merged.keywords { + if !updated_keeper.keywords.contains(keyword) { + updated_keeper.keywords.push(keyword.clone()); + } + } + + self.storage.save_topic(&updated_keeper)?; + merged_ids.insert(merged.topic_id.clone()); + merged_count += 1; + + info!( + keeper_id = %keeper.topic_id, + merged_id = %merged.topic_id, + similarity = similarity, + "Merged similar topics" + ); + } + } + } + + info!( + threshold = threshold, + merged_count = merged_count, + "Merged similar topics" + ); + Ok(merged_count) + } + + /// Get lifecycle statistics. + /// + /// # Returns + /// + /// Current lifecycle statistics including topic counts and timestamps. + #[instrument(skip(self))] + pub fn get_lifecycle_stats(&self) -> Result { + let stats = self.storage.get_stats()?; + + // Count active vs pruned topics + let all_topics = self.list_all_topics()?; + let active_count = all_topics.iter().filter(|t| t.is_active()).count(); + let archived_count = all_topics.len() - active_count; + + Ok(LifecycleStats { + active_topics: active_count, + archived_topics: archived_count, + total_relationships: stats.relationship_count as usize, + last_extraction: self.last_extraction, + last_prune: self.last_prune, + }) + } + + /// List all topics including pruned ones. + fn list_all_topics(&self) -> Result, TopicsError> { + // TopicStorage.list_topics() only returns active topics, + // so we need to scan the storage directly + let prefix = b"topic:"; + let mut topics = Vec::new(); + + for (_, value) in self + .storage + .storage() + .prefix_iterator(crate::storage::CF_TOPICS, prefix)? + { + let topic: Topic = serde_json::from_slice(&value)?; + topics.push(topic); + } + + Ok(topics) + } + + /// Get the timestamp of the last extraction cycle. + pub fn last_extraction(&self) -> Option> { + self.last_extraction + } + + /// Get the timestamp of the last prune operation. + pub fn last_prune(&self) -> Option> { + self.last_prune + } + + /// Set the last extraction timestamp (for restoring state). + pub fn set_last_extraction(&mut self, timestamp: DateTime) { + self.last_extraction = Some(timestamp); + } + + /// Set the last prune timestamp (for restoring state). + pub fn set_last_prune(&mut self, timestamp: DateTime) { + self.last_prune = Some(timestamp); + } + + /// Resurrect a pruned topic (set status back to Active). + /// + /// # Arguments + /// + /// * `topic_id` - ID of the topic to resurrect + /// + /// # Returns + /// + /// `Ok(true)` if the topic was resurrected, `Ok(false)` if it was already active, + /// or an error if the topic was not found. + #[instrument(skip(self))] + pub fn resurrect_topic(&self, topic_id: &str) -> Result { + let topic = self + .storage + .get_topic(topic_id)? + .ok_or_else(|| TopicsError::NotFound(topic_id.to_string()))?; + + if topic.is_active() { + debug!(topic_id = %topic_id, "Topic already active"); + return Ok(false); + } + + let mut resurrected = topic; + resurrected.status = TopicStatus::Active; + resurrected.last_mentioned_at = Utc::now(); + self.storage.save_topic(&resurrected)?; + + info!(topic_id = %topic_id, "Resurrected topic"); + Ok(true) + } + + /// Archive a topic (set status to Pruned). + /// + /// # Arguments + /// + /// * `topic_id` - ID of the topic to archive + /// + /// # Returns + /// + /// `Ok(true)` if the topic was archived, `Ok(false)` if it was already archived, + /// or an error if the topic was not found. + #[instrument(skip(self))] + pub fn archive_topic(&self, topic_id: &str) -> Result { + let topic = self + .storage + .get_topic(topic_id)? + .ok_or_else(|| TopicsError::NotFound(topic_id.to_string()))?; + + if topic.status == TopicStatus::Pruned { + debug!(topic_id = %topic_id, "Topic already archived"); + return Ok(false); + } + + let mut archived = topic; + archived.status = TopicStatus::Pruned; + self.storage.save_topic(&archived)?; + + info!(topic_id = %topic_id, "Archived topic"); + Ok(true) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::Topic; + use memory_storage::Storage; + use std::sync::Arc; + use tempfile::TempDir; + + fn create_test_storage() -> (TempDir, Arc) { + let dir = TempDir::new().unwrap(); + let storage = Storage::open(dir.path()).unwrap(); + (dir, Arc::new(storage)) + } + + fn create_test_topic(id: &str, label: &str, embedding: Vec) -> Topic { + Topic::new(id.to_string(), label.to_string(), embedding) + } + + #[test] + fn test_lifecycle_stats_default() { + let stats = LifecycleStats::default(); + assert_eq!(stats.active_topics, 0); + assert_eq!(stats.archived_topics, 0); + assert_eq!(stats.total_relationships, 0); + assert!(stats.last_extraction.is_none()); + assert!(stats.last_prune.is_none()); + } + + #[test] + fn test_lifecycle_stats_total_topics() { + let stats = LifecycleStats { + active_topics: 10, + archived_topics: 5, + total_relationships: 20, + last_extraction: None, + last_prune: None, + }; + assert_eq!(stats.total_topics(), 15); + } + + #[test] + fn test_manager_new() { + let (_dir, storage) = create_test_storage(); + let topic_storage = TopicStorage::new(storage); + let manager = TopicLifecycleManager::new(&topic_storage); + + assert!(manager.last_extraction().is_none()); + assert!(manager.last_prune().is_none()); + } + + #[test] + fn test_manager_with_importance_config() { + let (_dir, storage) = create_test_storage(); + let topic_storage = TopicStorage::new(storage); + let config = ImportanceConfig { + half_life_days: 60, + recency_boost: 3.0, + }; + let manager = TopicLifecycleManager::with_importance_config(&topic_storage, config); + + assert!(manager.last_extraction().is_none()); + } + + #[test] + fn test_run_extraction_cycle() { + let (_dir, storage) = create_test_storage(); + let topic_storage = TopicStorage::new(storage); + let mut manager = TopicLifecycleManager::new(&topic_storage); + + let result = manager.run_extraction_cycle(); + assert!(result.is_ok()); + assert!(manager.last_extraction().is_some()); + } + + #[test] + fn test_refresh_importance_scores() { + let (_dir, storage) = create_test_storage(); + let topic_storage = TopicStorage::new(storage); + + // Add a topic + let topic = create_test_topic("t1", "Test Topic", vec![0.1, 0.2, 0.3]); + topic_storage.save_topic(&topic).unwrap(); + + let mut manager = TopicLifecycleManager::new(&topic_storage); + let result = manager.refresh_importance_scores(); + + assert!(result.is_ok()); + } + + #[test] + fn test_prune_stale_topics() { + let (_dir, storage) = create_test_storage(); + let topic_storage = TopicStorage::new(storage); + + // Create an old topic + let mut old_topic = create_test_topic("t1", "Old Topic", vec![0.1, 0.2, 0.3]); + old_topic.last_mentioned_at = Utc::now() - Duration::days(100); + topic_storage.save_topic(&old_topic).unwrap(); + + // Create a recent topic + let recent_topic = create_test_topic("t2", "Recent Topic", vec![0.4, 0.5, 0.6]); + topic_storage.save_topic(&recent_topic).unwrap(); + + let mut manager = TopicLifecycleManager::new(&topic_storage); + let pruned = manager.prune_stale_topics(90).unwrap(); + + assert_eq!(pruned, 1); + assert!(manager.last_prune().is_some()); + + // Verify the old topic is pruned + let topic = topic_storage.get_topic("t1").unwrap().unwrap(); + assert_eq!(topic.status, TopicStatus::Pruned); + + // Verify the recent topic is still active + let topic = topic_storage.get_topic("t2").unwrap().unwrap(); + assert_eq!(topic.status, TopicStatus::Active); + } + + #[test] + fn test_merge_similar_topics() { + let (_dir, storage) = create_test_storage(); + let topic_storage = TopicStorage::new(storage); + + // Create similar topics (same embedding = similarity 1.0) + let mut topic1 = create_test_topic("t1", "Topic One", vec![1.0, 0.0, 0.0]); + topic1.importance_score = 0.9; + topic1.node_count = 5; + topic1.keywords = vec!["rust".to_string()]; + topic_storage.save_topic(&topic1).unwrap(); + + let mut topic2 = create_test_topic("t2", "Topic Two", vec![1.0, 0.0, 0.0]); + topic2.importance_score = 0.5; + topic2.node_count = 3; + topic2.keywords = vec!["memory".to_string()]; + topic_storage.save_topic(&topic2).unwrap(); + + // Create a different topic + let topic3 = create_test_topic("t3", "Different", vec![0.0, 0.0, 1.0]); + topic_storage.save_topic(&topic3).unwrap(); + + let mut manager = TopicLifecycleManager::new(&topic_storage); + let merged = manager.merge_similar_topics(0.95).unwrap(); + + assert_eq!(merged, 1); + + // Topic1 should be kept (higher importance) + let kept = topic_storage.get_topic("t1").unwrap().unwrap(); + assert_eq!(kept.status, TopicStatus::Active); + assert_eq!(kept.node_count, 8); // 5 + 3 + assert!(kept.keywords.contains(&"rust".to_string())); + assert!(kept.keywords.contains(&"memory".to_string())); + + // Topic2 should be pruned + let pruned = topic_storage.get_topic("t2").unwrap().unwrap(); + assert_eq!(pruned.status, TopicStatus::Pruned); + + // Topic3 should be unchanged + let different = topic_storage.get_topic("t3").unwrap().unwrap(); + assert_eq!(different.status, TopicStatus::Active); + } + + #[test] + fn test_merge_similar_topics_invalid_threshold() { + let (_dir, storage) = create_test_storage(); + let topic_storage = TopicStorage::new(storage); + let mut manager = TopicLifecycleManager::new(&topic_storage); + + let result = manager.merge_similar_topics(1.5); + assert!(result.is_err()); + + let result = manager.merge_similar_topics(-0.1); + assert!(result.is_err()); + } + + #[test] + fn test_get_lifecycle_stats() { + let (_dir, storage) = create_test_storage(); + let topic_storage = TopicStorage::new(storage); + + // Add topics + let active = create_test_topic("t1", "Active", vec![0.1, 0.2]); + topic_storage.save_topic(&active).unwrap(); + + let mut pruned = create_test_topic("t2", "Pruned", vec![0.3, 0.4]); + pruned.status = TopicStatus::Pruned; + topic_storage.save_topic(&pruned).unwrap(); + + let manager = TopicLifecycleManager::new(&topic_storage); + let stats = manager.get_lifecycle_stats().unwrap(); + + assert_eq!(stats.active_topics, 1); + assert_eq!(stats.archived_topics, 1); + assert_eq!(stats.total_topics(), 2); + } + + #[test] + fn test_resurrect_topic() { + let (_dir, storage) = create_test_storage(); + let topic_storage = TopicStorage::new(storage); + + let mut pruned = create_test_topic("t1", "Pruned Topic", vec![0.1, 0.2]); + pruned.status = TopicStatus::Pruned; + topic_storage.save_topic(&pruned).unwrap(); + + let manager = TopicLifecycleManager::new(&topic_storage); + let resurrected = manager.resurrect_topic("t1").unwrap(); + + assert!(resurrected); + + let topic = topic_storage.get_topic("t1").unwrap().unwrap(); + assert_eq!(topic.status, TopicStatus::Active); + } + + #[test] + fn test_resurrect_active_topic() { + let (_dir, storage) = create_test_storage(); + let topic_storage = TopicStorage::new(storage); + + let active = create_test_topic("t1", "Active Topic", vec![0.1, 0.2]); + topic_storage.save_topic(&active).unwrap(); + + let manager = TopicLifecycleManager::new(&topic_storage); + let resurrected = manager.resurrect_topic("t1").unwrap(); + + assert!(!resurrected); // Already active + } + + #[test] + fn test_resurrect_nonexistent_topic() { + let (_dir, storage) = create_test_storage(); + let topic_storage = TopicStorage::new(storage); + let manager = TopicLifecycleManager::new(&topic_storage); + + let result = manager.resurrect_topic("nonexistent"); + assert!(result.is_err()); + } + + #[test] + fn test_archive_topic() { + let (_dir, storage) = create_test_storage(); + let topic_storage = TopicStorage::new(storage); + + let active = create_test_topic("t1", "Active Topic", vec![0.1, 0.2]); + topic_storage.save_topic(&active).unwrap(); + + let manager = TopicLifecycleManager::new(&topic_storage); + let archived = manager.archive_topic("t1").unwrap(); + + assert!(archived); + + let topic = topic_storage.get_topic("t1").unwrap().unwrap(); + assert_eq!(topic.status, TopicStatus::Pruned); + } + + #[test] + fn test_archive_already_archived_topic() { + let (_dir, storage) = create_test_storage(); + let topic_storage = TopicStorage::new(storage); + + let mut pruned = create_test_topic("t1", "Pruned Topic", vec![0.1, 0.2]); + pruned.status = TopicStatus::Pruned; + topic_storage.save_topic(&pruned).unwrap(); + + let manager = TopicLifecycleManager::new(&topic_storage); + let archived = manager.archive_topic("t1").unwrap(); + + assert!(!archived); // Already archived + } + + #[test] + fn test_set_timestamps() { + let (_dir, storage) = create_test_storage(); + let topic_storage = TopicStorage::new(storage); + let mut manager = TopicLifecycleManager::new(&topic_storage); + + let extraction_time = Utc::now() - Duration::hours(1); + let prune_time = Utc::now() - Duration::hours(2); + + manager.set_last_extraction(extraction_time); + manager.set_last_prune(prune_time); + + assert_eq!(manager.last_extraction(), Some(extraction_time)); + assert_eq!(manager.last_prune(), Some(prune_time)); + } +} diff --git a/crates/memory-topics/src/llm_labeler.rs b/crates/memory-topics/src/llm_labeler.rs new file mode 100644 index 0000000..596dde8 --- /dev/null +++ b/crates/memory-topics/src/llm_labeler.rs @@ -0,0 +1,323 @@ +//! LLM-enhanced topic labeling with keyword fallback. +//! +//! Provides LLM-based label generation with automatic fallback to +//! keyword-based labeling when LLM is unavailable or fails. + +use crate::config::LabelingConfig; +use crate::error::TopicsError; +use crate::labeling::{ClusterDocument, KeywordLabeler, TopicLabel, TopicLabeler}; + +/// Trait for LLM completion. +/// +/// Implement this trait to provide LLM-based label generation. +/// The implementation should handle API calls, rate limiting, and error handling. +pub trait LlmClient: Send + Sync { + /// Generate a completion for the given prompt. + /// + /// Returns the generated text or an error. + fn complete(&self, prompt: &str) -> Result; +} + +/// LLM-enhanced topic labeler with keyword fallback. +/// +/// Uses an optional LLM client for sophisticated label generation, +/// falling back to keyword-based labeling when: +/// - LLM client is not provided +/// - LLM call fails +/// - `fallback_to_keywords` is configured +pub struct LlmLabeler { + /// Optional LLM client + llm: Option, + /// Keyword-based fallback labeler + keyword_fallback: KeywordLabeler, + /// Configuration + config: LabelingConfig, +} + +impl LlmLabeler { + /// Create a new LLM labeler with an optional client. + pub fn new(llm: Option, config: LabelingConfig) -> Self { + let keyword_fallback = KeywordLabeler::new(config.clone()); + Self { + llm, + keyword_fallback, + config, + } + } + + /// Create an LLM labeler with a client. + pub fn with_llm(llm: L, config: LabelingConfig) -> Self { + Self::new(Some(llm), config) + } + + /// Create an LLM labeler without a client (keyword-only). + pub fn without_llm(config: LabelingConfig) -> Self { + Self::new(None, config) + } + + /// Generate a prompt for the LLM. + fn generate_prompt(&self, documents: &[ClusterDocument]) -> String { + let samples: Vec<&str> = documents + .iter() + .take(5) // Limit context size + .map(|d| d.text.as_str()) + .collect(); + + let sample_text = samples.join("\n---\n"); + + format!( + r#"Generate a concise topic label (2-5 words) for the following cluster of related documents. +The label should capture the main theme or concept. + +Documents: +{} + +Respond with ONLY the topic label, nothing else."#, + sample_text + ) + } + + /// Parse LLM response into a label. + fn parse_response(&self, response: &str) -> String { + // Clean up response: trim, remove quotes, limit length + let cleaned = response.trim().trim_matches('"').trim_matches('\'').trim(); + + // Truncate if needed + if cleaned.len() > self.config.max_label_length { + if let Some(last_space) = cleaned[..self.config.max_label_length].rfind(' ') { + return cleaned[..last_space].to_string(); + } + return cleaned[..self.config.max_label_length].to_string(); + } + + cleaned.to_string() + } + + /// Label using LLM. + fn label_with_llm( + &self, + llm: &L, + documents: &[ClusterDocument], + ) -> Result { + let prompt = self.generate_prompt(documents); + let response = llm.complete(&prompt)?; + let label = self.parse_response(&response); + + // Get keywords from fallback for metadata + let keyword_result = self.keyword_fallback.label_cluster(documents)?; + + Ok(TopicLabel::new( + label, + keyword_result.keywords, + 0.85, // LLM labels get higher default confidence + )) + } +} + +impl TopicLabeler for LlmLabeler { + fn label_cluster(&self, documents: &[ClusterDocument]) -> Result { + // If LLM is not enabled or not available, use keywords + if !self.config.use_llm || self.llm.is_none() { + return self.keyword_fallback.label_cluster(documents); + } + + // Try LLM labeling + if let Some(ref llm) = self.llm { + match self.label_with_llm(llm, documents) { + Ok(label) => return Ok(label), + Err(e) => { + tracing::warn!("LLM labeling failed: {}, falling back to keywords", e); + + // Fall back to keywords if configured + if self.config.fallback_to_keywords { + return self.keyword_fallback.label_cluster(documents); + } + + return Err(e); + } + } + } + + // Should not reach here, but fallback just in case + self.keyword_fallback.label_cluster(documents) + } +} + +/// A no-op LLM client for testing and keyword-only mode. +/// +/// Always returns an error, forcing fallback to keyword labeling. +pub struct NoOpLlmClient; + +impl LlmClient for NoOpLlmClient { + fn complete(&self, _prompt: &str) -> Result { + Err(TopicsError::InvalidConfig("No LLM configured".to_string())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Test LLM client that returns a fixed response. + struct MockLlmClient { + response: String, + } + + impl MockLlmClient { + fn new(response: &str) -> Self { + Self { + response: response.to_string(), + } + } + } + + impl LlmClient for MockLlmClient { + fn complete(&self, _prompt: &str) -> Result { + Ok(self.response.clone()) + } + } + + /// Test LLM client that always fails. + struct FailingLlmClient; + + impl LlmClient for FailingLlmClient { + fn complete(&self, _prompt: &str) -> Result { + Err(TopicsError::Embedding("LLM API error".to_string())) + } + } + + fn make_doc(id: &str, text: &str) -> ClusterDocument { + ClusterDocument::new(id.to_string(), text.to_string()) + } + + #[test] + fn test_llm_labeler_with_mock() { + let config = LabelingConfig::default(); + let mock = MockLlmClient::new("Machine Learning"); + let labeler = LlmLabeler::with_llm(mock, config); + + let docs = vec![ + make_doc("d1", "deep learning neural networks"), + make_doc("d2", "machine learning algorithms"), + ]; + + let result = labeler.label_cluster(&docs).unwrap(); + assert_eq!(result.label, "Machine Learning"); + assert!(result.confidence > 0.8); + } + + #[test] + fn test_llm_labeler_fallback_on_failure() { + let config = LabelingConfig { + use_llm: true, + fallback_to_keywords: true, + ..Default::default() + }; + let labeler = LlmLabeler::with_llm(FailingLlmClient, config); + + let docs = vec![ + make_doc("d1", "rust programming systems"), + make_doc("d2", "rust memory safety"), + ]; + + // Should fall back to keywords and succeed + let result = labeler.label_cluster(&docs).unwrap(); + assert!(!result.label.is_empty()); + } + + #[test] + fn test_llm_labeler_without_llm() { + let config = LabelingConfig::default(); + let labeler: LlmLabeler = LlmLabeler::without_llm(config); + + let docs = vec![make_doc("d1", "rust programming language")]; + + // Should use keyword labeling + let result = labeler.label_cluster(&docs).unwrap(); + assert!(!result.label.is_empty()); + } + + #[test] + fn test_llm_labeler_disabled() { + let config = LabelingConfig { + use_llm: false, + ..Default::default() + }; + let mock = MockLlmClient::new("Should Not Use This"); + let labeler = LlmLabeler::with_llm(mock, config); + + let docs = vec![make_doc("d1", "python scripting automation")]; + + // Should use keyword labeling since LLM is disabled + let result = labeler.label_cluster(&docs).unwrap(); + assert_ne!(result.label, "Should Not Use This"); + } + + #[test] + fn test_llm_labeler_no_fallback() { + let config = LabelingConfig { + use_llm: true, + fallback_to_keywords: false, + ..Default::default() + }; + let labeler = LlmLabeler::with_llm(FailingLlmClient, config); + + let docs = vec![make_doc("d1", "some text")]; + + // Should fail without fallback + let result = labeler.label_cluster(&docs); + assert!(result.is_err()); + } + + #[test] + fn test_generate_prompt() { + let config = LabelingConfig::default(); + let labeler: LlmLabeler = LlmLabeler::without_llm(config); + + let docs = vec![ + make_doc("d1", "First document about rust"), + make_doc("d2", "Second document about programming"), + ]; + + let prompt = labeler.generate_prompt(&docs); + assert!(prompt.contains("First document")); + assert!(prompt.contains("Second document")); + assert!(prompt.contains("2-5 words")); + } + + #[test] + fn test_parse_response_clean() { + let config = LabelingConfig::default(); + let labeler: LlmLabeler = LlmLabeler::without_llm(config); + + assert_eq!( + labeler.parse_response("Machine Learning"), + "Machine Learning" + ); + assert_eq!( + labeler.parse_response(" Rust Programming "), + "Rust Programming" + ); + assert_eq!(labeler.parse_response("\"Quoted Label\""), "Quoted Label"); + } + + #[test] + fn test_parse_response_truncate() { + let config = LabelingConfig { + max_label_length: 20, + ..Default::default() + }; + let labeler: LlmLabeler = LlmLabeler::without_llm(config); + + let long_response = "This is a very long topic label that needs truncation"; + let parsed = labeler.parse_response(long_response); + assert!(parsed.len() <= 20); + } + + #[test] + fn test_noop_client() { + let client = NoOpLlmClient; + let result = client.complete("test prompt"); + assert!(result.is_err()); + } +} diff --git a/crates/memory-topics/src/relationships.rs b/crates/memory-topics/src/relationships.rs new file mode 100644 index 0000000..b0c5804 --- /dev/null +++ b/crates/memory-topics/src/relationships.rs @@ -0,0 +1,987 @@ +//! Topic relationship tracking and graph building. +//! +//! This module provides tools for building and managing relationships between topics, +//! enabling the construction of a topic graph that captures semantic, co-occurrence, +//! and hierarchical connections. +//! +//! ## Relationship Types +//! +//! - **Co-occurrence**: Topics that appear together in the same documents +//! - **Semantic**: Topics with similar embedding vectors +//! - **Hierarchical**: Parent/child relationships between topics +//! +//! ## Usage +//! +//! ```rust,ignore +//! use memory_topics::relationships::{RelationshipBuilder, TopicGraphBuilder}; +//! use memory_topics::types::RelationshipType; +//! +//! // Build a single relationship +//! let rel = RelationshipBuilder::new("topic-a", "topic-b") +//! .relationship_type(RelationshipType::Semantic) +//! .strength(0.85) +//! .build() +//! .unwrap(); +//! +//! // Build a topic graph +//! let mut builder = TopicGraphBuilder::new(); +//! builder.add_co_occurrence("topic-1", "topic-2", "doc-123"); +//! builder.add_co_occurrence("topic-1", "topic-2", "doc-456"); // Strengthens existing +//! let relationships = builder.build(); +//! ``` + +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use tracing::{debug, instrument}; + +use crate::error::TopicsError; +use crate::similarity::cosine_similarity; +use crate::types::{Embedding, RelationshipType, TopicId, TopicRelationship}; + +/// Default strength increase per co-occurrence evidence. +const CO_OCCURRENCE_STRENGTH_DELTA: f32 = 0.1; + +/// Default threshold for semantic similarity relationships. +const DEFAULT_SEMANTIC_THRESHOLD: f32 = 0.75; + +/// Builder for constructing `TopicRelationship` instances with validation. +#[derive(Debug, Clone)] +pub struct RelationshipBuilder { + source_id: TopicId, + target_id: TopicId, + relationship_type: Option, + strength: Option, + evidence_count: Option, + created_at: Option>, + updated_at: Option>, +} + +impl RelationshipBuilder { + /// Create a new relationship builder with source and target topics. + pub fn new(source_id: impl Into, target_id: impl Into) -> Self { + Self { + source_id: source_id.into(), + target_id: target_id.into(), + relationship_type: None, + strength: None, + evidence_count: None, + created_at: None, + updated_at: None, + } + } + + /// Set the relationship type. + pub fn relationship_type(mut self, rel_type: RelationshipType) -> Self { + self.relationship_type = Some(rel_type); + self + } + + /// Set the relationship strength (0.0 - 1.0). + pub fn strength(mut self, strength: f32) -> Self { + self.strength = Some(strength); + self + } + + /// Set the evidence count. + pub fn evidence_count(mut self, count: u32) -> Self { + self.evidence_count = Some(count); + self + } + + /// Set the created timestamp. + pub fn created_at(mut self, timestamp: DateTime) -> Self { + self.created_at = Some(timestamp); + self + } + + /// Set the updated timestamp. + pub fn updated_at(mut self, timestamp: DateTime) -> Self { + self.updated_at = Some(timestamp); + self + } + + /// Build the relationship, validating all fields. + /// + /// # Errors + /// + /// Returns an error if: + /// - Source and target are the same topic + /// - Relationship type is not specified + pub fn build(self) -> Result { + // Validate source != target + if self.source_id == self.target_id { + return Err(TopicsError::InvalidInput( + "Source and target topics must be different".to_string(), + )); + } + + // Validate relationship type is set + let rel_type = self.relationship_type.ok_or_else(|| { + TopicsError::InvalidInput("Relationship type must be specified".to_string()) + })?; + + let strength = self.strength.unwrap_or(0.5); + let evidence_count = self.evidence_count.unwrap_or(1); + let now = Utc::now(); + let created_at = self.created_at.unwrap_or(now); + let updated_at = self.updated_at.unwrap_or(now); + + Ok(TopicRelationship::with_timestamps( + self.source_id, + self.target_id, + rel_type, + strength, + evidence_count, + created_at, + updated_at, + )) + } +} + +/// Key for tracking relationships in the graph builder. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct RelationshipKey { + source_id: TopicId, + target_id: TopicId, + rel_type: RelationshipType, +} + +impl RelationshipKey { + fn new(source_id: TopicId, target_id: TopicId, rel_type: RelationshipType) -> Self { + Self { + source_id, + target_id, + rel_type, + } + } + + /// Create a canonical key where source < target to handle bidirectional relationships. + fn canonical(topic_a: &TopicId, topic_b: &TopicId, rel_type: RelationshipType) -> Self { + if topic_a <= topic_b { + Self::new(topic_a.clone(), topic_b.clone(), rel_type) + } else { + Self::new(topic_b.clone(), topic_a.clone(), rel_type) + } + } +} + +/// Tracks relationship evidence during graph building. +#[derive(Debug, Clone)] +struct RelationshipEvidence { + source_id: TopicId, + target_id: TopicId, + rel_type: RelationshipType, + strength: f32, + evidence_count: u32, + document_ids: Vec, + created_at: DateTime, + updated_at: DateTime, +} + +impl RelationshipEvidence { + fn new( + source_id: TopicId, + target_id: TopicId, + rel_type: RelationshipType, + initial_strength: f32, + ) -> Self { + let now = Utc::now(); + Self { + source_id, + target_id, + rel_type, + strength: initial_strength.clamp(0.0, 1.0), + evidence_count: 1, + document_ids: Vec::new(), + created_at: now, + updated_at: now, + } + } + + fn add_evidence(&mut self, doc_id: Option<&str>, strength_delta: f32) { + self.evidence_count = self.evidence_count.saturating_add(1); + self.strength = (self.strength + strength_delta).clamp(0.0, 1.0); + self.updated_at = Utc::now(); + + if let Some(id) = doc_id { + if !self.document_ids.contains(&id.to_string()) { + self.document_ids.push(id.to_string()); + } + } + } + + fn into_relationship(self) -> TopicRelationship { + TopicRelationship::with_timestamps( + self.source_id, + self.target_id, + self.rel_type, + self.strength, + self.evidence_count, + self.created_at, + self.updated_at, + ) + } +} + +/// Builder for constructing a topic relationship graph. +/// +/// The `TopicGraphBuilder` accumulates relationship evidence and produces +/// a set of `TopicRelationship` instances that can be stored. +/// +/// ## Features +/// +/// - Tracks co-occurrence of topics in documents +/// - Computes semantic relationships from embeddings +/// - Supports explicit hierarchical relationships +/// - Handles bidirectional relationships (A->B and B->A are tracked once) +/// - Strengthens relationships as more evidence is added +#[derive(Debug, Default)] +pub struct TopicGraphBuilder { + relationships: HashMap, + co_occurrence_strength_delta: f32, + semantic_threshold: f32, +} + +impl TopicGraphBuilder { + /// Create a new topic graph builder with default settings. + pub fn new() -> Self { + Self { + relationships: HashMap::new(), + co_occurrence_strength_delta: CO_OCCURRENCE_STRENGTH_DELTA, + semantic_threshold: DEFAULT_SEMANTIC_THRESHOLD, + } + } + + /// Create a builder with custom settings. + pub fn with_settings(co_occurrence_strength_delta: f32, semantic_threshold: f32) -> Self { + Self { + relationships: HashMap::new(), + co_occurrence_strength_delta, + semantic_threshold, + } + } + + /// Set the strength delta added per co-occurrence evidence. + pub fn set_co_occurrence_strength_delta(&mut self, delta: f32) { + self.co_occurrence_strength_delta = delta; + } + + /// Set the threshold for semantic similarity relationships. + pub fn set_semantic_threshold(&mut self, threshold: f32) { + self.semantic_threshold = threshold; + } + + /// Track co-occurring topics from the same document. + /// + /// When topics appear together in a document, they are considered related. + /// Repeated co-occurrences strengthen the relationship. + /// + /// # Arguments + /// + /// * `topic_a` - First topic ID + /// * `topic_b` - Second topic ID + /// * `doc_id` - Document where they co-occur + #[instrument(skip(self))] + pub fn add_co_occurrence(&mut self, topic_a: &TopicId, topic_b: &TopicId, doc_id: &str) { + if topic_a == topic_b { + debug!("Ignoring self-relationship"); + return; + } + + let key = RelationshipKey::canonical(topic_a, topic_b, RelationshipType::CoOccurrence); + + self.relationships + .entry(key) + .and_modify(|e| { + e.add_evidence(Some(doc_id), self.co_occurrence_strength_delta); + }) + .or_insert_with(|| { + let mut evidence = RelationshipEvidence::new( + topic_a.clone(), + topic_b.clone(), + RelationshipType::CoOccurrence, + self.co_occurrence_strength_delta, + ); + evidence.document_ids.push(doc_id.to_string()); + evidence + }); + + debug!( + topic_a = %topic_a, + topic_b = %topic_b, + doc_id = %doc_id, + "Added co-occurrence relationship" + ); + } + + /// Compute semantic relationships from topic embeddings. + /// + /// Topics with embedding similarity above the threshold are related. + /// + /// # Arguments + /// + /// * `embeddings` - Slice of (topic_id, embedding) pairs + /// * `threshold` - Minimum similarity for relationship (overrides builder default if provided) + #[instrument(skip(self, embeddings))] + pub fn compute_semantic_relationships( + &mut self, + embeddings: &[(TopicId, Embedding)], + threshold: Option, + ) { + let threshold = threshold.unwrap_or(self.semantic_threshold); + let n = embeddings.len(); + + debug!( + count = n, + threshold = threshold, + "Computing semantic relationships" + ); + + for i in 0..n { + for j in (i + 1)..n { + let (topic_a, emb_a) = &embeddings[i]; + let (topic_b, emb_b) = &embeddings[j]; + + if topic_a == topic_b { + continue; + } + + let similarity = cosine_similarity(emb_a, emb_b); + + if similarity >= threshold { + let key = + RelationshipKey::canonical(topic_a, topic_b, RelationshipType::Semantic); + + self.relationships + .entry(key) + .and_modify(|e| { + // Update strength to max of existing and new similarity + if similarity > e.strength { + e.strength = similarity; + e.updated_at = Utc::now(); + } + e.evidence_count = e.evidence_count.saturating_add(1); + }) + .or_insert_with(|| { + RelationshipEvidence::new( + topic_a.clone(), + topic_b.clone(), + RelationshipType::Semantic, + similarity, + ) + }); + + debug!( + topic_a = %topic_a, + topic_b = %topic_b, + similarity = similarity, + "Added semantic relationship" + ); + } + } + } + } + + /// Set an explicit hierarchical relationship (parent/child). + /// + /// # Arguments + /// + /// * `parent` - Parent topic ID + /// * `child` - Child topic ID + #[instrument(skip(self))] + pub fn set_hierarchy(&mut self, parent: &TopicId, child: &TopicId) { + if parent == child { + debug!("Ignoring self-hierarchy"); + return; + } + + // For hierarchy, direction matters: parent -> child + let key = RelationshipKey::new( + parent.clone(), + child.clone(), + RelationshipType::Hierarchical, + ); + + self.relationships + .entry(key) + .and_modify(|e| { + e.add_evidence(None, 0.0); + }) + .or_insert_with(|| { + RelationshipEvidence::new( + parent.clone(), + child.clone(), + RelationshipType::Hierarchical, + 1.0, // Hierarchical relationships have full strength + ) + }); + + debug!( + parent = %parent, + child = %child, + "Set hierarchical relationship" + ); + } + + /// Get topics related to the given topic. + /// + /// # Arguments + /// + /// * `topic_id` - Topic to find relationships for + /// * `limit` - Maximum number of related topics to return + /// + /// # Returns + /// + /// Vector of (related_topic_id, strength) pairs, sorted by strength descending. + pub fn get_related_topics(&self, topic_id: &TopicId, limit: usize) -> Vec<(TopicId, f32)> { + let mut related: Vec<(TopicId, f32)> = self + .relationships + .values() + .filter_map(|evidence| { + if evidence.source_id == *topic_id { + Some((evidence.target_id.clone(), evidence.strength)) + } else if evidence.target_id == *topic_id { + Some((evidence.source_id.clone(), evidence.strength)) + } else { + None + } + }) + .collect(); + + // Sort by strength descending + related.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + + related.truncate(limit); + related + } + + /// Strengthen an existing relationship by a delta amount. + /// + /// # Arguments + /// + /// * `source` - Source topic ID + /// * `target` - Target topic ID + /// * `delta` - Amount to strengthen (can be negative to weaken) + /// + /// # Returns + /// + /// `true` if the relationship was found and updated, `false` otherwise. + pub fn strengthen_relationship( + &mut self, + source: &TopicId, + target: &TopicId, + delta: f32, + ) -> bool { + // Try all relationship types + for rel_type in RelationshipType::all() { + // Try canonical key first + let key = RelationshipKey::canonical(source, target, *rel_type); + if let Some(evidence) = self.relationships.get_mut(&key) { + evidence.strength = (evidence.strength + delta).clamp(0.0, 1.0); + evidence.updated_at = Utc::now(); + return true; + } + + // For hierarchical, also try direct key + if *rel_type == RelationshipType::Hierarchical { + let direct_key = RelationshipKey::new(source.clone(), target.clone(), *rel_type); + if let Some(evidence) = self.relationships.get_mut(&direct_key) { + evidence.strength = (evidence.strength + delta).clamp(0.0, 1.0); + evidence.updated_at = Utc::now(); + return true; + } + } + } + false + } + + /// Get the number of relationships tracked. + pub fn relationship_count(&self) -> usize { + self.relationships.len() + } + + /// Check if a relationship exists between two topics. + pub fn has_relationship(&self, topic_a: &TopicId, topic_b: &TopicId) -> bool { + for rel_type in RelationshipType::all() { + let key = RelationshipKey::canonical(topic_a, topic_b, *rel_type); + if self.relationships.contains_key(&key) { + return true; + } + // For hierarchical, check both directions + if *rel_type == RelationshipType::Hierarchical { + let direct_key_ab = + RelationshipKey::new(topic_a.clone(), topic_b.clone(), *rel_type); + let direct_key_ba = + RelationshipKey::new(topic_b.clone(), topic_a.clone(), *rel_type); + if self.relationships.contains_key(&direct_key_ab) + || self.relationships.contains_key(&direct_key_ba) + { + return true; + } + } + } + false + } + + /// Build the final list of relationships. + /// + /// Consumes the builder and returns all accumulated relationships. + pub fn build(self) -> Vec { + self.relationships + .into_values() + .map(|e| e.into_relationship()) + .collect() + } + + /// Get relationships without consuming the builder. + pub fn get_relationships(&self) -> Vec { + self.relationships + .values() + .map(|e| { + TopicRelationship::with_timestamps( + e.source_id.clone(), + e.target_id.clone(), + e.rel_type, + e.strength, + e.evidence_count, + e.created_at, + e.updated_at, + ) + }) + .collect() + } + + /// Clear all tracked relationships. + pub fn clear(&mut self) { + self.relationships.clear(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ==================== RelationshipBuilder Tests ==================== + + #[test] + fn test_relationship_builder_basic() { + let rel = RelationshipBuilder::new("topic-a", "topic-b") + .relationship_type(RelationshipType::Semantic) + .strength(0.85) + .build() + .unwrap(); + + assert_eq!(rel.source_id, "topic-a"); + assert_eq!(rel.target_id, "topic-b"); + assert_eq!(rel.relationship_type, RelationshipType::Semantic); + assert!((rel.strength - 0.85).abs() < f32::EPSILON); + assert_eq!(rel.evidence_count, 1); + } + + #[test] + fn test_relationship_builder_all_fields() { + let created = Utc::now() - chrono::Duration::days(10); + let updated = Utc::now() - chrono::Duration::days(1); + + let rel = RelationshipBuilder::new("source", "target") + .relationship_type(RelationshipType::CoOccurrence) + .strength(0.7) + .evidence_count(5) + .created_at(created) + .updated_at(updated) + .build() + .unwrap(); + + assert_eq!(rel.evidence_count, 5); + assert_eq!(rel.created_at, created); + assert_eq!(rel.updated_at, updated); + } + + #[test] + fn test_relationship_builder_defaults() { + let rel = RelationshipBuilder::new("a", "b") + .relationship_type(RelationshipType::Hierarchical) + .build() + .unwrap(); + + // Default strength is 0.5 + assert!((rel.strength - 0.5).abs() < f32::EPSILON); + // Default evidence count is 1 + assert_eq!(rel.evidence_count, 1); + } + + #[test] + fn test_relationship_builder_strength_clamping() { + let rel_high = RelationshipBuilder::new("a", "b") + .relationship_type(RelationshipType::Semantic) + .strength(1.5) + .build() + .unwrap(); + assert!((rel_high.strength - 1.0).abs() < f32::EPSILON); + + let rel_low = RelationshipBuilder::new("a", "b") + .relationship_type(RelationshipType::Semantic) + .strength(-0.5) + .build() + .unwrap(); + assert!(rel_low.strength.abs() < f32::EPSILON); + } + + #[test] + fn test_relationship_builder_error_same_topic() { + let result = RelationshipBuilder::new("topic-x", "topic-x") + .relationship_type(RelationshipType::Semantic) + .build(); + + assert!(result.is_err()); + match result { + Err(TopicsError::InvalidInput(msg)) => { + assert!(msg.contains("different")); + } + _ => panic!("Expected InvalidInput error"), + } + } + + #[test] + fn test_relationship_builder_error_no_type() { + let result = RelationshipBuilder::new("a", "b").strength(0.5).build(); + + assert!(result.is_err()); + match result { + Err(TopicsError::InvalidInput(msg)) => { + assert!(msg.contains("type")); + } + _ => panic!("Expected InvalidInput error"), + } + } + + // ==================== TopicGraphBuilder Tests ==================== + + #[test] + fn test_graph_builder_new() { + let builder = TopicGraphBuilder::new(); + assert_eq!(builder.relationship_count(), 0); + } + + #[test] + fn test_graph_builder_with_settings() { + let builder = TopicGraphBuilder::with_settings(0.2, 0.9); + assert!((builder.co_occurrence_strength_delta - 0.2).abs() < f32::EPSILON); + assert!((builder.semantic_threshold - 0.9).abs() < f32::EPSILON); + } + + #[test] + fn test_add_co_occurrence_single() { + let mut builder = TopicGraphBuilder::new(); + builder.add_co_occurrence(&"topic-1".to_string(), &"topic-2".to_string(), "doc-1"); + + assert_eq!(builder.relationship_count(), 1); + + let rels = builder.build(); + assert_eq!(rels.len(), 1); + assert_eq!(rels[0].relationship_type, RelationshipType::CoOccurrence); + assert_eq!(rels[0].evidence_count, 1); + } + + #[test] + fn test_add_co_occurrence_strengthens() { + let mut builder = TopicGraphBuilder::new(); + let topic_a = "topic-1".to_string(); + let topic_b = "topic-2".to_string(); + + builder.add_co_occurrence(&topic_a, &topic_b, "doc-1"); + builder.add_co_occurrence(&topic_a, &topic_b, "doc-2"); + builder.add_co_occurrence(&topic_a, &topic_b, "doc-3"); + + assert_eq!(builder.relationship_count(), 1); + + let rels = builder.build(); + assert_eq!(rels.len(), 1); + assert_eq!(rels[0].evidence_count, 3); + // Strength should be 3 * 0.1 = 0.3 + assert!((rels[0].strength - 0.3).abs() < f32::EPSILON); + } + + #[test] + fn test_add_co_occurrence_bidirectional() { + let mut builder = TopicGraphBuilder::new(); + let topic_a = "topic-1".to_string(); + let topic_b = "topic-2".to_string(); + + // Adding in either order should create one relationship + builder.add_co_occurrence(&topic_a, &topic_b, "doc-1"); + builder.add_co_occurrence(&topic_b, &topic_a, "doc-2"); + + assert_eq!(builder.relationship_count(), 1); + + let rels = builder.build(); + assert_eq!(rels[0].evidence_count, 2); + } + + #[test] + fn test_add_co_occurrence_ignores_self() { + let mut builder = TopicGraphBuilder::new(); + builder.add_co_occurrence(&"topic-1".to_string(), &"topic-1".to_string(), "doc-1"); + + assert_eq!(builder.relationship_count(), 0); + } + + #[test] + fn test_compute_semantic_relationships() { + let mut builder = TopicGraphBuilder::new(); + + // Similar vectors (high similarity) + let embeddings = vec![ + ("topic-1".to_string(), vec![1.0, 0.0, 0.0]), + ("topic-2".to_string(), vec![0.9, 0.1, 0.0]), + ("topic-3".to_string(), vec![0.0, 0.0, 1.0]), // Orthogonal, won't match + ]; + + builder.compute_semantic_relationships(&embeddings, Some(0.8)); + + // Should only create relationship between topic-1 and topic-2 + assert_eq!(builder.relationship_count(), 1); + + let rels = builder.build(); + assert_eq!(rels[0].relationship_type, RelationshipType::Semantic); + assert!(rels[0].strength >= 0.8); + } + + #[test] + fn test_compute_semantic_relationships_uses_default_threshold() { + let mut builder = TopicGraphBuilder::with_settings(0.1, 0.5); + + let embeddings = vec![ + ("topic-1".to_string(), vec![1.0, 0.0]), + ("topic-2".to_string(), vec![0.6, 0.8]), // Similarity ~0.6 + ]; + + builder.compute_semantic_relationships(&embeddings, None); + + // Should create relationship since 0.6 > 0.5 (default threshold) + assert_eq!(builder.relationship_count(), 1); + } + + #[test] + fn test_set_hierarchy() { + let mut builder = TopicGraphBuilder::new(); + builder.set_hierarchy(&"parent-topic".to_string(), &"child-topic".to_string()); + + assert_eq!(builder.relationship_count(), 1); + + let rels = builder.build(); + assert_eq!(rels[0].relationship_type, RelationshipType::Hierarchical); + assert_eq!(rels[0].source_id, "parent-topic"); + assert_eq!(rels[0].target_id, "child-topic"); + assert!((rels[0].strength - 1.0).abs() < f32::EPSILON); + } + + #[test] + fn test_set_hierarchy_ignores_self() { + let mut builder = TopicGraphBuilder::new(); + builder.set_hierarchy(&"topic".to_string(), &"topic".to_string()); + + assert_eq!(builder.relationship_count(), 0); + } + + #[test] + fn test_get_related_topics() { + let mut builder = TopicGraphBuilder::new(); + let main = "main-topic".to_string(); + let related_1 = "related-1".to_string(); + let related_2 = "related-2".to_string(); + let unrelated = "unrelated".to_string(); + + // Create relationships with different strengths + builder.add_co_occurrence(&main, &related_1, "doc-1"); + builder.add_co_occurrence(&main, &related_1, "doc-2"); // strength 0.2 + builder.add_co_occurrence(&main, &related_2, "doc-3"); // strength 0.1 + builder.add_co_occurrence(&related_1, &unrelated, "doc-4"); // not related to main + + let related = builder.get_related_topics(&main, 10); + + assert_eq!(related.len(), 2); + // Should be sorted by strength descending + assert_eq!(related[0].0, related_1); + assert_eq!(related[1].0, related_2); + } + + #[test] + fn test_get_related_topics_limit() { + let mut builder = TopicGraphBuilder::new(); + let main = "main".to_string(); + + for i in 0..10 { + builder.add_co_occurrence(&main, &format!("topic-{}", i), "doc"); + } + + let related = builder.get_related_topics(&main, 5); + assert_eq!(related.len(), 5); + } + + #[test] + fn test_strengthen_relationship() { + let mut builder = TopicGraphBuilder::new(); + let topic_a = "topic-a".to_string(); + let topic_b = "topic-b".to_string(); + + builder.add_co_occurrence(&topic_a, &topic_b, "doc-1"); + + // Initial strength is 0.1 + let initial_rels = builder.get_relationships(); + let initial_strength = initial_rels[0].strength; + + // Strengthen by 0.25 + let updated = builder.strengthen_relationship(&topic_a, &topic_b, 0.25); + assert!(updated); + + let rels = builder.get_relationships(); + assert!((rels[0].strength - (initial_strength + 0.25)).abs() < f32::EPSILON); + } + + #[test] + fn test_strengthen_relationship_clamping() { + let mut builder = TopicGraphBuilder::new(); + let topic_a = "topic-a".to_string(); + let topic_b = "topic-b".to_string(); + + builder.add_co_occurrence(&topic_a, &topic_b, "doc-1"); + + // Try to strengthen beyond 1.0 + builder.strengthen_relationship(&topic_a, &topic_b, 10.0); + let rels = builder.get_relationships(); + assert!((rels[0].strength - 1.0).abs() < f32::EPSILON); + + // Try to weaken below 0.0 + builder.strengthen_relationship(&topic_a, &topic_b, -10.0); + let rels = builder.get_relationships(); + assert!(rels[0].strength.abs() < f32::EPSILON); + } + + #[test] + fn test_strengthen_relationship_not_found() { + let mut builder = TopicGraphBuilder::new(); + let result = builder.strengthen_relationship( + &"nonexistent-a".to_string(), + &"nonexistent-b".to_string(), + 0.1, + ); + assert!(!result); + } + + #[test] + fn test_has_relationship() { + let mut builder = TopicGraphBuilder::new(); + let topic_a = "topic-a".to_string(); + let topic_b = "topic-b".to_string(); + let topic_c = "topic-c".to_string(); + + builder.add_co_occurrence(&topic_a, &topic_b, "doc-1"); + + assert!(builder.has_relationship(&topic_a, &topic_b)); + assert!(builder.has_relationship(&topic_b, &topic_a)); // Bidirectional + assert!(!builder.has_relationship(&topic_a, &topic_c)); + } + + #[test] + fn test_clear() { + let mut builder = TopicGraphBuilder::new(); + builder.add_co_occurrence(&"a".to_string(), &"b".to_string(), "doc"); + builder.add_co_occurrence(&"c".to_string(), &"d".to_string(), "doc"); + + assert_eq!(builder.relationship_count(), 2); + + builder.clear(); + + assert_eq!(builder.relationship_count(), 0); + } + + #[test] + fn test_build_consumes_builder() { + let mut builder = TopicGraphBuilder::new(); + builder.add_co_occurrence(&"a".to_string(), &"b".to_string(), "doc"); + + let rels = builder.build(); + assert_eq!(rels.len(), 1); + // builder is now consumed + } + + #[test] + fn test_get_relationships_preserves_builder() { + let mut builder = TopicGraphBuilder::new(); + builder.add_co_occurrence(&"a".to_string(), &"b".to_string(), "doc"); + + let rels1 = builder.get_relationships(); + let rels2 = builder.get_relationships(); + + assert_eq!(rels1.len(), rels2.len()); + } + + #[test] + fn test_multiple_relationship_types_same_topics() { + let mut builder = TopicGraphBuilder::new(); + let topic_a = "topic-a".to_string(); + let topic_b = "topic-b".to_string(); + + // Same topics can have different relationship types + builder.add_co_occurrence(&topic_a, &topic_b, "doc-1"); + builder.set_hierarchy(&topic_a, &topic_b); + + let embeddings = vec![ + (topic_a.clone(), vec![1.0, 0.0]), + (topic_b.clone(), vec![0.9, 0.1]), + ]; + builder.compute_semantic_relationships(&embeddings, Some(0.5)); + + // Should have 3 different relationship types + assert_eq!(builder.relationship_count(), 3); + } + + #[test] + fn test_integration_complex_graph() { + let mut builder = TopicGraphBuilder::with_settings(0.15, 0.7); + + let topics = vec![ + ("rust".to_string(), vec![1.0, 0.0, 0.0]), + ("programming".to_string(), vec![0.9, 0.1, 0.0]), + ("memory".to_string(), vec![0.8, 0.2, 0.0]), + ("databases".to_string(), vec![0.0, 1.0, 0.0]), + ("sql".to_string(), vec![0.1, 0.9, 0.0]), + ]; + + // Add co-occurrences + builder.add_co_occurrence(&topics[0].0, &topics[1].0, "doc-rust-programming"); + builder.add_co_occurrence(&topics[0].0, &topics[2].0, "doc-rust-memory"); + builder.add_co_occurrence(&topics[3].0, &topics[4].0, "doc-db-sql"); + + // Add hierarchy + builder.set_hierarchy(&topics[1].0, &topics[0].0); // programming -> rust + + // Compute semantic relationships + builder.compute_semantic_relationships(&topics, None); + + let rels = builder.build(); + + // Verify we have multiple relationship types + let co_occur_count = rels + .iter() + .filter(|r| r.relationship_type == RelationshipType::CoOccurrence) + .count(); + let semantic_count = rels + .iter() + .filter(|r| r.relationship_type == RelationshipType::Semantic) + .count(); + let hier_count = rels + .iter() + .filter(|r| r.relationship_type == RelationshipType::Hierarchical) + .count(); + + assert!( + co_occur_count > 0, + "Should have co-occurrence relationships" + ); + assert!(semantic_count > 0, "Should have semantic relationships"); + assert_eq!(hier_count, 1, "Should have one hierarchical relationship"); + } +} diff --git a/crates/memory-topics/src/similarity.rs b/crates/memory-topics/src/similarity.rs new file mode 100644 index 0000000..23302ff --- /dev/null +++ b/crates/memory-topics/src/similarity.rs @@ -0,0 +1,198 @@ +//! Vector similarity functions. +//! +//! Pure Rust implementations without external dependencies. + +/// Calculate cosine similarity between two vectors. +/// +/// Returns value in [-1.0, 1.0] where 1.0 = identical direction. +/// +/// # Panics +/// Panics if vectors have different dimensions. +pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 { + assert_eq!(a.len(), b.len(), "Vectors must have same dimension"); + + let dot_product: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum(); + let norm_a: f32 = a.iter().map(|x| x * x).sum::().sqrt(); + let norm_b: f32 = b.iter().map(|x| x * x).sum::().sqrt(); + + if norm_a == 0.0 || norm_b == 0.0 { + return 0.0; + } + + dot_product / (norm_a * norm_b) +} + +/// Calculate the centroid of multiple embeddings. +/// +/// Returns a normalized vector representing the center of the cluster. +pub fn calculate_centroid(embeddings: &[&[f32]]) -> Vec { + if embeddings.is_empty() { + return Vec::new(); + } + + let dim = embeddings[0].len(); + let n = embeddings.len() as f32; + let mut centroid = vec![0.0f32; dim]; + + for embedding in embeddings { + assert_eq!( + embedding.len(), + dim, + "All embeddings must have same dimension" + ); + for (i, &val) in embedding.iter().enumerate() { + centroid[i] += val; + } + } + + // Average + for val in centroid.iter_mut() { + *val /= n; + } + + // Normalize + normalize(&mut centroid); + + centroid +} + +/// Normalize a vector to unit length in place. +pub fn normalize(v: &mut [f32]) { + let norm: f32 = v.iter().map(|x| x * x).sum::().sqrt(); + if norm > 0.0 { + for val in v.iter_mut() { + *val /= norm; + } + } +} + +/// Calculate pairwise distances between embeddings. +/// +/// Returns a distance matrix where distance = 1 - cosine_similarity. +pub fn pairwise_distances(embeddings: &[Vec]) -> Vec> { + let n = embeddings.len(); + let mut distances = vec![vec![0.0f64; n]; n]; + + for i in 0..n { + for j in (i + 1)..n { + let sim = cosine_similarity(&embeddings[i], &embeddings[j]); + let dist = (1.0 - sim) as f64; + distances[i][j] = dist; + distances[j][i] = dist; + } + } + + distances +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_cosine_similarity_identical() { + let a = vec![1.0, 0.0, 0.0]; + let b = vec![1.0, 0.0, 0.0]; + assert!((cosine_similarity(&a, &b) - 1.0).abs() < 0.001); + } + + #[test] + fn test_cosine_similarity_orthogonal() { + let a = vec![1.0, 0.0]; + let b = vec![0.0, 1.0]; + assert!(cosine_similarity(&a, &b).abs() < 0.001); + } + + #[test] + fn test_cosine_similarity_opposite() { + let a = vec![1.0, 0.0]; + let b = vec![-1.0, 0.0]; + assert!((cosine_similarity(&a, &b) + 1.0).abs() < 0.001); + } + + #[test] + fn test_cosine_similarity_similar() { + let a = vec![0.8, 0.6]; + let b = vec![0.6, 0.8]; + let sim = cosine_similarity(&a, &b); + assert!(sim > 0.9); // Should be similar + } + + #[test] + fn test_cosine_similarity_zero_vector() { + let a = vec![0.0, 0.0]; + let b = vec![1.0, 0.0]; + assert!(cosine_similarity(&a, &b).abs() < 0.001); + } + + #[test] + fn test_calculate_centroid() { + let e1 = vec![1.0, 0.0, 0.0]; + let e2 = vec![0.0, 1.0, 0.0]; + let embeddings: Vec<&[f32]> = vec![&e1, &e2]; + let centroid = calculate_centroid(&embeddings); + // Average of [1,0,0] and [0,1,0] = [0.5, 0.5, 0] normalized + let expected_norm = (0.5f32.powi(2) * 2.0).sqrt(); + assert!((centroid[0] - 0.5 / expected_norm).abs() < 0.001); + assert!((centroid[1] - 0.5 / expected_norm).abs() < 0.001); + assert!(centroid[2].abs() < 0.001); + } + + #[test] + fn test_calculate_centroid_empty() { + let embeddings: Vec<&[f32]> = vec![]; + let centroid = calculate_centroid(&embeddings); + assert!(centroid.is_empty()); + } + + #[test] + fn test_calculate_centroid_single() { + let e1 = vec![3.0, 4.0]; + let embeddings: Vec<&[f32]> = vec![&e1]; + let centroid = calculate_centroid(&embeddings); + // Single embedding normalized: [3,4]/5 = [0.6, 0.8] + assert!((centroid[0] - 0.6).abs() < 0.001); + assert!((centroid[1] - 0.8).abs() < 0.001); + } + + #[test] + fn test_normalize() { + let mut v = vec![3.0, 4.0]; + normalize(&mut v); + assert!((v[0] - 0.6).abs() < 0.001); + assert!((v[1] - 0.8).abs() < 0.001); + } + + #[test] + fn test_normalize_zero_vector() { + let mut v = vec![0.0, 0.0]; + normalize(&mut v); + assert!((v[0]).abs() < 0.001); + assert!((v[1]).abs() < 0.001); + } + + #[test] + fn test_pairwise_distances() { + let embeddings = vec![vec![1.0, 0.0], vec![0.0, 1.0], vec![1.0, 0.0]]; + let distances = pairwise_distances(&embeddings); + assert!((distances[0][2]).abs() < 0.001); // Identical + assert!((distances[0][1] - 1.0).abs() < 0.001); // Orthogonal + } + + #[test] + fn test_pairwise_distances_self() { + let embeddings = vec![vec![1.0, 0.0], vec![0.0, 1.0]]; + let distances = pairwise_distances(&embeddings); + // Self-distance should be 0 + assert!((distances[0][0]).abs() < 0.001); + assert!((distances[1][1]).abs() < 0.001); + } + + #[test] + #[should_panic(expected = "Vectors must have same dimension")] + fn test_cosine_similarity_different_dimensions() { + let a = vec![1.0, 0.0]; + let b = vec![1.0, 0.0, 0.0]; + cosine_similarity(&a, &b); + } +} diff --git a/crates/memory-topics/src/storage.rs b/crates/memory-topics/src/storage.rs new file mode 100644 index 0000000..5ea3ca7 --- /dev/null +++ b/crates/memory-topics/src/storage.rs @@ -0,0 +1,546 @@ +//! Topic storage operations. +//! +//! Manages topics, links, and relationships in RocksDB column families. + +use std::sync::Arc; + +use chrono::{DateTime, Utc}; +use memory_storage::Storage; +use tracing::{debug, info, instrument}; + +use crate::error::TopicsError; +use crate::importance::ImportanceScorer; +use crate::types::{ + RelationshipType, Topic, TopicLink, TopicRelationship, TopicStats, TopicStatus, +}; + +/// Column family names (must match memory-storage) +pub const CF_TOPICS: &str = "topics"; +pub const CF_TOPIC_LINKS: &str = "topic_links"; +pub const CF_TOPIC_RELS: &str = "topic_rels"; +/// Column family for topic relationships (alias for CF_TOPIC_RELS) +pub const CF_TOPIC_RELATIONSHIPS: &str = "topic_rels"; + +/// Key format for topics: topic:{topic_id} +pub fn topic_key(topic_id: &str) -> String { + format!("topic:{}", topic_id) +} + +/// Key format for topic links: link:{topic_id}:{node_id} +pub fn topic_link_key(topic_id: &str, node_id: &str) -> String { + format!("link:{}:{}", topic_id, node_id) +} + +/// Secondary index: node:{node_id}:{topic_id} +pub fn node_topic_key(node_id: &str, topic_id: &str) -> String { + format!("node:{}:{}", node_id, topic_id) +} + +/// Key format for relationships: rel:{from}:{type}:{to} +pub fn relationship_key(from_id: &str, rel_type: &str, to_id: &str) -> String { + format!("rel:{}:{}:{}", from_id, rel_type, to_id) +} + +/// Topic storage interface. +pub struct TopicStorage { + storage: Arc, +} + +impl TopicStorage { + /// Create a new topic storage wrapper. + pub fn new(storage: Arc) -> Self { + Self { storage } + } + + /// Get underlying storage. + pub fn storage(&self) -> &Arc { + &self.storage + } + + // --- Topic CRUD --- + + /// Save a topic. + #[instrument(skip(self, topic), fields(topic_id = %topic.topic_id))] + pub fn save_topic(&self, topic: &Topic) -> Result<(), TopicsError> { + let key = topic_key(&topic.topic_id); + let value = serde_json::to_vec(topic)?; + self.storage.put(CF_TOPICS, key.as_bytes(), &value)?; + debug!("Saved topic"); + Ok(()) + } + + /// Get a topic by ID. + #[instrument(skip(self))] + pub fn get_topic(&self, topic_id: &str) -> Result, TopicsError> { + let key = topic_key(topic_id); + match self.storage.get(CF_TOPICS, key.as_bytes())? { + Some(bytes) => { + let topic: Topic = serde_json::from_slice(&bytes)?; + Ok(Some(topic)) + } + None => Ok(None), + } + } + + /// Delete a topic and its links. + #[instrument(skip(self))] + pub fn delete_topic(&self, topic_id: &str) -> Result<(), TopicsError> { + // Delete topic + let key = topic_key(topic_id); + self.storage.delete(CF_TOPICS, key.as_bytes())?; + + // Delete links (would need iteration in production) + // For now, links are cleaned up separately + debug!("Deleted topic"); + Ok(()) + } + + /// List all active topics. + pub fn list_topics(&self) -> Result, TopicsError> { + let prefix = b"topic:"; + let mut topics = Vec::new(); + + for (_, value) in self.storage.prefix_iterator(CF_TOPICS, prefix)? { + let topic: Topic = serde_json::from_slice(&value)?; + if topic.status == TopicStatus::Active { + topics.push(topic); + } + } + + Ok(topics) + } + + /// List topics sorted by importance score (descending). + pub fn list_topics_by_importance(&self, limit: usize) -> Result, TopicsError> { + let mut topics = self.list_topics()?; + topics.sort_by(|a, b| { + b.importance_score + .partial_cmp(&a.importance_score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + topics.truncate(limit); + Ok(topics) + } + + // --- Topic Links --- + + /// Save a topic-node link. + #[instrument(skip(self, link), fields(topic_id = %link.topic_id, node_id = %link.node_id))] + pub fn save_link(&self, link: &TopicLink) -> Result<(), TopicsError> { + let value = serde_json::to_vec(link)?; + + // Primary key: topic -> nodes + let primary_key = topic_link_key(&link.topic_id, &link.node_id); + self.storage + .put(CF_TOPIC_LINKS, primary_key.as_bytes(), &value)?; + + // Secondary key: node -> topics + let secondary_key = node_topic_key(&link.node_id, &link.topic_id); + self.storage + .put(CF_TOPIC_LINKS, secondary_key.as_bytes(), &value)?; + + debug!("Saved topic link"); + Ok(()) + } + + /// Get links for a topic. + pub fn get_links_for_topic(&self, topic_id: &str) -> Result, TopicsError> { + let prefix = format!("link:{}:", topic_id); + let mut links = Vec::new(); + + for (_, value) in self + .storage + .prefix_iterator(CF_TOPIC_LINKS, prefix.as_bytes())? + { + let link: TopicLink = serde_json::from_slice(&value)?; + links.push(link); + } + + // Sort by relevance descending + links.sort_by(|a, b| { + b.relevance + .partial_cmp(&a.relevance) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + Ok(links) + } + + /// Get topics for a node. + pub fn get_topics_for_node(&self, node_id: &str) -> Result, TopicsError> { + let prefix = format!("node:{}:", node_id); + let mut links = Vec::new(); + + for (_, value) in self + .storage + .prefix_iterator(CF_TOPIC_LINKS, prefix.as_bytes())? + { + let link: TopicLink = serde_json::from_slice(&value)?; + links.push(link); + } + + Ok(links) + } + + // --- Relationships --- + + /// Save a topic relationship. + #[instrument(skip(self, rel), fields(source = %rel.source_id, target = %rel.target_id))] + pub fn save_relationship(&self, rel: &TopicRelationship) -> Result<(), TopicsError> { + let key = relationship_key(&rel.source_id, rel.relationship_type.code(), &rel.target_id); + let value = serde_json::to_vec(rel)?; + self.storage.put(CF_TOPIC_RELS, key.as_bytes(), &value)?; + debug!("Saved relationship"); + Ok(()) + } + + /// Store a topic relationship (alias for save_relationship). + #[instrument(skip(self, rel), fields(source = %rel.source_id, target = %rel.target_id))] + pub fn store_relationship(&self, rel: &TopicRelationship) -> Result<(), TopicsError> { + self.save_relationship(rel) + } + + /// Get relationships for a topic. + pub fn get_relationships(&self, topic_id: &str) -> Result, TopicsError> { + self.get_relationships_filtered(topic_id, None) + } + + /// Get relationships for a topic, optionally filtered by type. + pub fn get_relationships_filtered( + &self, + topic_id: &str, + rel_type: Option, + ) -> Result, TopicsError> { + let prefix = match rel_type { + Some(rt) => format!("rel:{}:{}:", topic_id, rt.code()), + None => format!("rel:{}:", topic_id), + }; + + let mut rels = Vec::new(); + + for (_, value) in self + .storage + .prefix_iterator(CF_TOPIC_RELS, prefix.as_bytes())? + { + let rel: TopicRelationship = serde_json::from_slice(&value)?; + rels.push(rel); + } + + // Sort by strength descending + rels.sort_by(|a, b| { + b.strength + .partial_cmp(&a.strength) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + Ok(rels) + } + + /// Get related topics with their relationship strength. + /// + /// Returns topics related to the given topic, optionally filtered by relationship type, + /// sorted by strength descending and limited to `limit` results. + pub fn get_related_topics( + &self, + topic_id: &str, + rel_type: Option, + limit: usize, + ) -> Result, TopicsError> { + let rels = self.get_relationships_filtered(topic_id, rel_type)?; + let related: Vec<(String, f32)> = rels + .into_iter() + .take(limit) + .map(|rel| (rel.target_id, rel.strength)) + .collect(); + Ok(related) + } + + /// Update the strength of an existing relationship. + /// + /// Returns an error if the relationship doesn't exist. + #[instrument(skip(self))] + pub fn update_relationship_strength( + &self, + source_id: &str, + target_id: &str, + rel_type: RelationshipType, + new_strength: f32, + ) -> Result<(), TopicsError> { + let key = relationship_key(source_id, rel_type.code(), target_id); + + // Load existing relationship + let bytes = self + .storage + .get(CF_TOPIC_RELS, key.as_bytes())? + .ok_or_else(|| { + TopicsError::NotFound(format!( + "Relationship {}->{}:{}", + source_id, + target_id, + rel_type.code() + )) + })?; + + let mut rel: TopicRelationship = serde_json::from_slice(&bytes)?; + rel.set_strength(new_strength); + + // Save updated relationship + let value = serde_json::to_vec(&rel)?; + self.storage.put(CF_TOPIC_RELS, key.as_bytes(), &value)?; + + debug!(new_strength = rel.strength, "Updated relationship strength"); + Ok(()) + } + + /// Get a specific relationship between two topics. + pub fn get_relationship( + &self, + source_id: &str, + target_id: &str, + rel_type: RelationshipType, + ) -> Result, TopicsError> { + let key = relationship_key(source_id, rel_type.code(), target_id); + match self.storage.get(CF_TOPIC_RELS, key.as_bytes())? { + Some(bytes) => { + let rel: TopicRelationship = serde_json::from_slice(&bytes)?; + Ok(Some(rel)) + } + None => Ok(None), + } + } + + /// Delete a relationship between two topics. + #[instrument(skip(self))] + pub fn delete_relationship( + &self, + source_id: &str, + target_id: &str, + rel_type: RelationshipType, + ) -> Result<(), TopicsError> { + let key = relationship_key(source_id, rel_type.code(), target_id); + self.storage.delete(CF_TOPIC_RELS, key.as_bytes())?; + debug!("Deleted relationship"); + Ok(()) + } + + /// Increment evidence count for a relationship and optionally strengthen it. + /// + /// If the relationship doesn't exist, creates a new one. + #[instrument(skip(self))] + pub fn record_relationship_evidence( + &self, + source_id: &str, + target_id: &str, + rel_type: RelationshipType, + strength_delta: f32, + ) -> Result<(), TopicsError> { + let key = relationship_key(source_id, rel_type.code(), target_id); + + let rel = match self.storage.get(CF_TOPIC_RELS, key.as_bytes())? { + Some(bytes) => { + let mut existing: TopicRelationship = serde_json::from_slice(&bytes)?; + existing.add_evidence(); + existing.strengthen(strength_delta); + existing + } + None => { + // Create new relationship + TopicRelationship::new( + source_id.to_string(), + target_id.to_string(), + rel_type, + strength_delta.clamp(0.0, 1.0), + ) + } + }; + + let value = serde_json::to_vec(&rel)?; + self.storage.put(CF_TOPIC_RELS, key.as_bytes(), &value)?; + + debug!( + evidence_count = rel.evidence_count, + strength = rel.strength, + "Recorded relationship evidence" + ); + Ok(()) + } + + // --- Statistics --- + + /// Get topic graph statistics. + pub fn get_stats(&self) -> Result { + let topics = self.list_topics()?; + let topic_count = topics.len() as u64; + + // Count links (approximate via prefix scan) + let link_count = self + .storage + .prefix_iterator(CF_TOPIC_LINKS, b"link:")? + .len() as u64; + + // Count relationships + let relationship_count = self.storage.prefix_iterator(CF_TOPIC_RELS, b"rel:")?.len() as u64; + + Ok(TopicStats { + topic_count, + link_count, + relationship_count, + last_extraction_ms: 0, // Set by extraction job + half_life_days: 30, // From config + similarity_threshold: 0.75, + }) + } + + // --- Importance Scoring --- + + /// Touch a topic to update its last_mentioned_at timestamp and recalculate importance. + /// + /// This updates the topic's timestamp to `now` and recalculates the importance + /// score using the provided scorer. The node count is NOT incremented. + /// + /// # Arguments + /// * `topic_id` - ID of the topic to touch + /// * `scorer` - Importance scorer for recalculation + /// * `now` - Current timestamp + /// + /// # Returns + /// * `Ok(())` on success + /// * `Err(TopicsError::NotFound)` if topic doesn't exist + #[instrument(skip(self, scorer))] + pub fn touch_topic( + &self, + topic_id: &str, + scorer: &ImportanceScorer, + now: DateTime, + ) -> Result<(), TopicsError> { + let mut topic = self + .get_topic(topic_id)? + .ok_or_else(|| TopicsError::NotFound(topic_id.to_string()))?; + + scorer.touch_topic(&mut topic, now); + self.save_topic(&topic)?; + + debug!(new_score = topic.importance_score, "Touched topic"); + Ok(()) + } + + /// Record a topic mention, incrementing node count and updating importance. + /// + /// This is used when a new node is linked to the topic. It: + /// - Increments the node count + /// - Updates last_mentioned_at to now + /// - Recalculates the importance score + /// + /// # Arguments + /// * `topic_id` - ID of the topic that was mentioned + /// * `scorer` - Importance scorer for recalculation + /// * `now` - Current timestamp + /// + /// # Returns + /// * `Ok(())` on success + /// * `Err(TopicsError::NotFound)` if topic doesn't exist + #[instrument(skip(self, scorer))] + pub fn record_topic_mention( + &self, + topic_id: &str, + scorer: &ImportanceScorer, + now: DateTime, + ) -> Result<(), TopicsError> { + let mut topic = self + .get_topic(topic_id)? + .ok_or_else(|| TopicsError::NotFound(topic_id.to_string()))?; + + scorer.on_topic_mentioned(&mut topic, now); + self.save_topic(&topic)?; + + debug!( + node_count = topic.node_count, + new_score = topic.importance_score, + "Recorded topic mention" + ); + Ok(()) + } + + /// Get top topics sorted by importance score. + /// + /// Alias for `list_topics_by_importance` with clearer naming. + pub fn get_top_topics(&self, limit: usize) -> Result, TopicsError> { + self.list_topics_by_importance(limit) + } + + /// Refresh importance scores for all topics. + /// + /// This is intended to be run as a periodic background job to ensure + /// topic scores reflect current decay. Only topics whose scores have + /// changed are persisted. + /// + /// # Arguments + /// * `scorer` - Importance scorer for recalculation + /// + /// # Returns + /// Number of topics that were updated + #[instrument(skip(self, scorer))] + pub fn refresh_importance_scores(&self, scorer: &ImportanceScorer) -> Result { + let now = Utc::now(); + let mut topics = self.list_topics()?; + let updated = scorer.recalculate_all(&mut topics, now); + + // Persist updated topics + for topic in &topics { + self.save_topic(topic)?; + } + + info!( + updated_count = updated, + total = topics.len(), + "Refreshed importance scores" + ); + Ok(updated) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_topic_key() { + assert_eq!(topic_key("abc123"), "topic:abc123"); + } + + #[test] + fn test_topic_link_key() { + assert_eq!(topic_link_key("t1", "n1"), "link:t1:n1"); + } + + #[test] + fn test_node_topic_key() { + assert_eq!(node_topic_key("n1", "t1"), "node:n1:t1"); + } + + #[test] + fn test_relationship_key() { + assert_eq!(relationship_key("t1", "sem", "t2"), "rel:t1:sem:t2"); + } + + #[test] + fn test_relationship_key_with_type() { + assert_eq!( + relationship_key("topic-a", RelationshipType::Hierarchical.code(), "topic-b"), + "rel:topic-a:hie:topic-b" + ); + } + + #[test] + fn test_relationship_key_co_occurrence() { + assert_eq!( + relationship_key("topic-x", RelationshipType::CoOccurrence.code(), "topic-y"), + "rel:topic-x:coo:topic-y" + ); + } + + #[test] + fn test_cf_topic_relationships_alias() { + assert_eq!(CF_TOPIC_RELATIONSHIPS, CF_TOPIC_RELS); + } +} diff --git a/crates/memory-topics/src/tfidf.rs b/crates/memory-topics/src/tfidf.rs new file mode 100644 index 0000000..459f4bb --- /dev/null +++ b/crates/memory-topics/src/tfidf.rs @@ -0,0 +1,358 @@ +//! TF-IDF (Term Frequency - Inverse Document Frequency) implementation. +//! +//! Pure Rust implementation for keyword extraction without external dependencies. + +use std::collections::{HashMap, HashSet}; + +/// TF-IDF calculator for keyword extraction. +/// +/// Computes term importance based on frequency within documents and rarity +/// across the document corpus. +pub struct TfIdf { + /// Term -> document count (how many documents contain this term) + doc_frequencies: HashMap, + /// Term -> total frequency across all documents + term_frequencies: HashMap, + /// Number of documents + doc_count: usize, +} + +impl TfIdf { + /// Create a new TF-IDF calculator from a corpus of documents. + pub fn new(documents: &[&str]) -> Self { + let mut doc_frequencies: HashMap = HashMap::new(); + let mut term_frequencies: HashMap = HashMap::new(); + let doc_count = documents.len(); + + for doc in documents { + let terms = tokenize(doc); + let unique_terms: HashSet<&String> = terms.iter().collect(); + + // Count document frequency (each term counted once per doc) + for term in unique_terms { + *doc_frequencies.entry(term.clone()).or_insert(0) += 1; + } + + // Count total term frequency + for term in terms { + *term_frequencies.entry(term).or_insert(0) += 1; + } + } + + Self { + doc_frequencies, + term_frequencies, + doc_count, + } + } + + /// Calculate TF-IDF score for a term. + /// + /// TF-IDF = TF * IDF + /// - TF (Term Frequency) = count of term / total terms + /// - IDF (Inverse Document Frequency) = log(N / df) where N = doc count, df = doc frequency + pub fn score(&self, term: &str) -> f32 { + let tf = self.term_frequency(term); + let idf = self.inverse_document_frequency(term); + tf * idf + } + + /// Calculate term frequency (normalized by total terms). + fn term_frequency(&self, term: &str) -> f32 { + let count = *self.term_frequencies.get(term).unwrap_or(&0) as f32; + let total: usize = self.term_frequencies.values().sum(); + if total == 0 { + return 0.0; + } + count / total as f32 + } + + /// Calculate inverse document frequency. + /// + /// Uses smoothed IDF: log((N + 1) / (df + 1)) + 1 + fn inverse_document_frequency(&self, term: &str) -> f32 { + let df = *self.doc_frequencies.get(term).unwrap_or(&0) as f32; + let n = self.doc_count as f32; + + if df == 0.0 { + return 0.0; + } + + // Smoothed IDF to avoid division by zero and extreme values + ((n + 1.0) / (df + 1.0)).ln() + 1.0 + } + + /// Get top N terms by TF-IDF score. + /// + /// Returns terms sorted by score (highest first). + pub fn top_terms(&self, n: usize) -> Vec<(String, f32)> { + let mut scores: Vec<(String, f32)> = self + .term_frequencies + .keys() + .map(|term| (term.clone(), self.score(term))) + .filter(|(_, score)| *score > 0.0) + .collect(); + + // Sort by score descending + scores.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + + scores.truncate(n); + scores + } + + /// Get all terms with their TF-IDF scores. + pub fn all_scores(&self) -> HashMap { + self.term_frequencies + .keys() + .map(|term| (term.clone(), self.score(term))) + .collect() + } + + /// Get document count. + pub fn doc_count(&self) -> usize { + self.doc_count + } + + /// Get unique term count. + pub fn term_count(&self) -> usize { + self.term_frequencies.len() + } +} + +/// Tokenize text into lowercase words. +/// +/// Filters out: +/// - Stop words (common English words) +/// - Single character tokens +/// - Numbers +fn tokenize(text: &str) -> Vec { + text.to_lowercase() + .split(|c: char| !c.is_alphanumeric()) + .filter(|s| !s.is_empty()) + .filter(|s| s.len() > 1) + .filter(|s| !is_stop_word(s)) + .filter(|s| !s.chars().all(|c| c.is_numeric())) + .map(String::from) + .collect() +} + +/// Check if a word is a stop word. +fn is_stop_word(word: &str) -> bool { + const STOP_WORDS: &[&str] = &[ + "a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "has", "he", "in", "is", + "it", "its", "of", "on", "or", "that", "the", "to", "was", "were", "will", "with", "this", + "they", "but", "have", "had", "what", "when", "where", "who", "which", "why", "how", "all", + "each", "every", "both", "few", "more", "most", "other", "some", "such", "no", "nor", + "not", "only", "own", "same", "so", "than", "too", "very", "can", "just", "should", "now", + "also", "been", "being", "do", "does", "did", "doing", "would", "could", "might", "must", + "shall", "about", "above", "after", "again", "against", "am", "any", "before", "below", + "between", "into", "through", "during", "out", "over", "under", "up", "down", "then", + "once", "here", "there", "if", "else", "while", "because", "until", "we", "you", "your", + "our", "their", "him", "her", "them", "me", "my", "myself", "itself", "those", "these", + "his", + ]; + + STOP_WORDS.contains(&word) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_tokenize_basic() { + let tokens = tokenize("Hello World"); + assert_eq!(tokens, vec!["hello", "world"]); + } + + #[test] + fn test_tokenize_removes_stop_words() { + let tokens = tokenize("the quick brown fox"); + assert!(!tokens.contains(&"the".to_string())); + assert!(tokens.contains(&"quick".to_string())); + assert!(tokens.contains(&"brown".to_string())); + assert!(tokens.contains(&"fox".to_string())); + } + + #[test] + fn test_tokenize_removes_single_chars() { + let tokens = tokenize("a b c rust"); + assert_eq!(tokens, vec!["rust"]); + } + + #[test] + fn test_tokenize_removes_numbers() { + let tokens = tokenize("rust 123 456 programming"); + assert_eq!(tokens, vec!["rust", "programming"]); + } + + #[test] + fn test_tokenize_handles_punctuation() { + let tokens = tokenize("rust, python, and java!"); + assert!(tokens.contains(&"rust".to_string())); + assert!(tokens.contains(&"python".to_string())); + assert!(tokens.contains(&"java".to_string())); + } + + #[test] + fn test_is_stop_word() { + assert!(is_stop_word("the")); + assert!(is_stop_word("and")); + assert!(is_stop_word("is")); + assert!(!is_stop_word("rust")); + assert!(!is_stop_word("programming")); + } + + #[test] + fn test_tfidf_new() { + let docs = vec!["rust programming", "python programming", "rust systems"]; + let tfidf = TfIdf::new(&docs); + + assert_eq!(tfidf.doc_count(), 3); + assert!(tfidf.term_count() > 0); + } + + #[test] + fn test_tfidf_score_common_term() { + let docs = vec!["rust programming", "python programming", "java programming"]; + let tfidf = TfIdf::new(&docs); + + // "programming" appears in all docs (3x), rust/python/java each appear once + // All have same TF (1/6 for each single occurrence, 3/6 for programming) + // programming: TF=0.5, IDF=ln(4/4)+1 = 1.0, score = 0.5 + // rust: TF=1/6, IDF=ln(4/2)+1 = 1.69, score = 0.28 + let prog_score = tfidf.score("programming"); + let rust_score = tfidf.score("rust"); + + // programming has higher TF which outweighs the IDF difference + assert!(prog_score > 0.0); + assert!(rust_score > 0.0); + // High frequency term dominates in TF-IDF + assert!(prog_score > rust_score); + } + + #[test] + fn test_tfidf_score_rare_term() { + let docs = vec![ + "machine learning algorithms", + "deep learning neural networks", + "machine learning models", + ]; + let tfidf = TfIdf::new(&docs); + + // "neural" only in one doc, "learning" in all + let neural_score = tfidf.score("neural"); + let learning_score = tfidf.score("learning"); + + // Neural should have higher IDF component + assert!(neural_score > 0.0); + assert!(learning_score > 0.0); + } + + #[test] + fn test_tfidf_score_nonexistent_term() { + let docs = vec!["rust programming"]; + let tfidf = TfIdf::new(&docs); + + let score = tfidf.score("nonexistent"); + assert!((score - 0.0).abs() < f32::EPSILON); + } + + #[test] + fn test_top_terms() { + let docs = vec![ + "rust rust rust systems", + "python scripting", + "rust memory safety", + ]; + let tfidf = TfIdf::new(&docs); + + let top = tfidf.top_terms(3); + assert!(!top.is_empty()); + assert!(top.len() <= 3); + + // Top terms should be sorted by score + for i in 1..top.len() { + assert!(top[i - 1].1 >= top[i].1); + } + } + + #[test] + fn test_top_terms_includes_rust() { + let docs = vec![ + "rust rust rust programming", + "rust systems programming", + "python scripting language", + ]; + let tfidf = TfIdf::new(&docs); + + let top = tfidf.top_terms(5); + let top_words: Vec<&str> = top.iter().map(|(w, _)| w.as_str()).collect(); + + // "rust" appears frequently but not in all docs, should rank high + assert!(top_words.contains(&"rust")); + } + + #[test] + fn test_all_scores() { + let docs = vec!["rust programming"]; + let tfidf = TfIdf::new(&docs); + + let scores = tfidf.all_scores(); + assert!(scores.contains_key("rust")); + assert!(scores.contains_key("programming")); + } + + #[test] + fn test_empty_corpus() { + let docs: Vec<&str> = vec![]; + let tfidf = TfIdf::new(&docs); + + assert_eq!(tfidf.doc_count(), 0); + assert_eq!(tfidf.term_count(), 0); + assert!(tfidf.top_terms(5).is_empty()); + } + + #[test] + fn test_single_document() { + let docs = vec!["rust memory safety ownership borrowing"]; + let tfidf = TfIdf::new(&docs); + + assert_eq!(tfidf.doc_count(), 1); + let top = tfidf.top_terms(5); + assert!(!top.is_empty()); + } + + #[test] + fn test_repeated_terms() { + let docs = vec!["rust rust rust rust", "python"]; + let tfidf = TfIdf::new(&docs); + + // "rust" repeated multiple times should have high TF + let rust_score = tfidf.score("rust"); + let python_score = tfidf.score("python"); + + // Both appear in only one doc, but rust has higher TF + assert!(rust_score > python_score); + } + + #[test] + fn test_idf_calculation() { + let docs = vec!["term1 term2", "term1 term3", "term1 term4"]; + let tfidf = TfIdf::new(&docs); + + // term1 in all 3 docs (TF=3/6=0.5), term2/3/4 in only 1 each (TF=1/6) + // IDF for term1: ln(4/4)+1 = 1.0 + // IDF for term2: ln(4/2)+1 = 1.69 + let score_common = tfidf.score("term1"); + let score_rare = tfidf.score("term2"); + + // term1 has 3x the TF with lower IDF, term2 has 1x TF with higher IDF + // 0.5 * 1.0 = 0.5 vs 0.167 * 1.69 = 0.28 + assert!(score_common > 0.0); + assert!(score_rare > 0.0); + + // In TF-IDF with global TF, frequent terms dominate + assert!(score_common > score_rare); + } +} diff --git a/crates/memory-topics/src/types.rs b/crates/memory-topics/src/types.rs new file mode 100644 index 0000000..c8bf75c --- /dev/null +++ b/crates/memory-topics/src/types.rs @@ -0,0 +1,467 @@ +//! Topic data types. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +/// A unique identifier for a topic. +pub type TopicId = String; + +/// An embedding vector. +pub type Embedding = Vec; + +/// A semantic topic extracted from TOC summaries. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Topic { + /// Unique identifier (ULID) + pub topic_id: TopicId, + /// Human-readable label (max 50 chars) + pub label: String, + /// Centroid embedding for similarity matching + pub embedding: Embedding, + /// Time-decayed importance score + pub importance_score: f64, + /// Number of linked TOC nodes + pub node_count: u32, + /// First occurrence timestamp + pub created_at: DateTime, + /// Most recent mention timestamp + pub last_mentioned_at: DateTime, + /// Active or pruned status + pub status: TopicStatus, + /// Keywords extracted from cluster + pub keywords: Vec, +} + +impl Topic { + /// Create a new topic with default values. + pub fn new(topic_id: String, label: String, embedding: Vec) -> Self { + let now = Utc::now(); + Self { + topic_id, + label, + embedding, + importance_score: 1.0, + node_count: 0, + created_at: now, + last_mentioned_at: now, + status: TopicStatus::Active, + keywords: Vec::new(), + } + } + + /// Check if topic is active. + pub fn is_active(&self) -> bool { + self.status == TopicStatus::Active + } +} + +/// Topic status. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum TopicStatus { + /// Topic is active and visible + Active, + /// Topic has been pruned due to inactivity + Pruned, +} + +/// Link between a topic and a TOC node. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TopicLink { + /// Topic identifier + pub topic_id: String, + /// TOC node identifier + pub node_id: String, + /// Relevance score (0.0 - 1.0) + pub relevance: f32, + /// When the link was created + pub created_at: DateTime, +} + +impl TopicLink { + /// Create a new topic-node link. + pub fn new(topic_id: String, node_id: String, relevance: f32) -> Self { + Self { + topic_id, + node_id, + relevance, + created_at: Utc::now(), + } + } +} + +/// Type of relationship between topics. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub enum RelationshipType { + /// Topics appear together in same documents + CoOccurrence, + /// Topics have similar embeddings + Semantic, + /// Parent/child hierarchical relationship + Hierarchical, +} + +impl RelationshipType { + /// Get short code for storage key. + pub fn code(&self) -> &'static str { + match self { + RelationshipType::CoOccurrence => "coo", + RelationshipType::Semantic => "sem", + RelationshipType::Hierarchical => "hie", + } + } + + /// Parse from code. + pub fn from_code(code: &str) -> Option { + match code { + "coo" => Some(RelationshipType::CoOccurrence), + "sem" => Some(RelationshipType::Semantic), + "hie" => Some(RelationshipType::Hierarchical), + _ => None, + } + } + + /// Get all relationship types. + pub fn all() -> &'static [RelationshipType] { + &[ + RelationshipType::CoOccurrence, + RelationshipType::Semantic, + RelationshipType::Hierarchical, + ] + } +} + +impl std::fmt::Display for RelationshipType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RelationshipType::CoOccurrence => write!(f, "co-occurrence"), + RelationshipType::Semantic => write!(f, "semantic"), + RelationshipType::Hierarchical => write!(f, "hierarchical"), + } + } +} + +/// Relationship between two topics. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TopicRelationship { + /// Source topic ID + pub source_id: TopicId, + /// Target topic ID + pub target_id: TopicId, + /// Type of relationship + pub relationship_type: RelationshipType, + /// Strength of relationship (0.0 - 1.0) + pub strength: f32, + /// Number of times relationship has been observed + pub evidence_count: u32, + /// When the relationship was first created + pub created_at: DateTime, + /// When the relationship was last updated + pub updated_at: DateTime, +} + +impl TopicRelationship { + /// Create a new relationship with current timestamp. + pub fn new( + source_id: TopicId, + target_id: TopicId, + relationship_type: RelationshipType, + strength: f32, + ) -> Self { + let now = Utc::now(); + Self { + source_id, + target_id, + relationship_type, + strength: strength.clamp(0.0, 1.0), + evidence_count: 1, + created_at: now, + updated_at: now, + } + } + + /// Create a new relationship with explicit timestamps. + pub fn with_timestamps( + source_id: TopicId, + target_id: TopicId, + relationship_type: RelationshipType, + strength: f32, + evidence_count: u32, + created_at: DateTime, + updated_at: DateTime, + ) -> Self { + Self { + source_id, + target_id, + relationship_type, + strength: strength.clamp(0.0, 1.0), + evidence_count, + created_at, + updated_at, + } + } + + /// Check if this relationship is between the given topics (in either direction). + pub fn connects(&self, topic_a: &TopicId, topic_b: &TopicId) -> bool { + (self.source_id == *topic_a && self.target_id == *topic_b) + || (self.source_id == *topic_b && self.target_id == *topic_a) + } + + /// Update the strength, clamping to valid range. + pub fn set_strength(&mut self, strength: f32) { + self.strength = strength.clamp(0.0, 1.0); + self.updated_at = Utc::now(); + } + + /// Increment the evidence count and update timestamp. + pub fn add_evidence(&mut self) { + self.evidence_count = self.evidence_count.saturating_add(1); + self.updated_at = Utc::now(); + } + + /// Strengthen the relationship by a delta amount. + pub fn strengthen(&mut self, delta: f32) { + self.set_strength(self.strength + delta); + } + + /// Weaken the relationship by a delta amount. + pub fn weaken(&mut self, delta: f32) { + self.set_strength(self.strength - delta); + } + + // Legacy compatibility aliases + /// Get the source topic ID (legacy alias for from_topic_id). + #[deprecated(note = "Use source_id instead")] + pub fn from_topic_id(&self) -> &TopicId { + &self.source_id + } + + /// Get the target topic ID (legacy alias for to_topic_id). + #[deprecated(note = "Use target_id instead")] + pub fn to_topic_id(&self) -> &TopicId { + &self.target_id + } + + /// Get the strength (legacy alias for score). + #[deprecated(note = "Use strength instead")] + pub fn score(&self) -> f32 { + self.strength + } +} + +/// Statistics about the topic graph. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct TopicStats { + /// Total number of topics + pub topic_count: u64, + /// Number of topic-node links + pub link_count: u64, + /// Number of topic relationships + pub relationship_count: u64, + /// Timestamp of last extraction (ms since epoch) + pub last_extraction_ms: i64, + /// Configured half-life in days + pub half_life_days: u32, + /// Configured similarity threshold + pub similarity_threshold: f32, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_topic_new() { + let topic = Topic::new( + "01HRQ7D5KQ".to_string(), + "Test Topic".to_string(), + vec![0.1, 0.2, 0.3], + ); + assert!(topic.is_active()); + assert_eq!(topic.node_count, 0); + assert!((topic.importance_score - 1.0).abs() < f64::EPSILON); + } + + #[test] + fn test_relationship_type_code() { + assert_eq!(RelationshipType::CoOccurrence.code(), "coo"); + assert_eq!(RelationshipType::Semantic.code(), "sem"); + assert_eq!(RelationshipType::Hierarchical.code(), "hie"); + assert_eq!( + RelationshipType::from_code("coo"), + Some(RelationshipType::CoOccurrence) + ); + assert_eq!( + RelationshipType::from_code("sem"), + Some(RelationshipType::Semantic) + ); + assert_eq!( + RelationshipType::from_code("hie"), + Some(RelationshipType::Hierarchical) + ); + } + + #[test] + fn test_relationship_type_all() { + let all = RelationshipType::all(); + assert_eq!(all.len(), 3); + assert!(all.contains(&RelationshipType::CoOccurrence)); + assert!(all.contains(&RelationshipType::Semantic)); + assert!(all.contains(&RelationshipType::Hierarchical)); + } + + #[test] + fn test_relationship_type_display() { + assert_eq!( + format!("{}", RelationshipType::CoOccurrence), + "co-occurrence" + ); + assert_eq!(format!("{}", RelationshipType::Semantic), "semantic"); + assert_eq!( + format!("{}", RelationshipType::Hierarchical), + "hierarchical" + ); + } + + #[test] + fn test_topic_link_new() { + let link = TopicLink::new("topic-123".to_string(), "node-456".to_string(), 0.85); + assert_eq!(link.topic_id, "topic-123"); + assert_eq!(link.node_id, "node-456"); + assert!((link.relevance - 0.85).abs() < f32::EPSILON); + } + + #[test] + fn test_topic_relationship_new() { + let rel = TopicRelationship::new( + "topic-a".to_string(), + "topic-b".to_string(), + RelationshipType::Semantic, + 0.92, + ); + assert_eq!(rel.source_id, "topic-a"); + assert_eq!(rel.target_id, "topic-b"); + assert_eq!(rel.relationship_type, RelationshipType::Semantic); + assert!((rel.strength - 0.92).abs() < f32::EPSILON); + assert_eq!(rel.evidence_count, 1); + } + + #[test] + fn test_topic_relationship_strength_clamping() { + // Test clamping on creation + let rel_high = TopicRelationship::new( + "a".to_string(), + "b".to_string(), + RelationshipType::CoOccurrence, + 1.5, + ); + assert!((rel_high.strength - 1.0).abs() < f32::EPSILON); + + let rel_low = TopicRelationship::new( + "a".to_string(), + "b".to_string(), + RelationshipType::CoOccurrence, + -0.5, + ); + assert!(rel_low.strength.abs() < f32::EPSILON); + + // Test clamping on set_strength + let mut rel = TopicRelationship::new( + "a".to_string(), + "b".to_string(), + RelationshipType::Semantic, + 0.5, + ); + rel.set_strength(2.0); + assert!((rel.strength - 1.0).abs() < f32::EPSILON); + rel.set_strength(-1.0); + assert!(rel.strength.abs() < f32::EPSILON); + } + + #[test] + fn test_topic_relationship_strengthen_weaken() { + let mut rel = TopicRelationship::new( + "a".to_string(), + "b".to_string(), + RelationshipType::Semantic, + 0.5, + ); + + rel.strengthen(0.2); + assert!((rel.strength - 0.7).abs() < f32::EPSILON); + + rel.weaken(0.3); + assert!((rel.strength - 0.4).abs() < f32::EPSILON); + + // Test clamping + rel.strengthen(1.0); + assert!((rel.strength - 1.0).abs() < f32::EPSILON); + + rel.weaken(2.0); + assert!(rel.strength.abs() < f32::EPSILON); + } + + #[test] + fn test_topic_relationship_add_evidence() { + let mut rel = TopicRelationship::new( + "a".to_string(), + "b".to_string(), + RelationshipType::CoOccurrence, + 0.5, + ); + assert_eq!(rel.evidence_count, 1); + + rel.add_evidence(); + assert_eq!(rel.evidence_count, 2); + + rel.add_evidence(); + assert_eq!(rel.evidence_count, 3); + } + + #[test] + fn test_topic_relationship_connects() { + let rel = TopicRelationship::new( + "topic-a".to_string(), + "topic-b".to_string(), + RelationshipType::Semantic, + 0.8, + ); + + assert!(rel.connects(&"topic-a".to_string(), &"topic-b".to_string())); + assert!(rel.connects(&"topic-b".to_string(), &"topic-a".to_string())); + assert!(!rel.connects(&"topic-a".to_string(), &"topic-c".to_string())); + assert!(!rel.connects(&"topic-c".to_string(), &"topic-d".to_string())); + } + + #[test] + fn test_topic_relationship_with_timestamps() { + let created = Utc::now() - chrono::Duration::days(10); + let updated = Utc::now() - chrono::Duration::days(1); + + let rel = TopicRelationship::with_timestamps( + "a".to_string(), + "b".to_string(), + RelationshipType::Hierarchical, + 0.9, + 5, + created, + updated, + ); + + assert_eq!(rel.evidence_count, 5); + assert_eq!(rel.created_at, created); + assert_eq!(rel.updated_at, updated); + } + + #[test] + fn test_relationship_type_from_code_invalid() { + assert_eq!(RelationshipType::from_code("invalid"), None); + assert_eq!(RelationshipType::from_code(""), None); + assert_eq!(RelationshipType::from_code("sim"), None); // Old code no longer valid + } + + #[test] + fn test_topic_status_equality() { + assert_eq!(TopicStatus::Active, TopicStatus::Active); + assert_ne!(TopicStatus::Active, TopicStatus::Pruned); + } +} diff --git a/crates/memory-types/src/config.rs b/crates/memory-types/src/config.rs index c17e79e..cb67e6a 100644 --- a/crates/memory-types/src/config.rs +++ b/crates/memory-types/src/config.rs @@ -91,6 +91,14 @@ pub struct Settings { /// Log level (trace, debug, info, warn, error) #[serde(default = "default_log_level")] pub log_level: String, + + /// Path to BM25 search index directory + #[serde(default = "default_search_index_path")] + pub search_index_path: String, + + /// Path to HNSW vector index directory + #[serde(default = "default_vector_index_path")] + pub vector_index_path: String, } fn default_db_path() -> String { @@ -113,6 +121,22 @@ fn default_log_level() -> String { "info".to_string() } +fn default_search_index_path() -> String { + ProjectDirs::from("", "", "agent-memory") + .map(|p| p.data_local_dir().join("bm25-index")) + .unwrap_or_else(|| PathBuf::from("./bm25-index")) + .to_string_lossy() + .to_string() +} + +fn default_vector_index_path() -> String { + ProjectDirs::from("", "", "agent-memory") + .map(|p| p.data_local_dir().join("vector-index")) + .unwrap_or_else(|| PathBuf::from("./vector-index")) + .to_string_lossy() + .to_string() +} + impl Default for Settings { fn default() -> Self { Self { @@ -123,6 +147,8 @@ impl Default for Settings { agent_id: None, summarizer: SummarizerSettings::default(), log_level: default_log_level(), + search_index_path: default_search_index_path(), + vector_index_path: default_vector_index_path(), } } } @@ -157,18 +183,16 @@ impl Settings { .map_err(|e| MemoryError::Config(e.to_string()))? .set_default("summarizer.model", default_summarizer_model()) .map_err(|e| MemoryError::Config(e.to_string()))? + .set_default("search_index_path", default_search_index_path()) + .map_err(|e| MemoryError::Config(e.to_string()))? + .set_default("vector_index_path", default_vector_index_path()) + .map_err(|e| MemoryError::Config(e.to_string()))? // 2. Default config file (~/.config/agent-memory/config.toml) - .add_source( - File::with_name(&default_config_path.to_string_lossy()) - .required(false) - ); + .add_source(File::with_name(&default_config_path.to_string_lossy()).required(false)); // 3. CLI-specified config file (higher precedence than default) if let Some(path) = cli_config_path { - builder = builder.add_source( - File::with_name(path) - .required(true) - ); + builder = builder.add_source(File::with_name(path).required(true)); } // 4. Environment variables (highest precedence before CLI flags) @@ -176,7 +200,7 @@ impl Settings { builder = builder.add_source( Environment::with_prefix("MEMORY") .separator("_") - .try_parsing(true) + .try_parsing(true), ); let config = builder @@ -207,7 +231,14 @@ impl Settings { /// Get user's home directory fn dirs_home() -> Option { ProjectDirs::from("", "", "agent-memory") - .map(|p| p.config_dir().parent().unwrap().parent().unwrap().to_path_buf()) + .map(|p| { + p.config_dir() + .parent() + .unwrap() + .parent() + .unwrap() + .to_path_buf() + }) .or_else(|| std::env::var("HOME").ok().map(PathBuf::from)) } diff --git a/crates/memory-types/src/event.rs b/crates/memory-types/src/event.rs index 3a0a9fd..9a75ce7 100644 --- a/crates/memory-types/src/event.rs +++ b/crates/memory-types/src/event.rs @@ -167,7 +167,8 @@ mod tests { EventType::ToolResult, EventRole::Tool, "File contents here".to_string(), - ).with_metadata(metadata); + ) + .with_metadata(metadata); assert_eq!(event.metadata.get("tool_name"), Some(&"Read".to_string())); } diff --git a/crates/memory-types/src/grip.rs b/crates/memory-types/src/grip.rs index 5298957..c0c207f 100644 --- a/crates/memory-types/src/grip.rs +++ b/crates/memory-types/src/grip.rs @@ -87,7 +87,8 @@ mod tests { "event-003".to_string(), Utc::now(), "segment_summarizer".to_string(), - ).with_toc_node("toc-day-20240115".to_string()); + ) + .with_toc_node("toc-day-20240115".to_string()); let bytes = grip.to_bytes().unwrap(); let decoded = Grip::from_bytes(&bytes).unwrap(); diff --git a/crates/memory-types/src/segment.rs b/crates/memory-types/src/segment.rs index bba2799..684fac9 100644 --- a/crates/memory-types/src/segment.rs +++ b/crates/memory-types/src/segment.rs @@ -65,7 +65,10 @@ impl Segment { /// Get all events (overlap + main) for summarization pub fn all_events(&self) -> Vec<&Event> { - self.overlap_events.iter().chain(self.events.iter()).collect() + self.overlap_events + .iter() + .chain(self.events.iter()) + .collect() } /// Serialize to JSON bytes @@ -101,13 +104,7 @@ mod tests { let start = events[0].timestamp; let end = events[1].timestamp; - let segment = Segment::new( - "seg-123".to_string(), - events.clone(), - start, - end, - 100, - ); + let segment = Segment::new("seg-123".to_string(), events.clone(), start, end, 100); assert_eq!(segment.events.len(), 2); assert_eq!(segment.token_count, 100); @@ -120,8 +117,8 @@ mod tests { let start = events[0].timestamp; let end = events[0].timestamp; - let segment = Segment::new("seg-123".to_string(), events, start, end, 50) - .with_overlap(overlap); + let segment = + Segment::new("seg-123".to_string(), events, start, end, 50).with_overlap(overlap); assert_eq!(segment.overlap_events.len(), 1); assert_eq!(segment.all_events().len(), 2); diff --git a/crates/memory-vector/Cargo.toml b/crates/memory-vector/Cargo.toml new file mode 100644 index 0000000..ae1675c --- /dev/null +++ b/crates/memory-vector/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "memory-vector" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Vector index for Agent Memory using HNSW algorithm" + +[dependencies] +# Vector search +usearch = { workspace = true } + +# Internal crates +memory-types = { workspace = true } +memory-embeddings = { workspace = true } + +# Storage +rocksdb = { workspace = true } + +# Async runtime +tokio = { workspace = true } + +# Logging +tracing = { workspace = true } + +# Error handling +thiserror = { workspace = true } + +# Serialization +serde = { workspace = true } +serde_json = { workspace = true } + +# Time +chrono = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +rand = { workspace = true } +tokio = { workspace = true, features = ["test-util", "macros", "rt-multi-thread"] } diff --git a/crates/memory-vector/src/error.rs b/crates/memory-vector/src/error.rs new file mode 100644 index 0000000..053c38c --- /dev/null +++ b/crates/memory-vector/src/error.rs @@ -0,0 +1,43 @@ +//! Vector index error types. + +use thiserror::Error; + +/// Errors that can occur during vector operations. +#[derive(Debug, Error)] +pub enum VectorError { + /// usearch index error + #[error("Index error: {0}")] + Index(String), + + /// Dimension mismatch + #[error("Dimension mismatch: expected {expected}, got {actual}")] + DimensionMismatch { expected: usize, actual: usize }, + + /// Vector not found + #[error("Vector not found: {0}")] + NotFound(u64), + + /// IO error + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + + /// Serialization error + #[error("Serialization error: {0}")] + Serialization(String), + + /// RocksDB error + #[error("Database error: {0}")] + Database(#[from] rocksdb::Error), + + /// Index is full + #[error("Index capacity reached: {0}")] + CapacityReached(usize), + + /// Index not initialized + #[error("Index not initialized")] + NotInitialized, + + /// Embedding error + #[error("Embedding error: {0}")] + Embedding(#[from] memory_embeddings::EmbeddingError), +} diff --git a/crates/memory-vector/src/hnsw.rs b/crates/memory-vector/src/hnsw.rs new file mode 100644 index 0000000..1d73d48 --- /dev/null +++ b/crates/memory-vector/src/hnsw.rs @@ -0,0 +1,337 @@ +//! HNSW index implementation using usearch. +//! +//! Parameters tuned for quality over speed: +//! - M = 16 (connections per layer) +//! - ef_construction = 200 (build-time quality) +//! - ef_search = 100 (search-time quality) + +use std::path::PathBuf; +use std::sync::RwLock; + +use memory_embeddings::Embedding; +use tracing::{debug, info}; +use usearch::{Index, IndexOptions, MetricKind, ScalarKind}; + +use crate::error::VectorError; +use crate::index::{IndexStats, SearchResult, VectorIndex}; + +/// HNSW index configuration +#[derive(Debug, Clone)] +pub struct HnswConfig { + /// Embedding dimension (must match model) + pub dimension: usize, + /// Number of connections per layer (M parameter) + pub connectivity: usize, + /// Build-time search depth (ef_construction) + pub expansion_add: usize, + /// Query-time search depth (ef_search) + pub expansion_search: usize, + /// Index file path + pub index_path: PathBuf, + /// Maximum capacity (for pre-allocation) + pub capacity: usize, +} + +impl Default for HnswConfig { + fn default() -> Self { + Self { + dimension: 384, // all-MiniLM-L6-v2 + connectivity: 16, + expansion_add: 200, + expansion_search: 100, + index_path: PathBuf::from("./vector-index"), + capacity: 1_000_000, + } + } +} + +impl HnswConfig { + pub fn new(dimension: usize, index_path: impl Into) -> Self { + Self { + dimension, + index_path: index_path.into(), + ..Default::default() + } + } + + pub fn with_connectivity(mut self, m: usize) -> Self { + self.connectivity = m; + self + } + + pub fn with_expansion(mut self, ef_add: usize, ef_search: usize) -> Self { + self.expansion_add = ef_add; + self.expansion_search = ef_search; + self + } + + pub fn with_capacity(mut self, capacity: usize) -> Self { + self.capacity = capacity; + self + } +} + +/// HNSW index wrapper around usearch. +pub struct HnswIndex { + index: RwLock, + config: HnswConfig, +} + +impl HnswIndex { + /// Create a new HNSW index or open existing one. + pub fn open_or_create(config: HnswConfig) -> Result { + let index_file = config.index_path.join("hnsw.usearch"); + + let options = IndexOptions { + dimensions: config.dimension, + metric: MetricKind::Cos, // Cosine similarity + quantization: ScalarKind::F32, + connectivity: config.connectivity, + expansion_add: config.expansion_add, + expansion_search: config.expansion_search, + multi: false, // Single vector per key + }; + + let index = if index_file.exists() { + info!(path = ?index_file, "Opening existing vector index"); + let idx = Index::new(&options).map_err(|e| VectorError::Index(e.to_string()))?; + idx.load( + index_file + .to_str() + .ok_or_else(|| VectorError::Index("Invalid path encoding".to_string()))?, + ) + .map_err(|e| VectorError::Index(format!("Failed to load: {}", e)))?; + idx + } else { + info!(path = ?index_file, dim = config.dimension, "Creating new vector index"); + std::fs::create_dir_all(&config.index_path)?; + let idx = Index::new(&options).map_err(|e| VectorError::Index(e.to_string()))?; + idx.reserve(config.capacity) + .map_err(|e| VectorError::Index(e.to_string()))?; + idx + }; + + Ok(Self { + index: RwLock::new(index), + config, + }) + } + + /// Get the index file path + pub fn index_file(&self) -> PathBuf { + self.config.index_path.join("hnsw.usearch") + } +} + +impl VectorIndex for HnswIndex { + fn dimension(&self) -> usize { + self.config.dimension + } + + fn len(&self) -> usize { + self.index.read().unwrap().size() + } + + #[allow(clippy::readonly_write_lock)] // usearch::Index uses interior mutability + fn add(&mut self, id: u64, embedding: &Embedding) -> Result<(), VectorError> { + if embedding.dimension() != self.config.dimension { + return Err(VectorError::DimensionMismatch { + expected: self.config.dimension, + actual: embedding.dimension(), + }); + } + + let index = self.index.write().unwrap(); + index + .add(id, &embedding.values) + .map_err(|e| VectorError::Index(e.to_string()))?; + + debug!(id = id, "Added vector"); + Ok(()) + } + + fn search(&self, query: &Embedding, k: usize) -> Result, VectorError> { + if query.dimension() != self.config.dimension { + return Err(VectorError::DimensionMismatch { + expected: self.config.dimension, + actual: query.dimension(), + }); + } + + let index = self.index.read().unwrap(); + let results = index + .search(&query.values, k) + .map_err(|e| VectorError::Index(e.to_string()))?; + + let search_results: Vec = results + .keys + .iter() + .zip(results.distances.iter()) + .map(|(&id, &dist)| SearchResult::new(id, 1.0 - dist)) // Convert distance to similarity + .collect(); + + debug!(k = k, found = search_results.len(), "Search complete"); + Ok(search_results) + } + + #[allow(clippy::readonly_write_lock)] // usearch::Index uses interior mutability + fn remove(&mut self, id: u64) -> Result { + let index = self.index.write().unwrap(); + let result = index + .remove(id) + .map_err(|e| VectorError::Index(e.to_string()))?; + + if result > 0 { + debug!(id = id, "Removed vector"); + Ok(true) + } else { + Ok(false) + } + } + + fn contains(&self, id: u64) -> bool { + let index = self.index.read().unwrap(); + index.contains(id) + } + + fn stats(&self) -> IndexStats { + let index = self.index.read().unwrap(); + let size_bytes = std::fs::metadata(self.index_file()) + .map(|m| m.len()) + .unwrap_or(0); + + IndexStats { + vector_count: index.size(), + dimension: self.config.dimension, + size_bytes, + available: true, + } + } + + fn save(&self) -> Result<(), VectorError> { + let index = self.index.read().unwrap(); + let path = self.index_file(); + let path_str = path + .to_str() + .ok_or_else(|| VectorError::Index("Invalid path encoding".to_string()))?; + index + .save(path_str) + .map_err(|e| VectorError::Index(format!("Failed to save: {}", e)))?; + + info!(path = ?path, vectors = index.size(), "Saved vector index"); + Ok(()) + } + + fn clear(&mut self) -> Result<(), VectorError> { + // Recreate empty index + let options = IndexOptions { + dimensions: self.config.dimension, + metric: MetricKind::Cos, + quantization: ScalarKind::F32, + connectivity: self.config.connectivity, + expansion_add: self.config.expansion_add, + expansion_search: self.config.expansion_search, + multi: false, + }; + + let new_index = Index::new(&options).map_err(|e| VectorError::Index(e.to_string()))?; + new_index + .reserve(self.config.capacity) + .map_err(|e| VectorError::Index(e.to_string()))?; + + *self.index.write().unwrap() = new_index; + info!("Cleared vector index"); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn random_embedding(dim: usize) -> Embedding { + use rand::Rng; + let mut rng = rand::rng(); + let values: Vec = (0..dim).map(|_| rng.random()).collect(); + Embedding::new(values) + } + + #[test] + fn test_create_index() { + let temp = TempDir::new().unwrap(); + let config = HnswConfig::new(384, temp.path()); + let index = HnswIndex::open_or_create(config).unwrap(); + assert_eq!(index.dimension(), 384); + assert_eq!(index.len(), 0); + } + + #[test] + fn test_add_and_search() { + let temp = TempDir::new().unwrap(); + let config = HnswConfig::new(64, temp.path()).with_capacity(100); + let mut index = HnswIndex::open_or_create(config).unwrap(); + + // Add some vectors + for i in 0..10 { + let emb = random_embedding(64); + index.add(i, &emb).unwrap(); + } + + assert_eq!(index.len(), 10); + + // Search + let query = random_embedding(64); + let results = index.search(&query, 5).unwrap(); + assert_eq!(results.len(), 5); + + // Results should be sorted by score (descending) + for i in 1..results.len() { + assert!(results[i - 1].score >= results[i].score); + } + } + + #[test] + fn test_save_and_load() { + let temp = TempDir::new().unwrap(); + let config = HnswConfig::new(64, temp.path()).with_capacity(100); + + // Create and populate + { + let mut index = HnswIndex::open_or_create(config.clone()).unwrap(); + for i in 0..5 { + index.add(i, &random_embedding(64)).unwrap(); + } + index.save().unwrap(); + } + + // Reopen + let index = HnswIndex::open_or_create(config).unwrap(); + assert_eq!(index.len(), 5); + } + + #[test] + fn test_dimension_mismatch() { + let temp = TempDir::new().unwrap(); + let config = HnswConfig::new(64, temp.path()); + let mut index = HnswIndex::open_or_create(config).unwrap(); + + let wrong_dim = random_embedding(32); + let result = index.add(0, &wrong_dim); + assert!(matches!(result, Err(VectorError::DimensionMismatch { .. }))); + } + + #[test] + fn test_remove() { + let temp = TempDir::new().unwrap(); + let config = HnswConfig::new(64, temp.path()).with_capacity(100); + let mut index = HnswIndex::open_or_create(config).unwrap(); + + index.add(42, &random_embedding(64)).unwrap(); + assert!(index.contains(42)); + + let removed = index.remove(42).unwrap(); + assert!(removed); + assert!(!index.contains(42)); + } +} diff --git a/crates/memory-vector/src/index.rs b/crates/memory-vector/src/index.rs new file mode 100644 index 0000000..65eec97 --- /dev/null +++ b/crates/memory-vector/src/index.rs @@ -0,0 +1,81 @@ +//! Vector index trait and types. +//! +//! Defines the interface for vector similarity search. + +use crate::error::VectorError; +use memory_embeddings::Embedding; + +/// Result of a vector search +#[derive(Debug, Clone)] +pub struct SearchResult { + /// Internal vector ID + pub vector_id: u64, + /// Distance/similarity score (lower = more similar for L2, higher = more similar for cosine) + pub score: f32, +} + +impl SearchResult { + pub fn new(vector_id: u64, score: f32) -> Self { + Self { vector_id, score } + } +} + +/// Index statistics +#[derive(Debug, Clone, Default)] +pub struct IndexStats { + /// Number of vectors in the index + pub vector_count: usize, + /// Embedding dimension + pub dimension: usize, + /// Index file size in bytes + pub size_bytes: u64, + /// Whether index is available for search + pub available: bool, +} + +/// Trait for vector indexes. +/// +/// Implementations must be thread-safe for concurrent read access. +pub trait VectorIndex: Send + Sync { + /// Get the embedding dimension + fn dimension(&self) -> usize; + + /// Get the number of vectors in the index + fn len(&self) -> usize; + + /// Check if the index is empty + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Add a vector with the given ID. + /// Returns error if ID already exists. + fn add(&mut self, id: u64, embedding: &Embedding) -> Result<(), VectorError>; + + /// Add multiple vectors in batch. + fn add_batch(&mut self, vectors: &[(u64, Embedding)]) -> Result<(), VectorError> { + for (id, emb) in vectors { + self.add(*id, emb)?; + } + Ok(()) + } + + /// Search for k nearest neighbors. + /// Returns results sorted by similarity (best first). + fn search(&self, query: &Embedding, k: usize) -> Result, VectorError>; + + /// Remove a vector by ID. + fn remove(&mut self, id: u64) -> Result; + + /// Check if a vector ID exists + fn contains(&self, id: u64) -> bool; + + /// Get index statistics + fn stats(&self) -> IndexStats; + + /// Save index to disk + fn save(&self) -> Result<(), VectorError>; + + /// Clear all vectors from the index + fn clear(&mut self) -> Result<(), VectorError>; +} diff --git a/crates/memory-vector/src/lib.rs b/crates/memory-vector/src/lib.rs new file mode 100644 index 0000000..bb26c2f --- /dev/null +++ b/crates/memory-vector/src/lib.rs @@ -0,0 +1,31 @@ +//! # memory-vector +//! +//! Vector index for Agent Memory using HNSW algorithm. +//! +//! This crate provides semantic similarity search by storing embeddings +//! in an HNSW (Hierarchical Navigable Small World) index via usearch. +//! +//! ## Features +//! - usearch-powered HNSW index with mmap persistence +//! - O(log n) approximate nearest neighbor search +//! - Metadata storage linking vector IDs to document IDs +//! - Configurable HNSW parameters (M, ef_construction, ef_search) +//! +//! ## Requirements +//! - FR-02: HNSW index via usearch +//! - FR-03: VectorTeleport RPC support +//! - Index lifecycle: prune/rebuild operations + +pub mod error; +pub mod hnsw; +pub mod index; +pub mod metadata; +pub mod pipeline; + +pub use error::VectorError; +pub use hnsw::{HnswConfig, HnswIndex}; +pub use index::{IndexStats, SearchResult, VectorIndex}; +pub use metadata::{DocType, VectorEntry, VectorMetadata, CF_VECTOR_META}; +pub use pipeline::{ + IndexableItem, IndexingStats, PipelineConfig, VectorIndexPipeline, VECTOR_INDEX_CHECKPOINT, +}; diff --git a/crates/memory-vector/src/metadata.rs b/crates/memory-vector/src/metadata.rs new file mode 100644 index 0000000..a617eeb --- /dev/null +++ b/crates/memory-vector/src/metadata.rs @@ -0,0 +1,373 @@ +//! Vector metadata storage. +//! +//! Maps internal vector IDs (u64) to document IDs (node_id or grip_id). +//! Stored in RocksDB for persistence and atomic updates. + +use std::path::Path; + +use rocksdb::{ColumnFamily, ColumnFamilyDescriptor, Options, DB}; +use serde::{Deserialize, Serialize}; +use tracing::{debug, info}; + +use crate::error::VectorError; + +/// Column family name for vector metadata +pub const CF_VECTOR_META: &str = "vector_meta"; + +/// Document type for vectors +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum DocType { + /// TOC node summary + TocNode, + /// Grip excerpt + Grip, +} + +impl DocType { + pub fn as_str(&self) -> &'static str { + match self { + DocType::TocNode => "toc_node", + DocType::Grip => "grip", + } + } +} + +/// Vector entry metadata +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VectorEntry { + /// Internal vector ID (key in HNSW index) + pub vector_id: u64, + /// Document type + pub doc_type: DocType, + /// Document ID (node_id or grip_id) + pub doc_id: String, + /// Timestamp when vector was created (ms since epoch) + pub created_at: i64, + /// Text that was embedded (truncated for storage) + pub text_preview: String, +} + +impl VectorEntry { + pub fn new( + vector_id: u64, + doc_type: DocType, + doc_id: impl Into, + created_at: i64, + text: &str, + ) -> Self { + const MAX_PREVIEW: usize = 200; + let text_preview = if text.len() > MAX_PREVIEW { + format!("{}...", &text[..MAX_PREVIEW]) + } else { + text.to_string() + }; + + Self { + vector_id, + doc_type, + doc_id: doc_id.into(), + created_at, + text_preview, + } + } +} + +/// Vector metadata storage using RocksDB. +pub struct VectorMetadata { + db: DB, +} + +impl VectorMetadata { + /// Open or create metadata storage. + pub fn open(path: impl AsRef) -> Result { + let path = path.as_ref(); + + let mut opts = Options::default(); + opts.create_if_missing(true); + opts.create_missing_column_families(true); + + let cf_opts = Options::default(); + let cf = ColumnFamilyDescriptor::new(CF_VECTOR_META, cf_opts); + + let db = DB::open_cf_descriptors(&opts, path, vec![cf])?; + + info!(path = ?path, "Opened vector metadata storage"); + Ok(Self { db }) + } + + /// Get the column family handle + fn cf(&self) -> &ColumnFamily { + self.db + .cf_handle(CF_VECTOR_META) + .expect("CF_VECTOR_META missing") + } + + /// Store vector entry metadata. + pub fn put(&self, entry: &VectorEntry) -> Result<(), VectorError> { + let key = entry.vector_id.to_be_bytes(); + let value = + serde_json::to_vec(entry).map_err(|e| VectorError::Serialization(e.to_string()))?; + + self.db.put_cf(self.cf(), key, value)?; + debug!(vector_id = entry.vector_id, doc_id = %entry.doc_id, "Stored metadata"); + Ok(()) + } + + /// Get vector entry by vector ID. + pub fn get(&self, vector_id: u64) -> Result, VectorError> { + let key = vector_id.to_be_bytes(); + match self.db.get_cf(self.cf(), key)? { + Some(bytes) => { + let entry: VectorEntry = serde_json::from_slice(&bytes) + .map_err(|e| VectorError::Serialization(e.to_string()))?; + Ok(Some(entry)) + } + None => Ok(None), + } + } + + /// Delete vector entry by vector ID. + pub fn delete(&self, vector_id: u64) -> Result<(), VectorError> { + let key = vector_id.to_be_bytes(); + self.db.delete_cf(self.cf(), key)?; + Ok(()) + } + + /// Get all entries for a document type. + pub fn get_by_type(&self, doc_type: DocType) -> Result, VectorError> { + let mut entries = Vec::new(); + let iter = self.db.iterator_cf(self.cf(), rocksdb::IteratorMode::Start); + + for item in iter { + let (_, value) = item?; + let entry: VectorEntry = serde_json::from_slice(&value) + .map_err(|e| VectorError::Serialization(e.to_string()))?; + if entry.doc_type == doc_type { + entries.push(entry); + } + } + + Ok(entries) + } + + /// Find vector ID for a document ID. + pub fn find_by_doc_id(&self, doc_id: &str) -> Result, VectorError> { + let iter = self.db.iterator_cf(self.cf(), rocksdb::IteratorMode::Start); + + for item in iter { + let (_, value) = item?; + let entry: VectorEntry = serde_json::from_slice(&value) + .map_err(|e| VectorError::Serialization(e.to_string()))?; + if entry.doc_id == doc_id { + return Ok(Some(entry)); + } + } + + Ok(None) + } + + /// Count total entries + pub fn count(&self) -> Result { + let iter = self.db.iterator_cf(self.cf(), rocksdb::IteratorMode::Start); + Ok(iter.count()) + } + + /// Get all entries. + /// + /// Returns all vector metadata entries in the store. + /// Use with caution on large indexes. + pub fn get_all(&self) -> Result, VectorError> { + let mut entries = Vec::new(); + let iter = self.db.iterator_cf(self.cf(), rocksdb::IteratorMode::Start); + + for item in iter { + let (_, value) = item?; + let entry: VectorEntry = serde_json::from_slice(&value) + .map_err(|e| VectorError::Serialization(e.to_string()))?; + entries.push(entry); + } + + Ok(entries) + } + + /// Clear all entries from metadata storage. + /// + /// Used during full rebuild operations. + pub fn clear(&self) -> Result<(), VectorError> { + // Collect all keys first to avoid iterator invalidation + let keys: Vec> = self + .db + .iterator_cf(self.cf(), rocksdb::IteratorMode::Start) + .filter_map(|item| item.ok().map(|(k, _)| k.to_vec())) + .collect(); + + // Delete each key + for key in keys { + self.db.delete_cf(self.cf(), &key)?; + } + + debug!("Cleared all vector metadata entries"); + Ok(()) + } + + /// Get the next available vector ID + pub fn next_vector_id(&self) -> Result { + let iter = self.db.iterator_cf(self.cf(), rocksdb::IteratorMode::End); + + if let Some(Ok((key, _))) = iter.into_iter().next() { + let id = u64::from_be_bytes(key[..8].try_into().unwrap()); + Ok(id + 1) + } else { + Ok(1) // Start from 1 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_put_and_get() { + let temp = TempDir::new().unwrap(); + let meta = VectorMetadata::open(temp.path()).unwrap(); + + let entry = VectorEntry::new( + 1, + DocType::TocNode, + "toc:day:2024-01-15", + 1705320000000, + "This is a test summary for the day", + ); + + meta.put(&entry).unwrap(); + + let retrieved = meta.get(1).unwrap().unwrap(); + assert_eq!(retrieved.vector_id, 1); + assert_eq!(retrieved.doc_id, "toc:day:2024-01-15"); + assert_eq!(retrieved.doc_type, DocType::TocNode); + } + + #[test] + fn test_find_by_doc_id() { + let temp = TempDir::new().unwrap(); + let meta = VectorMetadata::open(temp.path()).unwrap(); + + for i in 0..5 { + let entry = VectorEntry::new( + i, + DocType::Grip, + format!("grip:123456:{}", i), + 1705320000000, + "Test excerpt", + ); + meta.put(&entry).unwrap(); + } + + let found = meta.find_by_doc_id("grip:123456:3").unwrap().unwrap(); + assert_eq!(found.vector_id, 3); + } + + #[test] + fn test_next_vector_id() { + let temp = TempDir::new().unwrap(); + let meta = VectorMetadata::open(temp.path()).unwrap(); + + assert_eq!(meta.next_vector_id().unwrap(), 1); + + let entry = VectorEntry::new(42, DocType::TocNode, "test", 0, "test"); + meta.put(&entry).unwrap(); + + assert_eq!(meta.next_vector_id().unwrap(), 43); + } + + #[test] + fn test_get_by_type() { + let temp = TempDir::new().unwrap(); + let meta = VectorMetadata::open(temp.path()).unwrap(); + + // Add mixed entries + meta.put(&VectorEntry::new(1, DocType::TocNode, "toc:1", 0, "toc")) + .unwrap(); + meta.put(&VectorEntry::new(2, DocType::Grip, "grip:1", 0, "grip")) + .unwrap(); + meta.put(&VectorEntry::new(3, DocType::TocNode, "toc:2", 0, "toc")) + .unwrap(); + + let toc_entries = meta.get_by_type(DocType::TocNode).unwrap(); + assert_eq!(toc_entries.len(), 2); + + let grip_entries = meta.get_by_type(DocType::Grip).unwrap(); + assert_eq!(grip_entries.len(), 1); + } + + #[test] + fn test_delete() { + let temp = TempDir::new().unwrap(); + let meta = VectorMetadata::open(temp.path()).unwrap(); + + let entry = VectorEntry::new(1, DocType::TocNode, "test", 0, "test"); + meta.put(&entry).unwrap(); + assert!(meta.get(1).unwrap().is_some()); + + meta.delete(1).unwrap(); + assert!(meta.get(1).unwrap().is_none()); + } + + #[test] + fn test_count() { + let temp = TempDir::new().unwrap(); + let meta = VectorMetadata::open(temp.path()).unwrap(); + + assert_eq!(meta.count().unwrap(), 0); + + for i in 0..5 { + let entry = VectorEntry::new(i, DocType::Grip, format!("grip:{}", i), 0, "test"); + meta.put(&entry).unwrap(); + } + + assert_eq!(meta.count().unwrap(), 5); + } + + #[test] + fn test_text_preview_truncation() { + let long_text = "x".repeat(500); + let entry = VectorEntry::new(1, DocType::TocNode, "test", 0, &long_text); + assert!(entry.text_preview.len() < 250); // 200 + "..." + assert!(entry.text_preview.ends_with("...")); + } + + #[test] + fn test_get_all() { + let temp = TempDir::new().unwrap(); + let meta = VectorMetadata::open(temp.path()).unwrap(); + + // Add some entries + for i in 0..3 { + let entry = VectorEntry::new(i, DocType::TocNode, format!("toc:{}", i), 0, "test"); + meta.put(&entry).unwrap(); + } + + let all = meta.get_all().unwrap(); + assert_eq!(all.len(), 3); + } + + #[test] + fn test_clear() { + let temp = TempDir::new().unwrap(); + let meta = VectorMetadata::open(temp.path()).unwrap(); + + // Add some entries + for i in 0..5 { + let entry = VectorEntry::new(i, DocType::Grip, format!("grip:{}", i), 0, "test"); + meta.put(&entry).unwrap(); + } + assert_eq!(meta.count().unwrap(), 5); + + // Clear all + meta.clear().unwrap(); + assert_eq!(meta.count().unwrap(), 0); + assert!(meta.get_all().unwrap().is_empty()); + } +} diff --git a/crates/memory-vector/src/pipeline.rs b/crates/memory-vector/src/pipeline.rs new file mode 100644 index 0000000..a882b38 --- /dev/null +++ b/crates/memory-vector/src/pipeline.rs @@ -0,0 +1,516 @@ +//! Outbox-driven vector indexing pipeline. +//! +//! Consumes TOC nodes and grips, generates embeddings, and adds to HNSW index +//! with checkpoint tracking for crash recovery. +//! +//! Requirements: FR-09 (Outbox-driven indexing), FR-10 (Checkpoint-based recovery) + +use std::sync::{Arc, RwLock}; + +use chrono::Utc; +use tracing::{debug, error, info, warn}; + +use memory_embeddings::EmbeddingModel; +use memory_types::TocNode; + +use crate::error::VectorError; +use crate::hnsw::HnswIndex; +use crate::index::VectorIndex; +use crate::metadata::{DocType, VectorEntry, VectorMetadata}; + +/// Checkpoint key for vector indexing +pub const VECTOR_INDEX_CHECKPOINT: &str = "vector_index_last_processed"; + +/// Statistics from indexing run +#[derive(Debug, Default, Clone)] +pub struct IndexingStats { + /// Number of entries processed + pub entries_processed: usize, + /// Number of vectors successfully added to index + pub vectors_added: usize, + /// Number of entries skipped (already indexed or empty) + pub vectors_skipped: usize, + /// Number of errors encountered + pub errors: usize, +} + +impl IndexingStats { + /// Merge another stats into this one + pub fn merge(&mut self, other: &IndexingStats) { + self.entries_processed += other.entries_processed; + self.vectors_added += other.vectors_added; + self.vectors_skipped += other.vectors_skipped; + self.errors += other.errors; + } +} + +/// Vector indexing pipeline configuration +#[derive(Debug, Clone)] +pub struct PipelineConfig { + /// Batch size for processing entries + pub batch_size: usize, + /// Maximum entries to process per run (0 = unlimited) + pub max_entries_per_run: usize, + /// Whether to continue on individual entry errors + pub continue_on_error: bool, +} + +impl Default for PipelineConfig { + fn default() -> Self { + Self { + batch_size: 32, + max_entries_per_run: 1000, + continue_on_error: true, + } + } +} + +/// Item to be indexed by the pipeline +#[derive(Debug, Clone)] +pub enum IndexableItem { + /// A TOC node to index + TocNode { + /// Node ID + node_id: String, + /// Node data + node: TocNode, + }, + /// A grip to index + Grip { + /// Grip ID + grip_id: String, + /// Excerpt text to embed + excerpt: String, + /// Timestamp when grip was created + created_at: i64, + }, +} + +impl IndexableItem { + /// Get the document ID for this item + pub fn doc_id(&self) -> &str { + match self { + IndexableItem::TocNode { node_id, .. } => node_id, + IndexableItem::Grip { grip_id, .. } => grip_id, + } + } + + /// Get the document type for this item + pub fn doc_type(&self) -> DocType { + match self { + IndexableItem::TocNode { .. } => DocType::TocNode, + IndexableItem::Grip { .. } => DocType::Grip, + } + } + + /// Get the text to embed for this item + pub fn text(&self) -> String { + match self { + IndexableItem::TocNode { node, .. } => extract_node_text(node), + IndexableItem::Grip { excerpt, .. } => excerpt.clone(), + } + } + + /// Get the created_at timestamp for this item + pub fn created_at(&self) -> i64 { + match self { + IndexableItem::TocNode { node, .. } => node.created_at.timestamp_millis(), + IndexableItem::Grip { created_at, .. } => *created_at, + } + } +} + +/// Extract searchable text from a TOC node. +fn extract_node_text(node: &TocNode) -> String { + let mut parts = Vec::new(); + + // Include title + if !node.title.is_empty() { + parts.push(node.title.clone()); + } + + // Include bullets + for bullet in &node.bullets { + parts.push(bullet.text.clone()); + } + + // Include keywords + if !node.keywords.is_empty() { + parts.push(node.keywords.join(" ")); + } + + parts.join(". ") +} + +/// Vector indexing pipeline. +/// +/// Processes items (TOC nodes and grips), generates embeddings, and adds to HNSW index. +/// Uses checkpoint tracking for crash recovery. +pub struct VectorIndexPipeline { + embedder: Arc, + index: Arc>, + metadata: Arc, + config: PipelineConfig, +} + +impl VectorIndexPipeline { + /// Create a new pipeline. + pub fn new( + embedder: Arc, + index: Arc>, + metadata: Arc, + config: PipelineConfig, + ) -> Self { + Self { + embedder, + index, + metadata, + config, + } + } + + /// Index a batch of items. + /// + /// Returns statistics about the indexing operation. + pub fn index_items(&self, items: &[IndexableItem]) -> Result { + let mut stats = IndexingStats::default(); + + if items.is_empty() { + debug!("No items to index"); + return Ok(stats); + } + + info!(count = items.len(), "Processing items for vector indexing"); + + // Process in batches + for batch in items.chunks(self.config.batch_size) { + match self.process_batch(batch) { + Ok(batch_stats) => { + stats.merge(&batch_stats); + } + Err(e) => { + error!(error = %e, "Batch processing failed"); + if !self.config.continue_on_error { + return Err(e); + } + stats.errors += batch.len(); + } + } + } + + // Save index after processing + { + let index = self + .index + .read() + .map_err(|e| VectorError::Index(format!("Failed to acquire read lock: {}", e)))?; + index.save()?; + } + + info!( + processed = stats.entries_processed, + added = stats.vectors_added, + skipped = stats.vectors_skipped, + errors = stats.errors, + "Vector indexing complete" + ); + + Ok(stats) + } + + /// Process a batch of items. + fn process_batch(&self, items: &[IndexableItem]) -> Result { + let mut stats = IndexingStats::default(); + + for item in items { + stats.entries_processed += 1; + + match self.process_item(item) { + Ok(true) => stats.vectors_added += 1, + Ok(false) => stats.vectors_skipped += 1, + Err(e) => { + warn!(doc_id = %item.doc_id(), error = %e, "Failed to process item"); + if self.config.continue_on_error { + stats.errors += 1; + } else { + return Err(e); + } + } + } + } + + Ok(stats) + } + + /// Process a single item. + /// + /// Returns true if vector was added, false if skipped. + fn process_item(&self, item: &IndexableItem) -> Result { + let doc_id = item.doc_id(); + + // Skip if already indexed + if self.metadata.find_by_doc_id(doc_id)?.is_some() { + debug!(doc_id = %doc_id, "Already indexed, skipping"); + return Ok(false); + } + + // Get text to embed + let text = item.text(); + + // Skip empty text + if text.trim().is_empty() { + debug!(doc_id = %doc_id, "Empty text, skipping"); + return Ok(false); + } + + // Generate embedding + let embedding = self.embedder.embed(&text)?; + + // Get next vector ID + let vector_id = self.metadata.next_vector_id()?; + + // Add to index + { + let mut index = self + .index + .write() + .map_err(|e| VectorError::Index(format!("Failed to acquire write lock: {}", e)))?; + index.add(vector_id, &embedding)?; + } + + // Store metadata + let meta_entry = VectorEntry::new( + vector_id, + item.doc_type(), + doc_id.to_string(), + item.created_at(), + &text, + ); + self.metadata.put(&meta_entry)?; + + debug!(vector_id = vector_id, doc_id = %doc_id, "Indexed vector"); + Ok(true) + } + + /// Index a single TOC node. + pub fn index_toc_node(&self, node: &TocNode) -> Result { + let item = IndexableItem::TocNode { + node_id: node.node_id.clone(), + node: node.clone(), + }; + self.process_item(&item) + } + + /// Index a single grip. + pub fn index_grip( + &self, + grip_id: &str, + excerpt: &str, + created_at: i64, + ) -> Result { + let item = IndexableItem::Grip { + grip_id: grip_id.to_string(), + excerpt: excerpt.to_string(), + created_at, + }; + self.process_item(&item) + } + + /// Rebuild entire vector index from scratch. + /// + /// Clears existing index and re-indexes all provided items. + pub fn rebuild(&self, items: &[IndexableItem]) -> Result { + info!("Starting full vector index rebuild"); + + // Clear index + { + let mut index = self + .index + .write() + .map_err(|e| VectorError::Index(format!("Failed to acquire write lock: {}", e)))?; + index.clear()?; + } + + // Clear metadata + self.metadata.clear()?; + + // Re-index all items + self.index_items(items) + } + + /// Prune old vectors based on age. + /// + /// Removes vectors older than age_days from the HNSW index. + /// Does NOT delete primary data (TOC nodes, grips remain in RocksDB). + pub fn prune(&self, age_days: u64) -> Result { + let cutoff_ms = Utc::now().timestamp_millis() - (age_days as i64 * 24 * 60 * 60 * 1000); + + info!( + age_days = age_days, + cutoff_ms = cutoff_ms, + "Pruning old vectors" + ); + + let all_entries = self.metadata.get_all()?; + let mut pruned = 0; + + for entry in all_entries { + if entry.created_at < cutoff_ms { + // Remove from HNSW index + { + let mut index = self.index.write().map_err(|e| { + VectorError::Index(format!("Failed to acquire write lock: {}", e)) + })?; + index.remove(entry.vector_id)?; + } + // Remove metadata + self.metadata.delete(entry.vector_id)?; + pruned += 1; + } + } + + if pruned > 0 { + let index = self + .index + .read() + .map_err(|e| VectorError::Index(format!("Failed to acquire read lock: {}", e)))?; + index.save()?; + } + + info!(pruned = pruned, "Prune complete"); + Ok(pruned) + } + + /// Get the current index statistics. + pub fn stats(&self) -> Result { + let index = self + .index + .read() + .map_err(|e| VectorError::Index(format!("Failed to acquire read lock: {}", e)))?; + Ok(index.stats()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use memory_embeddings::{Embedding, EmbeddingError}; + + // Mock embedder for testing + struct MockEmbedder { + dimension: usize, + } + + impl MockEmbedder { + #[allow(dead_code)] + fn new(dimension: usize) -> Self { + Self { dimension } + } + } + + impl EmbeddingModel for MockEmbedder { + fn info(&self) -> &memory_embeddings::ModelInfo { + // Return a static reference - in real code you'd store this + // For testing, we'll panic if this is called + unimplemented!("info() not needed for tests") + } + + fn embed(&self, _text: &str) -> Result { + // Return a simple embedding of the correct dimension + let values: Vec = (0..self.dimension).map(|i| (i as f32) / 100.0).collect(); + Ok(Embedding::new(values)) + } + } + + #[test] + fn test_extract_node_text() { + use memory_types::{TocBullet, TocLevel}; + + let mut node = TocNode::new( + "test-node".to_string(), + TocLevel::Day, + "Test Day".to_string(), + Utc::now(), + Utc::now(), + ); + node.bullets.push(TocBullet::new("First bullet")); + node.bullets.push(TocBullet::new("Second bullet")); + node.keywords = vec!["keyword1".to_string(), "keyword2".to_string()]; + + let text = extract_node_text(&node); + + assert!(text.contains("Test Day")); + assert!(text.contains("First bullet")); + assert!(text.contains("Second bullet")); + assert!(text.contains("keyword1")); + assert!(text.contains("keyword2")); + } + + #[test] + fn test_indexable_item_toc_node() { + use memory_types::TocLevel; + + let node = TocNode::new( + "toc:day:2024-01-15".to_string(), + TocLevel::Day, + "Test Day".to_string(), + Utc::now(), + Utc::now(), + ); + + let item = IndexableItem::TocNode { + node_id: node.node_id.clone(), + node: node.clone(), + }; + + assert_eq!(item.doc_id(), "toc:day:2024-01-15"); + assert_eq!(item.doc_type(), DocType::TocNode); + assert!(!item.text().is_empty()); + } + + #[test] + fn test_indexable_item_grip() { + let item = IndexableItem::Grip { + grip_id: "grip:123".to_string(), + excerpt: "Test excerpt content".to_string(), + created_at: 1705320000000, + }; + + assert_eq!(item.doc_id(), "grip:123"); + assert_eq!(item.doc_type(), DocType::Grip); + assert_eq!(item.text(), "Test excerpt content"); + assert_eq!(item.created_at(), 1705320000000); + } + + #[test] + fn test_indexing_stats_merge() { + let mut stats1 = IndexingStats { + entries_processed: 10, + vectors_added: 8, + vectors_skipped: 1, + errors: 1, + }; + + let stats2 = IndexingStats { + entries_processed: 5, + vectors_added: 4, + vectors_skipped: 1, + errors: 0, + }; + + stats1.merge(&stats2); + + assert_eq!(stats1.entries_processed, 15); + assert_eq!(stats1.vectors_added, 12); + assert_eq!(stats1.vectors_skipped, 2); + assert_eq!(stats1.errors, 1); + } + + #[test] + fn test_pipeline_config_default() { + let config = PipelineConfig::default(); + assert_eq!(config.batch_size, 32); + assert_eq!(config.max_entries_per_run, 1000); + assert!(config.continue_on_error); + } +} diff --git a/docs/API.md b/docs/API.md index d6846b1..a2ef20f 100644 --- a/docs/API.md +++ b/docs/API.md @@ -372,6 +372,774 @@ message Grip { --- +## Phase 10.5: Agentic TOC Search + +### SearchNode + +Search a specific TOC node for term matches. + +**Request:** +```protobuf +message SearchNodeRequest { + string node_id = 1; // Node to search + string query = 2; // Search terms + repeated SearchField fields = 3; // Fields to search + float min_score = 4; // Minimum match score (0.0-1.0, default 0.3) +} +``` + +**Response:** +```protobuf +message SearchNodeResponse { + repeated SearchMatch matches = 1; // Matching content with scores + int32 total_matches = 2; // Total number of matches +} +``` + +**Errors:** +- `INVALID_ARGUMENT`: Empty node_id or query +- `NOT_FOUND`: Node does not exist +- `INVALID_ARGUMENT`: min_score outside 0.0-1.0 range + +**Example:** +```bash +grpcurl -plaintext -d '{ + "node_id": "toc:day:2026-01-15", + "query": "rust async", + "fields": ["SEARCH_FIELD_TITLE", "SEARCH_FIELD_SUMMARY", "SEARCH_FIELD_KEYWORDS"], + "min_score": 0.5 +}' localhost:50051 memory.MemoryService/SearchNode +``` + +--- + +### SearchChildren + +Search children of a TOC node recursively. + +**Request:** +```protobuf +message SearchChildrenRequest { + string parent_id = 1; // Parent node + string query = 2; // Search terms + int32 max_depth = 3; // Max levels to search (default 3) + int32 limit = 4; // Max results (default 20) + float min_score = 5; // Minimum match score (default 0.3) +} +``` + +**Response:** +```protobuf +message SearchChildrenResponse { + repeated NodeMatch matches = 1; // Matching nodes with scores + int32 nodes_searched = 2; // Number of nodes searched +} +``` + +**Errors:** +- `INVALID_ARGUMENT`: Empty parent_id or query +- `NOT_FOUND`: Parent node does not exist +- `INVALID_ARGUMENT`: max_depth or limit less than 1 + +**Example:** +```bash +grpcurl -plaintext -d '{ + "parent_id": "toc:month:2026-01", + "query": "memory optimization", + "max_depth": 2, + "limit": 10, + "min_score": 0.4 +}' localhost:50051 memory.MemoryService/SearchChildren +``` + +--- + +## Phase 11: BM25 Teleport + +### GetTeleportStatus + +Check BM25 index availability and stats. + +**Request:** +```protobuf +message GetTeleportStatusRequest {} +``` + +**Response:** +```protobuf +message GetTeleportStatusResponse { + bool available = 1; // True if index is ready + int64 document_count = 2; // Number of indexed documents + int64 size_bytes = 3; // Index size in bytes + int64 last_commit = 4; // Last commit timestamp (epoch ms) +} +``` + +**Example:** +```bash +grpcurl -plaintext localhost:50051 memory.MemoryService/GetTeleportStatus +``` + +--- + +### TeleportSearch + +BM25 keyword search across indexed documents. + +**Request:** +```protobuf +message TeleportSearchRequest { + string query = 1; // Search query + int32 limit = 2; // Max results (default 20) + repeated DocType doc_types = 3; // Filter by doc type (default both) +} +``` + +**Response:** +```protobuf +message TeleportSearchResponse { + repeated TeleportResult results = 1; // Matches with BM25 scores + int64 query_time_ms = 2; // Query execution time +} +``` + +**Errors:** +- `INVALID_ARGUMENT`: Empty query +- `UNAVAILABLE`: BM25 index not available + +**Example:** +```bash +grpcurl -plaintext -d '{ + "query": "tokio async runtime", + "limit": 10, + "doc_types": ["DOC_TYPE_TOC_NODE", "DOC_TYPE_GRIP"] +}' localhost:50051 memory.MemoryService/TeleportSearch +``` + +--- + +## Phase 12: Vector Teleport + +### GetVectorIndexStatus + +Check vector index availability. + +**Request:** +```protobuf +message GetVectorIndexStatusRequest {} +``` + +**Response:** +```protobuf +message GetVectorIndexStatusResponse { + bool available = 1; // True if index is ready + int64 vector_count = 2; // Number of indexed vectors + int32 dimension = 3; // Vector dimension (e.g., 384, 768) + int64 size_bytes = 4; // Index size in bytes +} +``` + +**Example:** +```bash +grpcurl -plaintext localhost:50051 memory.MemoryService/GetVectorIndexStatus +``` + +--- + +### VectorTeleport + +Semantic similarity search using embeddings. + +**Request:** +```protobuf +message VectorTeleportRequest { + string query = 1; // Natural language query + int32 limit = 2; // Max results (default 20) +} +``` + +**Response:** +```protobuf +message VectorTeleportResponse { + repeated VectorResult results = 1; // Matches with cosine similarity scores + int64 query_time_ms = 2; // Query execution time +} +``` + +**Errors:** +- `INVALID_ARGUMENT`: Empty query +- `UNAVAILABLE`: Vector index not available + +**Example:** +```bash +grpcurl -plaintext -d '{ + "query": "how to handle errors in async Rust", + "limit": 15 +}' localhost:50051 memory.MemoryService/VectorTeleport +``` + +--- + +### HybridSearch + +Combined BM25 + vector search using Reciprocal Rank Fusion (RRF k=60). + +**Request:** +```protobuf +message HybridSearchRequest { + string query = 1; // Search query + int32 limit = 2; // Max results (default 20) + float bm25_weight = 3; // BM25 contribution (default 0.5) + float vector_weight = 4; // Vector contribution (default 0.5) +} +``` + +**Response:** +```protobuf +message HybridSearchResponse { + repeated HybridResult results = 1; // Fused results with combined scores + bool bm25_available = 2; // True if BM25 was used + bool vector_available = 3; // True if vector was used +} +``` + +**Errors:** +- `INVALID_ARGUMENT`: Empty query +- `INVALID_ARGUMENT`: Weights must be between 0.0 and 1.0 +- `UNAVAILABLE`: Neither index available + +**Example:** +```bash +grpcurl -plaintext -d '{ + "query": "async error handling patterns", + "limit": 20, + "bm25_weight": 0.4, + "vector_weight": 0.6 +}' localhost:50051 memory.MemoryService/HybridSearch +``` + +--- + +## Phase 14: Topic Graph + +### GetTopicGraphStatus + +Check topic graph availability. + +**Request:** +```protobuf +message GetTopicGraphStatusRequest {} +``` + +**Response:** +```protobuf +message GetTopicGraphStatusResponse { + bool available = 1; // True if graph is ready + int64 topic_count = 2; // Number of topics + int64 relationship_count = 3; // Number of relationships + int64 last_update = 4; // Last update timestamp (epoch ms) +} +``` + +**Example:** +```bash +grpcurl -plaintext localhost:50051 memory.MemoryService/GetTopicGraphStatus +``` + +--- + +### GetTopicsByQuery + +Find topics matching a query. + +**Request:** +```protobuf +message GetTopicsByQueryRequest { + string query = 1; // Search query + int32 limit = 2; // Max results (default 10) +} +``` + +**Response:** +```protobuf +message GetTopicsByQueryResponse { + repeated Topic topics = 1; // Matching topics +} +``` + +**Errors:** +- `INVALID_ARGUMENT`: Empty query +- `UNAVAILABLE`: Topic graph not available + +**Example:** +```bash +grpcurl -plaintext -d '{ + "query": "rust memory management", + "limit": 5 +}' localhost:50051 memory.MemoryService/GetTopicsByQuery +``` + +--- + +### GetRelatedTopics + +Get topics related to a given topic. + +**Request:** +```protobuf +message GetRelatedTopicsRequest { + string topic_id = 1; // Topic ID to find relations for + repeated RelationshipType relationship_types = 2; // Filter by relationship type +} +``` + +**Response:** +```protobuf +message GetRelatedTopicsResponse { + repeated RelatedTopic related = 1; // Related topics with relationship info +} +``` + +**Errors:** +- `INVALID_ARGUMENT`: Empty topic_id +- `NOT_FOUND`: Topic does not exist +- `UNAVAILABLE`: Topic graph not available + +**Example:** +```bash +grpcurl -plaintext -d '{ + "topic_id": "topic:rust-async", + "relationship_types": ["RELATIONSHIP_TYPE_SIMILAR", "RELATIONSHIP_TYPE_CHILD"] +}' localhost:50051 memory.MemoryService/GetRelatedTopics +``` + +--- + +### GetTopTopics + +Get top topics by importance score with time decay. + +**Request:** +```protobuf +message GetTopTopicsRequest { + int32 limit = 1; // Max results (default 10) + int32 since_days = 2; // Look back window in days (default 30) +} +``` + +**Response:** +```protobuf +message GetTopTopicsResponse { + repeated RankedTopic topics = 1; // Topics ranked by importance +} +``` + +**Example:** +```bash +grpcurl -plaintext -d '{ + "limit": 10, + "since_days": 7 +}' localhost:50051 memory.MemoryService/GetTopTopics +``` + +--- + +## gRPC Service: SchedulerService + +The SchedulerService provides management operations for background scheduler jobs. + +Proto file: `proto/scheduler.proto` + +### ListJobs + +List all registered scheduler jobs. + +**Request:** +```protobuf +message ListJobsRequest {} +``` + +**Response:** +```protobuf +message ListJobsResponse { + repeated JobInfo jobs = 1; // All registered jobs +} +``` + +**Example:** +```bash +grpcurl -plaintext localhost:50051 scheduler.SchedulerService/ListJobs +``` + +--- + +### GetJob + +Get status of a specific job. + +**Request:** +```protobuf +message GetJobRequest { + string job_name = 1; // Name of the job +} +``` + +**Response:** +```protobuf +message GetJobResponse { + JobInfo job = 1; // Job details including status +} +``` + +**Errors:** +- `INVALID_ARGUMENT`: Empty job_name +- `NOT_FOUND`: Job does not exist + +**Example:** +```bash +grpcurl -plaintext -d '{ + "job_name": "segment_summarizer" +}' localhost:50051 scheduler.SchedulerService/GetJob +``` + +--- + +### PauseJob + +Pause a scheduler job. + +**Request:** +```protobuf +message PauseJobRequest { + string job_name = 1; // Name of the job to pause +} +``` + +**Response:** +```protobuf +message PauseJobResponse { + bool success = 1; // True if paused successfully + string message = 2; // Status message +} +``` + +**Errors:** +- `INVALID_ARGUMENT`: Empty job_name +- `NOT_FOUND`: Job does not exist +- `FAILED_PRECONDITION`: Job already paused + +**Example:** +```bash +grpcurl -plaintext -d '{ + "job_name": "segment_summarizer" +}' localhost:50051 scheduler.SchedulerService/PauseJob +``` + +--- + +### ResumeJob + +Resume a paused scheduler job. + +**Request:** +```protobuf +message ResumeJobRequest { + string job_name = 1; // Name of the job to resume +} +``` + +**Response:** +```protobuf +message ResumeJobResponse { + bool success = 1; // True if resumed successfully + string message = 2; // Status message +} +``` + +**Errors:** +- `INVALID_ARGUMENT`: Empty job_name +- `NOT_FOUND`: Job does not exist +- `FAILED_PRECONDITION`: Job not paused + +**Example:** +```bash +grpcurl -plaintext -d '{ + "job_name": "segment_summarizer" +}' localhost:50051 scheduler.SchedulerService/ResumeJob +``` + +--- + +## Data Types + +### Search Types (Phase 10.5) + +#### SearchField + +```protobuf +enum SearchField { + SEARCH_FIELD_UNSPECIFIED = 0; + SEARCH_FIELD_TITLE = 1; + SEARCH_FIELD_SUMMARY = 2; + SEARCH_FIELD_BULLETS = 3; + SEARCH_FIELD_KEYWORDS = 4; +} +``` + +| Value | Description | +|-------|-------------| +| `TITLE` | Search in node titles | +| `SUMMARY` | Search in summaries | +| `BULLETS` | Search in bullet text | +| `KEYWORDS` | Search in keywords | + +#### SearchMatch + +```protobuf +message SearchMatch { + string field = 1; // Field where match was found + string text = 2; // Matched text + float score = 3; // Match score (0.0-1.0) + repeated string highlights = 4; // Highlighted snippets +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `field` | string | Field name (title, summary, etc.) | +| `text` | string | Full text of matching content | +| `score` | float | Match relevance score | +| `highlights` | string[] | Text snippets with match highlighting | + +#### NodeMatch + +```protobuf +message NodeMatch { + string node_id = 1; // ID of matching node + TocNode node = 2; // Full node data + repeated SearchMatch matches = 3; // Matches within this node + float aggregate_score = 4; // Combined score for all matches +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `node_id` | string | TOC node identifier | +| `node` | TocNode | Full node data | +| `matches` | SearchMatch[] | Individual field matches | +| `aggregate_score` | float | Combined relevance score | + +### Teleport Types (Phase 11) + +#### DocType + +```protobuf +enum DocType { + DOC_TYPE_UNSPECIFIED = 0; + DOC_TYPE_TOC_NODE = 1; + DOC_TYPE_GRIP = 2; +} +``` + +| Value | Description | +|-------|-------------| +| `TOC_NODE` | Table of Contents node | +| `GRIP` | Provenance anchor | + +#### TeleportResult + +```protobuf +message TeleportResult { + string doc_id = 1; // Document ID + DocType doc_type = 2; // Document type + string text = 3; // Document text + float bm25_score = 4; // BM25 relevance score + repeated string highlights = 5; // Highlighted snippets +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `doc_id` | string | Unique document identifier | +| `doc_type` | DocType | Type of document | +| `text` | string | Document content | +| `bm25_score` | float | BM25 relevance score | +| `highlights` | string[] | Highlighted match snippets | + +### Vector Types (Phase 12) + +#### VectorResult + +```protobuf +message VectorResult { + string doc_id = 1; // Document ID + DocType doc_type = 2; // Document type + string text = 3; // Document text + float similarity = 4; // Cosine similarity score (0.0-1.0) +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `doc_id` | string | Unique document identifier | +| `doc_type` | DocType | Type of document | +| `text` | string | Document content | +| `similarity` | float | Cosine similarity to query | + +#### HybridResult + +```protobuf +message HybridResult { + string doc_id = 1; // Document ID + DocType doc_type = 2; // Document type + string text = 3; // Document text + float combined_score = 4; // RRF combined score + float bm25_score = 5; // BM25 component score + float vector_score = 6; // Vector component score + int32 bm25_rank = 7; // Rank in BM25 results + int32 vector_rank = 8; // Rank in vector results +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `doc_id` | string | Unique document identifier | +| `doc_type` | DocType | Type of document | +| `text` | string | Document content | +| `combined_score` | float | RRF fused score | +| `bm25_score` | float | BM25 score (if available) | +| `vector_score` | float | Similarity score (if available) | +| `bm25_rank` | int32 | Position in BM25 ranking | +| `vector_rank` | int32 | Position in vector ranking | + +### Topic Graph Types (Phase 14) + +#### Topic + +```protobuf +message Topic { + string topic_id = 1; // Unique topic identifier + string name = 2; // Topic name + string description = 3; // Topic description + float importance = 4; // Importance score (0.0-1.0) + int64 first_seen = 5; // First occurrence (epoch ms) + int64 last_seen = 6; // Last occurrence (epoch ms) + int32 occurrence_count = 7; // Number of occurrences +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `topic_id` | string | Unique identifier (e.g., "topic:rust-async") | +| `name` | string | Human-readable topic name | +| `description` | string | Brief topic description | +| `importance` | float | Computed importance score | +| `first_seen` | int64 | First occurrence timestamp | +| `last_seen` | int64 | Most recent occurrence | +| `occurrence_count` | int32 | Total occurrence count | + +#### RelationshipType + +```protobuf +enum RelationshipType { + RELATIONSHIP_TYPE_UNSPECIFIED = 0; + RELATIONSHIP_TYPE_SIMILAR = 1; + RELATIONSHIP_TYPE_CHILD = 2; + RELATIONSHIP_TYPE_PARENT = 3; + RELATIONSHIP_TYPE_SEQUENTIAL = 4; +} +``` + +| Value | Description | +|-------|-------------| +| `SIMILAR` | Topics are semantically similar | +| `CHILD` | Topic is a subtopic | +| `PARENT` | Topic is a parent topic | +| `SEQUENTIAL` | Topics often appear in sequence | + +#### RelatedTopic + +```protobuf +message RelatedTopic { + Topic topic = 1; // The related topic + RelationshipType relationship = 2; // Type of relationship + float strength = 3; // Relationship strength (0.0-1.0) +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `topic` | Topic | The related topic | +| `relationship` | RelationshipType | How topics are related | +| `strength` | float | Relationship strength | + +#### RankedTopic + +```protobuf +message RankedTopic { + Topic topic = 1; // The topic + float rank_score = 2; // Time-decayed importance score + int32 rank = 3; // Position in ranking +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `topic` | Topic | The topic | +| `rank_score` | float | Score with time decay applied | +| `rank` | int32 | Position in ranking (1-based) | + +### Scheduler Types + +#### JobStatus + +```protobuf +enum JobStatus { + JOB_STATUS_UNSPECIFIED = 0; + JOB_STATUS_RUNNING = 1; + JOB_STATUS_PAUSED = 2; + JOB_STATUS_IDLE = 3; + JOB_STATUS_ERROR = 4; +} +``` + +| Value | Description | +|-------|-------------| +| `RUNNING` | Job is currently executing | +| `PAUSED` | Job is paused | +| `IDLE` | Job is waiting for next scheduled run | +| `ERROR` | Job encountered an error | + +#### JobInfo + +```protobuf +message JobInfo { + string name = 1; // Job name + string description = 2; // Job description + JobStatus status = 3; // Current status + string schedule = 4; // Cron schedule expression + int64 last_run = 5; // Last run timestamp (epoch ms) + int64 next_run = 6; // Next scheduled run (epoch ms) + int32 run_count = 7; // Total successful runs + int32 error_count = 8; // Total error runs + optional string last_error = 9; // Last error message (if any) +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `name` | string | Unique job identifier | +| `description` | string | Human-readable description | +| `status` | JobStatus | Current job status | +| `schedule` | string | Cron expression (e.g., "0 */5 * * * *") | +| `last_run` | int64 | Last execution timestamp | +| `next_run` | int64 | Next scheduled execution | +| `run_count` | int32 | Successful execution count | +| `error_count` | int32 | Failed execution count | +| `last_error` | string | Most recent error message | + +--- + ## Health Check The service exposes a standard gRPC health check endpoint. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f6fe1b2..9cb05f4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -3,29 +3,45 @@ ## Component Overview ``` -┌─────────────────────────────────────────────────────────────┐ -│ Hook Handler │ -│ (captures conversation events from Claude Code hooks) │ -└────────────────────────┬────────────────────────────────────┘ - │ IngestEvent RPC - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ memory-daemon │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐ │ -│ │ gRPC │ │ TOC │ │ Summarizer │ │ -│ │ Server │ │ Builder │ │ (LLM API) │ │ -│ └──────┬──────┘ └──────┬──────┘ └──────────┬──────────┘ │ -│ │ │ │ │ -│ └────────────────┼─────────────────────┘ │ -│ ▼ │ -│ ┌───────────────────────────────────────────────────────┐ │ -│ │ memory-storage │ │ -│ │ (RocksDB) │ │ -│ │ ┌─────────┐ ┌─────────┐ ┌─────────┐ ┌─────────────┐ │ │ -│ │ │ Events │ │ TOC │ │ Grips │ │ Checkpoints │ │ │ -│ │ └─────────┘ └─────────┘ └─────────┘ └─────────────┘ │ │ -│ └───────────────────────────────────────────────────────┘ │ -└─────────────────────────────────────────────────────────────┘ +┌─────────────────────────────────────────────────────────────────┐ +│ Hook Handler │ +│ (captures conversation events from Claude Code hooks) │ +└───────────────────────────┬─────────────────────────────────────┘ + │ IngestEvent RPC + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ memory-daemon │ +│ ┌───────────┐ ┌───────────┐ ┌────────────┐ ┌─────────────────┐ │ +│ │ gRPC │ │ TOC │ │ Summarizer │ │ Scheduler │ │ +│ │ Server │ │ Builder │ │ (LLM API) │ │ (Background Jobs)│ │ +│ └─────┬─────┘ └─────┬─────┘ └──────┬─────┘ └────────┬────────┘ │ +│ │ │ │ │ │ +│ └─────────────┴──────────────┴────────────────┘ │ +│ │ │ +│ ┌────────────────────────────┼────────────────────────────────┐ │ +│ │ Search Layer │ │ +│ │ ┌─────────────────────┐ │ ┌────────────────────────────┐ │ │ +│ │ │ BM25 Full-Text │ │ │ Vector HNSW Index │ │ │ +│ │ │ (Tantivy) │ │ │ (usearch) │ │ │ +│ │ └─────────────────────┘ │ └────────────────────────────┘ │ │ +│ └───────────────────────────┼─────────────────────────────────┘ │ +│ │ │ +│ ┌───────────────────────────┼─────────────────────────────────┐ │ +│ │ Topic Graph Layer │ │ +│ │ ┌──────────────┐ ┌───────────────┐ ┌─────────────────────┐ │ │ +│ │ │ HDBSCAN │ │ LLM Labels │ │ Topic Relationships │ │ │ +│ │ │ Clustering │ │ & Importance │ │ & Scoring │ │ │ +│ │ └──────────────┘ └───────────────┘ └─────────────────────┘ │ │ +│ └───────────────────────────┼─────────────────────────────────┘ │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────────────┐│ +│ │ memory-storage ││ +│ │ (RocksDB) ││ +│ │ ┌───────┐ ┌─────┐ ┌───────┐ ┌───────────┐ ┌───────────────┐ ││ +│ │ │Events │ │ TOC │ │ Grips │ │Checkpoints│ │ Topics/Vectors│ ││ +│ │ └───────┘ └─────┘ └───────┘ └───────────┘ └───────────────┘ ││ +│ └──────────────────────────────────────────────────────────────┘│ +└─────────────────────────────────────────────────────────────────┘ ``` ## Crate Structure @@ -35,28 +51,74 @@ | `memory-types` | Domain models (Event, TocNode, Grip, Settings) | None (leaf crate) | | `memory-storage` | RocksDB persistence layer | memory-types | | `memory-toc` | Segmentation, summarization, TOC building | memory-types, memory-storage | -| `memory-service` | gRPC service implementation | memory-types, memory-storage | +| `memory-search` | Tantivy BM25 full-text search (indexes TOC nodes and grips) | memory-types | +| `memory-embeddings` | Candle ML model loading with all-MiniLM-L6-v2 (384-dim embeddings) | memory-types | +| `memory-vector` | usearch HNSW vector index wrapper | memory-types, memory-embeddings | +| `memory-indexing` | Outbox consumer pipeline with checkpoint-based crash recovery | memory-types, memory-storage, memory-search, memory-vector | +| `memory-topics` | HDBSCAN topic clustering, LLM labeling, importance scoring, relationships | memory-types, memory-storage, memory-embeddings | +| `memory-scheduler` | tokio-cron-scheduler wrapper for background jobs (rollup, compaction, indexing) | memory-types, memory-storage, memory-toc, memory-indexing | +| `memory-service` | gRPC service implementation | memory-types, memory-storage, memory-toc, memory-search, memory-vector, memory-topics, memory-scheduler | | `memory-client` | Client library for hook handlers | memory-types, memory-service | | `memory-daemon` | CLI binary | All crates | ### Dependency Graph ``` -memory-types - ↑ - ├─────────────────────┬───────────────────┐ - │ │ │ -memory-storage memory-toc memory-service - ↑ ↑ ↑ - ├─────────────────────┤ │ - │ │ │ - │ memory-client────────────┘ - │ │ - └─────────────────────┴───────────────────┐ - │ - memory-daemon + memory-types + ↑ + ┌───────────┬───────────────┼───────────────┬───────────────┐ + │ │ │ │ │ +memory-storage memory-toc memory-search memory-embeddings │ + ↑ ↑ ↑ ↑ │ + │ │ │ │ │ + │ │ │ memory-vector │ + │ │ │ ↑ │ + │ │ │ │ │ + │ │ memory-indexing───────┘ │ + │ │ ↑ │ + │ │ │ │ + │ │ memory-topics─────────────────────────┘ + │ │ ↑ + │ │ │ + ├───────────┴───────────────┤ + │ │ + memory-scheduler │ + │ │ + └───────────────────────────┤ + │ + memory-service + ↑ + │ + memory-client + │ + ▼ + memory-daemon ``` +### Crate Responsibilities + +**Core Layer:** +- `memory-types`: Shared domain models, traits, and error types +- `memory-storage`: RocksDB column families, atomic writes, range scans + +**TOC Layer:** +- `memory-toc`: Segmentation rules, LLM summarization, hierarchical TOC building + +**Search Layer:** +- `memory-search`: Tantivy index management, BM25 ranking, teleport search +- `memory-embeddings`: all-MiniLM-L6-v2 model loading via Candle, embedding generation +- `memory-vector`: usearch HNSW index, approximate nearest neighbor queries + +**Pipeline Layer:** +- `memory-indexing`: Outbox consumer, incremental indexing, checkpoint recovery +- `memory-topics`: HDBSCAN clustering, topic labeling, importance scoring, topic relationships +- `memory-scheduler`: Cron-based job scheduling (rollup, compaction, index maintenance) + +**API Layer:** +- `memory-service`: gRPC handlers, request validation, response mapping +- `memory-client`: Hook handler integration, event mapping +- `memory-daemon`: CLI, configuration loading, graceful shutdown + ## Data Flow ### Event Ingestion @@ -114,6 +176,59 @@ memory-storage memory-toc memory-service 6. ExpandGrip provides source evidence for verification ``` +### Search Pipeline + +``` +1. TeleportSearch or VectorTeleport called with query + │ +2. Query routed to appropriate index: + ├── BM25: Tantivy tokenizes and ranks by term frequency + └── Vector: Candle generates embedding, usearch finds nearest neighbors + │ +3. Results merged and deduplicated (for HybridSearch) + │ +4. Top-K results returned with scores and node references + │ +5. Client can ExpandGrip for full context +``` + +### Topic Graph Construction + +``` +1. Outbox indexing job collects new embeddings + │ +2. HDBSCAN clusters embeddings into topic groups + │ +3. LLM generates labels for each cluster + │ +4. Importance scoring based on: + ├── Cluster size (more members = higher importance) + ├── Recency (recent content boosts importance) + └── Centrality (well-connected topics score higher) + │ +5. Topic relationships computed: + ├── Co-occurrence in TOC nodes + └── Embedding similarity between cluster centroids + │ +6. Topics and relationships persisted to RocksDB +``` + +### Indexing Pipeline + +``` +1. Outbox consumer reads pending entries + │ +2. For each entry: + ├── Extract text from TOC node summaries and grips + ├── Index text in Tantivy (BM25) + ├── Generate embedding via Candle + └── Add embedding to usearch HNSW index + │ +3. Update checkpoint for crash recovery + │ +4. Periodic compaction and optimization +``` + ## Storage Schema ### Column Families @@ -126,6 +241,22 @@ memory-storage memory-toc memory-service | `grips` | `grip:{ts_ms:013}:{ulid}` | Grip JSON | Excerpt provenance | | `outbox` | `out:{sequence:016}` | OutboxEntry JSON | Pending TOC updates | | `checkpoints` | `chk:{job_name}` | Checkpoint bytes | Job crash recovery | +| `bm25_index` | `bm25:meta` | Index metadata | Pointer to Tantivy directory path | +| `vector_index` | `vec:meta` | Index metadata | Pointer to usearch directory path | +| `topics` | `topic:{topic_id}` | Topic JSON | Topic clusters with labels and importance | +| `topic_links` | `tlink:{topic_id}:{node_id}` | Link JSON | Topic-to-TOC node associations | +| `topic_rels` | `trel:{topic_id_a}:{topic_id_b}` | Relationship JSON | Inter-topic relationships and strength | + +### External Index Directories + +Some indexes are managed outside RocksDB for specialized libraries: + +| Index | Location | Library | Purpose | +|-------|----------|---------|---------| +| Tantivy BM25 | `{db_path}/tantivy/` | Tantivy | Full-text search with BM25 ranking | +| usearch HNSW | `{db_path}/usearch/` | usearch | Approximate nearest neighbor vectors | + +RocksDB column families store metadata and pointers to these external directories. The indexing pipeline coordinates writes to ensure consistency between RocksDB and external indexes. ### Key Design @@ -190,10 +321,49 @@ On Startup: | `overlap_minutes` | 5 | Context overlap in segments | | `overlap_tokens` | 500 | Token overlap in segments | +## Background Jobs + +The scheduler manages periodic maintenance tasks via tokio-cron-scheduler. + +### Job Types + +| Job | Schedule | Purpose | +|-----|----------|---------| +| `outbox_processor` | Every 30s | Process pending outbox entries for TOC updates | +| `index_sync` | Every 5m | Sync new content to BM25 and vector indexes | +| `topic_refresh` | Every 1h | Re-cluster embeddings and update topic graph | +| `rollup` | Daily 3am | Roll up day nodes into week summaries | +| `compaction` | Weekly Sun 4am | Optimize RocksDB and Tantivy indexes | + +### Job Lifecycle + +``` +1. Scheduler starts on daemon init + │ +2. Jobs registered with cron expressions + │ +3. Each job execution: + ├── Load checkpoint from CF_CHECKPOINTS + ├── Process work from checkpoint position + ├── Write progress checkpoints periodically + └── Write final checkpoint on completion + │ +4. On shutdown: graceful stop with checkpoint flush +``` + +### Job Control API + +- `ListJobs`: Returns all registered jobs with status and next run time +- `GetJob`: Returns detailed status for a single job +- `PauseJob`: Stops scheduled execution (in-flight work completes) +- `ResumeJob`: Resumes scheduled execution from checkpoint + ## gRPC Service ### Endpoints +#### Core TOC Operations + | RPC | Request | Response | Purpose | |-----|---------|----------|---------| | `IngestEvent` | `IngestEventRequest` | `IngestEventResponse` | Store conversation event | @@ -202,6 +372,41 @@ On Startup: | `BrowseToc` | `BrowseTocRequest` | `BrowseTocResponse` | Paginated children | | `GetEvents` | `GetEventsRequest` | `GetEventsResponse` | Events in time range | | `ExpandGrip` | `ExpandGripRequest` | `ExpandGripResponse` | Context around grip | +| `SearchNode` | `SearchNodeRequest` | `SearchNodeResponse` | Search within a TOC node | +| `SearchChildren` | `SearchChildrenRequest` | `SearchChildrenResponse` | Search node's children | + +#### BM25 Full-Text Search + +| RPC | Request | Response | Purpose | +|-----|---------|----------|---------| +| `TeleportSearch` | `TeleportSearchRequest` | `TeleportSearchResponse` | BM25 search across all content | +| `GetTeleportStatus` | `GetTeleportStatusRequest` | `GetTeleportStatusResponse` | BM25 index health and stats | + +#### Vector Search + +| RPC | Request | Response | Purpose | +|-----|---------|----------|---------| +| `VectorTeleport` | `VectorTeleportRequest` | `VectorTeleportResponse` | Semantic similarity search | +| `HybridSearch` | `HybridSearchRequest` | `HybridSearchResponse` | Combined BM25 + vector search | +| `GetVectorIndexStatus` | `GetVectorIndexStatusRequest` | `GetVectorIndexStatusResponse` | Vector index health and stats | + +#### Topic Graph + +| RPC | Request | Response | Purpose | +|-----|---------|----------|---------| +| `GetTopicGraphStatus` | `GetTopicGraphStatusRequest` | `GetTopicGraphStatusResponse` | Topic graph health and stats | +| `GetTopicsByQuery` | `GetTopicsByQueryRequest` | `GetTopicsByQueryResponse` | Find topics matching query | +| `GetRelatedTopics` | `GetRelatedTopicsRequest` | `GetRelatedTopicsResponse` | Get related topics by ID | +| `GetTopTopics` | `GetTopTopicsRequest` | `GetTopTopicsResponse` | Get most important topics | + +#### Scheduler Management + +| RPC | Request | Response | Purpose | +|-----|---------|----------|---------| +| `ListJobs` | `ListJobsRequest` | `ListJobsResponse` | List all scheduled jobs | +| `GetJob` | `GetJobRequest` | `GetJobResponse` | Get job status and schedule | +| `PauseJob` | `PauseJobRequest` | `PauseJobResponse` | Pause a scheduled job | +| `ResumeJob` | `ResumeJobRequest` | `ResumeJobResponse` | Resume a paused job | ### Health Check diff --git a/docs/COGNITIVE_ARCHITECTURE.md b/docs/COGNITIVE_ARCHITECTURE.md index 6d61d50..aba6172 100644 --- a/docs/COGNITIVE_ARCHITECTURE.md +++ b/docs/COGNITIVE_ARCHITECTURE.md @@ -1,7 +1,8 @@ # Agent Memory Cognitive Architecture -**Version:** 1.0 -**Date:** 2026-02-01 +**Version:** 2.0 +**Date:** 2026-02-02 +**Status:** All cognitive layers (0-5) fully implemented --- @@ -21,10 +22,10 @@ Agent Memory implements a 6-layer cognitive hierarchy, where each layer provides |-------|------------|----------------|------|---------| | **0** | Raw Events | RocksDB CF_EVENTS | Always present | Immutable truth | | **1** | TOC Hierarchy | RocksDB CF_TOC_NODES | Always present | Time-based navigation | -| **2** | Agentic TOC Search | SearchNode/SearchChildren (Phase 10.5) | Always works | Index-free term matching | -| **3** | Lexical Teleport | BM25/Tantivy (Phase 11) | Configurable | Keyword grounding | -| **4** | Semantic Teleport | Vector/HNSW (Phase 12) | Configurable | Embedding similarity | -| **5** | Conceptual Discovery | Topic Graph (Phase 13+) | Optional | Pattern and concept enrichment | +| **2** | Agentic TOC Search | SearchNode/SearchChildren ✓ | Always works | Index-free term matching | +| **3** | Lexical Teleport | BM25/Tantivy ✓ | Configurable | Keyword grounding | +| **4** | Semantic Teleport | Vector/HNSW ✓ | Configurable | Embedding similarity | +| **5** | Conceptual Discovery | Topic Graph ✓ | Optional | Pattern and concept enrichment | **Hybrid Mode** (not a layer): Score fusion of layers 3+4 when both are enabled. diff --git a/docs/plans/configuration-wizard-skills-plan.md b/docs/plans/configuration-wizard-skills-plan.md new file mode 100644 index 0000000..b8adb5b --- /dev/null +++ b/docs/plans/configuration-wizard-skills-plan.md @@ -0,0 +1,724 @@ +# Configuration Wizard Skills Plan + +## Overview + +Create three new configuration wizard skills for agent-memory that use the AskUserQuestion interactive pattern to guide users through complex configuration scenarios. + +**Location:** `plugins/memory-setup-plugin/skills/` + +**Skills to Create:** +1. `memory-storage` - Storage, retention, and performance tuning +2. `memory-llm` - LLM provider deep configuration +3. `memory-agents` - Multi-agent and team configuration + +--- + +## Feature/Config Coverage Matrix + +This matrix ensures all memory-daemon configuration options are covered by the wizard skills. + +### Coverage by Skill + +| Config Section | Option | memory-setup | memory-storage | memory-llm | memory-agents | Gap? | +|---------------|--------|--------------|----------------|------------|---------------|------| +| **[storage]** | path | Basic | Deep | - | - | No | +| | write_buffer_size_mb | - | Advanced | - | - | No | +| | max_background_jobs | - | Advanced | - | - | No | +| **[server]** | host | Advanced | - | - | - | No | +| | port | Advanced | - | - | - | No | +| | timeout_secs | - | - | - | - | **Yes** | +| **[summarizer]** | provider | Basic | - | Deep | - | No | +| | model | Basic | - | Deep + Discovery | - | No | +| | api_key | Basic | - | Deep + Test | - | No | +| | api_endpoint | - | - | Deep | - | No | +| | max_tokens | - | - | Advanced | - | No | +| | temperature | - | - | Advanced | - | No | +| **[toc]** | segment_min_tokens | Advanced | - | - | - | No | +| | segment_max_tokens | Advanced | - | - | - | No | +| | time_gap_minutes | Advanced | - | - | - | No | +| | overlap_tokens | - | - | - | - | **Yes** | +| | overlap_minutes | - | - | - | - | **Yes** | +| **[rollup]** | min_age_hours | - | Advanced | - | - | No | +| | schedule | - | Advanced | - | - | No | +| **[logging]** | level | - | - | - | - | **Yes** | +| | format | - | - | - | - | **Yes** | +| | file | - | - | - | - | **Yes** | +| **[agents]** | mode | - | - | - | Deep | No (New) | +| | storage_strategy | - | - | - | Deep | No (New) | +| | agent_id | - | - | - | Deep | No (New) | +| | query_scope | - | - | - | Deep | No (New) | +| **[retention]** | policy | - | Deep | - | - | No (New) | +| | cleanup_schedule | - | Advanced | - | - | No (New) | +| | archive_strategy | - | Advanced | - | - | No (New) | +| | gdpr_mode | - | Deep | - | - | No (New) | + +### Gap Resolution + +| Gap | Resolution | +|-----|------------| +| server.timeout_secs | Add to memory-setup --advanced mode | +| toc.overlap_tokens | Add to memory-setup --advanced mode | +| toc.overlap_minutes | Add to memory-setup --advanced mode | +| logging.* | Create new skill: memory-logging OR add to memory-setup --advanced | + +**Recommendation:** Add logging options to memory-setup --advanced rather than creating a 4th skill. + +--- + +## Skill 1: memory-storage + +### Purpose +Configure storage paths, data retention policies, cleanup schedules, and performance tuning. + +### Commands + +| Command | Purpose | +|---------|---------| +| `/memory-storage` | Interactive storage wizard | +| `/memory-storage --minimal` | Use defaults, minimal questions | +| `/memory-storage --advanced` | Show all options including cron and performance | + +### Question Flow (6 Steps) + +``` +State Detection + | + v ++------------------+ +| Step 1: Storage | <- Skip if path exists (unless --fresh) +| Path | ++--------+---------+ + | + v ++------------------+ +| Step 2: Retention| <- Skip if policy configured +| Policy | ++--------+---------+ + | + v ++------------------+ +| Step 3: Cleanup | <- --advanced only +| Schedule | ++--------+---------+ + | + v ++------------------+ +| Step 4: Archive | <- --advanced only +| Strategy | ++--------+---------+ + | + v ++------------------+ +| Step 5: GDPR | <- Show if EU locale detected +| Mode | ++--------+---------+ + | + v ++------------------+ +| Step 6: Perf | <- --advanced only +| Tuning | ++--------+---------+ + | + v + Execution +``` + +### Questions with AskUserQuestion Format + +**Step 1: Storage Path** +``` +question: "Where should agent-memory store conversation data?" +header: "Storage" +options: + - label: "~/.memory-store (Recommended)" + description: "Standard user location, works on all platforms" + - label: "~/.local/share/agent-memory/db" + description: "XDG-compliant location for Linux" + - label: "Custom path" + description: "Specify a custom storage location" +multiSelect: false +``` + +**Step 2: Retention Policy** +``` +question: "How long should conversation data be retained?" +header: "Retention" +options: + - label: "Forever (Recommended)" + description: "Keep all data permanently for maximum historical context" + - label: "90 days" + description: "Quarter retention, good balance of history and storage" + - label: "30 days" + description: "One month retention, lower storage usage" + - label: "7 days" + description: "Short-term memory only, minimal storage" +multiSelect: false +``` + +**Step 3: Cleanup Schedule** (--advanced) +``` +question: "When should automatic cleanup run?" +header: "Schedule" +options: + - label: "Daily at 3 AM (Recommended)" + description: "Runs during off-hours, catches expired data quickly" + - label: "Weekly on Sunday" + description: "Less frequent cleanup, lower system impact" + - label: "Disabled" + description: "Manual cleanup only with memory-daemon admin cleanup" + - label: "Custom cron" + description: "Specify a custom cron expression" +multiSelect: false +``` + +**Step 4: Archive Strategy** (--advanced) +``` +question: "How should old data be archived before deletion?" +header: "Archive" +options: + - label: "Compress to archive (Recommended)" + description: "Saves space, data recoverable from ~/.memory-archive/" + - label: "Export to JSON" + description: "Human-readable backup before deletion" + - label: "No archive" + description: "Delete directly (irreversible)" +multiSelect: false +``` + +**Step 5: GDPR Mode** +``` +question: "Enable GDPR-compliant deletion mode?" +header: "GDPR" +options: + - label: "No (Recommended)" + description: "Standard retention with tombstones" + - label: "Yes" + description: "Complete data removal, audit logging, export-before-delete" +multiSelect: false +``` + +**Step 6: Performance Tuning** (--advanced) +``` +question: "Configure storage performance parameters?" +header: "Performance" +options: + - label: "Balanced (Recommended)" + description: "64MB write buffer, 4 background jobs - works for most users" + - label: "Low memory" + description: "16MB write buffer, 1 background job - for constrained systems" + - label: "High performance" + description: "128MB write buffer, 8 background jobs - for heavy usage" + - label: "Custom" + description: "Specify write_buffer_size_mb and max_background_jobs" +multiSelect: false +``` + +### State Detection + +```bash +# Current storage path +grep -A5 '\[storage\]' ~/.config/memory-daemon/config.toml 2>/dev/null | grep path + +# Storage usage +du -sh ~/.memory-store 2>/dev/null + +# Available disk space +df -h ~/.memory-store 2>/dev/null | tail -1 + +# Retention configured? +grep retention ~/.config/memory-daemon/config.toml 2>/dev/null + +# Archive exists? +ls ~/.memory-archive 2>/dev/null +``` + +### Config Changes + +**New/updated sections in config.toml:** + +```toml +[storage] +path = "~/.memory-store" +write_buffer_size_mb = 64 +max_background_jobs = 4 + +[retention] +policy = "forever" # or "days:30", "days:90", etc. +cleanup_schedule = "0 3 * * *" +archive_strategy = "compress" +archive_path = "~/.memory-archive" +gdpr_mode = false +``` + +### Validation + +1. Path exists or can be created +2. Write permissions verified +3. Minimum 100MB free disk space +4. Cron expression valid (if custom) +5. Archive path writable (if archiving enabled) + +--- + +## Skill 2: memory-llm + +### Purpose +Deep configuration for LLM providers including model discovery, cost estimation, API testing, and custom endpoints. + +### Commands + +| Command | Purpose | +|---------|---------| +| `/memory-llm` | Interactive LLM wizard | +| `/memory-llm --test` | Test current API key only | +| `/memory-llm --discover` | List available models | +| `/memory-llm --estimate` | Show cost estimation | + +### Question Flow (7 Steps) + +``` +State Detection + | + v ++------------------+ +| Step 1: Provider | <- Always ask (core decision) ++--------+---------+ + | + v ++------------------+ +| Step 2: Model | <- Show discovered models +| Discovery | ++--------+---------+ + | + v ++------------------+ +| Step 3: API Key | <- Skip if env var set ++--------+---------+ + | + v ++------------------+ +| Step 4: Test | <- Always run to verify +| Connection | ++--------+---------+ + | + v ++------------------+ +| Step 5: Cost | <- Informational, no question +| Estimation | ++--------+---------+ + | + v ++------------------+ +| Step 6: Quality | <- --advanced only +| Tradeoffs | ++--------+---------+ + | + v ++------------------+ +| Step 7: Budget | <- --advanced only +| Optimization | ++--------+---------+ + | + v + Execution +``` + +### Questions with AskUserQuestion Format + +**Step 1: Provider Selection** +``` +question: "Which LLM provider should generate summaries?" +header: "Provider" +options: + - label: "OpenAI (Recommended)" + description: "GPT models - fast, reliable, good price/performance" + - label: "Anthropic" + description: "Claude models - high quality summaries" + - label: "Ollama (Local)" + description: "Private, runs on your machine, no API costs" + - label: "None" + description: "Disable summarization entirely" +multiSelect: false +``` + +**Step 2: Model Selection** (dynamic based on discovery) +``` +question: "Which model should be used for summarization?" +header: "Model" +options: + - label: "gpt-4o-mini (Recommended)" + description: "Fast and cost-effective at $0.15/1M tokens" + - label: "gpt-4o" + description: "Best quality at $5/1M tokens" + - label: "gpt-4-turbo" + description: "Previous generation at $10/1M tokens" +multiSelect: false +``` + +**Step 3: API Key** +``` +question: "How should the API key be configured?" +header: "API Key" +options: + - label: "Use existing environment variable (Recommended)" + description: "OPENAI_API_KEY is already set" + - label: "Enter new key" + description: "Provide a new API key" + - label: "Test existing key" + description: "Verify the current key works" +multiSelect: false +``` + +**Step 6: Quality/Latency Tradeoffs** (--advanced) +``` +question: "Configure quality vs latency tradeoff?" +header: "Quality" +options: + - label: "Balanced (Recommended)" + description: "temperature=0.3, max_tokens=512 - good for most uses" + - label: "Deterministic" + description: "temperature=0.0 - consistent, reproducible summaries" + - label: "Creative" + description: "temperature=0.7 - more variation in summaries" + - label: "Custom" + description: "Specify temperature and max_tokens manually" +multiSelect: false +``` + +**Step 7: Token Budget** (--advanced) +``` +question: "Configure token budget optimization?" +header: "Budget" +options: + - label: "Balanced (Recommended)" + description: "Standard summarization, ~$0.02/month typical usage" + - label: "Economical" + description: "Shorter summaries, lower cost" + - label: "Detailed" + description: "Longer summaries, higher cost" + - label: "Custom" + description: "Set specific token limits" +multiSelect: false +``` + +### State Detection + +```bash +# API keys set? +[ -n "$OPENAI_API_KEY" ] && echo "OPENAI: set" || echo "OPENAI: not set" +[ -n "$ANTHROPIC_API_KEY" ] && echo "ANTHROPIC: set" || echo "ANTHROPIC: not set" + +# Current config +grep -A10 '\[summarizer\]' ~/.config/memory-daemon/config.toml 2>/dev/null + +# Test connectivity +curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + https://api.openai.com/v1/models + +# Check Ollama +curl -s http://localhost:11434/api/tags 2>/dev/null +``` + +### Config Changes + +```toml +[summarizer] +provider = "openai" +model = "gpt-4o-mini" +# api_key loaded from OPENAI_API_KEY env var +# api_endpoint = "https://api.openai.com/v1" # for custom endpoints +max_tokens = 512 +temperature = 0.3 +budget_mode = "balanced" +``` + +### Validation + +1. API key format valid (sk- prefix for OpenAI, sk-ant- for Anthropic) +2. Live API test successful +3. Selected model available +4. Rate limits verified + +--- + +## Skill 3: memory-agents + +### Purpose +Configure multi-agent memory settings including store isolation, agent tagging, and cross-agent permissions. + +### Commands + +| Command | Purpose | +|---------|---------| +| `/memory-agents` | Interactive multi-agent wizard | +| `/memory-agents --single` | Configure for single user mode | +| `/memory-agents --team` | Configure for team use | + +### Question Flow (6 Steps) + +``` +State Detection + | + v ++------------------+ +| Step 1: Usage | <- Always ask (core decision) +| Mode | ++--------+---------+ + | + v ++------------------+ +| Step 2: Storage | <- Skip if single user +| Strategy | ++--------+---------+ + | + v ++------------------+ +| Step 3: Agent | <- Always ask +| Identifier | ++--------+---------+ + | + v ++------------------+ +| Step 4: Query | <- Skip if single user or separate stores +| Scope | ++--------+---------+ + | + v ++------------------+ +| Step 5: Storage | <- --advanced, if separate stores +| Organization | ++--------+---------+ + | + v ++------------------+ +| Step 6: Team | <- If team mode selected +| Settings | ++--------+---------+ + | + v + Execution +``` + +### Questions with AskUserQuestion Format + +**Step 1: Usage Mode** +``` +question: "How will agent-memory be used?" +header: "Mode" +options: + - label: "Single user (Recommended)" + description: "One person, one agent (Claude Code)" + - label: "Single user, multiple agents" + description: "One person using Claude Code, Cursor, etc." + - label: "Team mode" + description: "Multiple users sharing memory on a team" +multiSelect: false +``` + +**Step 2: Storage Strategy** +``` +question: "How should agent data be stored?" +header: "Storage" +options: + - label: "Unified store with tags (Recommended)" + description: "Single database, agents identified by tag, easy cross-query" + - label: "Separate stores per agent" + description: "Complete isolation, cannot query across agents" +multiSelect: false +``` + +**Step 3: Agent Identifier** +``` +question: "Choose your agent identifier (tags all events from this instance):" +header: "Agent ID" +options: + - label: "claude-code (Recommended)" + description: "Standard identifier for Claude Code" + - label: "claude-code-{hostname}" + description: "Unique per machine for multi-machine setups" + - label: "{username}-claude" + description: "User-specific for shared machines" + - label: "Custom" + description: "Specify a custom identifier" +multiSelect: false +``` + +**Step 4: Cross-Agent Query Permissions** +``` +question: "What data should queries return?" +header: "Query Scope" +options: + - label: "Own events only (Recommended)" + description: "Query only this agent's data" + - label: "All agents" + description: "Query all agents' data (read-only)" + - label: "Specified agents" + description: "Query specific agents' data" +multiSelect: false +``` + +### State Detection + +```bash +# Current multi-agent config +grep -A5 'agents' ~/.config/memory-daemon/config.toml 2>/dev/null + +# Current agent_id +grep 'agent_id' ~/.config/memory-daemon/config.toml 2>/dev/null + +# Detect other agents +ls ~/.memory-store/agents/ 2>/dev/null + +# Hostname and user +hostname +whoami +``` + +### Config Changes + +**New section in config.toml:** + +```toml +[agents] +mode = "single" # single, multi, team +storage_strategy = "unified" # unified, separate +agent_id = "claude-code" +query_scope = "own" # own, all, or comma-separated list + +[team] +name = "default" +storage_path = "~/.memory-store/team/" +shared = false +``` + +### Validation + +1. Agent ID valid (no spaces, 3-50 chars) +2. Agent ID unique in unified store +3. Storage path writable +4. Team path accessible (if shared) + +--- + +## Implementation Tasks + +### Phase 1: File Structure + +Create skill directories: +``` +plugins/memory-setup-plugin/skills/ +├── memory-setup/ # Existing +├── memory-storage/ +│ ├── SKILL.md +│ └── references/ +│ ├── retention-policies.md +│ ├── gdpr-compliance.md +│ └── archive-strategies.md +├── memory-llm/ +│ ├── SKILL.md +│ └── references/ +│ ├── provider-comparison.md +│ ├── model-selection.md +│ ├── cost-estimation.md +│ └── custom-endpoints.md +└── memory-agents/ + ├── SKILL.md + └── references/ + ├── storage-strategies.md + ├── team-setup.md + └── agent-identifiers.md +``` + +### Phase 2: SKILL.md Creation + +For each skill: +1. Create SKILL.md with YAML frontmatter +2. Define commands table +3. Document question flow +4. Add state detection commands +5. Document config changes +6. Add output formatting (success/partial/error) +7. Add cross-skill navigation hints + +### Phase 3: Reference Documentation + +Create reference files for each skill with detailed explanations of options. + +### Phase 4: Plugin Integration + +Update `marketplace.json` to include new skills: +```json +{ + "plugins": [{ + "skills": [ + "./skills/memory-setup", + "./skills/memory-storage", + "./skills/memory-llm", + "./skills/memory-agents" + ] + }] +} +``` + +### Phase 5: Update memory-setup + +Add missing advanced options: +- server.timeout_secs +- toc.overlap_tokens +- toc.overlap_minutes +- logging.level, format, file + +--- + +## Verification + +After implementation, verify: + +1. **Skill Discovery** + ```bash + # Skills should appear in Claude Code + /memory-storage --help + /memory-llm --help + /memory-agents --help + ``` + +2. **Question Flow** + - Run each skill with no config + - Run each skill with existing config (should skip) + - Run each skill with --fresh (should ask all) + - Run each skill with --advanced (should show extra options) + +3. **Config Generation** + - Verify config.toml updated correctly + - Verify new sections added properly + - Verify existing sections preserved + +4. **Cross-Skill Navigation** + - Each skill suggests related skills at completion + - No circular dependencies + +5. **Coverage Check** + - All config options from coverage matrix are addressable + - No gaps remain + +--- + +## Files to Create + +| File | Purpose | +|------|---------| +| `plugins/memory-setup-plugin/skills/memory-storage/SKILL.md` | Storage wizard skill | +| `plugins/memory-setup-plugin/skills/memory-llm/SKILL.md` | LLM wizard skill | +| `plugins/memory-setup-plugin/skills/memory-agents/SKILL.md` | Multi-agent wizard skill | +| `plugins/memory-setup-plugin/skills/memory-storage/references/*.md` | Storage reference docs | +| `plugins/memory-setup-plugin/skills/memory-llm/references/*.md` | LLM reference docs | +| `plugins/memory-setup-plugin/skills/memory-agents/references/*.md` | Agent reference docs | + +## Files to Modify + +| File | Change | +|------|--------| +| `plugins/memory-setup-plugin/.claude-plugin/marketplace.json` | Add new skill paths | +| `plugins/memory-setup-plugin/skills/memory-setup/SKILL.md` | Add missing advanced options | +| `plugins/memory-setup-plugin/skills/memory-setup/references/wizard-questions.md` | Add missing questions | diff --git a/docs/plans/qa-agent-release-skill-ci-setup.md b/docs/plans/qa-agent-release-skill-ci-setup.md new file mode 100644 index 0000000..bd62d0b --- /dev/null +++ b/docs/plans/qa-agent-release-skill-ci-setup.md @@ -0,0 +1,272 @@ +# Plan: QA Agent, Release Skill, and CI/CD Setup for agent-memory + +## Summary + +Set up comprehensive Rust QA infrastructure for the agent-memory project: +1. Install external skills for Rust testing and cargo assistance +2. Create local `qa-rust-agent` in `.claude/agents/` +3. Create local `releasing-rust` skill in `.claude/skills/` +4. Create GitHub Actions CI workflow for PR checks +5. Create GitHub Actions release workflow for multi-platform builds + +All agents and skills are **project-local** (stored in `.claude/`), not global plugins. + +--- + +## Phase 1: Install External Skills + +### 1.1 Install rust-testing skill +```bash +skilz install attunehq/hurry/rust-testing -p --agent claude +``` + +### 1.2 Install rust-cargo-assistant skill +```bash +skilz install CuriousLearner/devkit/rust-cargo-assistant -b --agent claude +``` + +**Note**: If these skills don't exist in the marketplace, create local equivalents in Phase 2. + +--- + +## Phase 2: Create Local QA Agent + +### File: `.claude/agents/qa-rust-agent.md` + +Create a Rust-specific QA agent that: +- Triggers after any `.rs` file changes +- Runs `cargo check`, `cargo clippy`, `cargo test` +- Validates documentation builds +- Enforces workspace lint rules (unwrap_used, expect_used, panic, todo = deny) +- Reports per-crate test results +- Blocks completion on test failures + +**Key Commands the Agent Will Execute:** +```bash +cargo check --all-features +cargo clippy --all-targets --all-features -- -D warnings +cargo test --all-features +cargo test -p memory-daemon --test integration_test -- --test-threads=1 +cargo doc --no-deps --all-features +``` + +--- + +## Phase 3: Create Local Skills + +### 3.1 Create `rust-testing` skill (if external not available) + +**Directory**: `.claude/skills/rust-testing/` + +**Files to create:** +- `SKILL.md` - Testing patterns, commands, fixtures +- `.skilz-manifest.yaml` - Local skill metadata + +**Content covers:** +- `cargo test` command variations +- Async test patterns with tokio +- Property-based testing with proptest +- Test fixtures with tempfile +- Integration test harness patterns +- Coverage with cargo-tarpaulin + +### 3.2 Create `releasing-rust` skill + +**Directory**: `.claude/skills/releasing-rust/` + +**Files to create:** +- `SKILL.md` - Release workflow, cross-compilation, versioning + +**Content covers:** + +#### Cross-Compilation Targets +| Platform | Target Triple | Runner | +|----------|---------------|--------| +| Linux x86_64 | x86_64-unknown-linux-gnu | ubuntu-latest | +| Linux ARM64 | aarch64-unknown-linux-gnu | ubuntu-latest + cross | +| macOS Intel | x86_64-apple-darwin | macos-13 | +| macOS Apple Silicon | aarch64-apple-darwin | macos-14 | +| Windows x86_64 | x86_64-pc-windows-msvc | windows-latest | + +#### Artifact Naming Convention +``` +memory-daemon-{version}-{platform}-{arch}.{ext} + +Examples: + memory-daemon-0.2.0-linux-x86_64.tar.gz + memory-daemon-0.2.0-macos-aarch64.tar.gz + memory-daemon-0.2.0-windows-x86_64.zip +``` + +#### Version Management +- Use `cargo set-version` (cargo-edit) +- Semantic versioning (MAJOR.MINOR.PATCH) +- Changelog generation with git-cliff + +#### Integration with other skills +- References `rust-cargo-assistant` for dependency management +- References `git-cli` for tagging +- References `mastering-github-cli` for release creation + +--- + +## Phase 4: Create GitHub CI Workflow + +### File: `.github/workflows/ci.yml` + +**Jobs:** +| Job | Purpose | Runs On | +|-----|---------|---------| +| `fmt` | Format check | ubuntu-latest | +| `clippy` | Lint check | ubuntu-latest | +| `test` | Run tests | ubuntu-latest, macos-latest | +| `build` | Build binaries | ubuntu-latest, macos-latest | +| `doc` | Documentation check | ubuntu-latest | + +**Triggers:** +- Push to `main` +- Pull requests targeting `main` + +**Dependencies installed:** +- `protobuf-compiler` (proto compilation) +- `libclang-dev` (RocksDB, usearch bindgen) + +**Caching:** +- Use `Swatinem/rust-cache@v2` with shared keys + +--- + +## Phase 5: Create GitHub Release Workflow + +### File: `.github/workflows/release.yml` + +**Triggers:** +- Push tags matching `v[0-9]+.[0-9]+.[0-9]+` +- Manual workflow_dispatch with version input + +**Build Matrix:** +```yaml +matrix: + include: + - target: x86_64-unknown-linux-gnu + os: ubuntu-latest + name: linux-x86_64 + - target: aarch64-unknown-linux-gnu + os: ubuntu-latest + name: linux-aarch64 + cross: true + - target: x86_64-apple-darwin + os: macos-13 + name: macos-x86_64 + - target: aarch64-apple-darwin + os: macos-14 + name: macos-aarch64 + - target: x86_64-pc-windows-msvc + os: windows-latest + name: windows-x86_64 +``` + +**Outputs:** +- Platform-specific archives (tar.gz for Unix, zip for Windows) +- SHA256 checksums file +- GitHub Release with all artifacts + +--- + +## Phase 6: Create Supporting Configuration Files + +### 6.1 `rust-toolchain.toml` +```toml +[toolchain] +channel = "1.83" +components = ["rustfmt", "clippy"] +``` + +### 6.2 `.rustfmt.toml` +```toml +edition = "2021" +max_width = 100 +tab_spaces = 4 +newline_style = "Unix" +imports_granularity = "Module" +group_imports = "StdExternalCrate" +``` + +### 6.3 `clippy.toml` +```toml +cognitive-complexity-threshold = 25 +too-many-lines-threshold = 200 +too-many-arguments-threshold = 7 +``` + +--- + +## Files to Create/Modify + +| File | Action | Purpose | +|------|--------|---------| +| `.claude/agents/qa-rust-agent.md` | Create | QA enforcement agent | +| `.claude/skills/rust-testing/SKILL.md` | Create | Testing patterns skill | +| `.claude/skills/rust-testing/.skilz-manifest.yaml` | Create | Skill metadata | +| `.claude/skills/releasing-rust/SKILL.md` | Create | Release workflow skill | +| `.claude/skills/releasing-rust/.skilz-manifest.yaml` | Create | Skill metadata | +| `.github/workflows/ci.yml` | Create | PR checks workflow | +| `.github/workflows/release.yml` | Create | Multi-platform release workflow | +| `rust-toolchain.toml` | Create | Pin Rust version | +| `.rustfmt.toml` | Create | Format configuration | +| `clippy.toml` | Create | Lint configuration | +| `CLAUDE.md` | Update | Document new agents/skills | +| `.claude/settings.local.json` | Update | Add new permissions | + +--- + +## Verification Plan + +### After Implementation: + +1. **Verify skill installation:** + ```bash + ls -la .claude/skills/ + ``` + +2. **Test CI workflow locally:** + ```bash + cargo fmt --all -- --check + cargo clippy --workspace --all-targets -- -D warnings + cargo test --workspace + ``` + +3. **Test release build:** + ```bash + cargo build --release --bin memory-daemon + ``` + +4. **Create test PR to verify CI:** + ```bash + git checkout -b test/ci-workflow + git add .github/ + git commit -m "ci: add PR checks workflow" + gh pr create --title "CI: Add PR checks" --body "Test CI workflow" + ``` + +5. **Verify PR checks run:** + ```bash + gh pr checks + ``` + +--- + +## Execution Order + +1. Install external skills (or note if unavailable) +2. Create `.claude/agents/` directory and `qa-rust-agent.md` +3. Create `.claude/skills/rust-testing/` with SKILL.md +4. Create `.claude/skills/releasing-rust/` with SKILL.md +5. Create `.github/workflows/` directory +6. Create `ci.yml` workflow +7. Create `release.yml` workflow +8. Create `rust-toolchain.toml`, `.rustfmt.toml`, `clippy.toml` +9. Update `CLAUDE.md` to document new infrastructure +10. Update `.claude/settings.local.json` with new permissions +11. Test locally with cargo commands +12. Create PR to test CI workflow diff --git a/docs/wiki-mapping.yml b/docs/wiki-mapping.yml new file mode 100644 index 0000000..3d5bb04 --- /dev/null +++ b/docs/wiki-mapping.yml @@ -0,0 +1,63 @@ +# Wiki Mapping File +# Maps source files to GitHub Wiki page names +# Organization: SpillwaveSolutions/agent-memory + +# Main documentation +docs: + README.md: "README" + USAGE.md: "Usage-Guide" + ARCHITECTURE.md: "Architecture" + API.md: "API-Reference" + INTEGRATION.md: "Integration-Guide" + COGNITIVE_ARCHITECTURE.md: "Cognitive-Architecture" + +# Planning documents +planning: + .planning/PROJECT.md: "Project-Overview" + .planning/ROADMAP.md: "Roadmap" + .planning/STATE.md: "Current-State" + .planning/REQUIREMENTS.md: "Requirements" + .planning/MILESTONES.md: "Milestones" + +# Phase documentation +phases: + .planning/phases/01/RESEARCH.md: "Phase-1-Foundation" + .planning/phases/02/RESEARCH.md: "Phase-2-TOC-Building" + .planning/phases/03/RESEARCH.md: "Phase-3-Grips-Provenance" + .planning/phases/05/RESEARCH.md: "Phase-5-Integration" + .planning/phases/06/RESEARCH.md: "Phase-6-End-to-End" + .planning/phases/07/RESEARCH.md: "Phase-7-CCH-Integration" + .planning/phases/08/RESEARCH.md: "Phase-8-CCH-Hook-Integration" + .planning/phases/09/RESEARCH.md: "Phase-9-Setup-Plugin" + .planning/phases/10/RESEARCH.md: "Phase-10-Background-Scheduler" + .planning/phases/10.5/RESEARCH.md: "Phase-10.5-Agentic-TOC-Search" + .planning/phases/11/RESEARCH.md: "Phase-11-BM25-Teleport" + .planning/phases/12/RESEARCH.md: "Phase-12-Vector-Teleport" + .planning/phases/13/RESEARCH.md: "Phase-13-Outbox-Index-Ingestion" + .planning/phases/14/RESEARCH.md: "Phase-14-Topic-Graph-Memory" + .planning/phases/15/RESEARCH.md: "Phase-15-Configuration-Wizard" + +# Design documents +design: + docs/design/01-architecture-overview.md: "Design-Architecture-Overview" + docs/design/02-data-flow-sequences.md: "Design-Data-Flow" + docs/design/03-domain-model.md: "Design-Domain-Model" + docs/design/04-state-machines.md: "Design-State-Machines" + docs/design/05-api-reference.md: "Design-API-Reference" + docs/design/06-toc-navigation-guide.md: "Design-TOC-Navigation" + docs/design/07-storage-architecture.md: "Design-Storage-Architecture" + docs/design/08-scheduler-design.md: "Design-Scheduler" + docs/design/09-getting-started.md: "Design-Getting-Started" + docs/design/10-architecture-decisions.md: "Design-Architecture-Decisions" + +# PRDs +prds: + docs/prds/agent-retrieval-policy-prd.md: "PRD-Agent-Retrieval-Policy" + docs/prds/agentic-toc-search-prd.md: "PRD-Agentic-TOC-Search" + docs/prds/bm25-teleport-prd.md: "PRD-BM25-Teleport" + docs/prds/hierarchical-vector-indexing-prd.md: "PRD-Vector-Indexing" + docs/prds/topic-graph-memory-prd.md: "PRD-Topic-Graph-Memory" + +# Wiki Home page (special handling) +wiki: + docs/wiki/Home.md: "Home" diff --git a/docs/wiki/Home.md b/docs/wiki/Home.md index 4deb421..0680dae 100644 --- a/docs/wiki/Home.md +++ b/docs/wiki/Home.md @@ -50,13 +50,21 @@ Agent Memory is a local, append-only conversational memory system for AI coding | [Phase 9](Phase-9-Setup-Plugin) | Setup & Installer Plugin | Interactive setup wizard plugin with commands | | [Phase 10](Phase-10-Background-Scheduler) | Background Scheduler | In-process Tokio cron scheduler for TOC rollups | -#### v2.0 Phases (In Progress) +#### v2.0 Phases (Complete) | Phase | Title | Description | |-------|-------|-------------| +| [Phase 10.5](Phase-10.5-Agentic-TOC-Search) | Agentic TOC Search | Index-free term matching via SearchNode/SearchChildren | | [Phase 11](Phase-11-BM25-Teleport) | BM25 Teleport (Tantivy) | Full-text search index for keyword-based teleportation | | [Phase 12](Phase-12-Vector-Teleport) | Vector Teleport (HNSW) | Semantic similarity search via local HNSW | | [Phase 13](Phase-13-Outbox-Index-Ingestion) | Outbox Index Ingestion | Event-driven index updates for rebuildable indexes | +| [Phase 14](Phase-14-Topic-Graph-Memory) | Topic Graph Memory | HDBSCAN clustering, LLM labeling, importance scoring | + +#### v2.1 Phases (Planned) + +| Phase | Title | Description | +|-------|-------|-------------| +| [Phase 15](Phase-15-Configuration-Wizard) | Configuration Wizard Skills | AskUserQuestion-based interactive config wizards | ## Architecture Overview @@ -68,14 +76,24 @@ Agent Memory is a local, append-only conversational memory system for AI coding +---------------------------------------------------------------+ | Memory Daemon | | +-------------+ +-------------+ +-----------------------+ | -| | Ingestion | | Query | | TOC Builder | | -| | Service | | Service | | (Background) | | +| | Ingestion | | Query | | Scheduler | | +| | Service | | Service | | (Background Jobs) | | | +-------------+ +-------------+ +-----------------------+ | | | | | +-----------------------------------------------------+ | +| | Search Layer | | +| | +----------------+ +---------------------------+ | | +| | | BM25 (Tantivy) | | Vector HNSW (usearch) | | | +| +-----------------------------------------------------+ | +| +-----------------------------------------------------+ | +| | Topic Graph Layer | | +| | +----------------+ +---------------------------+ | | +| | | HDBSCAN | | LLM Labels & Importance | | | +| +-----------------------------------------------------+ | +| +-----------------------------------------------------+ | | | Storage Layer (RocksDB) | | | | +--------+ +----------+ +-------+ +------------+ | | -| | | Events | | TOC Nodes| | Grips | | Checkpoints| | | +| | | Events | | TOC Nodes| | Grips | | Topics | | | | +-----------------------------------------------------+ | +---------------------------------------------------------------+ ``` @@ -113,7 +131,10 @@ Agents navigate the TOC hierarchy level by level: | Storage | RocksDB | | API | gRPC (tonic) | | Summarizer | Pluggable (API or local LLM) | -| Search (v2) | Tantivy (BM25), HNSW (vector) | +| BM25 Search | Tantivy (full-text indexing) | +| Vector Search | usearch HNSW + Candle (all-MiniLM-L6-v2) | +| Topic Clustering | HDBSCAN (density-based) | +| Scheduler | tokio-cron-scheduler | ## Quick Links @@ -123,4 +144,4 @@ Agents navigate the TOC hierarchy level by level: --- -*Last updated: 2026-01-31* +*Last updated: 2026-02-02* diff --git a/plugins/memory-query-plugin/skills/memory-query/SKILL.md b/plugins/memory-query-plugin/skills/memory-query/SKILL.md index a80a761..31e1047 100644 --- a/plugins/memory-query-plugin/skills/memory-query/SKILL.md +++ b/plugins/memory-query-plugin/skills/memory-query/SKILL.md @@ -55,6 +55,73 @@ Year → Month → Week → Day → Segment - `toc:week:2026-W04` - `toc:day:2026-01-30` +## Search-Based Navigation + +Use search RPCs to efficiently find relevant content without scanning everything. + +### Search Workflow + +1. **Search at root level** - Find which time periods are relevant: + ```bash + memory-daemon query search --query "JWT authentication" + # Returns: Year/Month nodes with relevance scores + ``` + +2. **Drill into best match** - Search children of matching period: + ```bash + memory-daemon query search --parent "toc:month:2026-01" --query "JWT authentication" + # Returns: Week nodes with matches + ``` + +3. **Continue until Segment level** - Extract evidence: + ```bash + memory-daemon query search --parent "toc:day:2026-01-30" --query "JWT" + # Returns: Segment nodes with bullet matches and grip IDs + ``` + +4. **Expand grip for verification**: + ```bash + memory-daemon query expand --grip-id "grip:..." --before 3 --after 3 + ``` + +### Search Command Reference + +```bash +# Search within a specific node +memory-daemon query search --node "toc:month:2026-01" --query "debugging" + +# Search children of a parent +memory-daemon query search --parent "toc:week:2026-W04" --query "JWT token" + +# Search root level (years) +memory-daemon query search --query "authentication" + +# Filter by fields (title, summary, bullets, keywords) +memory-daemon query search --query "JWT" --fields "title,bullets" --limit 20 +``` + +### Agent Navigation Loop + +When answering "find discussions about X": + +1. Parse query for time hints ("last week", "in January", "yesterday") +2. Start at appropriate level based on hints, or root if no hints +3. Use `search_children` to find relevant nodes at each level +4. Drill into highest-scoring matches +5. At Segment level, extract bullets with grip IDs +6. Offer to expand grips for full context + +Example path: +``` +Query: "What JWT discussions happened last week?" +-> SearchChildren(parent="toc:week:2026-W04", query="JWT") + -> Day 2026-01-30 (score: 0.85) +-> SearchChildren(parent="toc:day:2026-01-30", query="JWT") + -> Segment abc123 (score: 0.92) +-> Return bullets from Segment with grip IDs +-> Offer: "Found 2 relevant points. Expand grip:xyz for context?" +``` + ## CLI Reference ```bash diff --git a/plugins/memory-query-plugin/skills/memory-query/references/command-reference.md b/plugins/memory-query-plugin/skills/memory-query/references/command-reference.md index 4352102..c6ebc1b 100644 --- a/plugins/memory-query-plugin/skills/memory-query/references/command-reference.md +++ b/plugins/memory-query-plugin/skills/memory-query/references/command-reference.md @@ -108,6 +108,60 @@ memory-daemon query --endpoint http://[::1]:50051 expand \ - `excerpt`: The referenced conversation segment - `after`: Events following the excerpt +## Search Commands + +### search + +Search TOC nodes for matching content. + +**Usage:** +```bash +memory-daemon query search --query [OPTIONS] +``` + +**Options:** +| Option | Description | Default | +|--------|-------------|---------| +| `--query`, `-q` | Search terms (required) | - | +| `--node` | Search within specific node | - | +| `--parent` | Search children of parent | - | +| `--fields` | Fields to search (comma-separated) | all | +| `--limit` | Maximum results | 10 | + +**Fields:** +- `title` - Node title +- `summary` - Derived from bullets +- `bullets` - Individual bullet points (includes grip IDs) +- `keywords` - Extracted keywords + +**Examples:** +```bash +# Search at root level +memory-daemon query search --query "authentication debugging" + +# Search within month +memory-daemon query search --node "toc:month:2026-01" --query "JWT" + +# Search week's children (days) +memory-daemon query search --parent "toc:week:2026-W04" --query "token refresh" + +# Search only in bullets and keywords +memory-daemon query search --query "OAuth" --fields "bullets,keywords" --limit 20 +``` + +**Output:** +``` +Search Results for children of toc:week:2026-W04 +Query: "token refresh" +Found: 2 nodes + +Node: toc:day:2026-01-30 (score=0.85) + Title: Thursday, January 30 + Matches: + - [bullets] Fixed JWT token refresh rotation + - [keywords] authentication +``` + ## Event Types | Type | Description | diff --git a/plugins/memory-query-plugin/skills/vector-search/SKILL.md b/plugins/memory-query-plugin/skills/vector-search/SKILL.md new file mode 100644 index 0000000..80f30fd --- /dev/null +++ b/plugins/memory-query-plugin/skills/vector-search/SKILL.md @@ -0,0 +1,253 @@ +--- +name: vector-search +description: | + Semantic vector search for agent-memory. Use when asked to "find similar discussions", "semantic search", "find related topics", "what's conceptually related to X", or when keyword search returns poor results. Provides vector similarity search and hybrid BM25+vector fusion. +license: MIT +metadata: + version: 1.0.0 + author: SpillwaveSolutions +--- + +# Vector Search Skill + +Semantic similarity search using vector embeddings in the agent-memory system. + +## When to Use + +| Use Case | Best Search Type | +|----------|------------------| +| Exact keyword match | BM25 (`teleport search`) | +| Conceptual similarity | Vector (`teleport vector-search`) | +| Best of both worlds | Hybrid (`teleport hybrid-search`) | +| Typos/synonyms | Vector or Hybrid | +| Technical terms | BM25 or Hybrid | + +## When Not to Use + +- Current session context (already in memory) +- Time-based queries (use TOC navigation instead) +- Counting or aggregation (not supported) + +## Quick Start + +| Command | Purpose | Example | +|---------|---------|---------| +| `teleport vector-search` | Semantic search | `teleport vector-search -q "authentication patterns"` | +| `teleport hybrid-search` | BM25 + Vector | `teleport hybrid-search -q "JWT token handling"` | +| `teleport vector-stats` | Index status | `teleport vector-stats` | + +## Prerequisites + +```bash +memory-daemon status # Check daemon +memory-daemon start # Start if needed +``` + +## Validation Checklist + +Before presenting results: +- [ ] Daemon running: `memory-daemon status` returns "running" +- [ ] Vector index available: `teleport vector-stats` shows `Status: Available` +- [ ] Query returns results: Check for non-empty `matches` array +- [ ] Scores are reasonable: 0.7+ is strong match, 0.5-0.7 moderate + +## Vector Search + +### Basic Usage + +```bash +# Simple semantic search +memory-daemon teleport vector-search -q "authentication patterns" + +# With filtering +memory-daemon teleport vector-search -q "debugging strategies" \ + --top-k 5 \ + --min-score 0.6 \ + --target toc +``` + +### Options + +| Option | Default | Description | +|--------|---------|-------------| +| `-q, --query` | required | Query text to embed and search | +| `--top-k` | 10 | Number of results to return | +| `--min-score` | 0.0 | Minimum similarity (0.0-1.0) | +| `--target` | all | Filter: all, toc, grip | +| `--addr` | http://[::1]:50051 | gRPC server address | + +### Output Format + +``` +Vector Search: "authentication patterns" +Top-K: 10, Min Score: 0.00, Target: all + +Found 3 results: +---------------------------------------------------------------------- +1. [toc_node] toc:segment:abc123 (score: 0.8542) + Implemented JWT authentication with refresh token rotation... + Time: 2026-01-30 14:32 + +2. [grip] grip:1738252800000:01JKXYZ (score: 0.7891) + The OAuth2 flow handles authentication through the identity... + Time: 2026-01-28 09:15 +``` + +## Hybrid Search + +Combines BM25 keyword matching with vector semantic similarity using Reciprocal Rank Fusion (RRF). + +### Basic Usage + +```bash +# Default hybrid mode (50/50 weights) +memory-daemon teleport hybrid-search -q "JWT authentication" + +# Favor vector semantics +memory-daemon teleport hybrid-search -q "similar topics" \ + --bm25-weight 0.3 \ + --vector-weight 0.7 + +# Favor keyword matching +memory-daemon teleport hybrid-search -q "exact_function_name" \ + --bm25-weight 0.8 \ + --vector-weight 0.2 +``` + +### Search Modes + +| Mode | Description | Use When | +|------|-------------|----------| +| `hybrid` | RRF fusion of both | Default, general purpose | +| `vector-only` | Only vector similarity | Conceptual queries, synonyms | +| `bm25-only` | Only keyword matching | Exact terms, debugging | + +```bash +# Force vector-only mode +memory-daemon teleport hybrid-search -q "similar concepts" --mode vector-only + +# Force BM25-only mode +memory-daemon teleport hybrid-search -q "exact_function" --mode bm25-only +``` + +### Options + +| Option | Default | Description | +|--------|---------|-------------| +| `-q, --query` | required | Search query | +| `--top-k` | 10 | Number of results | +| `--mode` | hybrid | hybrid, vector-only, bm25-only | +| `--bm25-weight` | 0.5 | BM25 weight in fusion | +| `--vector-weight` | 0.5 | Vector weight in fusion | +| `--target` | all | Filter: all, toc, grip | +| `--addr` | http://[::1]:50051 | gRPC server address | + +### Output Format + +``` +Hybrid Search: "JWT authentication" +Mode: hybrid, BM25 Weight: 0.50, Vector Weight: 0.50 + +Mode used: hybrid (BM25: yes, Vector: yes) + +Found 5 results: +---------------------------------------------------------------------- +1. [toc_node] toc:segment:abc123 (score: 0.9234) + JWT token validation and refresh handling... + Time: 2026-01-30 14:32 +``` + +## Index Statistics + +```bash +memory-daemon teleport vector-stats +``` + +Output: +``` +Vector Index Statistics +---------------------------------------- +Status: Available +Vectors: 1523 +Dimension: 384 +Last Indexed: 2026-01-30T15:42:31Z +Index Path: ~/.local/share/agent-memory/vector.idx +Index Size: 2.34 MB +``` + +## Search Strategy + +### Decision Flow + +``` +User Query + | + v ++-- Contains exact terms/function names? --> BM25 Search +| ++-- Conceptual/semantic query? --> Vector Search +| ++-- Mixed or unsure? --> Hybrid Search (default) +``` + +### Recommended Workflows + +**Finding related discussions:** +```bash +# Start with hybrid for broad coverage +memory-daemon teleport hybrid-search -q "error handling patterns" + +# If too noisy, increase min-score or switch to vector +memory-daemon teleport vector-search -q "error handling patterns" --min-score 0.7 +``` + +**Debugging with exact terms:** +```bash +# Use BM25 for exact matches +memory-daemon teleport search "ConnectionTimeout" + +# Or hybrid with BM25 bias +memory-daemon teleport hybrid-search -q "ConnectionTimeout" --bm25-weight 0.8 +``` + +**Exploring concepts:** +```bash +# Pure semantic search for conceptual exploration +memory-daemon teleport vector-search -q "best practices for testing" +``` + +## Error Handling + +| Error | Resolution | +|-------|------------| +| Connection refused | `memory-daemon start` | +| Vector index unavailable | Wait for index build or check disk space | +| No results | Lower `--min-score`, try hybrid mode, broaden query | +| Slow response | Reduce `--top-k`, check index size | + +## Advanced + +### Tuning Weights + +The hybrid search uses Reciprocal Rank Fusion (RRF): +- Higher BM25 weight: Better for exact keyword matches +- Higher vector weight: Better for semantic similarity +- Equal weights (0.5/0.5): Balanced for general queries + +### Combining with TOC Navigation + +After finding relevant documents via vector search: + +```bash +# Get vector search results +memory-daemon teleport vector-search -q "authentication" +# Returns: toc:segment:abc123 + +# Navigate to get full context +memory-daemon query node --node-id "toc:segment:abc123" + +# Expand grip for details +memory-daemon query expand --grip-id "grip:..." --before 3 --after 3 +``` + +See [Command Reference](references/command-reference.md) for full CLI options. diff --git a/plugins/memory-query-plugin/skills/vector-search/references/command-reference.md b/plugins/memory-query-plugin/skills/vector-search/references/command-reference.md new file mode 100644 index 0000000..400b907 --- /dev/null +++ b/plugins/memory-query-plugin/skills/vector-search/references/command-reference.md @@ -0,0 +1,226 @@ +# Vector Search Command Reference + +Complete CLI reference for vector search commands. + +## teleport vector-search + +Semantic similarity search using vector embeddings. + +### Synopsis + +```bash +memory-daemon teleport vector-search [OPTIONS] --query +``` + +### Options + +| Option | Short | Default | Description | +|--------|-------|---------|-------------| +| `--query` | `-q` | required | Query text to embed and search | +| `--top-k` | | 10 | Maximum number of results to return | +| `--min-score` | | 0.0 | Minimum similarity score threshold (0.0-1.0) | +| `--target` | | all | Filter by document type: all, toc, grip | +| `--addr` | | http://[::1]:50051 | gRPC server address | + +### Examples + +```bash +# Basic semantic search +memory-daemon teleport vector-search -q "authentication patterns" + +# With minimum score threshold +memory-daemon teleport vector-search -q "debugging" --min-score 0.6 + +# Search only TOC nodes +memory-daemon teleport vector-search -q "testing strategies" --target toc + +# Search only grips (excerpts) +memory-daemon teleport vector-search -q "error messages" --target grip + +# Limit results +memory-daemon teleport vector-search -q "best practices" --top-k 5 + +# Custom endpoint +memory-daemon teleport vector-search -q "query" --addr http://localhost:9999 +``` + +### Output Fields + +| Field | Description | +|-------|-------------| +| doc_type | Type of document: toc_node or grip | +| doc_id | Document identifier | +| score | Similarity score (0.0-1.0, higher is better) | +| text_preview | Truncated preview of matched content | +| timestamp | Document creation time | + +--- + +## teleport hybrid-search + +Combined BM25 keyword + vector semantic search with RRF fusion. + +### Synopsis + +```bash +memory-daemon teleport hybrid-search [OPTIONS] --query +``` + +### Options + +| Option | Short | Default | Description | +|--------|-------|---------|-------------| +| `--query` | `-q` | required | Search query | +| `--top-k` | | 10 | Maximum number of results | +| `--mode` | | hybrid | Search mode: hybrid, vector-only, bm25-only | +| `--bm25-weight` | | 0.5 | Weight for BM25 in fusion (0.0-1.0) | +| `--vector-weight` | | 0.5 | Weight for vector in fusion (0.0-1.0) | +| `--target` | | all | Filter by document type: all, toc, grip | +| `--addr` | | http://[::1]:50051 | gRPC server address | + +### Search Modes + +| Mode | Description | +|------|-------------| +| `hybrid` | Combines BM25 and vector with RRF fusion | +| `vector-only` | Uses only vector similarity (ignores BM25 index) | +| `bm25-only` | Uses only BM25 keyword matching (ignores vector index) | + +### Examples + +```bash +# Default hybrid search +memory-daemon teleport hybrid-search -q "JWT authentication" + +# Vector-only mode +memory-daemon teleport hybrid-search -q "similar concepts" --mode vector-only + +# BM25-only mode for exact keywords +memory-daemon teleport hybrid-search -q "ConnectionError" --mode bm25-only + +# Favor semantic matching +memory-daemon teleport hybrid-search -q "related topics" \ + --bm25-weight 0.3 \ + --vector-weight 0.7 + +# Favor keyword matching +memory-daemon teleport hybrid-search -q "function_name" \ + --bm25-weight 0.8 \ + --vector-weight 0.2 + +# Filter to grip documents only +memory-daemon teleport hybrid-search -q "debugging" --target grip +``` + +### Output Fields + +| Field | Description | +|-------|-------------| +| mode_used | Actual mode used (may differ if index unavailable) | +| bm25_available | Whether BM25 index was available | +| vector_available | Whether vector index was available | +| matches | List of ranked results | + +--- + +## teleport vector-stats + +Display vector index statistics. + +### Synopsis + +```bash +memory-daemon teleport vector-stats [OPTIONS] +``` + +### Options + +| Option | Default | Description | +|--------|---------|-------------| +| `--addr` | http://[::1]:50051 | gRPC server address | + +### Examples + +```bash +# Show vector index stats +memory-daemon teleport vector-stats + +# Custom endpoint +memory-daemon teleport vector-stats --addr http://localhost:9999 +``` + +### Output Fields + +| Field | Description | +|-------|-------------| +| Status | Whether index is available for searches | +| Vectors | Number of vectors in the index | +| Dimension | Embedding dimension (e.g., 384 for MiniLM) | +| Last Indexed | Timestamp of last index update | +| Index Path | File path to index on disk | +| Index Size | Size of index file | + +--- + +## teleport stats + +Display BM25 index statistics (for comparison). + +### Synopsis + +```bash +memory-daemon teleport stats [OPTIONS] +``` + +### Options + +| Option | Default | Description | +|--------|---------|-------------| +| `--addr` | http://[::1]:50051 | gRPC server address | + +--- + +## teleport search + +BM25 keyword search (non-vector). + +### Synopsis + +```bash +memory-daemon teleport search [OPTIONS] +``` + +### Options + +| Option | Short | Default | Description | +|--------|-------|---------|-------------| +| `` | | required | Search keywords | +| `--doc-type` | `-t` | all | Filter: all, toc, grip | +| `--limit` | `-n` | 10 | Maximum results | +| `--addr` | | http://[::1]:50051 | gRPC server address | + +### Examples + +```bash +# Basic BM25 search +memory-daemon teleport search "authentication" + +# Filter to TOC nodes +memory-daemon teleport search "JWT" -t toc + +# Limit results +memory-daemon teleport search "debugging" -n 5 +``` + +--- + +## Comparison: When to Use Each + +| Scenario | Recommended Command | +|----------|---------------------| +| Exact function/variable name | `teleport search` (BM25) | +| Conceptual query | `teleport vector-search` | +| General purpose | `teleport hybrid-search` | +| Error messages | `teleport search` or `hybrid --bm25-weight 0.8` | +| Finding similar topics | `teleport vector-search` | +| Technical documentation | `teleport hybrid-search` | diff --git a/proto/memory.proto b/proto/memory.proto index 65b51e1..fea3bde 100644 --- a/proto/memory.proto +++ b/proto/memory.proto @@ -42,6 +42,44 @@ service MemoryService { // Resume a paused job rpc ResumeJob(ResumeJobRequest) returns (ResumeJobResponse); + + // Search RPCs (Phase 10.5) + + // Search within a single TOC node + rpc SearchNode(SearchNodeRequest) returns (SearchNodeResponse); + + // Search across children of a parent node + rpc SearchChildren(SearchChildrenRequest) returns (SearchChildrenResponse); + + // Teleport RPCs (TEL-01 through TEL-04) + + // Search for TOC nodes or grips by keyword using BM25 ranking + rpc TeleportSearch(TeleportSearchRequest) returns (TeleportSearchResponse); + + // Vector RPCs (Phase 12 - VEC-01 through VEC-03) + + // Vector semantic search using HNSW index + rpc VectorTeleport(VectorTeleportRequest) returns (VectorTeleportResponse); + + // Hybrid BM25 + vector search using RRF fusion + rpc HybridSearch(HybridSearchRequest) returns (HybridSearchResponse); + + // Get vector index status and statistics + rpc GetVectorIndexStatus(GetVectorIndexStatusRequest) returns (VectorIndexStatus); + + // Topic Graph RPCs (Phase 14 - TOPIC-08) + + // Get topic graph status and statistics + rpc GetTopicGraphStatus(GetTopicGraphStatusRequest) returns (GetTopicGraphStatusResponse); + + // Get topics matching a query + rpc GetTopicsByQuery(GetTopicsByQueryRequest) returns (GetTopicsByQueryResponse); + + // Get topics related to a specific topic + rpc GetRelatedTopics(GetRelatedTopicsRequest) returns (GetRelatedTopicsResponse); + + // Get top topics by importance score + rpc GetTopTopics(GetTopTopicsRequest) returns (GetTopTopicsResponse); } // Role of the message author @@ -66,6 +104,15 @@ enum EventType { EVENT_TYPE_SESSION_END = 8; } +// Fields to search within a TOC node +enum SearchField { + SEARCH_FIELD_UNSPECIFIED = 0; + SEARCH_FIELD_TITLE = 1; + SEARCH_FIELD_SUMMARY = 2; + SEARCH_FIELD_BULLETS = 3; + SEARCH_FIELD_KEYWORDS = 4; +} + // A conversation event to be stored. // // Per ING-02: Includes session_id, timestamp, role, text, metadata. @@ -319,3 +366,320 @@ message ResumeJobResponse { bool success = 1; optional string error = 2; } + +// ===== Search Messages (Phase 10.5) ===== + +// Search within a single node +message SearchNodeRequest { + // Node ID to search within + string node_id = 1; + // Search terms (space-separated, case-insensitive) + string query = 2; + // Fields to search (empty = all fields) + repeated SearchField fields = 3; + // Max matches to return (default: 10) + int32 limit = 4; + // Optional token budget for response control + int32 token_budget = 5; +} + +message SearchMatch { + // Which field matched + SearchField field = 1; + // Matching text snippet + string text = 2; + // If bullet, the supporting grip IDs + repeated string grip_ids = 3; + // Term overlap score (0.0-1.0) + float score = 4; +} + +message SearchNodeResponse { + // Whether any matches found + bool matched = 1; + // Individual matches + repeated SearchMatch matches = 2; + // Node that was searched + string node_id = 3; + // Level of the searched node + TocLevel level = 4; +} + +// Search children of a parent node +message SearchChildrenRequest { + // Parent node ID (empty string for root/year level) + string parent_id = 1; + // Search terms (space-separated, case-insensitive) + string query = 2; + // Level to search at (ignored if parent_id provided - uses children's level) + TocLevel child_level = 3; + // Fields to search (empty = all fields) + repeated SearchField fields = 4; + // Max nodes to return (default: 10) + int32 limit = 5; + // Optional token budget for response control + int32 token_budget = 6; +} + +message SearchNodeResult { + // Node ID of the matching node + string node_id = 1; + // Node title for display + string title = 2; + // Level of this node + TocLevel level = 3; + // Matches within this node + repeated SearchMatch matches = 4; + // Aggregate relevance score + float relevance_score = 5; +} + +message SearchChildrenResponse { + // Matching nodes with their matches + repeated SearchNodeResult results = 1; + // Whether more results available + bool has_more = 2; +} + +// ===== Teleport Search Messages (TEL-01 through TEL-04) ===== + +// Document type filter for teleport search +enum TeleportDocType { + TELEPORT_DOC_TYPE_UNSPECIFIED = 0; // Search all types + TELEPORT_DOC_TYPE_TOC_NODE = 1; // TOC nodes only + TELEPORT_DOC_TYPE_GRIP = 2; // Grips only +} + +// Request for teleport search +message TeleportSearchRequest { + // Search query (keywords) + string query = 1; + // Filter by document type (optional) + TeleportDocType doc_type = 2; + // Maximum results to return (default: 10) + int32 limit = 3; +} + +// A single teleport search result +message TeleportSearchResult { + // Document ID (node_id or grip_id) + string doc_id = 1; + // Document type + TeleportDocType doc_type = 2; + // BM25 relevance score + float score = 3; + // Keywords from the document (if available) + optional string keywords = 4; + // Timestamp in milliseconds + optional int64 timestamp_ms = 5; +} + +// Response from teleport search +message TeleportSearchResponse { + // Ranked search results + repeated TeleportSearchResult results = 1; + // Total documents in index + uint64 total_docs = 2; +} + +// ===== Vector Search Messages (Phase 12 - VEC-01 through VEC-03) ===== + +// Target type filter for vector search +enum VectorTargetType { + VECTOR_TARGET_TYPE_UNSPECIFIED = 0; + VECTOR_TARGET_TYPE_TOC_NODE = 1; + VECTOR_TARGET_TYPE_GRIP = 2; + VECTOR_TARGET_TYPE_ALL = 3; +} + +// Time range filter for searches +message TimeRange { + // Start time in milliseconds (inclusive) + int64 start_ms = 1; + // End time in milliseconds (exclusive) + int64 end_ms = 2; +} + +// Request for vector semantic search +message VectorTeleportRequest { + // Query text to embed and search + string query = 1; + // Maximum results to return (default: 10) + int32 top_k = 2; + // Minimum similarity score 0.0-1.0 (default: 0.0) + float min_score = 3; + // Optional time range filter + optional TimeRange time_filter = 4; + // Target type filter + VectorTargetType target = 5; +} + +// A vector search match +message VectorMatch { + // Document ID (node_id or grip_id) + string doc_id = 1; + // Document type ("toc_node" or "grip") + string doc_type = 2; + // Similarity score 0.0-1.0 + float score = 3; + // Preview of matched text + string text_preview = 4; + // Document timestamp in milliseconds + int64 timestamp_ms = 5; +} + +// Response from vector search +message VectorTeleportResponse { + // Ranked matches by similarity + repeated VectorMatch matches = 1; + // Index status at time of search + optional VectorIndexStatus index_status = 2; +} + +// Search mode for hybrid search +enum HybridMode { + HYBRID_MODE_UNSPECIFIED = 0; + HYBRID_MODE_VECTOR_ONLY = 1; + HYBRID_MODE_BM25_ONLY = 2; + HYBRID_MODE_HYBRID = 3; +} + +// Request for hybrid BM25 + vector search +message HybridSearchRequest { + // Search query + string query = 1; + // Maximum results to return (default: 10) + int32 top_k = 2; + // Search mode + HybridMode mode = 3; + // Weight for BM25 in fusion (default: 0.5) + float bm25_weight = 4; + // Weight for vector in fusion (default: 0.5) + float vector_weight = 5; + // Optional time range filter + optional TimeRange time_filter = 6; + // Target type filter + VectorTargetType target = 7; +} + +// Response from hybrid search +message HybridSearchResponse { + // Ranked matches by fused score + repeated VectorMatch matches = 1; + // Actual mode used for this search + HybridMode mode_used = 2; + // Whether BM25 index was available + bool bm25_available = 3; + // Whether vector index was available + bool vector_available = 4; +} + +// Request for vector index status +message GetVectorIndexStatusRequest {} + +// Vector index status and statistics +message VectorIndexStatus { + // Whether index is available for search + bool available = 1; + // Number of vectors in the index + int64 vector_count = 2; + // Embedding dimension + int32 dimension = 3; + // Last index update timestamp (RFC3339) + string last_indexed = 4; + // Index file path + string index_path = 5; + // Index size in bytes + int64 size_bytes = 6; +} + +// ===== Topic Graph Messages (Phase 14 - TOPIC-08) ===== + +// Request for topic graph status +message GetTopicGraphStatusRequest {} + +// Response with topic graph status +message GetTopicGraphStatusResponse { + // Total number of topics + uint64 topic_count = 1; + // Total number of relationships between topics + uint64 relationship_count = 2; + // Last update timestamp (RFC3339) + string last_updated = 3; + // Whether the topic graph is available + bool available = 4; +} + +// A semantic topic from the topic graph +message Topic { + // Unique topic identifier + string id = 1; + // Human-readable label + string label = 2; + // Time-decayed importance score (0.0 - 1.0+) + float importance_score = 3; + // Keywords associated with this topic + repeated string keywords = 4; + // When the topic was first created (RFC3339) + string created_at = 5; + // Most recent mention timestamp (RFC3339) + string last_mention = 6; +} + +// A relationship between two topics +message TopicRelationship { + // Source topic ID + string source_id = 1; + // Target topic ID + string target_id = 2; + // Relationship type: "co-occurrence", "semantic", "hierarchical" + string relationship_type = 3; + // Relationship strength (0.0 - 1.0) + float strength = 4; +} + +// Request for topics by query +message GetTopicsByQueryRequest { + // Search query (keywords to match against topic labels/keywords) + string query = 1; + // Maximum results to return (default: 10) + uint32 limit = 2; +} + +// Response with matching topics +message GetTopicsByQueryResponse { + // Topics matching the query + repeated Topic topics = 1; +} + +// Request for related topics +message GetRelatedTopicsRequest { + // Topic ID to find related topics for + string topic_id = 1; + // Optional: filter by relationship type ("co-occurrence", "semantic", "hierarchical") + string relationship_type = 2; + // Maximum results to return (default: 10) + uint32 limit = 3; +} + +// Response with related topics +message GetRelatedTopicsResponse { + // Topics related to the requested topic + repeated Topic related_topics = 1; + // Relationships between the source topic and related topics + repeated TopicRelationship relationships = 2; +} + +// Request for top topics by importance +message GetTopTopicsRequest { + // Maximum results to return (default: 10) + uint32 limit = 1; + // Look back window in days for importance calculation (default: 30) + uint32 days = 2; +} + +// Response with top topics +message GetTopTopicsResponse { + // Topics sorted by importance score (descending) + repeated Topic topics = 1; +}