From 26e3817b15ae893be1f4619073335cbab2e43fad Mon Sep 17 00:00:00 2001 From: Louis-David Coulombe Date: Tue, 10 Jun 2025 23:46:03 -0400 Subject: [PATCH 01/15] Add AI generated documentation using https://github.com/The-Pocket/PocketFlow-Tutorial-Codebase-Knowledge --- ...hedule_blueprint___schedule_schedule___.md | 280 +++++++++++++++ .../02_http_api_endpoints___routing_.md | 261 ++++++++++++++ ...ement_service___api_schedulesservice___.md | 265 ++++++++++++++ ..._core_services_aggregator___api_core___.md | 205 +++++++++++ ...duler_____scheduler_defaultscheduler___.md | 288 +++++++++++++++ ...schedule_persistence_and_communication_.md | 314 ++++++++++++++++ ..._scheduler_dailyschedulestoredefault___.md | 307 ++++++++++++++++ ...heduler_schedulepostprocessordefault___.md | 248 +++++++++++++ ...e___scheduler_deadletterqueuedefault___.md | 232 ++++++++++++ ...monitor___api_instrumentationservice___.md | 335 ++++++++++++++++++ documentation/index.md | 81 +++++ 11 files changed, 2816 insertions(+) create mode 100644 documentation/01_schedule_blueprint___schedule_schedule___.md create mode 100644 documentation/02_http_api_endpoints___routing_.md create mode 100644 documentation/03_schedule_management_service___api_schedulesservice___.md create mode 100644 documentation/04_core_services_aggregator___api_core___.md create mode 100644 documentation/05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md create mode 100644 documentation/06_kafka_based_schedule_persistence_and_communication_.md create mode 100644 documentation/07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md create mode 100644 documentation/08_post_execution_handler___scheduler_schedulepostprocessordefault___.md create mode 100644 documentation/09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md create mode 100644 documentation/10_application_health_monitor___api_instrumentationservice___.md create mode 100644 documentation/index.md diff --git a/documentation/01_schedule_blueprint___schedule_schedule___.md b/documentation/01_schedule_blueprint___schedule_schedule___.md new file mode 100644 index 00000000..7553b2d3 --- /dev/null +++ b/documentation/01_schedule_blueprint___schedule_schedule___.md @@ -0,0 +1,280 @@ +# Chapter 1: Schedule Blueprint (`schedule.Schedule`) + +Welcome to the `scheduler` project! If you've ever needed a program to do something automatically at a specific time, like sending a daily report or checking for updates every hour, then you're in the right place. This project helps you do just that. + +At the heart of any scheduling system is the need to clearly define *what* task to do, *when* to do it, and *how often* it should repeat. In our `scheduler` project, this detailed instruction sheet is called a **`schedule.Schedule`**. + +Think of `schedule.Schedule` as a **recipe card** or a detailed instruction manual for a task. Just like a recipe tells you the ingredients, steps, and cooking time, a `schedule.Schedule` tells our system everything it needs to know to perform an action correctly and consistently. + +## What Problem Does the `schedule.Schedule` Solve? + +Imagine you're building an e-commerce website. A common feature is to send a reminder email to a customer if they add items to their shopping cart but don't complete the purchase within, say, 24 hours. + +To make this happen automatically, you need to tell the system: +1. **What to do:** Send an email. +2. **To whom:** The specific customer. +3. **With what content:** The reminder message and a link to their cart. +4. **When to do it:** 24 hours after they last updated their cart. +5. **How often:** Just once for that specific abandoned cart. + +The `schedule.Schedule` is the way we package all this information into a neat, understandable format for our `scheduler` system. + +## Meet the `schedule.Schedule`: Your Task's Recipe Card + +Let's break down what goes into this "recipe card." A `schedule.Schedule` primarily defines three things: + +1. **The Task (WHAT):** What action needs to be performed? This usually involves sending some data (a "payload") to a specific destination (a "topic"). +2. **The Timing (WHEN):** When should this task be executed for the first time? This includes the date, time, and timezone. +3. **The Repetition (HOW OFTEN):** Should this task repeat? If so, how frequently (e.g., hourly, daily, weekly, or a custom interval)? + +## Dissecting the Blueprint: Key Ingredients of `schedule.Schedule` + +In our Go codebase, the `schedule.Schedule` is a data structure (a `struct`). Let's look at its main fields. You can find the full definition in `api/pkg/schedule/schedule.go`. + +Here's a simplified view of its structure: + +```go +// Simplified from: api/pkg/schedule/schedule.go +package schedule + +import "time" + +// Schedule holds all details for a task. +type Schedule struct { + // === WHEN to run? === + LocalExecutionTime time.Time // The exact date and time for the task. + TimeLocation string // The timezone for LocalExecutionTime (e.g., "America/New_York"). + + // === HOW OFTEN to repeat? === + Recurrence *Recurrence // Details about repetition (if any). + + // === WHAT task to perform? === + TargetTopic string // Where the task's message/result should go (e.g., "email-sending-service"). + TargetKey string // A specific key for the message (e.g., "user123-reminder"). + TargetPayload []byte // The actual data/content for the task (e.g., the email body). + + // === HOW to identify this blueprint? === + Key string // Your unique name for this schedule definition. + + // ID string // An internal, system-generated ID. You don't usually set this. +} + +// Recurrence defines how a schedule repeats. +type Recurrence struct { + Scheme string // Type of repetition (e.g., "DAILY", "WEEKLY"). + Metadata map[string]string // Extra info for complex schemes (e.g., day of month for "MONTHLY"). +} +``` + +Let's explore these "ingredients": + +* **`LocalExecutionTime` (When):** This is a `time.Time` object in Go. It specifies the exact date and time the task should run. For example, `2024-07-15T10:00:00`. +* **`TimeLocation` (When):** This is a string like `"America/New_York"` or `"UTC"`. It's crucial because `LocalExecutionTime` is interpreted according to this timezone. 10:00 AM in New York is different from 10:00 AM in London! +* **`Recurrence` (How Often):** This points to another small structure, `Recurrence`, which we'll discuss more soon. It tells the system if the task should run just once (`RecurrenceNone`) or repeat (e.g., `RecurrenceDaily`, `RecurrenceWeekly`). +* **`TargetTopic` (What):** Think of this as the address or channel where the outcome of your task is sent. For example, if your task is "send an email," the `TargetTopic` might be something like `"email-notifications"`. Other parts of your system listen to this topic. +* **`TargetKey` (What):** When the task's message is sent to the `TargetTopic`, this `TargetKey` helps identify or organize it. For example, if sending a password reset email, the `TargetKey` might be the `userID`. +* **`TargetPayload` (What):** This is the actual content or data for your task, represented as a slice of bytes (`[]byte`). For an email task, this could be the email's subject and body, perhaps formatted as JSON. +* **`Key` (Identity):** This is a unique string you provide to name your schedule definition. For example, `"monthly-invoice-generation"` or `"user-activity-report-user123"`. You'll use this `Key` if you ever want to update or delete this specific schedule blueprint. +* **`ID` (Identity - Internal):** The `ID` field (commented out in the simplified struct above for clarity, but present in the actual code as `json:"-"`) is typically generated by the system itself (e.g., when you save a schedule using the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md)). It's an internal unique identifier. For recurring tasks, a new `ID` might be generated for each upcoming run derived from the same `Key`. You generally don't set this `ID` yourself when creating a new schedule. + +## Example: Crafting an Abandoned Cart Reminder Blueprint + +Let's go back to our e-commerce example: sending a reminder email for an abandoned cart 24 hours later. Here's how you might define a `schedule.Schedule` for a specific user's abandoned cart (let's say cart ID "cart-xyz-789" for user "customer-567"). + +```go +package main + +import ( + "fmt" + "time" + + // This is the actual package for Schedule struct + "github.com/nestoca/scheduler/api/pkg/schedule" +) + +func createAbandonedCartReminder() *schedule.Schedule { + // Calculate execution time: 24 hours from now + executionTime := time.Now().Add(24 * time.Hour) + + // Define the timezone. Let's use UTC for simplicity. + // In a real app, you might load a specific store's timezone: + // loc, err := time.LoadLocation("America/Denver") + // if err != nil { /* handle error */ } + loc := time.UTC + + // The payload for our task (e.g., what to include in the email) + // Usually, this would be JSON data. + payload := []byte(fmt.Sprintf(`{"userId": "customer-567", "cartId": "cart-xyz-789", "message": "You left items in your cart!"}`)) + + // This is a one-time reminder for this specific cart. + reminderBlueprint := &schedule.Schedule{ + // WHEN + LocalExecutionTime: schedule.DeriveScheduleExecutionTimeFromEntry(executionTime, loc), // Helper to set time with location + TimeLocation: loc.String(), // e.g., "UTC" or "America/Denver" + + // HOW OFTEN (None means it runs only once) + Recurrence: &schedule.Recurrence{ + Scheme: schedule.RecurrenceNone, // This task does not repeat + Metadata: nil, // No extra info needed for RecurrenceNone + }, + + // WHAT + TargetTopic: "cart-reminder-notifications", // Where to send the task info + TargetKey: "cart-xyz-789", // Unique key for this specific cart reminder task + TargetPayload: payload, // The actual data + + // IDENTITY + Key: "abandoned-cart-reminder-cart-xyz-789", // Your unique name for this blueprint + } + + return reminderBlueprint +} + +func main() { + myReminder := createAbandonedCartReminder() + fmt.Printf("Blueprint created for Key: %s at %s in %s\n", + myReminder.Key, + myReminder.LocalExecutionTime.Format(time.RFC3339), + myReminder.TimeLocation) +} +``` + +When you run this (conceptually), you'd create an object in memory. This object is the "recipe card." +The `schedule.DeriveScheduleExecutionTimeFromEntry()` function is a helper from `api/pkg/schedule/schedule.go` that ensures the time is correctly associated with the location, stripping monotonic clock readings if any, which is good practice for scheduled times. + +## Understanding Recurrence: Making Tasks Repeat + +What if you want a task to run every day, or every Monday at 9 AM? That's where the `Recurrence` part of the `schedule.Schedule` comes in. + +The `Recurrence` struct has two main parts: +* `Scheme`: A string indicating the type of repetition. +* `Metadata`: A map for any extra information needed by certain schemes. + +Here are some common recurrence schemes (defined as constants in `api/pkg/schedule/schedule.go`): + +```mermaid +graph TD + RecurrenceTypes["Recurrence Schemes"] -->|Defines repetition| RecurrenceStruct["Recurrence { Scheme, Metadata }"] + + RecurrenceStruct --> RecurrenceNone["RecurrenceNone (Execute Once)"] + RecurrenceStruct --> SimpleRepeats["Simple Repeats"] + SimpleRepeats --> RecurrenceHourly["RecurrenceHourly"] + SimpleRepeats --> RecurrenceDaily["RecurrenceDaily"] + SimpleRepeats --> RecurrenceWeekly["RecurrenceWeekly"] + SimpleRepeats --> RecurrenceBiWeekly["RecurrenceBiWeekly (Every 2 Weeks)"] + SimpleRepeats --> RecurrenceYearly["RecurrenceYearly"] + + RecurrenceStruct --> ComplexRepeats["Complex Repeats (needs Metadata)"] + ComplexRepeats --> RecurrenceMonthly["RecurrenceMonthly"] + RecurrenceMonthly --> RMDay["Metadata: MONTHLY_PAYMENT_DAY (e.g., '15' for 15th)"] + ComplexRepeats --> RecurrenceSemiMonthly["RecurrenceSemiMonthly"] + RecurrenceSemiMonthly --> RSMDays["Metadata: SEMIMONTHLY_PAYMENT_DAYS (e.g., '1_15' for 1st and 15th)"] + ComplexRepeats --> RecurrenceCustom["RecurrenceCustom"] + RecurrenceCustom --> RCDelay["Metadata: CUSTOM_DELTA (e.g., '3h30m' for every 3.5 hours)"] +``` + +**Examples of setting up `Recurrence`:** + +1. **A task that runs daily:** + ```go + dailyRecurrence := &schedule.Recurrence{ + Scheme: schedule.RecurrenceDaily, + // Metadata is usually nil or empty for daily + } + ``` + +2. **A task that runs on the 10th of every month:** + ```go + monthlyRecurrence := &schedule.Recurrence{ + Scheme: schedule.RecurrenceMonthly, + Metadata: map[string]string{ + schedule.RecurrenceMetadataMonthlyPaymentDay: "10", // Run on the 10th + }, + } + ``` + +3. **A task that runs every 2 hours and 30 minutes (custom interval):** + ```go + customRecurrence := &schedule.Recurrence{ + Scheme: schedule.RecurrenceCustom, + Metadata: map[string]string{ + schedule.RecurrenceMetadataCustomDelta: "2h30m", // Standard Go duration string + }, + } + ``` +When you define a recurring schedule, the `LocalExecutionTime` you provide is typically the *first* time the task should run. The system then uses the `Recurrence` settings to figure out all future execution times. + +## How the System Uses This Blueprint + +So you've carefully crafted your `schedule.Schedule` "recipe card." What happens next? + +1. **Submission:** You typically send this blueprint to the `scheduler` system. This often happens by sending it as a JSON payload to an API endpoint. We'll learn more about this in [Chapter 2: HTTP API Endpoints & Routing](02_http_api_endpoints___routing_.md). + + Here's how a part of our example blueprint might look as JSON: + ```json + { + "localExecutionTime": "2024-07-16T10:30:00Z", // Example: Time in ISO 8601 format + "timeLocation": "UTC", + "recurrence": { + "scheme": "NONE", // Matches RecurrenceNone + "metadata": null + }, + "key": "abandoned-cart-reminder-cart-xyz-789", + "targetTopic": "cart-reminder-notifications", + "targetKey": "cart-xyz-789", + "targetPayload": "eyJ1c2VySWQiOiAiY3VzdG9tZXItNTY3IiwgImNhcnRJZCI6ICJjYXJ0LXh5ei03ODkiLCAibWVzc2FnZSI6ICJZb3UgbGVmdCBpdGVtcyBpbiB5b3VyIGNhcnQhIn0=" // Payload is base64 encoded + } + ``` + (Note: `[]byte` payloads are often base64 encoded in JSON.) + + The system decodes this JSON back into a `schedule.Schedule` object. For instance, the `getSchedule` function in `api/cmd/scheduler-api/schedules/save.go` does this: + ```go + // Simplified from api/cmd/scheduler-api/schedules/save.go + func getSchedule(r *http.Request) (*schedule.Schedule, error) { + p := new(schedule.Schedule) + // Decodes the JSON from the request body into our Schedule struct + if err := json.NewDecoder(r.Body).Decode(p); err != nil { + // ... handle error ... + return nil, err + } + return p, nil + } + ``` + +2. **Validation & Storage:** The system, often via a component like the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md), will validate your blueprint. For example, it checks if `TimeLocation` is valid or if `Recurrence` metadata makes sense for the chosen scheme. If valid, it's stored, often in a reliable message queue like Kafka (more in [Chapter 6: Kafka-based Schedule Persistence and Communication](06_kafka_based_schedule_persistence_and_communication_.md)). + + When preparing to store the schedule (e.g., in Kafka), it might be converted into a specific event format. The `toUpsertScheduleEvent` function in `api/pkg/schedule/kafka/producer.go` shows how a `schedule.Schedule` object is transformed into an `UpsertedEvent`, converting `LocalExecutionTime` to a string: + ```go + // Simplified from api/pkg/schedule/kafka/producer.go + func toUpsertScheduleEvent(s *schedule.Schedule) *schedule.UpsertedEvent { + return &schedule.UpsertedEvent{ + ID: s.ID, // System-generated ID + LocalExecutionTime: s.LocalExecutionTime.Format(schedule.ScheduleExecutionTimeFormat), // "2006-01-02 15:04" + TimeLocation: s.TimeLocation, + Recurrence: s.Recurrence, + TargetTopic: s.TargetTopic, + TargetKey: s.TargetKey, + TargetPayload: s.TargetPayload, + } + } + ``` + +3. **Execution:** Later, the core [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) picks up these stored blueprints. When a schedule's `LocalExecutionTime` arrives, the engine triggers the defined task by sending the `TargetPayload` to the `TargetTopic` with the `TargetKey`. If it's a recurring task, the engine also calculates its next execution time. + +## Why is the Blueprint So Important? + +The `schedule.Schedule` blueprint is fundamental because: +* **Clarity:** It provides a clear, unambiguous definition of what needs to be done. +* **Consistency:** It ensures that all tasks are understood and handled uniformly by the system. +* **Automation:** It's the contract that allows the `scheduler` to reliably automate your tasks. +* **Single Source of Truth:** For any given scheduled job, its `schedule.Schedule` definition tells the complete story. + +## Conclusion + +You've now learned about the most basic and crucial concept in our `scheduler` project: the `schedule.Schedule` blueprint. It's the detailed "recipe card" that specifies what task to run, when to run it, and how often it should repeat. Understanding its components (`LocalExecutionTime`, `TimeLocation`, `Recurrence`, `TargetTopic`, `TargetKey`, `TargetPayload`, and `Key`) is key to using the scheduler effectively. + +Now that you understand what a 'Schedule Blueprint' looks like, you're probably wondering how you tell the `scheduler` system about it. How do you submit this recipe card? That's exactly what we'll cover in the next chapter, where we dive into the [HTTP API Endpoints & Routing](02_http_api_endpoints___routing_.md). + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/02_http_api_endpoints___routing_.md b/documentation/02_http_api_endpoints___routing_.md new file mode 100644 index 00000000..3040ab9b --- /dev/null +++ b/documentation/02_http_api_endpoints___routing_.md @@ -0,0 +1,261 @@ +# Chapter 2: HTTP API Endpoints & Routing + +In [Chapter 1: Schedule Blueprint (`schedule.Schedule`)](01_schedule_blueprint___schedule_schedule___.md), we learned all about the `schedule.Schedule` – our "recipe card" for tasks. We know how to define *what* task to run, *when* to run it, and *how often*. But a recipe card sitting on your desk doesn't cook the meal by itself, right? We need a way to submit this recipe to the chef – our `scheduler` system! + +This chapter is all about how we communicate with the `scheduler` from the outside world. How do we send it new schedules? How do we ask it to delete old ones? How do we check if it's feeling okay? The answer is through **HTTP API Endpoints & Routing**. + +## The Problem: Talking to the Scheduler + +Imagine our `scheduler` is a busy office building. Inside, different departments handle various tasks: one department files new schedules, another shreds old ones, and yet another gives updates on the building's operational status. + +If you, as an external client (another program, or even a person using a tool), want to interact with this office, you can't just barge in and shout your request. You need: +1. **Specific Entrances (Endpoints):** You need to know which "door" or "reception desk" to go to for your specific request. Want to submit a new schedule? That's one desk. Want to check the system's health? That's a different desk. +2. **A Directory Service (Routing):** Once you're at the right entrance, someone needs to guide your request to the correct internal department (the code that actually handles it). + +HTTP API Endpoints and Routing provide exactly this structure for web-based services like our `scheduler`. + +## Meet the "Reception Desks": HTTP API Endpoints + +An **HTTP API Endpoint** is like a specific web address (URL) that you can send requests to. Each endpoint is designed for a particular type of interaction. "API" stands for Application Programming Interface – it's a way for programs to talk to each other. "HTTP" is the protocol used on the web, the same one your browser uses to fetch web pages. + +Think of these endpoints as different reception desks in our "scheduler office building": +* A desk for "New Schedule Submissions" +* A desk for "Schedule Deletions" +* A desk for "System Status Inquiries" + +When you send a request to one of these endpoints, you're essentially handing over a form (your data, like a `schedule.Schedule` in JSON format) or asking a question. + +## The "Directory Service": Routing + +Once your request arrives at the `scheduler`'s main entrance (its network address), **Routing** is the process that looks at the specific endpoint you targeted (e.g., `/api/schedule` or `/api/status`) and the type of request you made (e.g., "create," "delete," "get information") and directs it to the correct "internal department" – a specific piece of code called a **handler function**. + +In our `scheduler` project, a function named `routes.GetRouter` is responsible for setting up all these "reception desks" and the "directory service." It acts like the building manager who decides where each desk goes and trains the receptionists (the router itself) to send visitors to the right place. + +## The "Office Workers": Handler Functions + +A **handler function** is a piece of Go code that actually does the work for a specific request. For example: +* If you send a request to create a new schedule, the router will pass it to a handler like `schedules.Save`. This handler takes the schedule details you sent, validates them, and then probably uses the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md) to store it. +* If you ask for the system status, the router directs your request to a handler like `status.Get`. This handler checks the system's health, perhaps using the [Application Health Monitor (`api.InstrumentationService`)](10_application_health_monitor___api_instrumentationservice___.md), and sends back a response. + +These handlers often interact with a central coordinator called the [Core Services Aggregator (`api.Core`)](04_core_services_aggregator___api_core___.md) to get access to various services they need to fulfill the request. + +## Our Main Use Case: Submitting a Daily Report Schedule + +Let's say you've crafted a `schedule.Schedule` blueprint for a daily report, as we discussed in Chapter 1. Now, you want to submit it to the `scheduler`. You'll do this by sending an HTTP request to a specific endpoint. + +**HTTP Methods: The Verbs of Your Request** + +When you make an HTTP request, you also specify an "HTTP method." These are like verbs telling the server what kind of action you want to perform on the resource identified by the endpoint. Common ones include: +* `POST`: Typically used to create a new resource (e.g., submit a new schedule). +* `GET`: Used to retrieve information (e.g., get system status, get a list of schedules). +* `DELETE`: Used to remove a resource (e.g., delete an existing schedule). +* `PUT`: Often used to update an existing resource entirely. + +**Submitting Our Schedule (POST Request)** + +To submit your new daily report schedule, you would typically send an HTTP `POST` request to an endpoint like `/api/schedule`. The body of this request would contain your `schedule.Schedule` blueprint, usually in JSON format. + +Let's imagine our schedule blueprint in JSON (simplified): +```json +// This is what we want to send +{ + "key": "daily-sales-report", + "localExecutionTime": "2024-08-01T09:00:00Z", + "timeLocation": "UTC", + "recurrence": { "scheme": "DAILY" }, + "targetTopic": "reports-generation", + "targetPayload": "eyJhY3Rpb24iOiAiZ2VuZXJhdGVfc2FsZXNfcmVwb3J0In0=" // Base64 for: {"action": "generate_sales_report"} +} +``` + +You could use a command-line tool like `curl` to send this (don't worry if `curl` is new to you, it's just a way to send HTTP requests): +```bash +# Example using curl (conceptual) +curl -X POST \ + http://your-scheduler-address/api/schedule \ + -H "Content-Type: application/json" \ + -d '{ + "key": "daily-sales-report", + "localExecutionTime": "2024-08-01T09:00:00Z", + "timeLocation": "UTC", + "recurrence": { "scheme": "DAILY" }, + "targetTopic": "reports-generation", + "targetPayload": "eyJhY3Rpb24iOiAiZ2VuZXJhdGVfc2FsZXNfcmVwb3J0In0=" + }' +``` +* `-X POST`: Specifies the HTTP POST method. +* `http://your-scheduler-address/api/schedule`: The endpoint URL. +* `-H "Content-Type: application/json"`: Tells the server we're sending JSON data. +* `-d '{...}'`: The actual JSON data (our schedule blueprint). + +**What Happens Next?** +1. The `scheduler`'s web server receives this `POST` request to `/api/schedule`. +2. The router sees `/api/schedule` and `POST`, and knows to send it to the `schedules.Save` handler function. +3. The `schedules.Save` handler (which we'll see more of later) reads the JSON data, converts it into a `schedule.Schedule` Go object, and then uses the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md) to save it. +4. If successful, the `scheduler` sends back an HTTP response, typically `201 Created`. + +**Other Common Interactions:** + +* **Deleting a Schedule (DELETE Request):** + To delete the "daily-sales-report" schedule, you might send: + ```bash + curl -X DELETE http://your-scheduler-address/api/schedule/daily-sales-report + ``` + The router would direct this to the `schedules.Delete` handler. If successful, you'd get a `200 OK` response. + +* **Checking System Status (GET Request):** + To check if the scheduler is running and its version: + ```bash + curl -X GET http://your-scheduler-address/api/status + ``` + The router would send this to the `status.Get` handler. You might get back a JSON response like: + ```json + { + "status": 200, + "time": "2024-07-15T12:30:00Z", + "version": "v1.2.3" + } + ``` + +## Under the Hood: How Routing Works in `scheduler` + +Let's peek into how the `scheduler` sets up these routes. The main file to look at is `api/cmd/scheduler-api/routes/routes.go`. + +**1. The Main Router Setup (`GetRouter`)** + +The `GetRouter` function is the starting point. It creates a new router object (using a popular Go library called `chi`) and configures it. + +```go +// Simplified from: api/cmd/scheduler-api/routes/routes.go +package routes + +import ( + // ... other imports + "github.com/go-chi/chi" + "github.com/nestoca/scheduler/api" + // ... handlers like status, schedules +) + +func GetRouter(c *api.Core) chi.Router { + r := chi.NewRouter() // Creates a new router instance + + // r.Use(...) // Applies some standard "middleware" (explained below) + r.Mount("/api", getAPIRoutes(c)) // All our API routes will start with /api + + return r +} +``` +* `chi.NewRouter()`: This creates the main router object. +* `r.Mount("/api", getAPIRoutes(c))`: This is key! It says that any URL starting with `/api` should be handled by another set of routes defined in the `getAPIRoutes` function. The `c *api.Core` is an instance of the [Core Services Aggregator (`api.Core`)](04_core_services_aggregator___api_core___.md), which provides access to all necessary services for the handlers. + +The `r.Use(...)` line (commented for brevity above, but present in the actual code) sets up "middleware." Middleware are like checkpoints that every request passes through before reaching the final handler. They can do things like log the request, check for authentication, or add information to the request. + +**2. Defining API-Specific Routes (`getAPIRoutes`)** + +The `getAPIRoutes` function defines the actual endpoints like `/status`, `/schedule`, etc. + +```go +// Simplified from: api/cmd/scheduler-api/routes/routes.go +func getAPIRoutes(core *api.Core) http.Handler { + r := chi.NewRouter() // A sub-router for /api paths + + // Status endpoint + r.Get("/status", status.Get(core.HealthService)) + + // Schedule endpoints + r.Route("/schedule", func(r chi.Router) { + // POST /api/schedule -> creates a new schedule + r.Post("/", schedules.Save(core.SchedulesService)) + // DELETE /api/schedule/{key} -> deletes a schedule by its key + r.Delete("/{key}", schedules.Delete(core.SchedulesService)) + // ... other schedule-related routes + }) + + // ... routes for instrumentation/metrics ... + + return r +} +``` +Let's break this down: +* `r.Get("/status", status.Get(core.HealthService))`: + * This says: If a `GET` request comes to `/api/status` (remember the `/api` prefix from `Mount`), then call the handler function returned by `status.Get(...)`. + * `status.Get` is a function (from `api/cmd/scheduler-api/status/get.go`) that takes the `core.HealthService` (an instance of [Application Health Monitor (`api.InstrumentationService`)](10_application_health_monitor___api_instrumentationservice___.md)) and returns the actual handler function. +* `r.Route("/schedule", func(r chi.Router) { ... })`: + * This groups all routes that start with `/api/schedule`. + * `r.Post("/", schedules.Save(core.SchedulesService))`: For a `POST` request to `/api/schedule/` (the `/` here means the base of the `/schedule` route), use the handler from `schedules.Save(...)`. This handler gets the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md) from the `core`. + * `r.Delete("/{key}", schedules.Delete(core.SchedulesService))`: For a `DELETE` request to `/api/schedule/some-schedule-key`, use the `schedules.Delete(...)` handler. The `{key}` part is a placeholder for the actual schedule key (e.g., "daily-sales-report"). The handler can extract this key. + +**A Visual Flow of a Request:** + +Here's how a request to create a schedule flows through the system: + +```mermaid +sequenceDiagram + participant Client as External Client + participant WebServer as Scheduler Web Server (Chi) + participant Router as Main Router (from GetRouter) + participant APIRouter as /api Sub-Router (from getAPIRoutes) + participant Handler as schedules.Save Handler + participant Svc as api.SchedulesService + + Client->>WebServer: POST /api/schedule (JSON data) + WebServer->>Router: Request for /api/schedule + Router->>APIRouter: Mounts to /api, passes to APIRouter for /schedule + APIRouter->>Handler: Matches POST / and calls schedules.Save + Handler->>Svc: Processes JSON, calls SaveSchedule() + Svc-->>Handler: Returns success/error + Handler-->>WebServer: Sends HTTP Response (e.g., 201 Created) + WebServer-->>Client: Delivers HTTP Response +``` + +**3. Inside a Handler Function (e.g., `schedules.Save`)** + +Let's look at a simplified piece of a handler function, like `schedules.Save` from `api/cmd/scheduler-api/schedules/save.go`: + +```go +// Simplified from: api/cmd/scheduler-api/schedules/save.go +package schedules + +// ... imports ... +import ( + "github.com/nestoca/scheduler/api" + "github.com/nestoca/scheduler/api/pkg/schedule" +) + +func Save(s api.SchedulesService) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // 1. Get the schedule data from the request body + scheduleBlueprint, err := getSchedule(r) + if err != nil { + // ... send an error response ... + return + } + + // 2. Use the SchedulesService to save it + err = s.SaveSchedule(r.Context(), scheduleBlueprint) + if err != nil { + // ... send an error response ... + return + } + + // 3. Send a success response + w.WriteHeader(http.StatusCreated) // 201 Created + } +} +``` +* `Save(s api.SchedulesService)`: This function takes the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md) as an argument (provided by `core.SchedulesService` when setting up the route) and returns the actual handler function `func(w http.ResponseWriter, r *http.Request)`. +* `getSchedule(r)`: This helper function (also in `save.go`, shown in Chapter 1 context) reads the JSON data from the request body (`r.Body`) and decodes it into a `*schedule.Schedule` object. +* `s.SaveSchedule(...)`: This is where the handler calls the actual service responsible for saving the schedule. +* `w.WriteHeader(http.StatusCreated)`: If everything is successful, it sends back an HTTP status code `201 Created` to the client. + +The other handlers like `schedules.Delete` or `status.Get` follow similar patterns: they receive the request, potentially extract information from it (like a URL parameter or request body), call the appropriate service from the `api.Core`, and then formulate an HTTP response. + +## Conclusion + +You've now seen how the `scheduler` opens its doors to the outside world! **HTTP API Endpoints** are the specific "reception desks" for different types of requests (create schedule, delete schedule, get status). **Routing** is the "directory service" (`routes.GetRouter` and `chi` router) that ensures your request reaches the correct **handler function** (like `schedules.Save` or `status.Get`). These handlers then do the actual work, often by using services provided by the [Core Services Aggregator (`api.Core`)](04_core_services_aggregator___api_core___.md). + +So, we can create a `schedule.Schedule` blueprint (Chapter 1) and we know how to send it to the scheduler via an HTTP API (this chapter). But what exactly happens when `schedules.Save` calls `s.SaveSchedule()`? What does the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md) do with our blueprint? That's what we'll explore in the next chapter! + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/03_schedule_management_service___api_schedulesservice___.md b/documentation/03_schedule_management_service___api_schedulesservice___.md new file mode 100644 index 00000000..6176106c --- /dev/null +++ b/documentation/03_schedule_management_service___api_schedulesservice___.md @@ -0,0 +1,265 @@ +# Chapter 3: Schedule Management Service (`api.SchedulesService`) + +In [Chapter 2: HTTP API Endpoints & Routing](02_http_api_endpoints___routing_.md), we saw how our `scheduler` system receives requests from the outside world, like a request to create a new schedule. We learned that an API handler function, such as `schedules.Save`, takes the schedule details and calls a service to do the actual work of saving it. That service is the star of this chapter: the **`api.SchedulesService`**. + +So, you've handed your `schedule.Schedule` blueprint (your "recipe card" from [Chapter 1: Schedule Blueprint (`schedule.Schedule`)](01_schedule_blueprint___schedule_schedule___.md)) to the API. What happens next? How does the system ensure this blueprint is valid and store it safely? + +## The Problem: Managing the "Recipe Card Collection" + +Imagine our `scheduler` system is a grand library for task recipes (our `schedule.Schedule` blueprints). We can't just throw new recipes onto a pile! We need a librarian or an administrator to: +1. **Check the recipe:** Is it complete? Does it make sense (e.g., is the cooking time in a valid timezone)? +2. **File it properly:** Store it in a way that it can be found and used later. +3. **Update it if needed:** If we change a recipe, the old one needs to be updated. +4. **Remove it when done:** If a recipe is no longer needed, it should be discarded. +5. **Help find future cooking times:** If it's a recipe that's made regularly (a recurring schedule), help figure out when it's next due. + +The `api.SchedulesService` is this librarian or administrator for all our "schedule blueprints." It's the central point for managing the lifecycle of these `schedule.Schedule` objects. + +## Meet the `api.SchedulesService`: Your Schedule Librarian + +The `api.SchedulesService` is a core component in our `scheduler`. It's an interface (a contract defining what it can do) and has a default implementation (`SchedulesServiceDefault`). Its main job is to provide a clean way for other parts of the system (especially the API handlers) to manage schedule definitions. + +Here are its key responsibilities: + +* **Creating New Schedules:** When you want to add a new task, you give the `schedule.Schedule` blueprint to this service. +* **Validating Schedule Details:** Before saving, it checks if the blueprint is valid. For example: + * Is the `TimeLocation` (timezone) a real, recognized timezone? + * If it's a recurring schedule (e.g., "monthly"), are the recurrence rules sensible (e.g., does it specify which day of the month)? + It uses a helper called a `LocationProvider` to check timezones. +* **Saving Schedules Persistently:** Once validated, it saves the schedule. "Persistently" means it's stored somewhere durable, so it won't be lost if the program restarts. This is typically done via another helper called a `ScheduleWriter`, which often writes to a reliable system like Kafka (more on this in [Chapter 6: Kafka-based Schedule Persistence and Communication](06_kafka_based_schedule_persistence_and_communication_.md)). +* **Calculating Next Execution Times:** For a given schedule (especially a recurring one), it can calculate when it should run next. +* **Deleting Schedules:** When a task is no longer needed, this service handles its removal. + +Let's look at the `SchedulesService` interface defined in `api/schedules.go`: +```go +// Simplified from: api/schedules.go +package api + +import ( + "context" + "time" + "github.com/nestoca/scheduler/api/pkg/schedule" +) + +// SchedulesService abstract representation of a SchedulesService +type SchedulesService interface { + SaveSchedule(ctx context.Context, schedule *schedule.Schedule) error + CalculateNextLocalExecutionTime(ctx context.Context, currentSchedule *schedule.Schedule) (time.Time, error) + DeleteSchedule(ctx context.Context, scheduleKey string) error + // Shutdown() error // For gracefully stopping the service +} +``` +This tells us that any `SchedulesService` must know how to `SaveSchedule`, `CalculateNextLocalExecutionTime`, and `DeleteSchedule`. + +## How the API Uses the `SchedulesService` + +Let's revisit our use case from Chapter 2: submitting a new schedule for a daily report. The API handler `schedules.Save` receives the JSON blueprint, converts it into a `schedule.Schedule` object, and then calls `SchedulesService.SaveSchedule()`. + +Here's a simplified snippet of how `schedules.Save` (from `api/cmd/scheduler-api/schedules/save.go`) uses the service: +```go +// Simplified from api/cmd/scheduler-api/schedules/save.go +package schedules + +import ( + "net/http" + "github.com/nestoca/scheduler/api" // Where SchedulesService lives + "github.com/nestoca/scheduler/api/pkg/schedule" +) + +func Save(s api.SchedulesService) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // 1. Get the schedule blueprint from the HTTP request + scheduleBlueprint, err := getSchedule(r) // Decodes JSON to schedule.Schedule + if err != nil { /* handle error */ return } + + // 2. Use the SchedulesService to save it! + err = s.SaveSchedule(r.Context(), scheduleBlueprint) + if err != nil { /* handle error */ return } + + // 3. Send a success response (e.g., HTTP 201 Created) + w.WriteHeader(http.StatusCreated) + } +} +``` +The `s` variable here is an instance of `api.SchedulesService`. The handler doesn't need to know *how* the schedule is validated or stored; it just trusts the `SchedulesService` to do it correctly. + +**Input to `SaveSchedule`:** +* `r.Context()`: A context object (common in Go for managing request lifecycles, timeouts, etc.). +* `scheduleBlueprint`: The `*schedule.Schedule` object we want to save. + +**What happens (at a high level) when `s.SaveSchedule()` is called:** +1. The `SchedulesService` validates the `scheduleBlueprint`. +2. If valid, it assigns a unique ID to this version of the schedule. +3. It then tells its `ScheduleWriter` to persist this blueprint. +4. If any step fails (e.g., validation error, storage error), it returns an error. Otherwise, it returns `nil` (success). + +## Under the Hood: A Look Inside `SchedulesServiceDefault` + +The `scheduler` project provides `SchedulesServiceDefault` as the standard implementation of the `SchedulesService` interface. It's defined in `api/schedules.go`. + +**1. What's Inside `SchedulesServiceDefault`?** + +It needs two helpers to do its job: +* `writer ScheduleWriter`: This is responsible for actually writing the schedule data to a persistent store (like Kafka). +* `locationProvider LocationProvider`: This helps validate timezones. + +```go +// Simplified from: api/schedules.go +package api + +// ... (ScheduleWriter and LocationProvider interfaces are also in this file) ... + +// SchedulesServiceDefault default implementation of SchedulesService +type SchedulesServiceDefault struct { + writer ScheduleWriter + locationProvider LocationProvider +} + +// NewSchedulesService creates a new SchedulesService +func NewSchedulesService(writer ScheduleWriter, locationProvider LocationProvider) *SchedulesServiceDefault { + // ... (nil check) ... + return &SchedulesServiceDefault{ + writer: writer, + locationProvider: locationProvider, + } +} +``` +When an instance of `SchedulesServiceDefault` is created (usually when the application starts, as part of setting up the [Core Services Aggregator (`api.Core`)](04_core_services_aggregator___api_core___.md)), it's given these two helpers. + +**2. The `SaveSchedule` Method in Detail** + +Let's follow the journey of a `schedule.Schedule` blueprint when `SaveSchedule` is called: + +```mermaid +sequenceDiagram + participant Handler as API Handler (schedules.Save) + participant Svc as SchedulesServiceDefault + participant Validator as s.isValid() method + participant LocProv as LocationProvider + participant Writer as ScheduleWriter + + Handler->>Svc: SaveSchedule(ctx, blueprint) + Svc->>Validator: isValid(blueprint)? + Validator->>LocProv: Valid(blueprint.TimeLocation)? + LocProv-->>Validator: true (e.g., "America/New_York" is valid) + Validator-->>Svc: validation result (ok) + Svc->>Svc: Generates unique blueprint.ID + Svc->>Writer: Save(ctx, blueprintWithID) + Writer-->>Svc: Save successful + Svc-->>Handler: Returns success (nil error) +``` + +Now, let's look at the simplified code for `SaveSchedule` in `api/schedules.go`: +```go +// Simplified from: api/schedules.go +func (s SchedulesServiceDefault) SaveSchedule(ctx context.Context, sch *schedule.Schedule) error { + if sch == nil { + return fmt.Errorf("supplied no schedule") + } + + // Step 1: Validate the schedule details + if err := s.isValid(sch); err != nil { + return err // If invalid, return the error + } + + // Step 2: Generate a unique ID for this schedule instance. + // This ID might change for each run of a recurring schedule. + sch.ID = uuid.NewString() + + // Step 3: Pass it to the writer to be saved. + return s.writer.Save(ctx, sch) +} +``` +* First, it checks if the schedule (`sch`) itself is provided. +* Then, it calls its own private method `s.isValid(sch)` to perform validation. +* If valid, it assigns a new unique `ID` (using `uuid.NewString()`) to the `sch.ID` field. This `ID` is an internal system identifier, different from the `Key` you provide. For recurring schedules, each future instance might get a new `ID`. +* Finally, it calls `s.writer.Save(ctx, sch)`, delegating the actual storage to the `ScheduleWriter`. + +**3. The Validation Logic: `isValid()`** + +The `isValid()` method (also in `SchedulesServiceDefault`) is crucial. It checks various parts of the `schedule.Schedule`. + +```go +// Simplified from: api/schedules.go +func (s SchedulesServiceDefault) isValid(sched *schedule.Schedule) error { + // Check 1: Is the TimeLocation valid? + if !s.locationProvider.Valid(sched.TimeLocation) { + return e.NewInvalidParameters("schedule time location") // Error if not valid + } + + if sched.Recurrence == nil { + return nil // No recurrence to validate + } + + // Check 2: Are recurrence rules valid? (Example for monthly) + switch sched.Recurrence.Scheme { + case schedule.RecurrenceMonthly: + dayStr, found := sched.Recurrence.Metadata[schedule.RecurrenceMetadataMonthlyPaymentDay] + if !found { + return e.NewInvalidParameters("missing day for monthly recurrence") + } + // ... (code to check if dayStr is a valid number 1-31) ... + // ... (other cases for RecurrenceDaily, RecurrenceWeekly, etc.) ... + case schedule.RecurrenceCustom: + // ... (check if custom delta is a valid duration string) ... + } + return nil // All good! +} +``` +* It uses the `s.locationProvider` to confirm that `sched.TimeLocation` (e.g., `"America/New_York"`) is a recognized timezone. +* If `sched.Recurrence` is set, it checks the `Scheme` and `Metadata`. For example, if `Scheme` is `schedule.RecurrenceMonthly`, it expects `Metadata` to contain `schedule.RecurrenceMetadataMonthlyPaymentDay` with a valid day number (1-31). + +If any check fails, `isValid()` returns an error, which then `SaveSchedule` returns to the API handler, and ultimately back to the client who tried to save the schedule. + +**4. Deleting a Schedule: `DeleteSchedule()`** + +Deleting is simpler. The API handler (`schedules.Delete`) gets the `scheduleKey` from the URL, and calls `SchedulesService.DeleteSchedule()`: + +```go +// Simplified from: api/schedules.go +func (s SchedulesServiceDefault) DeleteSchedule(ctx context.Context, scheduleKey string) error { + if len(scheduleKey) == 0 { + return fmt.Errorf("supplied invalid key") + } + // Just pass the key to the writer to handle deletion. + return s.writer.Delete(ctx, scheduleKey) +} +``` +The `SchedulesService` mostly just delegates this to its `ScheduleWriter`. The `ScheduleWriter` would then be responsible for telling the persistence layer (e.g., Kafka) to mark this schedule (identified by its `scheduleKey`) for deletion. + +**5. Calculating Next Execution Time: `CalculateNextLocalExecutionTime()`** + +This method helps figure out when a schedule should run next, based on its current definition and recurrence rules. + +```go +// Simplified from: api/schedules.go +func (s SchedulesServiceDefault) CalculateNextLocalExecutionTime( + ctx context.Context, + currentSchedule *schedule.Schedule, +) (time.Time, error) { + if currentSchedule == nil { /* error */ } + if err := s.isValid(currentSchedule); err != nil { /* error */ } // Must be valid + + // Delegates to a dedicated calculator function + return scheduler.CalculateNextLocalExecutionTime(ctx, currentSchedule, s.locationProvider) +} +``` +* It first validates the provided `currentSchedule`. +* Then, it calls a specialized function `scheduler.CalculateNextLocalExecutionTime` (from `api/pkg/scheduler/calculator.go`, not shown here). This function contains the logic to interpret `LocalExecutionTime`, `TimeLocation`, and `Recurrence` rules, using the `locationProvider` for timezone calculations, to determine the next run time. This is very useful for recurring tasks. + +## Why is the `SchedulesService` So Important? + +* **Centralized Logic:** It keeps all the rules about what makes a schedule valid and how to manage its lifecycle in one place. +* **Abstraction:** API Handlers (and other potential clients of this service) don't need to worry about the nitty-gritty details of validation or how schedules are stored. They just talk to the `SchedulesService`. +* **Testability:** Because it's a well-defined interface, it's easier to test this component in isolation. +* **Flexibility:** If we wanted to change *how* schedules are stored (e.g., move from Kafka to a database), we'd primarily change the `ScheduleWriter` implementation. The `SchedulesService` interface and its interaction with API handlers could remain largely the same. + +## Conclusion + +The `api.SchedulesService` acts as the diligent administrator for all our `schedule.Schedule` blueprints. It's responsible for receiving new schedule definitions, validating them thoroughly (using helpers like a `LocationProvider`), ensuring they are saved persistently (via a `ScheduleWriter`), calculating their future execution times, and handling their deletion. This service ensures that our "library of recipes" is well-organized, correct, and ready for the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) to use. + +We've seen that the `SchedulesService` relies on helpers like `ScheduleWriter` and `LocationProvider`. But how does it get these helpers? And how do the API handlers from Chapter 2 get an instance of `SchedulesService`? This is where the [Core Services Aggregator (`api.Core`)](04_core_services_aggregator___api_core___.md) comes in, which we'll explore in the next chapter! + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/04_core_services_aggregator___api_core___.md b/documentation/04_core_services_aggregator___api_core___.md new file mode 100644 index 00000000..9d107444 --- /dev/null +++ b/documentation/04_core_services_aggregator___api_core___.md @@ -0,0 +1,205 @@ +# Chapter 4: Core Services Aggregator (`api.Core`) + +In [Chapter 3: Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md), we learned how the `SchedulesService` acts as a librarian for our schedule blueprints. We saw it validates and saves schedules, often relying on other helpers. You might have wondered: how do different parts of our `scheduler` application, like the API handlers from [Chapter 2: HTTP API Endpoints & Routing](02_http_api_endpoints___routing_.md), get access to the `SchedulesService` or other essential tools? This chapter introduces the component that makes this possible: the **`api.Core`**. + +## The Problem: Accessing Specialized Tools + +Imagine you're building a complex machine, like a car. The car has many specialized systems: an engine, brakes, a steering system, and an electronics system for the radio and dashboard. For the car to work, these systems need to be connected and accessible. The driver (representing a user or another part of our application) needs a way to use these systems without knowing every single detail of how they are built. They just need a steering wheel, an accelerator pedal, and a brake pedal. + +In our `scheduler` application, we have several specialized "tools" or "services": +* The `SchedulesService` to manage schedule definitions. +* A `HealthService` to check if the application is running correctly. +* An `InstrumentationService` to gather performance metrics. +* The `Scheduler` engine itself, which actually runs the tasks. + +How can components like our API handlers easily get a reference to the specific service they need? If every handler had to create these services itself, it would be messy and inefficient. We need a central place that holds all these tools and makes them available. + +## Meet `api.Core`: The Scheduler's Main Toolbox + +Think of `api.Core` as the main **toolbox** or **control panel** for our `scheduler` application. It's a Go struct that doesn't perform tasks directly. Instead, its primary job is to **hold and provide access to all the essential, specialized services** needed for the application to operate. + +It's the central hub that connects various parts of the application. Components like the web API handlers (which we met in Chapter 2) can ask `api.Core` for the specific service they need to perform their job. + +Here's a look at the structure of `api.Core` from `api/core.go`: +```go +// Simplified from: api/core.go +package api + +// Core represents the entrypoint to call the business logic. +type Core struct { + HealthService HealthService + InstrumentationService InstrumentationService + Scheduler Scheduler + SchedulesService SchedulesService +} +``` +As you can see, an `api.Core` object simply contains fields for each of the major services: +* `HealthService`: Used for health checkups (e.g., for the `/api/status` endpoint). +* `InstrumentationService`: Used for collecting performance metrics ([Application Health Monitor (`api.InstrumentationService`)](10_application_health_monitor___api_instrumentationservice___.md)). +* `Scheduler`: This is the actual [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) responsible for running scheduled tasks. +* `SchedulesService`: The [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md) we learned about in the previous chapter. + +## How `api.Core` is Used: Connecting Handlers to Services + +Let's revisit how our API routes and handlers are set up. In [Chapter 2: HTTP API Endpoints & Routing](02_http_api_endpoints___routing_.md), we saw a function `routes.GetRouter` that configures all the API endpoints. This function receives an `*api.Core` instance. + +Here's a simplified snippet from `api/cmd/scheduler-api/routes/routes.go` showing how `api.Core` is used: +```go +// Simplified from: api/cmd/scheduler-api/routes/routes.go +package routes + +import ( + "github.com/go-chi/chi" + "github.com/nestoca/scheduler/api" // Where api.Core is defined + // ... imports for status, schedules handlers +) + +// GetRouter configures routes using a core. +func GetRouter(c *api.Core) chi.Router { + r := chi.NewRouter() + // ... + r.Mount("/api", getAPIRoutes(c)) // Pass core to API route setup + return r +} +``` +The `GetRouter` function takes `c *api.Core` as an argument. This `c` is our "toolbox." It's then passed to `getAPIRoutes`. + +Inside `getAPIRoutes`, the individual services from the `api.Core` instance are given to the specific handlers that need them: +```go +// Simplified from: api/cmd/scheduler-api/routes/routes.go +func getAPIRoutes(core *api.Core) http.Handler { + r := chi.NewRouter() + + // Status endpoint needs the HealthService + r.Get("/status", status.Get(core.HealthService)) + + // Schedule endpoints need the SchedulesService + r.Route("/schedule", func(r chi.Router) { + r.Post("/", schedules.Save(core.SchedulesService)) + r.Delete("/{key}", schedules.Delete(core.SchedulesService)) + // ... other schedule routes ... + }) + + return r +} +``` +Look closely: +* `status.Get(core.HealthService)`: The `status.Get` handler function is created and given `core.HealthService` (the `HealthService` tool from our `api.Core` toolbox). +* `schedules.Save(core.SchedulesService)`: The `schedules.Save` handler function is given `core.SchedulesService` (the `SchedulesService` tool from the toolbox). + +This way, the `schedules.Save` handler doesn't need to know how to create or find a `SchedulesService`; it's simply handed one. The `api.Core` acts as a central provider. + +## Under the Hood: Creating and Populating `api.Core` + +So, where does this `api.Core` instance come from? How is the toolbox filled with its tools? + +Typically, the `api.Core` instance is created once when the application starts up. There's a special function, `FromEnv()`, located in `api/pkg/core/core.go`, that is responsible for building all the necessary services and then packaging them into an `api.Core` struct. + +**A Visual Analogy:** + +Imagine a workshop preparing for a big project. +1. First, specialized tools are forged or assembled (this is like `FromEnv()` creating each service like `HealthService`, `SchedulesService`, etc.). +2. Then, all these tools are placed into a central toolbox (`api.Core`). +3. When a worker (an API handler) needs a tool, they get it from this central toolbox. + +```mermaid +graph TD + subgraph "Initialization (e.g., in api/pkg/core/core.go's FromEnv)" + S1[HealthService instance] + S2[InstrumentationService instance] + S3[Scheduler Engine instance] + S4[SchedulesService instance] + end + + Toolbox["api.Core (The Main Toolbox)"] + style Toolbox fill:#ccf,stroke:#333,stroke-width:2px + + S1 --> Toolbox + S2 --> Toolbox + S3 --> Toolbox + S4 --> Toolbox + + subgraph "Usage by API Handlers" + SaveHandler["schedules.Save Handler"] + StatusHandler["status.Get Handler"] + end + + Toolbox -->|Provides SchedulesService| SaveHandler + Toolbox -->|Provides HealthService| StatusHandler +``` + +**The `FromEnv()` Function (Highly Simplified)** + +The `FromEnv()` function in `api/pkg/core/core.go` does a lot of work to initialize each service. This often involves reading configuration from environment variables (hence "FromEnv"), setting up connections to things like Kafka (which we'll see in [Chapter 6: Kafka-based Schedule Persistence and Communication](06_kafka_based_schedule_persistence_and_communication_.md)), and more. + +For our purposes, the most important part is the end, where it assembles the `api.Core` struct: + +```go +// Highly simplified concept from: api/pkg/core/core.go +package core + +import ( + "github.com/nestoca/scheduler/api" + // ... many other imports for creating actual services ... +) + +// FromEnv creates the default instances of all services. +func FromEnv() *api.Core { + // 1. Create HealthService instance (details omitted) + healthSvc := api.NewHealthService() + + // 2. Create InstrumentationService instance (details omitted) + metricsStore := instrumentation.NewMetricsDataStore() // Example helper + instrumentationSvc := api.NewInstrumentationService(metricsStore) + + // 3. Create SchedulesService instance (details omitted) + // This involves creating a ScheduleWriter, LocationProvider, etc. + // scheduleProducer := kafkaschedules.NewScheduleProducer(...) + // locationProvider := location.NewCachedProvider() + // schedulesSvc := api.NewSchedulesService(scheduleProducer, locationProvider) + var schedulesSvc api.SchedulesService // Placeholder for actual creation + // ... (actual complex creation of schedulesSvc here) ... + + + // 4. Create Scheduler engine instance (details omitted) + // This involves creating stores, post-processors, DLQs, etc. + // store := scheduler.NewDailyScheduleConsumer(...) + // postProcessor := scheduler.NewSchedulePostProcessor(...) + // dlq := scheduler.NewDeadLetterQueue(...) + // actor := scheduler.NewConcurrentScheduleActor(...) + // schedulerEngine := scheduler.NewScheduler(store, postProcessor, dlq, actor) + var schedulerEngine api.Scheduler // Placeholder for actual creation + // ... (actual complex creation of schedulerEngine here) ... + + + // 5. Assemble and return the api.Core struct + return &api.Core{ + HealthService: healthSvc, + InstrumentationService: instrumentationSvc, + Scheduler: schedulerEngine, // The main scheduling engine + SchedulesService: schedulesSvc, // Our schedule manager + } +} +``` +**Important:** The snippet above is *heavily* simplified. The actual `FromEnv()` function is much more complex because creating each service (like `SchedulesService` or `Scheduler`) involves setting up its own dependencies. For example, creating `SchedulesService` requires providing it with a `ScheduleWriter` and a `LocationProvider`, as we saw in Chapter 3. + +The key takeaway is that `FromEnv()` acts as a "factory" that produces a fully populated `api.Core` instance. This instance is then typically passed to the parts of the application that need to set up routes or initialize other components. In our `scheduler-api` command, `main.go` would call `core.FromEnv()` and then pass the result to `routes.GetRouter()`. + +## Why is `api.Core` Important? + +The `api.Core` aggregator plays a vital role: + +1. **Centralization:** It provides a single, well-defined place to get access to all core application services. If you need a service, you know where to look. +2. **Dependency Management (Simplified):** It helps manage dependencies. Components like API handlers don't need to create services themselves; they receive them. This is a form of "dependency injection." +3. **Organization:** It keeps the application startup and wiring of components cleaner. The `main` function (or equivalent startup code) creates the `api.Core`, and then passes this single "toolbox" around, rather than passing many individual services. +4. **Testability:** When testing a component that uses services (e.g., an API handler), it's easier to provide "mock" or "fake" versions of these services if they are accessed through a central point like `api.Core`. You could create an `api.Core` with mock services for your tests. + +## Conclusion + +You've now learned about `api.Core`, the central toolbox or control panel of our `scheduler` application. It doesn't do the heavy lifting itself, but it holds and provides convenient access to all the specialized services that do, such as `SchedulesService`, `HealthService`, `InstrumentationService`, and the `Scheduler` engine. This makes it easy for components like API handlers to get the tools they need and helps keep our application organized. + +One of the most important tools held within `api.Core` is the `Scheduler` itself – the engine that actually picks up schedules and makes sure tasks get triggered. In the next chapter, we'll dive into this crucial component: [Chapter 5: Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md). + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md b/documentation/05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md new file mode 100644 index 00000000..21e6c3fb --- /dev/null +++ b/documentation/05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md @@ -0,0 +1,288 @@ +# Chapter 5: Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`) + +In [Chapter 4: Core Services Aggregator (`api.Core`)](04_core_services_aggregator___api_core___.md), we saw how `api.Core` acts as a central toolbox, holding all the essential services our `scheduler` application needs. One of the most critical tools in that toolbox is the `api.Scheduler` itself. This chapter focuses on this "engine" of our system, specifically its main implementation, `scheduler.DefaultScheduler`. + +So far, we've learned how to: +1. Define a task with its timing: a `schedule.Schedule` blueprint ([Chapter 1: Schedule Blueprint (`schedule.Schedule`)](01_schedule_blueprint___schedule_schedule___.md)). +2. Send this blueprint to our system: via [HTTP API Endpoints & Routing](02_http_api_endpoints___routing_.md). +3. Have the blueprint validated and prepared for storage: by the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md). + +But how does a stored blueprint actually cause something to happen at the scheduled time? Who is constantly watching the clock and saying, "Aha! Time to run this task!"? That's the job of the Task Execution Engine. + +## The Problem: Making Scheduled Tasks Actually Happen + +Imagine you've written down a list of chores on a to-do list with specific times for each: +* "Take out trash - Tuesday 8:00 AM" +* "Water plants - Wednesday 6:00 PM" +* "Send weekly email summary - Friday 9:00 AM, repeat weekly" + +Having this list is great, but it doesn't do the chores. You need someone (or something) to: +1. Constantly check the list and the current time. +2. When a chore's time arrives, actually *start* doing it. +3. If it's a repeating chore, remember to schedule it for the next time. +4. If something goes wrong while doing a chore, decide what to do (e.g., try again later). + +In our `scheduler` project, the **Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)** is this diligent worker. It's like an automated conductor for an orchestra, ensuring each instrument plays at the right time, or the engine of a car, providing the power to move forward. + +## Meet the `api.Scheduler` and `scheduler.DefaultScheduler` + +The `api.Scheduler` is an interface (a contract) defining what a task execution engine should be able to do. It's very simple: + +```go +// Simplified from: api/scheduler.go +package api + +import "context" + +// Scheduler Abstract representation of a scheduler +type Scheduler interface { + StartBlocking(ctx context.Context) // Starts the engine's main work loop + // ForceGenerationChange(ctx context.Context) // A way to signal internal changes +} +``` +* `StartBlocking(ctx context.Context)`: This is the main command to tell the engine to begin its work of monitoring and executing tasks. It's "blocking" because it will typically run continuously until the application is told to stop. + +The primary implementation of this interface is `scheduler.DefaultScheduler` (found in `api/pkg/scheduler/scheduler.go`). This is the component that does the actual work. + +**What `scheduler.DefaultScheduler` Does:** + +Think of `scheduler.DefaultScheduler` as the main coordinator on the factory floor. It doesn't build everything itself, but it directs other specialized workers and machines: + +1. **Monitors for Due Tasks:** It continuously asks a component called the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) (let's call it the `ScheduleStore` for short): "Are there any tasks due *right now*?" +2. **Triggers Task Execution:** When the `ScheduleStore` says, "Yes, this task is due!", the `DefaultScheduler` tells another component (an `ScheduleActor`) to actually perform the task. "Performing the task" in our system usually means sending a message (the `TargetPayload` from our `schedule.Schedule` blueprint) to a specific Kafka topic (the `TargetTopic`). +3. **Handles Post-Execution:** After a task is attempted: + * If it was successful, the `DefaultScheduler` passes the task to the [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md). This handler figures out if the task needs to be rescheduled (if it's recurring) or if it's done for good. + * If it failed, the `DefaultScheduler` sends it to the [Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md). This queue will try to run the task again later. +4. **Keeps Time:** It also manages a daily timer to know when a new day starts, which is important for the `ScheduleStore` to load tasks for the current day. + +It ensures tasks are picked up and processed in a timely and orderly manner. + +## How the Engine Starts and Works + +When our `scheduler` application starts up (as seen in `api/cmd/scheduler-api/api.go`), it gets an instance of the `api.Scheduler` from the `api.Core` toolbox and tells it to start: + +```go +// Simplified from: api/cmd/scheduler-api/api.go +func main() { + // ... setup code ... + c := core.FromEnv() // Gets the api.Core toolbox + + // ... + // Run Scheduler async + log.L.Infof("[START] starting scheduler") + startScheduler(ctx, c.Scheduler, &wg, schedulerError) // c.Scheduler is our engine + // ... +} + +func startScheduler(ctx context.Context, s api.Scheduler, /*...*/) { + // ... + go func() { // Runs in a separate goroutine (like a parallel thread) + // ... + s.StartBlocking(ctx) // This kicks off the engine! + }() +} +``` +The `s.StartBlocking(ctx)` call is what makes the `scheduler.DefaultScheduler` spring to life. + +**The Main Work Loop (Conceptual):** + +Once started, `scheduler.DefaultScheduler` enters a continuous loop. Here's a simplified idea of what it's doing: + +```mermaid +graph TD + A[Start Engine] --> B{Anything to do?}; + + subgraph "Engine's Continuous Loop" + B -- "Daily Timer Ticked (Midnight!)" --> C[Notify ScheduleStore: New Day!]; + C --> B; + + B -- "ScheduleStore: Task X is Due!" --> D[Get Task X]; + D --> E{Attempt Task X via ScheduleActor}; + E -- "Success!" --> F[Send Task X to PostProcessor]; + F --> B; + E -- "Failure!" --> G[Send Task X to DeadLetterQueue]; + G --> B; + + B -- "DeadLetterQueue: Task Y (retried) is Done" --> H[Send Task Y to PostProcessor]; + H --> B; + + B -- "Application Shutting Down" --> I[Stop]; + end +``` +This loop continues, making the scheduler responsive to due tasks, daily rollovers, and retry completions. + +## Under the Hood: The `scheduler.DefaultScheduler` and its Helpers + +The `scheduler.DefaultScheduler` doesn't work in isolation. It relies on several helper components, which are given to it when it's created. This creation happens inside `api/pkg/core/core.go` as part of the `FromEnv()` function we saw in [Chapter 4: Core Services Aggregator (`api.Core`)](04_core_services_aggregator___api_core___.md). + +```go +// Simplified from: api/pkg/core/core.go +// Inside FromEnv() function: + +// ... (store, postProcessor, dlq, actor are created first) ... +// store is an instance of scheduler.ScheduleStore +// postProcessor is an instance of scheduler.Queue (for post-processing) +// dlq is an instance of scheduler.Queue (for dead letters) +// actor is an instance of scheduler.ScheduleActor + +schedulerEngine := scheduler.NewScheduler(store, postProcessor, dlq, actor) + +// This schedulerEngine is then put into api.Core +// return &api.Core{ +// Scheduler: schedulerEngine, +// // ... other services +// } +``` + +The `scheduler.NewScheduler` function (which creates our `DefaultScheduler`) takes these key helpers: +* `store ScheduleStore`: This is the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md). The engine asks it for the next task that's due. +* `schedulePostProcessor Queue`: This is the [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md). Tasks go here after successful execution (or after exhausting retries) for final processing like rescheduling or deletion. +* `deadLetterQueue Queue`: This is the [Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md). Tasks that fail execution are sent here to be retried later. +* `actor ScheduleActor`: This component is responsible for the actual "execution" of the task – typically, sending a Kafka message. + +Let's look at a very simplified structure of `DefaultScheduler` and its `StartBlocking` method from `api/pkg/scheduler/scheduler.go`: + +```go +// Simplified from: api/pkg/scheduler/scheduler.go +package scheduler + +// ... (imports for context, sync, time, schedule package) ... + +type DefaultScheduler struct { + store ScheduleStore // Gets due tasks + deadLetterQueue Queue // Handles failed tasks for retry + schedulePostProcessor Queue // Handles tasks after execution/retry + actor ScheduleActor // Performs the task action + + dailyTimer *time.Timer // For midnight signal + // ... (mutex for synchronization) ... +} + +func NewScheduler(store ScheduleStore, schedulePostProcessor Queue, deadLetterQueue Queue, actor ScheduleActor) *DefaultScheduler { + // ... (nil checks) ... + return &DefaultScheduler{ + store: store, + deadLetterQueue: deadLetterQueue, + schedulePostProcessor: schedulePostProcessor, + actor: actor, + dailyTimer: time.NewTimer(calculateTimeToMidnight()), // Sets timer for next midnight + } +} +``` +This shows how the `DefaultScheduler` holds onto its helpers. The `dailyTimer` is set to fire at the next UTC midnight. + +Now, a simplified look at the `StartBlocking` method: +```go +// Simplified from: api/pkg/scheduler/scheduler.go +func (s *DefaultScheduler) StartBlocking(ctx context.Context) { + // ... (logging, waitgroup setup for graceful shutdown) ... + + // Start helper components (they run their own loops) + // s.startLongLastingProcess(ctx, &wg, s.store.RunBlocking) + // s.startLongLastingProcess(ctx, &wg, s.deadLetterQueue.RunBlocking) + // s.startLongLastingProcess(ctx, &wg, s.schedulePostProcessor.RunBlocking) + +readLoop: + for { + select { // Waits for any of these events + case <-s.dailyTimer.C: + // Midnight reached! + s.store.MidnightReachedChannel() <- struct{}{} // Signal the store + s.dailyTimer.Reset(calculateTimeToMidnight()) // Reset timer for next midnight + + case sched := <-s.store.ProduceChannel(): // A task is due from the store! + log.G(ctx).Tracef("[SCHED] Task %s ready", sched.Key) + err := s.actor.IssueSchedule(ctx, sched) // Tell actor to run it + if err == nil { + // Success! Send to post-processor for cleanup/rescheduling + s.schedulePostProcessor.PushChannel() <- sched + } else { + // Failure! Send to dead-letter queue for retry + s.deadLetterQueue.PushChannel() <- sched + } + + case sched := <-s.deadLetterQueue.ExitChannel(): // Task finished retries (success or fail) + // Send to post-processor for final handling + s.schedulePostProcessor.PushChannel() <- sched + + case err := <-s.store.ConsumeErrorChannel(): // An error from the store + log.G(ctx).Errorf("[SCHED] Store error: %s", err.Error()) + + case <-ctx.Done(): // Application is shutting down + break readLoop // Exit the loop + } + } +} +``` + +**Let's break down the `select` block, the heart of the engine:** + +1. `case <-s.dailyTimer.C:` + * The `dailyTimer` (which was set to fire at midnight) has gone off. + * The engine sends a signal to the `s.store.MidnightReachedChannel()`. This tells the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) that it's a new day, so it might need to load schedules for this new day. + * The `dailyTimer` is reset for the *next* midnight. + +2. `case sched := <-s.store.ProduceChannel():` + * The engine receives a `schedule.Schedule` (named `sched`) from the `s.store.ProduceChannel()`. This means the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) has determined this task is due *right now*. + * It calls `s.actor.IssueSchedule(ctx, sched)`. The `ScheduleActor` attempts to "issue" the task (e.g., send its payload to a Kafka topic). + * If `IssueSchedule` returns no error (`err == nil`), the task was successfully issued. It's then sent to the `s.schedulePostProcessor.PushChannel()`. The [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md) will take care of it (e.g., if it's a daily task, it will calculate the next run time and save a new schedule instance for tomorrow). + * If `IssueSchedule` returns an error, the task failed. It's sent to the `s.deadLetterQueue.PushChannel()`. The [Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md) will try to run it again later. + +3. `case sched := <-s.deadLetterQueue.ExitChannel():` + * A task has "exited" the [Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md). This means the DLQ has finished its retry attempts for this task (either it eventually succeeded, or it reached the maximum number of retries). + * This task is then sent to the `s.schedulePostProcessor.PushChannel()` for final processing, just like a task that succeeded on its first try. The post-processor will handle its removal or logging as appropriate. + +4. `case err := <-s.store.ConsumeErrorChannel():` + * If the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) has a problem (e.g., trouble reading from its data source like Kafka), it can send an error here. The engine logs this error. + +5. `case <-ctx.Done():` + * The `context` (`ctx`) has been "canceled." This is the signal for the application to shut down gracefully. + * The `break readLoop` statement exits the `for` loop, and the `StartBlocking` method will eventually return, stopping the engine. + +**A Visual Summary of Interactions:** + +Here's how `DefaultScheduler` coordinates with its main helpers for a typical task lifecycle: + +```mermaid +sequenceDiagram + participant Store as ScheduleStore + participant Engine as DefaultScheduler + participant Actor as ScheduleActor + participant PostProc as SchedulePostProcessor + participant DLQ as DeadLetterQueue + + Store->>Engine: Task `S` is due! (via ProduceChannel) + Engine->>Actor: Issue Task `S` + alt Task S succeeds + Actor-->>Engine: Success! + Engine->>PostProc: Handle Task `S` (via PushChannel) + else Task S fails + Actor-->>Engine: Failed! + Engine->>DLQ: Retry Task `S` (via PushChannel) + DLQ-->>Engine: Task `S` finished retries (via ExitChannel) + Engine->>PostProc: Handle Task `S` (via PushChannel) + end +``` + +## Why is the Task Execution Engine So Important? + +* **The Heartbeat:** It's the component that makes the scheduler "live." Without it, schedules would just be data sitting in storage. +* **Orchestration:** It masterfully coordinates several other specialized components (`ScheduleStore`, `ScheduleActor`, `SchedulePostProcessor`, `DeadLetterQueue`) to ensure the entire lifecycle of a task is managed. +* **Reliability:** By integrating with a dead-letter queue, it provides a mechanism for retrying failed tasks, increasing the overall reliability of the system. +* **Timeliness:** Its core loop, driven by signals from the `ScheduleStore` and internal timers, ensures tasks are picked up when they are due. + +## Conclusion + +The Task Execution Engine, primarily `scheduler.DefaultScheduler`, is the automated conductor at the heart of our `scheduler` project. It doesn't do everything itself but intelligently uses other components: +* It gets due tasks from the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md). +* It uses a `ScheduleActor` to trigger the task's action (like sending a message). +* It sends successfully completed (or fully retried) tasks to the [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md) for cleanup and rescheduling. +* It sends failed tasks to the [Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md). + +This orchestration ensures that the "recipe cards" (`schedule.Schedule` blueprints) we create are actually acted upon at the right times. + +Now, we've seen tasks being "issued" or "produced," often involving Kafka. But how exactly are schedules stored in Kafka, and how do components like the `ScheduleStore` or `ScheduleActor` use Kafka to communicate? That's what we'll explore in the next chapter: [Chapter 6: Kafka-based Schedule Persistence and Communication](06_kafka_based_schedule_persistence_and_communication_.md). + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/06_kafka_based_schedule_persistence_and_communication_.md b/documentation/06_kafka_based_schedule_persistence_and_communication_.md new file mode 100644 index 00000000..6f30de23 --- /dev/null +++ b/documentation/06_kafka_based_schedule_persistence_and_communication_.md @@ -0,0 +1,314 @@ +# Chapter 6: Kafka-based Schedule Persistence and Communication + +Welcome to Chapter 6! In [Chapter 5: Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md), we saw how the scheduler engine orchestrates task execution. We learned that it gets tasks from a `ScheduleStore` and uses an `ScheduleActor` to issue tasks. But where does the `ScheduleStore` get its information? And how do different parts of our `scheduler` system stay in sync about all the schedule definitions, especially if components restart or if we have multiple instances running? + +This chapter dives into **Apache Kafka**, the technology our `scheduler` uses as a robust "event bus" (a way to send messages) and a "persistence layer" (a way to store data reliably) for schedules. + +## The Problem: Reliable Schedule Storage and Announcements + +Imagine our `scheduler` system is a busy train station. +* **New Schedules are like New Train Routes:** When someone wants to schedule a new task (like our daily report from [Chapter 1: Schedule Blueprint (`schedule.Schedule`)](01_schedule_blueprint___schedule_schedule___.md)), it's like announcing a new train route. +* **Updates are Timetable Changes:** If a schedule changes, it's like a train timetable update. +* **Deletions are Route Cancellations:** If a schedule is removed, the route is cancelled. + +We need a central, super-reliable announcement system and archive for all these train routes and timetable changes. +1. **Durability:** Announcements must not be lost, even if the station master (one part of our program) goes home and a new one comes on duty (the program restarts). +2. **Broadcasting:** All relevant departments in the station (different parts of our `scheduler` system) need to hear these announcements. For example, the department that displays "upcoming trains" (our [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md)) needs to know about new routes and changes. +3. **Order:** It's helpful if announcements are kept in order. + +This is where Kafka comes in. + +## Meet Kafka: Our Reliable, Distributed Postal Service + +Think of **Kafka** as a highly reliable, distributed postal service combined with a public archive of all sent mail. +* **Topics:** A Kafka **topic** is like a specific mailbox or a public bulletin board dedicated to a certain kind of information. For our schedules, we'll have a topic, let's call it `schedules-log`. +* **Messages:** Each piece of information (a new schedule, an update, a deletion) is sent as a **message** (a "letter" or a "bulletin post") to this topic. +* **Producers:** Components that *write* or *send* messages to a Kafka topic are called **Producers**. They are like people dropping letters into the mailbox. +* **Consumers:** Components that *read* or *receive* messages from a Kafka topic are called **Consumers**. They are like people checking the mailbox or reading the bulletin board. + +Kafka ensures that once a message is successfully sent to a topic, it's stored durably (it won't get lost easily) and can be read by any authorized consumer, even much later. + +## Key Players in Our Kafka-Powered Scheduler + +In our `scheduler` project, we have specific components that interact with Kafka: + +1. **`ScheduleProducer` (The Mail Sender for Schedules)** + * Found in: `api/pkg/schedule/kafka/producer.go` + * This component is responsible for writing schedule information to our Kafka `schedules-log` topic. + * When you create a new schedule (via the API and the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md)), the `SchedulesService` uses the `ScheduleProducer` to send a "new schedule" message to Kafka. + * It also sends messages for updates or deletions. + +2. **`ScheduleConsumer` (The Mail Reader for Schedules)** + * Found in: `api/pkg/schedule/kafka/consumer.go` + * This component reads the schedule messages from the Kafka `schedules-log` topic. + * Other parts of the system, like the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) (which we'll see in the next chapter), use a `ScheduleConsumer` to learn about all active schedules. + +3. **Kafka Messages: The "Letters"** + * A message typically contains the details of a `schedule.Schedule` blueprint, packaged in a standard format. + * Our system converts the `schedule.Schedule` object into a byte array (a sequence of 0s and 1s) to send it over Kafka, and then converts it back when reading. This is called **serialization** and **deserialization**. + +4. **"Tombstones": Marking Schedules for Deletion** + * How do you delete something from Kafka? You can't just erase a message that's already written. + * Instead, Kafka uses a special kind of message called a **tombstone**. If we want to delete a schedule with `Key: "daily-report"`, the `ScheduleProducer` sends a message to Kafka with `Key: "daily-report"` but with an empty (or `nil`) body. + * Consumers see this "tombstone" message and understand that the schedule identified by `"daily-report"` should be considered deleted. + +5. **`ConsumerGroup`: A Team of Mail Sorters** + * Kafka allows multiple `ScheduleConsumer` instances to work together as a **Consumer Group**. + * Imagine having several mail sorters for the same `schedules-log` mailbox. Kafka cleverly divides the work among them. + * **Benefits:** + * **Scalability:** If there are many messages, more consumers can share the load. + * **Fault Tolerance:** If one consumer in the group crashes, another one can take over its work, ensuring no messages are missed. + * The `api/pkg/schedule/kafka/client/consumer_group.go` file provides the underlying mechanism for this, ensuring our `ScheduleConsumer` can be part of such a resilient team. + +## How It Works: Storing and Communicating Schedules + +Let's trace how a schedule makes its way into Kafka and how it's read. + +### 1. Saving a New Schedule (or Updating an Existing One) + +When you submit a new `schedule.Schedule` blueprint (or update an existing one): +1. The [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md) validates it. +2. It then calls `Save()` on our `ScheduleProducer`. + +Here's a simplified look at what `ScheduleProducer.Save()` does: +```go +// Simplified from: api/pkg/schedule/kafka/producer.go +func (p *ScheduleProducer) Save(ctx context.Context, s *schedule.Schedule) error { + // 1. Convert our schedule (s) into an "UpsertedEvent" format. + // "Upserted" means it's for an insert (new) or update. + upsertData := toUpsertScheduleEvent(s) // From producer.go + + // 2. Package this data into a standard "event" envelope. + // This helps track changes systematically. + event := &es.Event{ + ID: uuid.New(), // A unique ID for this event itself + Type: /* type like "schedule.UpsertedEvent" */, + Data: upsertData, // Our schedule details + Created: time.Now().UTC(), + } + + // 3. Create the actual Kafka "letter" (message). + // s.Key (e.g., "daily-report") is the address label for this schedule. + // p.dataStoreTopic is the main mailbox for all schedule definitions. + // p.eventHandler helps turn the `event` into bytes. + kafkaMessage, err := ToDataStoreMessage(s.Key, event, p.eventHandler, p.dataStoreTopic) // From converter.go + if err != nil { return err } + + // 4. Send the letter! The `p.producer` is a generic Kafka message writer. + return p.producer.WriteMessage(ctx, *kafkaMessage) +} +``` +* `toUpsertScheduleEvent(s)`: This helper (also in `producer.go`) takes your `schedule.Schedule` object and converts some fields (like `LocalExecutionTime`) into strings, preparing it for storage. + ```go + // Simplified from: api/pkg/schedule/kafka/producer.go + func toUpsertScheduleEvent(s *schedule.Schedule) *schedule.UpsertedEvent { + return &schedule.UpsertedEvent{ // This is defined in schedule/events.go + ID: s.ID, // The schedule's system-generated ID + LocalExecutionTime: s.LocalExecutionTime.Format("2006-01-02 15:04"), // Time as string + TimeLocation: s.TimeLocation, + Recurrence: s.Recurrence, + TargetTopic: s.TargetTopic, + TargetKey: s.TargetKey, + TargetPayload: s.TargetPayload, + } + } + ``` +* `ToDataStoreMessage(...)`: This helper from `converter.go` actually creates the Kafka message structure. The `eventHandler` turns the `event` struct into bytes (`[]byte`). + ```go + // Simplified from: api/pkg/schedule/kafka/converter.go + func ToDataStoreMessage(key string, event *es.Event, eventHandler es.EventHandler, dataStoreTopic string) (*kafkaclient.Message, error) { + // Convert the 'event' (which contains our schedule data) into bytes + eventBytes, /* schemaID */ _, err := eventHandler.Marshal(event) + if err != nil { return nil, err } + // ... (schema ID might be added to eventBytes for versioning) ... + + return &kafkaclient.Message{ + Topic: dataStoreTopic, // The mailbox name + Key: key, // The schedule's unique name (e.g., "daily-report") + Value: eventBytes, // The schedule data, as bytes + }, nil + } + ``` + +**Visualized: Saving a Schedule** +```mermaid +sequenceDiagram + participant SMService as Schedule Management Service + participant SProducer as ScheduleProducer + participant KafkaTopic as Kafka Topic (schedules-log) + + SMService->>SProducer: Save(scheduleBlueprint) + SProducer->>SProducer: scheduleBlueprint -> UpsertedEvent + SProducer->>SProducer: UpsertedEvent -> es.Event + SProducer->>SProducer: es.Event -> Kafka Message (Key, Value bytes) + SProducer->>KafkaTopic: Write Message +``` +Now, the schedule blueprint is safely stored in Kafka! + +### 2. Deleting a Schedule + +When `ScheduleProducer.Delete()` is called: +```go +// Simplified from: api/pkg/schedule/kafka/producer.go +func (p *ScheduleProducer) Delete(ctx context.Context, scheduleKey string) error { + // 1. Create a special "tombstone" message. + // It has the scheduleKey, but its Value (body) is nil. + tombstoneMessage, err := ToDataStoreTombstoneMessage(scheduleKey, p.dataStoreTopic) // From converter.go + if err != nil { return err } + + // 2. Send this tombstone message to Kafka. + return p.producer.WriteMessage(ctx, *tombstoneMessage) +} +``` +* `ToDataStoreTombstoneMessage(...)` from `converter.go`: + ```go + // Simplified from: api/pkg/schedule/kafka/converter.go + func ToDataStoreTombstoneMessage(key string, dataStoreTopic string) (*kafkaclient.Message, error) { + return &kafkaclient.Message{ + Topic: dataStoreTopic, + Key: key, // The schedule's unique name + Value: nil, // Empty body signifies a tombstone! + }, nil + } + ``` +Kafka and its consumers understand this `nil` value as a "delete" instruction for the given `key`. + +### 3. Reading Schedules (The `ScheduleConsumer` in Action) + +Components like the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) need to get the list of schedules. They use the `ScheduleConsumer`. +```go +// Simplified from: api/pkg/schedule/kafka/consumer.go +func (c *ScheduleConsumer) GetNextSchedule(ctx context.Context) (*schedule.ConsumedSchedule, error) { + // 1. Read the next raw message from the Kafka topic. + // This `c.consumer` is a generic Kafka message reader. + kafkaMessage, err := c.consumer.ReadFirstAvailableMessageNonBlocking(ctx) + if err != nil { return nil, err } // e.g., no new messages right now + + // 2. Convert the raw Kafka message back into our schedule event format. + // `c.eventHandler` helps decode the message bytes. + scheduleKey, event, err := FromDataStoreKafkaMessage(kafkaMessage, c.eventHandler) // From converter.go + if err != nil { return nil, err } + + if event == nil { + // It's a tombstone! This schedule (identified by scheduleKey) was deleted. + // Return a special marker for deletion. + return schedule.NewConsumedSchedule( /* metadata */, &schedule.Schedule{Key: scheduleKey, TargetTopic: schedule.TombstoneIdentifier}), nil + } + + // 3. If it's not a tombstone, it's an event (e.g., UpsertedEvent). + // Extract the schedule data from the event. + s, err := fromUpsertScheduleEvent(scheduleKey, event.Data.(*schedule.UpsertedEvent)) // From consumer.go + if err != nil { return nil, err } + + return schedule.NewConsumedSchedule( /* metadata */, s), nil +} +``` +* `FromDataStoreKafkaMessage(...)` from `converter.go`: + ```go + // Simplified from: api/pkg/schedule/kafka/converter.go + func FromDataStoreKafkaMessage(message kafkaclient.Message, eventHandler es.EventHandler) (string, *es.Event, error) { + if message.Value == nil { + // This is a tombstone message (deletion)! + return message.Key, nil, nil + } + + // ... (schema ID might be removed from message.Value before unmarshalling) ... + // Convert the bytes back into an `es.Event` struct + event, err := eventHandler.Unmarshal(message.Value, /* schemaID */) + if err != nil { return "", nil, err } + + return message.Key, event, nil + } + ``` +* `fromUpsertScheduleEvent(...)` from `consumer.go`: + ```go + // Simplified from: api/pkg/schedule/kafka/consumer.go + func fromUpsertScheduleEvent(key string, eventData *schedule.UpsertedEvent) (*schedule.Schedule, error) { + // Convert string time back to time.Time object + execTime, _ := time.Parse("2006-01-02 15:04", eventData.LocalExecutionTime) + + return &schedule.Schedule{ + ID: eventData.ID, + LocalExecutionTime: execTime, + TimeLocation: eventData.TimeLocation, + Recurrence: eventData.Recurrence, + Key: key, // The key comes from the Kafka message itself + TargetTopic: eventData.TargetTopic, + TargetKey: eventData.TargetKey, + TargetPayload: eventData.TargetPayload, + }, nil + } + ``` + +**Visualized: Reading a Schedule** +```mermaid +sequenceDiagram + participant SConsumer as ScheduleConsumer + participant KafkaTopic as Kafka Topic (schedules-log) + participant DailyStore as Daily Schedule Store + + DailyStore->>SConsumer: GetNextSchedule() + SConsumer->>KafkaTopic: Read Message + KafkaTopic-->>SConsumer: Kafka Message (Key, Value bytes) + SConsumer->>SConsumer: Kafka Message -> es.Event (or Tombstone) + alt Tombstone + SConsumer->>SConsumer: Mark schedule as deleted + else UpsertedEvent + SConsumer->>SConsumer: es.Event -> schedule.Schedule + end + SConsumer-->>DailyStore: ConsumedSchedule (or deletion signal) +``` + +### 4. Communicating Task Execution (Bonus Kafka Use!) + +Kafka isn't just for storing schedule *definitions*. It's also used to send the actual task *payloads* when it's time for a task to run. +The [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md), through its `ScheduleActor`, uses `ScheduleProducer.IssueOnTargetTopic()`: + +```go +// Simplified from: api/pkg/schedule/kafka/producer.go +func (p *ScheduleProducer) IssueOnTargetTopic(ctx context.Context, s *schedule.Schedule) error { + // 1. Create a Kafka message destined for the schedule's `TargetTopic`. + // The body of this message is the schedule's `TargetPayload`. + targetMessage, err := ToTargetMessage(s) // From converter.go + if err != nil { return err } + + // 2. Send this "task payload" message. + // Another system (or another part of this system) will be listening + // on `s.TargetTopic` to actually perform the work. + return p.producer.WriteMessage(ctx, *targetMessage) +} +``` +* `ToTargetMessage(...)` from `converter.go`: + ```go + // Simplified from: api/pkg/schedule/kafka/converter.go + func ToTargetMessage(s *schedule.Schedule) (*kafkaclient.Message, error) { + return &kafkaclient.Message{ + Topic: s.TargetTopic, // The topic specified in the schedule blueprint + Key: s.TargetKey, // The key specified in the schedule blueprint + Value: s.TargetPayload, // The actual work data! + Headers: { /* schedule ID, timestamp, etc. */ }, + }, nil + } + ``` +This decouples the scheduler (which decides *when* to run) from the worker that actually *does* the task. + +## Why Kafka? The Advantages for Our Scheduler + +Using Kafka as the backbone for schedule persistence and communication offers several benefits: + +* **Durability & Reliability:** Schedule definitions are safely stored and won't be lost if a component restarts. +* **Decoupling:** The `ScheduleProducer` doesn't need to know who the `ScheduleConsumers` are, and vice-versa. They only need to agree on the Kafka topic and message format. This makes the system more flexible. +* **Scalability:** Kafka is designed to handle a high volume of messages and many consumers. If our scheduler gets very busy, Kafka can keep up. +* **Fault Tolerance:** With consumer groups, if one instance of a `ScheduleConsumer` fails, others can take over, ensuring continuous processing. +* **Central Log:** The Kafka topic acts as a chronological log of all changes to schedules. This can be useful for auditing or debugging. +* **Asynchronous Communication:** Components can send messages without waiting for an immediate response, which can improve performance and responsiveness. + +## Conclusion + +You've now seen how Apache Kafka serves as the reliable "postal service" and "archive" for our `scheduler` system. The `ScheduleProducer` writes schedule information (new tasks, updates, and "tombstone" deletions) as messages to a Kafka topic. The `ScheduleConsumer` (often as part of a resilient `ConsumerGroup`) reads these messages. This ensures that schedule definitions are durably stored and can be processed by different parts of the scheduler system, keeping everything in sync even if components restart or scale out. Kafka is also used to send the actual task payloads to their designated target topics when it's time for execution. + +But how does a component like the `ScheduleConsumer` intelligently manage all these schedule messages from Kafka to figure out which tasks are due *today* or *right now*? That's where the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) comes in, and we'll explore it in the next chapter! + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md b/documentation/07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md new file mode 100644 index 00000000..564c57bd --- /dev/null +++ b/documentation/07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md @@ -0,0 +1,307 @@ +# Chapter 7: Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`) + +In [Chapter 6: Kafka-based Schedule Persistence and Communication](06_kafka_based_schedule_persistence_and_communication_.md), we learned how our `scheduler` system uses Kafka as a reliable way to store all the "recipe cards" for tasks (`schedule.Schedule` blueprints). Kafka acts like a giant, permanent library of every schedule we've ever defined. But if the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) had to sift through this entire library every second to see what's due, it would be very inefficient! + +This chapter introduces the **`scheduler.DailyScheduleStoreDefault`**, our "Daily Task Organizer." It's like a smart personal assistant who focuses only on what needs to be done *today*. + +## The Problem: Focusing on Today's Tasks + +Imagine you have a master calendar with all your appointments for the entire year (that's like our Kafka topic). Every morning, you don't want to re-read the whole year's calendar. Instead, you'd probably: +1. Look at the master calendar once in the morning. +2. Write down *only today's* appointments on a small notepad, in order of time. +3. Throughout the day, you just glance at this notepad to see what's next. +4. At the end of the day (or at midnight), you throw away the notepad and prepare a new one for tomorrow. + +The `DailyScheduleStoreDefault` does exactly this for our `scheduler`. It needs to: +* Efficiently find out which tasks are relevant for the current day. +* Keep them sorted by time. +* Make a task available precisely when its scheduled time arrives. +* Clear out old tasks and prepare for the next day (the "midnight reset"). + +## Meet the `scheduler.DailyScheduleStoreDefault`: Your Daily Planner + +The `scheduler.DailyScheduleStoreDefault` acts like a meticulous daily planner or a short-term memory for the scheduler. Its main job is to take all the schedule definitions it reads from Kafka (via a `ScheduleReader`) and organize a focused list of tasks specifically for the current day. + +Here's what it does: + +1. **Reads from Kafka:** It continuously reads schedule messages (new schedules, updates, deletions) from the Kafka topic where all schedules are stored. It uses a helper called a `ScheduleReader` for this. +2. **Filters for "Today":** It looks at each schedule's execution time and decides if it falls within the current day's active window. This "window" is typically from a little before "now" (e.g., 5 minutes in the past, to catch anything slightly delayed) up to the next midnight. +3. **Maintains an Ordered List:** It keeps an internal, sorted list of all tasks scheduled for this "today" window, ordered by their execution time. This way, it always knows which task is next. +4. **Provides Due Tasks:** When a task's scheduled time arrives, this organizer makes it available to the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) for execution. It does this by sending the task on a Go channel (a way for different parts of a Go program to communicate). +5. **Handles Midnight Reset:** When it's notified that midnight has passed (usually by the Task Execution Engine), it clears its internal list of "today's" tasks and starts building a new list for the new day by re-reading from Kafka. + +Think of it as the scheduler's highly efficient secretary, ensuring the boss (the Task Execution Engine) only sees the tasks that are immediately relevant. + +## How It Works: A Day in the Life of the Organizer + +Let's follow the `DailyScheduleStoreDefault` through its daily routine: + +1. **Morning Prep (Initialization or after Midnight):** + * The organizer starts with a blank slate. + * It begins reading *all* schedule definitions from the Kafka topic using its `ScheduleReader`. This is like getting the master list of all possible tasks. + * For each schedule, it checks if its `LocalExecutionTime` falls within the current day's active window. + * If a schedule is for "today," it's added to an internal, sorted list. If it's a deletion message (a "tombstone" from Kafka), the corresponding task is removed. + +2. **During the Day (Continuous Operation):** + * The organizer has a timer set for the *very next* task in its sorted list. + * When this timer goes off, it means that task is now due! + * It takes this due task and sends it to the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) through a special communication pipe called a Go channel (`ProduceChannel`). + * The task is then removed from its internal list of "today's" tasks. + * The organizer then looks at the *new* next task in its list and resets its timer for that task's execution time. + * It also continues to listen for new schedule messages from Kafka. If a new task for "today" is added or an existing one is updated/deleted, the internal list and the timer are adjusted accordingly. + +3. **Midnight! (The Reset):** + * The [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) sends a signal to the organizer: "It's midnight!" + * The organizer stops its current timer, throws away its entire internal list of "today's" tasks. + * It then tells its `ScheduleReader` to "reset," meaning it should start reading all schedules from the beginning of the Kafka topic again. + * The process then repeats from "Morning Prep" for the new day. + +This cycle ensures that the scheduler is always working with an up-to-date, focused list of tasks. + +## Under the Hood: A Peek Inside `DailyScheduleStoreDefault` + +The `DailyScheduleStoreDefault` is defined in `api/pkg/scheduler/daily_schedule_store.go`. Let's look at its key parts and how it's created. + +**1. Key Components Held by `DailyScheduleStoreDefault`:** + +When a `DailyScheduleStoreDefault` is created (usually by the `NewDailyScheduleConsumer` function), it's given several helpers: + +```go +// Simplified from: api/pkg/scheduler/daily_schedule_store.go +package scheduler + +// ... (other imports) ... +import ( + "github.com/nestoca/scheduler/api/pkg/schedule" +) + +type DailyScheduleStoreDefault struct { + executionTimer *storeTimer // A timer for the next due schedule + scheduleReader ScheduleReader // Reads schedules from Kafka + + // Channel to send due schedules to the Task Execution Engine + produceChannel chan *schedule.Schedule + // Channel to receive the "midnight has arrived" signal + midnightReachedChannel chan struct{} + // Channel to send very old/problematic schedules to the Dead Letter Queue + deadLetterChannel chan<- *schedule.Schedule + + // Internal storage for today's schedules, conceptually a sorted list. + // (Actual implementation uses 'dailySchedules' map for Kafka partitions) + dailySchedules dailyScheduleStore + + locationProvider LocationProvider // Helps with timezone calculations + // ... (other fields for metrics, retry logic, etc.) ... +} + +// scheduleEntry stores a schedule and its specific execution time. +type scheduleEntry struct { + schedule *schedule.Schedule + localExecutionTime time.Time // The actual time.Time object for execution + destination string // e.g., "OUTPUT" or "DEAD_LETTER_QUEUE" +} + +// dailyScheduleStore is a map where keys are Kafka partition IDs and values +// are partitions, each holding a list of scheduleEntry. +// For simplicity, think of this as managing one combined sorted list. +type dailyScheduleStore map[int]*dailyScheduleStorePartition +type dailyScheduleStorePartition struct { + entries []scheduleEntry // Sorted list of tasks for this partition for today + // ... (fields for readiness tracking) ... +} +``` +* `executionTimer`: A smart timer that's always set to fire when the *next* schedule in its internal list is due. +* `scheduleReader`: An instance of the `ScheduleReader` interface (which we learned about in [Chapter 6: Kafka-based Schedule Persistence and Communication](06_kafka_based_schedule_persistence_and_communication_.md)) to fetch schedule definitions. +* `produceChannel`: This is how it "produces" or "outputs" due schedules to the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md). +* `midnightReachedChannel`: It listens on this channel for a signal indicating a new day has begun. +* `deadLetterChannel`: If it encounters a schedule that is too old (e.g., its execution time was hours ago and missed), it sends it here. +* `dailySchedules`: This is its internal "notepad" for today's tasks. Although the actual structure is a bit more complex to handle Kafka partitions efficiently, you can think of it as maintaining one big, sorted list of `scheduleEntry` items for the current day. Each `scheduleEntry` holds a `schedule.Schedule` and its precisely calculated `localExecutionTime`. +* `locationProvider`: Essential for correctly interpreting timezones from `schedule.Schedule` blueprints. + +**2. The Main Loop (`RunBlocking`)** + +The `RunBlocking` method is the heart of the `DailyScheduleStoreDefault`. It runs in a continuous loop, waiting for different events: + +```go +// Simplified from: api/pkg/scheduler/daily_schedule_store.go +func (s *DailyScheduleStoreDefault) RunBlocking(ctx context.Context) { + // ... (setup) ... + for { // Infinite loop + select { + case <-s.executionTimer.C: // Timer for the next schedule fired! + log.G(ctx).Tracef("[Store] Execution time of schedule reached") + s.sendNextScheduleForConsumption() // Send it out + + case <-s.midnightReachedChannel: // Midnight signal received! + log.G(ctx).Tracef("[Store] Midnight boundary reached, we must reset") + s.reset(s.consumeErrorChannel, true) // Clear list, prepare for new day + + case <-s.readRetryTimer.C: // Time to try reading a new schedule from Kafka + s.readRetryTimer.Stop() // Stop this timer first + sched, err := s.scheduleReader.GetNextSchedule(ctx) // Get a schedule + if err == nil && sched != nil { + s.manageScheduleEntry(sched, s.consumeErrorChannel) // Process it + } + // ... (handle errors, reset readRetryTimer for next read attempt) ... + + case <-ctx.Done(): // Application is shutting down + return // Exit the loop + // ... (other cases like new generation events) ... + } + } +} +``` +* **`<-s.executionTimer.C`**: The `executionTimer` has fired. This means the task at the top of its internal "today's list" is due. The `sendNextScheduleForConsumption()` method is called. +* **`<-s.midnightReachedChannel`**: A signal arrives telling the store it's midnight. The `reset()` method is called to clear everything for the new day. +* **`<-s.readRetryTimer.C`**: It's time to attempt to read another message from the `scheduleReader` (which gets data from Kafka). If a schedule is read, `manageScheduleEntry()` is called. This timer ensures the store periodically checks Kafka for new or updated schedules. +* **`<-ctx.Done()`**: The application is shutting down, so the loop exits. + +**3. Processing a Schedule (`manageScheduleEntry`)** + +When a new schedule (`sched`) is read from Kafka, `manageScheduleEntry` decides what to do: + +```go +// Simplified logic of manageScheduleEntry from api/pkg/scheduler/daily_schedule_store.go +func (s *DailyScheduleStoreDefault) manageScheduleEntry( + consumedSched *schedule.ConsumedSchedule, + errChan chan<- error) { + + actualSchedule := consumedSched.Schedule() // The schedule.Schedule blueprint + + // If this schedule already exists in our list, remove the old one first. + idx, found := s.findKeyMatch(consumedSched) + if found { + s.deleteEntry(consumedSched.Partition(), idx, actualSchedule) + } + + // If it's a "tombstone" (delete message), we're done with it. + if isTombstone(actualSchedule) { + s.updateExecTimerAfterAction("deletion") // May need to update timer + return + } + + // Add the schedule to our internal "today's list" if it's relevant. + // This involves checking if it's within time boundaries for today. + // addEntry returns true if the new entry is now the *very next* one. + if s.addEntry(consumedSched.Partition(), actualSchedule, s.lowBoundaryFunc, s.highBoundaryFunc) { + s.updateExecTimerAfterAction("insertion") // Update timer if next task changed + } +} +``` +* It first checks if this schedule (by its `Key`) is already in the internal list. If so, the old version is removed (`deleteEntry`). +* If the message from Kafka is a "tombstone" (meaning the schedule was deleted), then nothing more is done for this key. +* Otherwise (`addEntry`), it checks if the schedule's `LocalExecutionTime` is within the "today" window (using `lowBoundaryFunc` and `highBoundaryFunc`). + * `lowBoundaryFunc`: Checks if the task is not too old (e.g., not older than 5 minutes ago). If it is, it might be sent to a dead-letter queue. + * `highBoundaryFunc`: Checks if the task is before the next midnight. + * If it's for today, it's inserted into the internal sorted list. +* `updateExecTimerAfterAction` is called to potentially reset the `executionTimer` if the newly added/deleted schedule affects which task is next. + +**4. Sending a Due Schedule (`sendNextScheduleForConsumption`)** + +When `executionTimer` fires: + +```go +// Simplified logic of sendNextScheduleForConsumption from api/pkg/scheduler/daily_schedule_store.go +func (s *DailyScheduleStoreDefault) sendNextScheduleForConsumption() { + // Find the very next schedule from our internal sorted list. + entry, partition := s.findNextScheduleToExecute() + if entry == nil { // No tasks left for today or store not ready + s.executionTimer.Stop() // Stop timer + return + } + + // Check if all parts of the store are ready (relevant for Kafka partitions). + // For simplicity, let's assume it's ready. + + // Send the schedule to the correct channel. + if entry.destination == scheduleDestinationDLQ { // Too old + s.deadLetterChannel <- entry.schedule + } else { // It's due now! + s.produceChannel <- entry.schedule // Send to Task Execution Engine + } + + // Remove this task from our internal list. + s.deleteEntry(partition, 0, entry.schedule) // 0 is the index of the first item + + // Set the timer for the NEW next schedule. + s.updateExecTimerAfterAction("execution") +} +``` +* It gets the task at the top of its sorted list (`findNextScheduleToExecute`). +* It sends this `schedule.Schedule` object to the `produceChannel` (if it's not too old). The [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) is listening on this channel. +* It removes the task from its list and then calls `updateExecTimerAfterAction` to set the `executionTimer` for the next task in the list. + +**5. Midnight Reset (`reset`)** + +When the midnight signal arrives: + +```go +// Simplified logic of reset from api/pkg/scheduler/daily_schedule_store.go +func (s *DailyScheduleStoreDefault) reset(errChan chan<- error, forceEndOfGeneration bool) { + s.executionTimer.Stop() // Stop the main timer + s.readRetryTimer.Stop() // Stop trying to read from Kafka for a moment + + // Clear the internal list of "today's" schedules. + clear(s.dailySchedules) // 'clear' is a Go built-in for maps + + // Tell the ScheduleReader to reset its position in Kafka, + // so it starts reading all schedules from the beginning for the new day. + if err := s.scheduleReader.Reset(); err != nil { + errChan <- err // Report error if reset fails + } + + // ... (reset metrics) ... + + // Start the Kafka reading process again for the new day. + s.readRetryTimer.Reset(s.readRetryStrategy.ResetSequence()) +} +``` +* It stops all its timers. +* It completely clears its `dailySchedules` internal storage (the "notepad" is wiped clean). +* Crucially, it calls `s.scheduleReader.Reset()`. This tells the component reading from Kafka to go back to the beginning of the topic and re-process all schedule definitions, so the store can build a fresh list for the new day. +* It then restarts its `readRetryTimer` to begin populating its list for the new day. + +**Visualizing the Flow:** + +Here's a simplified diagram of how the `DailyScheduleStoreDefault` interacts: + +```mermaid +sequenceDiagram + participant Reader as ScheduleReader (from Kafka) + participant Store as DailyScheduleStoreDefault + participant Engine as Task Execution Engine + + Store->>Reader: GetNextSchedule() + Reader-->>Store: ConsumedSchedule (e.g., new_task_for_today) + Store->>Store: manageScheduleEntry(new_task_for_today)
(Adds to internal sorted list) + Store->>Store: updateExecTimerAfterAction()
(Sets executionTimer for new_task_for_today) + + Note over Store,Engine: Time passes... executionTimer for new_task_for_today fires! + + Store->>Store: sendNextScheduleForConsumption() + Store->>Engine: produceChannel <- new_task_for_today + Store->>Store: (Removes new_task_for_today from list) + Store->>Store: updateExecTimerAfterAction()
(Sets timer for the *next* task in list) + + Note over Store,Engine: Midnight signal arrives! + Engine->>Store: midnightReachedChannel <- signal + Store->>Store: reset() (Clears internal list) + Store->>Reader: Reset() (Start reading Kafka from beginning) +``` + +## Why is the Daily Task Organizer So Important? + +* **Efficiency:** It prevents the core [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) from having to constantly scan *all* schedules in Kafka. It only deals with a small, relevant subset. +* **Focus:** It maintains a clear, short-term view of tasks due "today," making the system responsive. +* **Timeliness:** By keeping an ordered list and using a precise timer, it ensures tasks are made available for execution exactly when they are scheduled. +* **Cleanliness:** The "midnight reset" provides a clean way to roll over to the next day, discarding completed or outdated tasks from the active set and rebuilding with fresh data. + +## Conclusion + +The `scheduler.DailyScheduleStoreDefault` is like the scheduler's dedicated daily assistant. It diligently reads all potential tasks from Kafka (via `ScheduleReader`), filters them to create an ordered to-do list for just the current day, and hands over tasks to the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) precisely when they are due. Its "midnight reset" ensures each day starts fresh. This component is crucial for keeping the scheduler efficient and focused on what needs to happen *now*. + +Now that a task has been picked up by the Daily Task Organizer and sent for execution, what happens *after* the Task Execution Engine attempts to run it? How are recurring tasks rescheduled for their next run? What if a task needs special handling after it's done? That's what we'll explore in the next chapter: [Chapter 8: Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md). + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/08_post_execution_handler___scheduler_schedulepostprocessordefault___.md b/documentation/08_post_execution_handler___scheduler_schedulepostprocessordefault___.md new file mode 100644 index 00000000..5276ca3d --- /dev/null +++ b/documentation/08_post_execution_handler___scheduler_schedulepostprocessordefault___.md @@ -0,0 +1,248 @@ +# Chapter 8: Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`) + +In [Chapter 7: Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md), we saw how our scheduler efficiently manages and provides tasks that are due for execution "today." The [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) then picks up these due tasks and attempts to run them. + +But what happens *after* a task has been successfully executed (or has completed its retries)? Is it just forgotten? What if it's a task that needs to run every day? This is where our **Post-Execution Handler**, the `scheduler.SchedulePostProcessorDefault`, steps in. + +## The Problem: What To Do After the Work is Done? + +Imagine you have a personal assistant who helps you with your daily tasks. +* If you had a one-time task like "Mail a birthday card to Alex," once it's done, your assistant should probably cross it off your list for good. +* But if you have a recurring task like "Water the plants every Monday," after your assistant does it this Monday, they need to remember to schedule it again for *next* Monday. + +The `scheduler.SchedulePostProcessorDefault` acts like this diligent administrative assistant or a cleanup crew for our tasks. After a task finishes its journey through the execution engine, this handler takes charge of the follow-up actions. + +## Meet `scheduler.SchedulePostProcessorDefault`: The After-Task Admin + +The `scheduler.SchedulePostProcessorDefault` (let's call it "PostProcessor" for short) is a crucial component that ensures tasks are handled correctly *after* they've been processed by the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md). + +Its main responsibilities are: + +1. **Handling One-Time Tasks:** If a task was a one-off event (meaning its `Recurrence` from the [Schedule Blueprint (`schedule.Schedule`)](01_schedule_blueprint___schedule_schedule___.md) is set to `RecurrenceNone` or its recurrence period has ended), the PostProcessor ensures this task is marked for deletion from the system. We don't want to keep old, completed one-time tasks cluttering our schedule list. +2. **Rescheduling Recurring Tasks:** If a task is meant to repeat (e.g., daily, weekly, hourly), the PostProcessor calculates the *next* time this task should run based on its `Recurrence` rules. It then updates the schedule definition, effectively re-scheduling it for the future. This "update" usually means creating a new `schedule.Schedule` instance with the new execution time and saving it (often back to Kafka, as we saw in [Chapter 6: Kafka-based Schedule Persistence and Communication](06_kafka_based_schedule_persistence_and_communication_.md)). + +It uses a helper component called a `SchedulePostProcessUpdater` to perform these deletions or updates. + +## How the PostProcessor Gets Involved + +As we saw in [Chapter 5: Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md), when the Task Execution Engine successfully issues a task, or when a task finishes its retries from the [Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md), it sends the `schedule.Schedule` object to the PostProcessor. + +Here's a simplified view of that interaction from the `DefaultScheduler`'s `StartBlocking` method: +```go +// Simplified from: api/pkg/scheduler/scheduler.go +// Inside DefaultScheduler's main loop: + +// ... (when a task `sched` is successfully issued by `s.actor`) ... +s.schedulePostProcessor.PushChannel() <- sched + +// ... (or when a task `sched` exits the deadLetterQueue) ... +s.schedulePostProcessor.PushChannel() <- sched +``` +The `s.schedulePostProcessor` is our `SchedulePostProcessorDefault`. It receives the just-processed schedule on its `PushChannel()`. + +## A Day in the Life of the PostProcessor + +Let's consider two scenarios: + +**Scenario 1: A One-Time Abandoned Cart Reminder** +1. An "Abandoned Cart Reminder" task (which is a one-time task) for `user123` is successfully executed. +2. The Task Execution Engine sends this `schedule.Schedule` object to the PostProcessor. +3. The PostProcessor looks at the schedule's `Recurrence` settings. It sees it's `RecurrenceNone`. +4. It then instructs its `SchedulePostProcessUpdater` to "delete" this schedule (e.g., `user123-cart-reminder`). The updater will typically send a "tombstone" message to Kafka. + +**Scenario 2: A Daily Sales Report Task** +1. The "Daily Sales Report" task (which runs daily at 9 AM) is successfully executed today. +2. The Task Execution Engine sends this `schedule.Schedule` to the PostProcessor. +3. The PostProcessor sees the `Recurrence` is `RecurrenceDaily`. +4. It calculates the next execution time: tomorrow at 9 AM. +5. It creates a new `schedule.Schedule` instance (or modifies the existing one) with this new `LocalExecutionTime` and a new unique internal `ID`. +6. It then tells its `SchedulePostProcessUpdater` to "update" (effectively, save) this new schedule instance. The updater will typically save this new blueprint to Kafka. + +## Under the Hood: Inside `SchedulePostProcessorDefault` + +The `SchedulePostProcessorDefault` is defined in `api/pkg/scheduler/schedule_postprocessor.go`. Let's explore its inner workings. + +**1. Key Components:** +The PostProcessor needs a few helpers: +```go +// Simplified from: api/pkg/scheduler/schedule_postprocessor.go +package scheduler + +type SchedulePostProcessorDefault struct { + postProcessUpdater SchedulePostProcessUpdater // To save or delete schedules + locationProvider LocationProvider // To help with time calculations + + postProcessInputChannel chan *schedule.Schedule // Receives schedules to process + scheduleQueue *schedule.Queue // Internal queue for schedules + // ... (timers, metrics, retry logic) ... +} +``` +* `postProcessUpdater`: This is an instance of `SchedulePostProcessUpdater`. It's the component that actually performs the save (for rescheduling) or delete operations. Often, this is an instance of `ConcurrentScheduleActor` which writes to Kafka. +* `locationProvider`: Used to correctly interpret timezones when calculating next execution times. +* `postProcessInputChannel`: The channel where it receives schedules from the Task Execution Engine. +* `scheduleQueue`: An internal queue to hold schedules waiting for post-processing. + +**2. The Main Loop (`RunBlocking`)** +The PostProcessor runs its own loop, much like other core components: +```go +// Simplified from: api/pkg/scheduler/schedule_postprocessor.go +func (s SchedulePostProcessorDefault) RunBlocking(ctx context.Context) { + for { + select { + case schedule := <-s.postProcessInputChannel: // New schedule arrived! + s.scheduleQueue.Enqueue(schedule) // Add to internal queue + // ... (logic to start/reset a processing timer if needed) ... + + case <-s.processTimer.timer.C: // Internal timer fired, time to process + // Calls a helper that uses s.selectActionAndPrepareQueue() + processQueueEntry(ctx, s.scheduleQueue, /*...other args...*/, s.selectActionAndPrepareQueue(ctx)) + + case <-ctx.Done(): // Application shutting down + return // Exit loop + // ... (other cases like generation changes) ... + } + } +} +``` +* When a schedule arrives on `postProcessInputChannel`, it's added to an internal `scheduleQueue`. +* A timer (`s.processTimer`) ensures that schedules in the queue are processed. When this timer fires, `processQueueEntry` is called, which eventually uses the logic from `selectActionAndPrepareQueue`. + +**3. The Decision Maker: `selectActionAndPrepareQueue`** +This is where the core logic resides for deciding what to do with a schedule: +```go +// Simplified from: api/pkg/scheduler/schedule_postprocessor.go +func (s *SchedulePostProcessorDefault) selectActionAndPrepareQueue(ctx context.Context) actionFunc { + sched := s.scheduleQueue.Dequeue() // Get next schedule from internal queue + if sched == nil { + return noOpAction // Nothing to do + } + defer s.scheduleQueue.Requeue(sched) // Put it back if processing fails & needs retry + + // Calculate when this schedule should run next. + nextLocalExecutionTime, err := CalculateNextLocalExecutionTime(ctx, sched, s.locationProvider) + if err != nil { + // Problem calculating next time, probably delete it. + return s.deleteScheduleProxy() + } + + if nextLocalExecutionTime.IsZero() { + // If next time is zero, it means no more recurrences. Delete it. + return s.deleteScheduleProxy() + } + + // Otherwise, it needs to be rescheduled (updated). + return s.updateScheduleProxy() +} +``` +* It takes a schedule (`sched`) from its internal queue. +* It calls `CalculateNextLocalExecutionTime` (a helper function also in this file) to determine the next run time. +* If `CalculateNextLocalExecutionTime` returns an error or a "zero" time (meaning no further runs), it decides to delete the schedule (`s.deleteScheduleProxy()`). +* Otherwise, it decides to update (reschedule) it (`s.updateScheduleProxy()`). + +**4. Calculating the Next Run: `CalculateNextLocalExecutionTime`** +This helper function checks the `sched.Recurrence` rules and current `sched.LocalExecutionTime` to figure out the next one. +```go +// Simplified from: api/pkg/scheduler/schedule_postprocessor.go +func CalculateNextLocalExecutionTime(ctx context.Context, sched *schedule.Schedule, provider LocationProvider) (time.Time, error) { + recurrenceScheme := schedule.RecurrenceNone + if sched.Recurrence != nil { + recurrenceScheme = sched.Recurrence.Scheme + } + + nextTime := time.Time{} // A "zero" time.Time + switch recurrenceScheme { + case schedule.RecurrenceDaily: + nextTime = sched.LocalExecutionTime.AddDate(0, 0, 1) // Add 1 day + case schedule.RecurrenceHourly: + nextTime = sched.LocalExecutionTime.Add(1 * time.Hour) // Add 1 hour + // ... (cases for RecurrenceWeekly, RecurrenceMonthly, RecurrenceCustom, etc.) ... + case schedule.RecurrenceNone: + // No recurrence, nextTime remains zero. + } + return nextTime, nil // Error handling omitted for simplicity +} +``` +* For `RecurrenceDaily`, it adds 1 day. For `RecurrenceHourly`, 1 hour, and so on. +* If `RecurrenceNone`, `nextTime` remains zero, signaling it's a one-time task. + +**5. Taking Action: `updateScheduleProxy` and `deleteScheduleProxy`** +These functions return the actual "action" to be performed on the schedule, using the `s.postProcessUpdater`. +```go +// Simplified from: api/pkg/scheduler/schedule_postprocessor.go +func (s *SchedulePostProcessorDefault) updateScheduleProxy() actionFunc { + return func(ctx context.Context, sched *schedule.Schedule) (*schedule.Schedule, error) { + // Recalculate next time (important for retries) + nextLocalExecutionTime, _ := CalculateNextLocalExecutionTime(ctx, sched, s.locationProvider) + + updateSchedule := sched.Clone() // Work on a copy + updateSchedule.LocalExecutionTime = nextLocalExecutionTime + updateSchedule.ID = uuid.NewString() // Give new instance a new system ID + + // Use the updater to save this new version + err := s.postProcessUpdater.UpdateSchedule(ctx, updateSchedule) + return updateSchedule, err + } +} + +func (s *SchedulePostProcessorDefault) deleteScheduleProxy() actionFunc { + return func(ctx context.Context, sched *schedule.Schedule) (*schedule.Schedule, error) { + // Use the updater to delete this schedule + err := s.postProcessUpdater.DeleteSchedule(ctx, sched) + return sched, err + } +} +``` +* `updateScheduleProxy`: Clones the original schedule, sets its `LocalExecutionTime` to the newly calculated next run time, assigns a fresh `ID`, and then calls `s.postProcessUpdater.UpdateSchedule()`. +* `deleteScheduleProxy`: Simply calls `s.postProcessUpdater.DeleteSchedule()`. + +**6. The `SchedulePostProcessUpdater` Interface** +This interface defines what the actual updater component must do: +```go +// From: api/pkg/scheduler/schedule_postprocessor.go +type SchedulePostProcessUpdater interface { + UpdateSchedule(ctx context.Context, schedule *schedule.Schedule) error + DeleteSchedule(ctx context.Context, schedule *schedule.Schedule) error +} +``` +As mentioned, the `ConcurrentScheduleActor` (from `api/pkg/scheduler/concurrent_schedule_actor.go`) often implements this. Its `UpdateSchedule` method calls `writer.Save()` and `DeleteSchedule` calls `writer.Delete()`. The `writer` is usually a `ScheduleProducer` that writes these changes to Kafka. + +**Visualizing the Post-Execution Flow:** +```mermaid +sequenceDiagram + participant Engine as Task Execution Engine + participant PostProc as SchedulePostProcessorDefault + participant Calculator as CalculateNextLocalExecutionTime + participant Updater as SchedulePostProcessUpdater (e.g., ConcurrentScheduleActor) + participant Kafka as Kafka + + Engine->>PostProc: PushChannel <- executedSchedule + PostProc->>PostProc: Enqueues executedSchedule + Note over PostProc: Timer fires for processing... + PostProc->>Calculator: CalculateNextTime(executedSchedule) + alt Recurring Task + Calculator-->>PostProc: nextRunTime (e.g., tomorrow 9 AM) + PostProc->>Updater: UpdateSchedule(newScheduleInstance for nextRunTime) + Updater->>Kafka: Save newScheduleInstance + else One-Time Task or Error + Calculator-->>PostProc: zeroTime or error + PostProc->>Updater: DeleteSchedule(executedSchedule) + Updater->>Kafka: Send Tombstone for executedSchedule + end +``` + +## Why is the Post-Execution Handler So Important? + +* **Automates Task Lifecycle:** It ensures that recurring tasks are automatically rescheduled, keeping the system running smoothly without manual intervention. +* **Keeps System Tidy:** It cleans up one-time tasks after they are done, preventing the accumulation of obsolete schedule definitions. +* **Centralizes Post-Execution Logic:** All the rules about what to do after a task runs are concentrated in this component, making the system easier to understand and maintain. +* **Enables Recurrence:** Without it, recurring schedules would just run once and then stop. This handler is the engine that drives continuous, repeated execution. + +## Conclusion + +The `scheduler.SchedulePostProcessorDefault` is the vital "after-action report" handler for our scheduler. Once a task has been executed by the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md), this PostProcessor steps in. It intelligently decides whether to mark the task for deletion (if it was a one-time event) or to calculate its next run time and re-submit it for future execution (if it's a recurring task). This is done using a `SchedulePostProcessUpdater` that typically writes these changes back to Kafka. This ensures the scheduler can handle both one-off jobs and continuously repeating tasks effectively. + +We've seen tasks get executed successfully and then handled by the PostProcessor. But what if a task *fails* during its execution attempt? How does our scheduler handle such failures and try again? That's the topic of our next chapter: [Chapter 9: Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md). + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md b/documentation/09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md new file mode 100644 index 00000000..2216ed9e --- /dev/null +++ b/documentation/09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md @@ -0,0 +1,232 @@ +# Chapter 9: Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`) + +In [Chapter 8: Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md), we learned how tasks are managed *after* they've been successfully executed or have finished their retry attempts. Recurring tasks get rescheduled, and one-time tasks get cleaned up. + +But what if a task *fails* during its initial execution attempt? Imagine our scheduler tries to send a daily report, but the email server is temporarily down. Do we just give up? Of course not! This is where the **Failed Task Retry Queue**, specifically `scheduler.DeadLetterQueueDefault`, comes into play. + +## The Problem: Dealing With Temporary Glitches + +Life isn't always perfect, and sometimes tasks fail. +* A network connection might blip for a few seconds. +* A service the task depends on might be briefly unavailable. +* There might be a temporary hiccup that prevents the task from completing right away. + +If our scheduler tried a task once and gave up immediately upon any error, many tasks would never get done. We need a way to handle these temporary failures gracefully and give tasks a second (or third, or fourth...) chance. + +Think of it like trying to call a friend. If the line is busy, you don't just give up forever. You wait a bit and try again. If it's still busy, you might wait a little longer before trying again. + +## Meet `scheduler.DeadLetterQueueDefault`: The Task ICU + +When a scheduled task encounters an error during its execution attempt by the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md), it's not immediately discarded. Instead, it's sent to this "Dead Letter Queue" (DLQ), specifically our `scheduler.DeadLetterQueueDefault`. + +Think of the DLQ as an **intensive care unit (ICU)** or a **special retry inbox** for tasks. +* It holds these failed tasks. +* It attempts to re-issue them later, often using a `RetryStrategy` with increasing delays (this is called "backoff"). +* This provides resilience against temporary issues that might have caused the initial failure, increasing the chances that tasks eventually get processed. + +If a task in the DLQ finally succeeds on a retry attempt, great! It then goes to the [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md) just like a task that succeeded on its first try. If, after several retries, the task still fails, it also exits the DLQ and goes to the Post-Execution Handler, which will then decide on its final fate (e.g., log it as permanently failed, or if recurring, schedule the *next* instance). + +## How a Task Lands in the DLQ + +Let's look back at the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md). When it tries to issue a schedule using its `ScheduleActor` (which actually performs the task, like sending a Kafka message): + +```go +// Simplified from: api/pkg/scheduler/scheduler.go +// Inside DefaultScheduler's main loop (StartBlocking method) + +// sched is the task to be processed +err := s.actor.IssueSchedule(ctx, sched) // Attempt to run the task +if err == nil { + // SUCCESS! Send to post-processor + s.schedulePostProcessor.PushChannel() <- sched +} else { + // FAILURE! Send to dead letter queue + log.G(ctx).Tracef("[SCHED] Process failure for key %s, sending schedule to be retried", sched.Key) + s.deadLetterQueue.PushChannel() <- sched // <--- This sends it to the DLQ! +} +``` +If `s.actor.IssueSchedule()` (the attempt to run the task) returns an error, the schedule `sched` is sent to `s.deadLetterQueue.PushChannel()`. This `s.deadLetterQueue` is our `scheduler.DeadLetterQueueDefault`. + +## A Day in the Life of a Failed Task (DLQ Journey) + +Here's what happens when a task enters the `DeadLetterQueueDefault`: + +1. **Admission to ICU:** The task arrives at the DLQ. +2. **Waiting Period:** The DLQ doesn't try to re-run it immediately. It uses a `RetryStrategy` to decide how long to wait. The first wait might be short (e.g., a few seconds). +3. **Retry Attempt:** After the wait, the DLQ attempts to "issue" the task again. It uses a `ScheduleIssuer` (often the same kind of actor that the main engine uses) to do this. +4. **Outcome:** + * **Success!** If the task runs successfully this time, it "exits" the DLQ and is sent to the [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md) for normal follow-up (like rescheduling if it's a recurring task). + * **Still Fails!** If the task fails again, the `RetryStrategy` calculates a *new, longer* waiting period (this is the "backoff"). The task goes back to step 2. +5. **Max Retries Reached:** The `RetryStrategy` usually has a limit on how many times a task can be retried. If this limit is reached and the task still hasn't succeeded, it's considered "terminally failed" *for this specific run*. It then "exits" the DLQ and is sent to the [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md) for final processing (e.g., logging the failure). + +**Visualizing the DLQ Flow:** + +```mermaid +sequenceDiagram + participant Engine as Task Execution Engine + participant DLQ as DeadLetterQueueDefault + participant Issuer as DLQ's ScheduleIssuer + participant PostProc as Post-Execution Handler + + Engine->>DLQ: Failed Task (via PushChannel) + DLQ->>DLQ: Add to internal queue, set timer (RetryStrategy) + + loop Retry Attempts + Note over DLQ: Timer fires... + DLQ->>Issuer: Issue Task (Retry Attempt) + alt Retry Successful + Issuer-->>DLQ: Success! + DLQ->>PostProc: Task (via ExitChannel) + break + else Retry Fails + Issuer-->>DLQ: Failed! + DLQ->>DLQ: Update retry count, set longer timer (RetryStrategy) + Note over DLQ: If max retries, then... + DLQ->>PostProc: Task (failed) (via ExitChannel) + break + end + end +``` + +## Under the Hood: Inside `scheduler.DeadLetterQueueDefault` + +The `scheduler.DeadLetterQueueDefault` is defined in `api/pkg/scheduler/dead_letter_queue.go`. Let's look at its key parts. + +**1. Key Components Held by `DeadLetterQueueDefault`:** +When a `DeadLetterQueueDefault` is created (usually by `NewDeadLetterQueue`), it's given: +```go +// Simplified from: api/pkg/scheduler/dead_letter_queue.go +package scheduler + +type DeadLetterQueueDefault struct { + issuer ScheduleIssuer // To re-attempt issuing the schedule + + deadLetterEnterChannel chan *schedule.Schedule // Receives failed tasks + deadLetterExitChannel chan *schedule.Schedule // Sends out tasks after retries + + processTimer *queryableTimer // Timer for the next retry attempt + retryStrategy RetryStrategy // Decides wait times and max retries + scheduleQueue *schedule.Queue // Internal list of tasks waiting for retry + // ... (metrics, newGenEventHandler for Kafka rebalancing) ... +} +``` +* `issuer ScheduleIssuer`: This is crucial. It's the component the DLQ uses to try and re-run the task. It's an interface, often implemented by `ConcurrentScheduleActor` (from `api/pkg/scheduler/concurrent_schedule_actor.go`), which can issue tasks (e.g., send a Kafka message). +* `deadLetterEnterChannel`: Failed tasks from the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) arrive here. +* `deadLetterExitChannel`: Tasks leave the DLQ through this channel after they either succeed on a retry or exhaust all retry attempts. The Task Execution Engine listens to this and sends them to the [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md). +* `processTimer`: A timer that's set according to the `retryStrategy` to trigger the next retry attempt for the task at the head of the `scheduleQueue`. +* `retryStrategy RetryStrategy`: An interface (defined in the same file) that dictates the retry logic. It provides methods like `GetNewDuration()` to get the next wait time and can signal if `ErrMaxRetriesReached`. An example implementation could be an exponential backoff strategy (wait longer after each failure). +* `scheduleQueue *schedule.Queue`: An internal queue holding the schedules that are currently in the DLQ, waiting for their next retry attempt. + +**2. Creation (`NewDeadLetterQueue`)** +```go +// Simplified from: api/pkg/scheduler/dead_letter_queue.go +func NewDeadLetterQueue( + issuer ScheduleIssuer, + newGenEventHandler EntityGenerationChangeHandler, // For Kafka changes + retryStrategy RetryStrategy, + aggregator MetricAggregator, // For metrics +) *DeadLetterQueueDefault { + if issuer == nil || retryStrategy == nil { /* ... panic ... */ } + + // ... (timer and metrics setup) ... + + return &DeadLetterQueueDefault{ + issuer: issuer, + retryStrategy: retryStrategy, + deadLetterEnterChannel: make(chan *schedule.Schedule, 10), + deadLetterExitChannel: make(chan *schedule.Schedule, 10), + scheduleQueue: schedule.NewQueue(), + processTimer: newQueryableTimer( /* ... */ ), + // ... (other fields) ... + } +} +``` +This function sets up the DLQ with its necessary helpers, most importantly the `issuer` to re-run tasks and the `retryStrategy` to manage the retry timings. + +**3. The Main Loop (`RunBlocking`)** +The `RunBlocking` method is the DLQ's engine room. It listens for incoming tasks and manages the retry process: +```go +// Simplified from: api/pkg/scheduler/dead_letter_queue.go +func (d *DeadLetterQueueDefault) RunBlocking(ctx context.Context) { + for { + select { + case schedule := <-d.deadLetterEnterChannel: // A failed task arrives! + log.G(ctx).Tracef("[DLQ] Received new schedule to enqueue: %s", schedule.Key) + d.scheduleQueue.Enqueue(schedule) // Add to internal queue + // If timer wasn't running, start it for the new task. + if !d.processTimer.running { + // updateTimerAfter sets timer based on retryStrategy for head of queue + updateTimerAfter(processEventNone, d.scheduleQueue, d.processTimer, d.retryStrategy) + } + + case <-d.processTimer.timer.C: // Timer for a retry attempt fired! + // processQueueEntry tries to run the task at head of queue + // It uses d.issueScheduleProxy() as the action. + // It returns the schedule if it succeeded OR if max retries hit. + if exitSched := processQueueEntry(ctx, d.scheduleQueue, d.processTimer, d.retryStrategy, d.issueScheduleProxy()); exitSched != nil { + // Task succeeded or max retries reached. Send to exit channel. + d.deadLetterExitChannel <- exitSched + } + // If exitSched is nil, it means retry failed but more retries are pending. + // processQueueEntry has already reset the timer for the next attempt. + + case <-ctx.Done(): // Application is shutting down + d.reset() // Clear queue, stop timer + return // Exit loop + // ... (case for newGenEventHandler for Kafka rebalancing - more advanced) ... + } + } +} +``` +* **`schedule := <-d.deadLetterEnterChannel`**: A task that failed its initial execution attempt arrives. It's added to the `d.scheduleQueue`. The `updateTimerAfter` helper function (not shown in detail, but part of `queue_processor.go` utilities) is called to ensure the `d.processTimer` is set to fire for this task's first retry attempt, according to the `d.retryStrategy`. +* **`<-d.processTimer.timer.C`**: The timer for a retry has elapsed. The `processQueueEntry` function (a shared helper, also used by the PostProcessor) is called. This function: + 1. Takes the task from the front of `d.scheduleQueue`. + 2. Calls the action function provided, which for the DLQ is `d.issueScheduleProxy()`. + 3. If `d.issueScheduleProxy()` (see below) succeeds, `processQueueEntry` returns the schedule. + 4. If `d.issueScheduleProxy()` fails, `processQueueEntry` consults the `d.retryStrategy`: + * If more retries are allowed, `processQueueEntry` resets the `d.processTimer` for the next attempt (with backoff) and returns `nil` (meaning the task stays in the DLQ). + * If max retries are reached, `processQueueEntry` returns the schedule (even though it failed). + 5. Back in `RunBlocking`, if `exitSched` is not `nil` (meaning success or max retries), the schedule is sent to `d.deadLetterExitChannel`. + +**4. Attempting the Retry (`issueScheduleProxy`)** +This small but vital function defines *what action* the `processQueueEntry` should try: +```go +// From: api/pkg/scheduler/dead_letter_queue.go +func (d *DeadLetterQueueDefault) issueScheduleProxy() actionFunc { + // actionFunc is: func(ctx context.Context, sched *schedule.Schedule) (*schedule.Schedule, error) + return func(ctx context.Context, sched *schedule.Schedule) (*schedule.Schedule, error) { + // Use the DLQ's issuer to try and run the schedule again. + return sched, d.issuer.IssueSchedule(ctx, sched) + } +} +``` +It simply calls `d.issuer.IssueSchedule(ctx, sched)`. If this returns `nil` (no error), the retry was successful! If it returns an error, the retry failed. + +**5. The `RetryStrategy` Interface** +This interface (defined in `dead_letter_queue.go`) is key to the "wait and try again, maybe longer" logic. +```go +// From: api/pkg/scheduler/dead_letter_queue.go +type RetryStrategy interface { + GetNewDuration() (time.Duration, error) // Gets next wait duration. Returns ErrMaxRetriesReached if done. + MustGetNewDuration() time.Duration // Similar, but panics on max retries. + ResetSequence() time.Duration // Resets retry count, returns first duration. +} +``` +The `scheduler` project might use an implementation like "exponential backoff" (e.g., wait 2s, then 4s, then 8s, etc., up to a maximum number of tries). This strategy is passed to `processQueueEntry` which uses it to manage the retry loop for each task. + +## Why is the Failed Task Retry Queue So Important? + +* **Resilience:** It makes the scheduler robust against temporary, transient failures. Many tasks that would otherwise be lost can succeed on a retry. +* **Automatic Recovery:** It automates the retry process. No manual intervention is needed for common temporary issues. +* **Controlled Retries:** The `RetryStrategy` ensures that retries don't happen too aggressively (which could overload a struggling downstream system) and that they don't go on forever. +* **Improved Task Completion Rates:** By giving tasks multiple chances, the overall success rate of task processing increases significantly. + +## Conclusion + +The `scheduler.DeadLetterQueueDefault` is our scheduler's "intensive care unit" for tasks that fail their initial execution. Instead of discarding them, it holds them and intelligently retries them using a `RetryStrategy` that often involves increasing delays (backoff). If a retry succeeds, the task moves on to post-processing. If all retries are exhausted, the task also moves to post-processing, but marked as failed. This mechanism is crucial for building a resilient and reliable scheduling system that can handle the inevitable hiccups of distributed environments. + +We've now covered how schedules are defined, submitted, managed, executed, handled post-execution, and retried upon failure. But how do we keep an eye on the overall health and performance of our scheduler application itself? That's what we'll explore in the final chapter: [Chapter 10: Application Health Monitor (`api.InstrumentationService`)](10_application_health_monitor___api_instrumentationservice___.md). + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/10_application_health_monitor___api_instrumentationservice___.md b/documentation/10_application_health_monitor___api_instrumentationservice___.md new file mode 100644 index 00000000..90a08a68 --- /dev/null +++ b/documentation/10_application_health_monitor___api_instrumentationservice___.md @@ -0,0 +1,335 @@ +# Chapter 10: Application Health Monitor (`api.InstrumentationService`) + +Welcome to the final chapter of our `scheduler` project tutorial! In [Chapter 9: Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md), we explored how our system handles tasks that fail, giving them a second chance through a retry mechanism. We've covered a lot, from defining schedules to executing them and handling various outcomes. Now, let's look at how we can keep an eye on the overall health and performance of our scheduler. + +## The Problem: Knowing How Your Scheduler is Doing + +Imagine you've built a complex machine, like our `scheduler`. It's running, processing tasks, handling retries, and more. But how do you know if it's running smoothly? +* Is it processing tasks quickly, or is there a growing backlog? +* How many tasks are currently waiting in the retry queue? +* Are tasks being successfully executed, or are many ending up deleted after post-processing? + +Without a way to see these "vital signs," troubleshooting problems or understanding the system's load becomes very difficult. It's like driving a car without a dashboard – you wouldn't know your speed, fuel level, or if the engine is overheating until it's too late! + +We need a system that acts like this dashboard, or like a doctor performing a checkup, providing real-time information about how the scheduler is performing. This is crucial for: +* **Observability:** Understanding what's happening inside the system. +* **Monitoring Health:** Quickly identifying if something is wrong. +* **Debugging:** Pinpointing bottlenecks or areas of high error rates. +* **Capacity Planning:** Knowing if the system is handling its current load or if it needs more resources. + +## Meet the `api.InstrumentationService`: Your Scheduler's Dashboard + +The `api.InstrumentationService` is our scheduler's built-in health and performance monitoring system. It provides a way to collect and expose various **metrics** – numerical data points that represent some aspect of the scheduler's operation. + +Think of it like this: +* Different parts of the scheduler (like the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) or the [Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md)) are like sensors in your car. +* These "sensors" report data (e.g., "one task entered the queue," "one task was processed"). +* This data is collected and stored by a component called `instrumentation.MetricsDataStore`. +* The `api.InstrumentationService` then acts as the interface to read this collected data, much like how your car's dashboard displays information from the sensors. + +Other systems, or even human operators, can then query this service (usually via API endpoints) to get a snapshot of the scheduler's current status. + +## Key Concepts of the Health Monitor + +Let's break down the main parts involved: + +1. **Metrics (The "Vital Signs"):** + These are specific counts or values that tell us something about the system. Our `scheduler` tracks various metrics, defined as constants in `api/pkg/instrumentation/supported_metrics.go`: + ```go + // From: api/pkg/instrumentation/supported_metrics.go + package instrumentation + + const ( + MetricDataStoreReadFromTopic = "metrics.dataStore.read" // Schedules read by DailyStore + MetricDataStoreStored = "metrics.dataStore.stored" // Schedules sent to DailyStore's output + MetricDataStoreExited = "metrics.dataStore.exited" // Schedules too old for DailyStore + + MetricDeadLetterQueueEntered = "metrics.dlq.entered" // Tasks entering DLQ + MetricDeadLetterQueueExited = "metrics.dlq.exited" // Tasks leaving DLQ (succeeded/failed) + + MetricPostProcessorEntered = "metrics.postProcessor.entered" // Tasks entering PostProcessor + MetricPostProcessorRescheduled = "metrics.postProcessor.rescheduled" // Tasks rescheduled + MetricPostProcessorDeleted = "metrics.postProcessor.deleted" // Tasks deleted by PostProcessor + ) + ``` + These are just examples. A real system might have many more, like "tasks_successfully_issued," "tasks_failed_permanently," etc. Each metric is a simple counter. + +2. **`instrumentation.MetricsDataStore` (The "Data Collector and Storage"):** + * Found in: `api/pkg/instrumentation/metrics_data_store.go` + * This component is responsible for actually storing the current values of all metrics. + * Other parts of the system (like the `DailyScheduleStoreDefault` or `DeadLetterQueueDefault`) "register" the metrics they care about with the `MetricsDataStore`. + * When an event happens (e.g., a task enters the DLQ), the relevant component tells the `MetricsDataStore` to `Increment()` the corresponding metric (e.g., `MetricDeadLetterQueueEntered`). + * The `MetricsDataStore` keeps all these counts in memory, protected by a mutex for safe access from multiple parts of the program. + +3. **`api.MetricExtractor` (The "Data Retriever Contract"):** + * Found in: `api/instrumentation.go` + * This is an interface that defines how metrics can be extracted (retrieved). + ```go + // From: api/instrumentation.go + package api + // ... + type MetricExtractor interface { + ListSupported(ctx context.Context) []string + Extract(ctx context.Context, metric string) (int, error) + ExtractAll(ctx context.Context) map[string]int + } + ``` + * The `instrumentation.MetricsDataStore` implements this interface, meaning it knows how to list available metrics, get a specific one, or get all of them. + +4. **`api.InstrumentationService` (The "Dashboard Interface"):** + * Found in: `api/instrumentation.go` + * This is the service that other parts of the application (like API handlers) will use to get metric data. + * It's a very simple service; its main job is to delegate requests to a `MetricExtractor` (which is typically our `MetricsDataStore`). + ```go + // From: api/instrumentation.go + package api + // ... + type InstrumentationService interface { + ListSupportedMetrics(ctx context.Context) []string + ExtractMetric(ctx context.Context, metric string) (int, error) + ExtractAllMetrics(ctx context.Context) map[string]int + } + + type InstrumentationServiceDefault struct { // The implementation + extractor MetricExtractor + } + // ... (NewInstrumentationService and method implementations) + ``` + +## How to Use the Health Monitor: Checking the Scheduler's Pulse + +You typically interact with the `api.InstrumentationService` through HTTP API endpoints, which we learned about in [Chapter 2: HTTP API Endpoints & Routing](02_http_api_endpoints___routing_.md). The `scheduler` provides a few endpoints specifically for metrics, usually under a path like `/api/instrumentation/metrics`. + +These endpoints are defined in `api/cmd/scheduler-api/routes/routes.go` and use handler functions from `api/cmd/scheduler-api/instrumentation/metrics.go`. + +Let's see how you might use them with a tool like `curl` (a command-line tool for making HTTP requests): + +**1. Listing Supported Metrics:** +You want to know what metrics are even available. +* **Request:** Send a `GET` request to `/api/instrumentation/metrics` (or a similar path depending on router setup). + ```bash + curl http://your-scheduler-address/api/instrumentation/metrics + ``` +* **Handler:** The `instrumentation.ListSupportedMetrics` handler (from `api/cmd/scheduler-api/instrumentation/metrics.go`) is called. It uses the `InstrumentationService`'s `ListSupportedMetrics` method. + ```go + // From: api/cmd/scheduler-api/instrumentation/metrics.go + func ListSupportedMetrics(s api.InstrumentationService) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + metrics := s.ListSupportedMetrics(r.Context()) // Calls service + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(metrics) // Sends response + } + } + ``` +* **Output (Example JSON):** + ```json + [ + "metrics.dataStore.read", + "metrics.dataStore.stored", + "metrics.dlq.entered", + "metrics.dlq.exited", + "metrics.postProcessor.rescheduled" + ] + ``` + This tells you the names of metrics you can query. + +**2. Getting a Specific Metric:** +You want to know the current value of `metrics.dlq.entered`. +* **Request:** Send a `GET` request to `/api/instrumentation/metrics/{metric_name}`. + ```bash + curl http://your-scheduler-address/api/instrumentation/metrics/metrics.dlq.entered + ``` +* **Handler:** The `instrumentation.GetMetric` handler is called. + ```go + // From: api/cmd/scheduler-api/instrumentation/metrics.go + func GetMetric(s api.InstrumentationService) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + metricName, _ := getMetricName(r) // Extracts "metrics.dlq.entered" + val, err := s.ExtractMetric(r.Context(), metricName) // Calls service + // ... (error handling) ... + json.NewEncoder(w).Encode(val) // Sends value + } + } + ``` +* **Output (Example JSON):** + ```json + 15 + ``` + This means 15 tasks have entered the Dead Letter Queue since the counter was last reset (or since the application started). + +**3. Getting All Metrics:** +You want a snapshot of all current metric values. +* **Request:** Send a `GET` request to `/api/instrumentation/metrics/all` (or similar). + ```bash + curl http://your-scheduler-address/api/instrumentation/metrics/all + ``` +* **Handler:** The `instrumentation.GetAllMetrics` handler is called. + ```go + // From: api/cmd/scheduler-api/instrumentation/metrics.go + func GetAllMetrics(s api.InstrumentationService) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + metricsMap := s.ExtractAllMetrics(r.Context()) // Calls service + // ... + json.NewEncoder(w).Encode(toArray(metricsMap)) // Sends array + } + } + ``` + The `toArray` helper converts the map of metrics into an array of objects for the JSON response. +* **Output (Example JSON):** + ```json + [ + { "name": "metrics.dataStore.read", "value": 1052 }, + { "name": "metrics.dataStore.stored", "value": 1000 }, + { "name": "metrics.dlq.entered", "value": 15 }, + { "name": "metrics.dlq.exited", "value": 12 }, + { "name": "metrics.postProcessor.rescheduled", "value": 980 } + ] + ``` + +This data gives you a rich overview of the scheduler's activity! + +## Under the Hood: How Metrics are Collected and Served + +Let's trace the journey of a metric from its origin to being displayed. + +**Step-by-Step Flow:** + +1. **Registration:** When a component (e.g., `DeadLetterQueueDefault`) starts, it tells the `MetricsDataStore` about the metrics it will manage. + * Example: `DeadLetterQueueDefault` calls `MetricsDataStore.RegisterForManagement("metrics.dlq.entered")`. + * The `MetricsDataStore` initializes this metric to `0`. + * It returns an `MetricIncrementor` and `MetricResetter` to the `DeadLetterQueueDefault`. + +2. **Incrementing:** When an event occurs (e.g., a task enters the `DeadLetterQueueDefault`), it uses its `MetricIncrementor` to update the count. + * Example: `DeadLetterQueueDefault` calls `dlqEnteredIncrementor.Increment()`. + * The `MetricsDataStore` (behind the scenes) safely increases the value of `"metrics.dlq.entered"`. + +3. **Extraction Request:** A user or monitoring tool makes an API call (e.g., `GET /api/instrumentation/metrics/all`). + +4. **Service Call:** The API handler calls the `api.InstrumentationService` (e.g., `ExtractAllMetrics()`). + +5. **Extractor Call:** The `InstrumentationServiceDefault` implementation simply calls its `MetricExtractor` (which is our `MetricsDataStore`). + ```go + // From: api/instrumentation.go + func (i *InstrumentationServiceDefault) ExtractAllMetrics(ctx context.Context) map[string]int { + return i.extractor.ExtractAll(ctx) // Delegates to extractor + } + ``` + +6. **Data Retrieval:** The `MetricsDataStore` retrieves the current values of all its stored metrics. + ```go + // From: api/pkg/instrumentation/metrics_data_store.go + func (s *MetricsDataStore) ExtractAll(ctx context.Context) map[string]int { + s.mutex.RLock() // Read lock for safety + defer s.mutex.RUnlock() + return s.metrics // Returns the internal map + } + ``` + +7. **Response:** The data flows back through the service and API handler to the client. + +**Visualizing the Metric Retrieval Flow:** + +```mermaid +sequenceDiagram + participant Client as External Client/Tool + participant APIEndpoint as /api/instrumentation/metrics/... + participant InstrService as InstrumentationService + participant MDS as MetricsDataStore (as MetricExtractor) + + Client->>APIEndpoint: GET /api/instrumentation/metrics/all + APIEndpoint->>InstrService: ExtractAllMetrics() + InstrService->>MDS: ExtractAll() + MDS-->>InstrService: map[string]int {metric_name: value, ...} + InstrService-->>APIEndpoint: map[string]int {...} + APIEndpoint-->>Client: JSON Response +``` + +### Diving into `MetricsDataStore` + +The `instrumentation.MetricsDataStore` is the heart of metric collection. +File: `api/pkg/instrumentation/metrics_data_store.go` + +1. **Structure:** + ```go + // From: api/pkg/instrumentation/metrics_data_store.go + type MetricsDataStore struct { + metrics map[string]int // Where the counts are stored + mutex sync.RWMutex // For thread-safe access + } + ``` + It's a simple map to hold metric names and their integer values, with a `sync.RWMutex` to prevent issues if multiple parts of the program try to read or write metrics at the same time. + +2. **Registering a Metric:** + ```go + // From: api/pkg/instrumentation/metrics_data_store.go + func (s *MetricsDataStore) RegisterForManagement(metric string) (MetricIncrementor, MetricResetter, error) { + s.mutex.Lock() // Exclusive lock for modification + defer s.mutex.Unlock() + + if _, found := s.metrics[metric]; found { + return nil, nil, fmt.Errorf("metric is already under management") + } + s.metrics[metric] = 0 // Initialize to zero + + // Return functions that can increment/reset this specific metric + inc := func() { s.increment(metric) } + res := func() { s.reset(metric) } + return metricIncrementAction(inc), metricResetAction(res), nil + } + ``` + When a component registers a metric (e.g., `MetricDeadLetterQueueEntered`), the `MetricsDataStore` adds it to its internal `metrics` map (initialized to 0). It then returns two small functions (wrapped as `MetricIncrementor` and `MetricResetter`) that are tied *specifically* to that metric name. + +3. **Incrementing a Metric:** + The `MetricIncrementor` returned by `RegisterForManagement` calls this internal `increment` method: + ```go + // From: api/pkg/instrumentation/metrics_data_store.go + func (s *MetricsDataStore) increment(metric string) { + s.mutex.Lock() // Exclusive lock + defer s.mutex.Unlock() + + val, found := s.metrics[metric] + if !found { return } // Should not happen if registered + s.metrics[metric] = val + 1 + } + ``` + This safely finds the metric in the map and adds 1 to its value. + +### How Other Components Update Metrics + +Components like `DeadLetterQueueDefault` or `DailyScheduleStoreDefault` integrate with this system. +The file `api/pkg/scheduler/metrics.go` shows a helper struct: +```go +// From: api/pkg/scheduler/metrics.go +package scheduler +import "github.com/nestoca/scheduler/api/pkg/instrumentation" + +type metricModifier struct { + instrumentation.MetricIncrementor + instrumentation.MetricResetter +} +``` +A component (e.g., `DeadLetterQueueDefault`) would typically: +1. Have fields for each `MetricIncrementor` it needs (e.g., `dlqEnteredCounter`). +2. During its initialization, it calls `MetricsDataStore.RegisterForManagement()` for each metric it's responsible for and stores the returned `MetricIncrementor`. +3. When a relevant event occurs (e.g., a schedule enters the DLQ), it calls `dlqEnteredCounter.Increment()`. + +This design keeps the `MetricsDataStore` as a central, simple collector, and lets individual components manage their own metric updates cleanly. + +## Why is the Application Health Monitor So Important? + +* **Visibility:** It shines a light into the "black box" of the scheduler, letting you see what it's doing. +* **Proactive Problem Detection:** By monitoring key metrics (e.g., queue sizes, error rates), you can often detect problems before they become critical. +* **Performance Tuning:** Metrics can help identify bottlenecks. If `metrics.dataStore.read` is very high but `metrics.dataStore.stored` is low, it might indicate a problem in the `DailyScheduleStoreDefault`'s filtering or processing. +* **Foundation for Alerting:** Monitoring systems can use these metrics to trigger alerts if values go outside expected ranges (e.g., "DLQ size > 100 for 5 minutes"). + +## Conclusion + +The `api.InstrumentationService`, powered by the `instrumentation.MetricsDataStore` and various `MetricIncrementor`s throughout the application, provides a vital "dashboard" for our `scheduler`. It collects key performance indicators (metrics) like queue lengths, items processed, and errors encountered. This data is then exposed via API endpoints, allowing operators and monitoring tools to observe the system's health, diagnose issues, and understand its performance characteristics. This capability is essential for running any robust application in a production environment. + +Congratulations on completing this tutorial series on the `scheduler` project! You've journeyed through defining tasks with `schedule.Schedule`, exposing functionality via [HTTP API Endpoints & Routing](02_http_api_endpoints___routing_.md), managing schedules with `api.SchedulesService`, understanding the `api.Core` aggregator, seeing the `api.Scheduler` execution engine in action, learning about [Kafka-based Schedule Persistence and Communication](06_kafka_based_schedule_persistence_and_communication_.md), how tasks are organized daily by `scheduler.DailyScheduleStoreDefault`, handled post-execution by `scheduler.SchedulePostProcessorDefault`, retried with `scheduler.DeadLetterQueueDefault`, and finally, how the system's health is monitored. We hope this gives you a solid foundation for understanding and working with the `scheduler`! + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/index.md b/documentation/index.md new file mode 100644 index 00000000..f77583a5 --- /dev/null +++ b/documentation/index.md @@ -0,0 +1,81 @@ +# Tutorial: scheduler + +This project implements a **task scheduling system**. It allows users to define *tasks* (what to do, like sending a Kafka message) and *schedules* (when and how often to do it). These schedules are reliably stored and communicated using **Kafka**. The system's *engine* then picks up due tasks, *executes* them by, for example, sending a message to a target Kafka topic, handles *recurring tasks* by rescheduling them, and manages *failed tasks* through a retry mechanism. It also offers *monitoring* of its operations through metrics. + + +**Source Repository:** [None](None) + +```mermaid +flowchart TD + A0["Core Services Aggregator (`api.Core`) +"] + A1["Schedule Blueprint (`schedule.Schedule`) +"] + A2["Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`) +"] + A3["Schedule Management Service (`api.SchedulesService`) +"] + A4["Kafka-based Schedule Persistence and Communication +"] + A5["Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`) +"] + A6["Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`) +"] + A7["Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`) +"] + A8["Application Health Monitor (`api.InstrumentationService`) +"] + A9["HTTP API Endpoints & Routing +"] + A9 -- "Accesses core services via" --> A0 + A9 -- "Manages schedules via" --> A3 + A9 -- "Exposes metrics from" --> A8 + A9 -- "Controls engine via" --> A2 + A0 -- "Holds instance of" --> A2 + A0 -- "Holds instance of" --> A3 + A0 -- "Holds instance of" --> A8 + A3 -- "Defines/Validates" --> A1 + A3 -- "Writes schedule definitions to" --> A4 + A5 -- "Reads schedule definitions ..." --> A4 + A5 -- "Organizes for daily execution" --> A1 + A2 -- "Gets due tasks from" --> A5 + A2 -- "Triggers execution of" --> A1 + A2 -- "Issues tasks to target topi..." --> A4 + A2 -- "Delegates post-execution to" --> A6 + A6 -- "Processes executed" --> A1 + A6 -- "Updates/Deletes schedule de..." --> A4 + A2 -- "Sends failed tasks to" --> A7 + A7 -- "Queues failed" --> A1 + A7 -- "Re-issues tasks via" --> A4 + A5 -- "Reports metrics to" --> A8 + A6 -- "Reports metrics to" --> A8 + A7 -- "Reports metrics to" --> A8 +``` + +## Chapters + +1. [Schedule Blueprint (`schedule.Schedule`) +](01_schedule_blueprint___schedule_schedule___.md) +2. [HTTP API Endpoints & Routing +](02_http_api_endpoints___routing_.md) +3. [Schedule Management Service (`api.SchedulesService`) +](03_schedule_management_service___api_schedulesservice___.md) +4. [Core Services Aggregator (`api.Core`) +](04_core_services_aggregator___api_core___.md) +5. [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`) +](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) +6. [Kafka-based Schedule Persistence and Communication +](06_kafka_based_schedule_persistence_and_communication_.md) +7. [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`) +](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) +8. [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`) +](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md) +9. [Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`) +](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md) +10. [Application Health Monitor (`api.InstrumentationService`) +](10_application_health_monitor___api_instrumentationservice___.md) + + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file From 144ceea355484908bbd2a5392b090f6e27cef976 Mon Sep 17 00:00:00 2001 From: Louis-David Coulombe Date: Sat, 21 Jun 2025 14:12:53 -0400 Subject: [PATCH 02/15] Moved corebanking doc --- ...hedule_blueprint___schedule_schedule___.md | 280 --------------- .../02_http_api_endpoints___routing_.md | 261 -------------- ...ement_service___api_schedulesservice___.md | 265 -------------- ..._core_services_aggregator___api_core___.md | 205 ----------- ...duler_____scheduler_defaultscheduler___.md | 288 --------------- ...schedule_persistence_and_communication_.md | 314 ---------------- ..._scheduler_dailyschedulestoredefault___.md | 307 ---------------- ...heduler_schedulepostprocessordefault___.md | 248 ------------- ...e___scheduler_deadletterqueuedefault___.md | 232 ------------ ...monitor___api_instrumentationservice___.md | 335 ------------------ documentation/index.md | 81 ----- nesto/corebanking/01_event_.md | 179 ++++++++++ nesto/corebanking/02_command_.md | 179 ++++++++++ nesto/corebanking/03_aggregate_.md | 241 +++++++++++++ nesto/corebanking/04_repository_.md | 229 ++++++++++++ nesto/corebanking/05_api_handler_.md | 225 ++++++++++++ nesto/corebanking/06_core_facade_.md | 160 +++++++++ nesto/corebanking/07_service_.md | 264 ++++++++++++++ nesto/corebanking/08_consumer_.md | 204 +++++++++++ nesto/corebanking/09_product_engine_.md | 265 ++++++++++++++ ...0_simulation_services_and_repositories_.md | 205 +++++++++++ nesto/corebanking/index.md | 78 ++++ 22 files changed, 2229 insertions(+), 2816 deletions(-) delete mode 100644 documentation/01_schedule_blueprint___schedule_schedule___.md delete mode 100644 documentation/02_http_api_endpoints___routing_.md delete mode 100644 documentation/03_schedule_management_service___api_schedulesservice___.md delete mode 100644 documentation/04_core_services_aggregator___api_core___.md delete mode 100644 documentation/05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md delete mode 100644 documentation/06_kafka_based_schedule_persistence_and_communication_.md delete mode 100644 documentation/07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md delete mode 100644 documentation/08_post_execution_handler___scheduler_schedulepostprocessordefault___.md delete mode 100644 documentation/09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md delete mode 100644 documentation/10_application_health_monitor___api_instrumentationservice___.md delete mode 100644 documentation/index.md create mode 100644 nesto/corebanking/01_event_.md create mode 100644 nesto/corebanking/02_command_.md create mode 100644 nesto/corebanking/03_aggregate_.md create mode 100644 nesto/corebanking/04_repository_.md create mode 100644 nesto/corebanking/05_api_handler_.md create mode 100644 nesto/corebanking/06_core_facade_.md create mode 100644 nesto/corebanking/07_service_.md create mode 100644 nesto/corebanking/08_consumer_.md create mode 100644 nesto/corebanking/09_product_engine_.md create mode 100644 nesto/corebanking/10_simulation_services_and_repositories_.md create mode 100644 nesto/corebanking/index.md diff --git a/documentation/01_schedule_blueprint___schedule_schedule___.md b/documentation/01_schedule_blueprint___schedule_schedule___.md deleted file mode 100644 index 7553b2d3..00000000 --- a/documentation/01_schedule_blueprint___schedule_schedule___.md +++ /dev/null @@ -1,280 +0,0 @@ -# Chapter 1: Schedule Blueprint (`schedule.Schedule`) - -Welcome to the `scheduler` project! If you've ever needed a program to do something automatically at a specific time, like sending a daily report or checking for updates every hour, then you're in the right place. This project helps you do just that. - -At the heart of any scheduling system is the need to clearly define *what* task to do, *when* to do it, and *how often* it should repeat. In our `scheduler` project, this detailed instruction sheet is called a **`schedule.Schedule`**. - -Think of `schedule.Schedule` as a **recipe card** or a detailed instruction manual for a task. Just like a recipe tells you the ingredients, steps, and cooking time, a `schedule.Schedule` tells our system everything it needs to know to perform an action correctly and consistently. - -## What Problem Does the `schedule.Schedule` Solve? - -Imagine you're building an e-commerce website. A common feature is to send a reminder email to a customer if they add items to their shopping cart but don't complete the purchase within, say, 24 hours. - -To make this happen automatically, you need to tell the system: -1. **What to do:** Send an email. -2. **To whom:** The specific customer. -3. **With what content:** The reminder message and a link to their cart. -4. **When to do it:** 24 hours after they last updated their cart. -5. **How often:** Just once for that specific abandoned cart. - -The `schedule.Schedule` is the way we package all this information into a neat, understandable format for our `scheduler` system. - -## Meet the `schedule.Schedule`: Your Task's Recipe Card - -Let's break down what goes into this "recipe card." A `schedule.Schedule` primarily defines three things: - -1. **The Task (WHAT):** What action needs to be performed? This usually involves sending some data (a "payload") to a specific destination (a "topic"). -2. **The Timing (WHEN):** When should this task be executed for the first time? This includes the date, time, and timezone. -3. **The Repetition (HOW OFTEN):** Should this task repeat? If so, how frequently (e.g., hourly, daily, weekly, or a custom interval)? - -## Dissecting the Blueprint: Key Ingredients of `schedule.Schedule` - -In our Go codebase, the `schedule.Schedule` is a data structure (a `struct`). Let's look at its main fields. You can find the full definition in `api/pkg/schedule/schedule.go`. - -Here's a simplified view of its structure: - -```go -// Simplified from: api/pkg/schedule/schedule.go -package schedule - -import "time" - -// Schedule holds all details for a task. -type Schedule struct { - // === WHEN to run? === - LocalExecutionTime time.Time // The exact date and time for the task. - TimeLocation string // The timezone for LocalExecutionTime (e.g., "America/New_York"). - - // === HOW OFTEN to repeat? === - Recurrence *Recurrence // Details about repetition (if any). - - // === WHAT task to perform? === - TargetTopic string // Where the task's message/result should go (e.g., "email-sending-service"). - TargetKey string // A specific key for the message (e.g., "user123-reminder"). - TargetPayload []byte // The actual data/content for the task (e.g., the email body). - - // === HOW to identify this blueprint? === - Key string // Your unique name for this schedule definition. - - // ID string // An internal, system-generated ID. You don't usually set this. -} - -// Recurrence defines how a schedule repeats. -type Recurrence struct { - Scheme string // Type of repetition (e.g., "DAILY", "WEEKLY"). - Metadata map[string]string // Extra info for complex schemes (e.g., day of month for "MONTHLY"). -} -``` - -Let's explore these "ingredients": - -* **`LocalExecutionTime` (When):** This is a `time.Time` object in Go. It specifies the exact date and time the task should run. For example, `2024-07-15T10:00:00`. -* **`TimeLocation` (When):** This is a string like `"America/New_York"` or `"UTC"`. It's crucial because `LocalExecutionTime` is interpreted according to this timezone. 10:00 AM in New York is different from 10:00 AM in London! -* **`Recurrence` (How Often):** This points to another small structure, `Recurrence`, which we'll discuss more soon. It tells the system if the task should run just once (`RecurrenceNone`) or repeat (e.g., `RecurrenceDaily`, `RecurrenceWeekly`). -* **`TargetTopic` (What):** Think of this as the address or channel where the outcome of your task is sent. For example, if your task is "send an email," the `TargetTopic` might be something like `"email-notifications"`. Other parts of your system listen to this topic. -* **`TargetKey` (What):** When the task's message is sent to the `TargetTopic`, this `TargetKey` helps identify or organize it. For example, if sending a password reset email, the `TargetKey` might be the `userID`. -* **`TargetPayload` (What):** This is the actual content or data for your task, represented as a slice of bytes (`[]byte`). For an email task, this could be the email's subject and body, perhaps formatted as JSON. -* **`Key` (Identity):** This is a unique string you provide to name your schedule definition. For example, `"monthly-invoice-generation"` or `"user-activity-report-user123"`. You'll use this `Key` if you ever want to update or delete this specific schedule blueprint. -* **`ID` (Identity - Internal):** The `ID` field (commented out in the simplified struct above for clarity, but present in the actual code as `json:"-"`) is typically generated by the system itself (e.g., when you save a schedule using the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md)). It's an internal unique identifier. For recurring tasks, a new `ID` might be generated for each upcoming run derived from the same `Key`. You generally don't set this `ID` yourself when creating a new schedule. - -## Example: Crafting an Abandoned Cart Reminder Blueprint - -Let's go back to our e-commerce example: sending a reminder email for an abandoned cart 24 hours later. Here's how you might define a `schedule.Schedule` for a specific user's abandoned cart (let's say cart ID "cart-xyz-789" for user "customer-567"). - -```go -package main - -import ( - "fmt" - "time" - - // This is the actual package for Schedule struct - "github.com/nestoca/scheduler/api/pkg/schedule" -) - -func createAbandonedCartReminder() *schedule.Schedule { - // Calculate execution time: 24 hours from now - executionTime := time.Now().Add(24 * time.Hour) - - // Define the timezone. Let's use UTC for simplicity. - // In a real app, you might load a specific store's timezone: - // loc, err := time.LoadLocation("America/Denver") - // if err != nil { /* handle error */ } - loc := time.UTC - - // The payload for our task (e.g., what to include in the email) - // Usually, this would be JSON data. - payload := []byte(fmt.Sprintf(`{"userId": "customer-567", "cartId": "cart-xyz-789", "message": "You left items in your cart!"}`)) - - // This is a one-time reminder for this specific cart. - reminderBlueprint := &schedule.Schedule{ - // WHEN - LocalExecutionTime: schedule.DeriveScheduleExecutionTimeFromEntry(executionTime, loc), // Helper to set time with location - TimeLocation: loc.String(), // e.g., "UTC" or "America/Denver" - - // HOW OFTEN (None means it runs only once) - Recurrence: &schedule.Recurrence{ - Scheme: schedule.RecurrenceNone, // This task does not repeat - Metadata: nil, // No extra info needed for RecurrenceNone - }, - - // WHAT - TargetTopic: "cart-reminder-notifications", // Where to send the task info - TargetKey: "cart-xyz-789", // Unique key for this specific cart reminder task - TargetPayload: payload, // The actual data - - // IDENTITY - Key: "abandoned-cart-reminder-cart-xyz-789", // Your unique name for this blueprint - } - - return reminderBlueprint -} - -func main() { - myReminder := createAbandonedCartReminder() - fmt.Printf("Blueprint created for Key: %s at %s in %s\n", - myReminder.Key, - myReminder.LocalExecutionTime.Format(time.RFC3339), - myReminder.TimeLocation) -} -``` - -When you run this (conceptually), you'd create an object in memory. This object is the "recipe card." -The `schedule.DeriveScheduleExecutionTimeFromEntry()` function is a helper from `api/pkg/schedule/schedule.go` that ensures the time is correctly associated with the location, stripping monotonic clock readings if any, which is good practice for scheduled times. - -## Understanding Recurrence: Making Tasks Repeat - -What if you want a task to run every day, or every Monday at 9 AM? That's where the `Recurrence` part of the `schedule.Schedule` comes in. - -The `Recurrence` struct has two main parts: -* `Scheme`: A string indicating the type of repetition. -* `Metadata`: A map for any extra information needed by certain schemes. - -Here are some common recurrence schemes (defined as constants in `api/pkg/schedule/schedule.go`): - -```mermaid -graph TD - RecurrenceTypes["Recurrence Schemes"] -->|Defines repetition| RecurrenceStruct["Recurrence { Scheme, Metadata }"] - - RecurrenceStruct --> RecurrenceNone["RecurrenceNone (Execute Once)"] - RecurrenceStruct --> SimpleRepeats["Simple Repeats"] - SimpleRepeats --> RecurrenceHourly["RecurrenceHourly"] - SimpleRepeats --> RecurrenceDaily["RecurrenceDaily"] - SimpleRepeats --> RecurrenceWeekly["RecurrenceWeekly"] - SimpleRepeats --> RecurrenceBiWeekly["RecurrenceBiWeekly (Every 2 Weeks)"] - SimpleRepeats --> RecurrenceYearly["RecurrenceYearly"] - - RecurrenceStruct --> ComplexRepeats["Complex Repeats (needs Metadata)"] - ComplexRepeats --> RecurrenceMonthly["RecurrenceMonthly"] - RecurrenceMonthly --> RMDay["Metadata: MONTHLY_PAYMENT_DAY (e.g., '15' for 15th)"] - ComplexRepeats --> RecurrenceSemiMonthly["RecurrenceSemiMonthly"] - RecurrenceSemiMonthly --> RSMDays["Metadata: SEMIMONTHLY_PAYMENT_DAYS (e.g., '1_15' for 1st and 15th)"] - ComplexRepeats --> RecurrenceCustom["RecurrenceCustom"] - RecurrenceCustom --> RCDelay["Metadata: CUSTOM_DELTA (e.g., '3h30m' for every 3.5 hours)"] -``` - -**Examples of setting up `Recurrence`:** - -1. **A task that runs daily:** - ```go - dailyRecurrence := &schedule.Recurrence{ - Scheme: schedule.RecurrenceDaily, - // Metadata is usually nil or empty for daily - } - ``` - -2. **A task that runs on the 10th of every month:** - ```go - monthlyRecurrence := &schedule.Recurrence{ - Scheme: schedule.RecurrenceMonthly, - Metadata: map[string]string{ - schedule.RecurrenceMetadataMonthlyPaymentDay: "10", // Run on the 10th - }, - } - ``` - -3. **A task that runs every 2 hours and 30 minutes (custom interval):** - ```go - customRecurrence := &schedule.Recurrence{ - Scheme: schedule.RecurrenceCustom, - Metadata: map[string]string{ - schedule.RecurrenceMetadataCustomDelta: "2h30m", // Standard Go duration string - }, - } - ``` -When you define a recurring schedule, the `LocalExecutionTime` you provide is typically the *first* time the task should run. The system then uses the `Recurrence` settings to figure out all future execution times. - -## How the System Uses This Blueprint - -So you've carefully crafted your `schedule.Schedule` "recipe card." What happens next? - -1. **Submission:** You typically send this blueprint to the `scheduler` system. This often happens by sending it as a JSON payload to an API endpoint. We'll learn more about this in [Chapter 2: HTTP API Endpoints & Routing](02_http_api_endpoints___routing_.md). - - Here's how a part of our example blueprint might look as JSON: - ```json - { - "localExecutionTime": "2024-07-16T10:30:00Z", // Example: Time in ISO 8601 format - "timeLocation": "UTC", - "recurrence": { - "scheme": "NONE", // Matches RecurrenceNone - "metadata": null - }, - "key": "abandoned-cart-reminder-cart-xyz-789", - "targetTopic": "cart-reminder-notifications", - "targetKey": "cart-xyz-789", - "targetPayload": "eyJ1c2VySWQiOiAiY3VzdG9tZXItNTY3IiwgImNhcnRJZCI6ICJjYXJ0LXh5ei03ODkiLCAibWVzc2FnZSI6ICJZb3UgbGVmdCBpdGVtcyBpbiB5b3VyIGNhcnQhIn0=" // Payload is base64 encoded - } - ``` - (Note: `[]byte` payloads are often base64 encoded in JSON.) - - The system decodes this JSON back into a `schedule.Schedule` object. For instance, the `getSchedule` function in `api/cmd/scheduler-api/schedules/save.go` does this: - ```go - // Simplified from api/cmd/scheduler-api/schedules/save.go - func getSchedule(r *http.Request) (*schedule.Schedule, error) { - p := new(schedule.Schedule) - // Decodes the JSON from the request body into our Schedule struct - if err := json.NewDecoder(r.Body).Decode(p); err != nil { - // ... handle error ... - return nil, err - } - return p, nil - } - ``` - -2. **Validation & Storage:** The system, often via a component like the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md), will validate your blueprint. For example, it checks if `TimeLocation` is valid or if `Recurrence` metadata makes sense for the chosen scheme. If valid, it's stored, often in a reliable message queue like Kafka (more in [Chapter 6: Kafka-based Schedule Persistence and Communication](06_kafka_based_schedule_persistence_and_communication_.md)). - - When preparing to store the schedule (e.g., in Kafka), it might be converted into a specific event format. The `toUpsertScheduleEvent` function in `api/pkg/schedule/kafka/producer.go` shows how a `schedule.Schedule` object is transformed into an `UpsertedEvent`, converting `LocalExecutionTime` to a string: - ```go - // Simplified from api/pkg/schedule/kafka/producer.go - func toUpsertScheduleEvent(s *schedule.Schedule) *schedule.UpsertedEvent { - return &schedule.UpsertedEvent{ - ID: s.ID, // System-generated ID - LocalExecutionTime: s.LocalExecutionTime.Format(schedule.ScheduleExecutionTimeFormat), // "2006-01-02 15:04" - TimeLocation: s.TimeLocation, - Recurrence: s.Recurrence, - TargetTopic: s.TargetTopic, - TargetKey: s.TargetKey, - TargetPayload: s.TargetPayload, - } - } - ``` - -3. **Execution:** Later, the core [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) picks up these stored blueprints. When a schedule's `LocalExecutionTime` arrives, the engine triggers the defined task by sending the `TargetPayload` to the `TargetTopic` with the `TargetKey`. If it's a recurring task, the engine also calculates its next execution time. - -## Why is the Blueprint So Important? - -The `schedule.Schedule` blueprint is fundamental because: -* **Clarity:** It provides a clear, unambiguous definition of what needs to be done. -* **Consistency:** It ensures that all tasks are understood and handled uniformly by the system. -* **Automation:** It's the contract that allows the `scheduler` to reliably automate your tasks. -* **Single Source of Truth:** For any given scheduled job, its `schedule.Schedule` definition tells the complete story. - -## Conclusion - -You've now learned about the most basic and crucial concept in our `scheduler` project: the `schedule.Schedule` blueprint. It's the detailed "recipe card" that specifies what task to run, when to run it, and how often it should repeat. Understanding its components (`LocalExecutionTime`, `TimeLocation`, `Recurrence`, `TargetTopic`, `TargetKey`, `TargetPayload`, and `Key`) is key to using the scheduler effectively. - -Now that you understand what a 'Schedule Blueprint' looks like, you're probably wondering how you tell the `scheduler` system about it. How do you submit this recipe card? That's exactly what we'll cover in the next chapter, where we dive into the [HTTP API Endpoints & Routing](02_http_api_endpoints___routing_.md). - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/02_http_api_endpoints___routing_.md b/documentation/02_http_api_endpoints___routing_.md deleted file mode 100644 index 3040ab9b..00000000 --- a/documentation/02_http_api_endpoints___routing_.md +++ /dev/null @@ -1,261 +0,0 @@ -# Chapter 2: HTTP API Endpoints & Routing - -In [Chapter 1: Schedule Blueprint (`schedule.Schedule`)](01_schedule_blueprint___schedule_schedule___.md), we learned all about the `schedule.Schedule` – our "recipe card" for tasks. We know how to define *what* task to run, *when* to run it, and *how often*. But a recipe card sitting on your desk doesn't cook the meal by itself, right? We need a way to submit this recipe to the chef – our `scheduler` system! - -This chapter is all about how we communicate with the `scheduler` from the outside world. How do we send it new schedules? How do we ask it to delete old ones? How do we check if it's feeling okay? The answer is through **HTTP API Endpoints & Routing**. - -## The Problem: Talking to the Scheduler - -Imagine our `scheduler` is a busy office building. Inside, different departments handle various tasks: one department files new schedules, another shreds old ones, and yet another gives updates on the building's operational status. - -If you, as an external client (another program, or even a person using a tool), want to interact with this office, you can't just barge in and shout your request. You need: -1. **Specific Entrances (Endpoints):** You need to know which "door" or "reception desk" to go to for your specific request. Want to submit a new schedule? That's one desk. Want to check the system's health? That's a different desk. -2. **A Directory Service (Routing):** Once you're at the right entrance, someone needs to guide your request to the correct internal department (the code that actually handles it). - -HTTP API Endpoints and Routing provide exactly this structure for web-based services like our `scheduler`. - -## Meet the "Reception Desks": HTTP API Endpoints - -An **HTTP API Endpoint** is like a specific web address (URL) that you can send requests to. Each endpoint is designed for a particular type of interaction. "API" stands for Application Programming Interface – it's a way for programs to talk to each other. "HTTP" is the protocol used on the web, the same one your browser uses to fetch web pages. - -Think of these endpoints as different reception desks in our "scheduler office building": -* A desk for "New Schedule Submissions" -* A desk for "Schedule Deletions" -* A desk for "System Status Inquiries" - -When you send a request to one of these endpoints, you're essentially handing over a form (your data, like a `schedule.Schedule` in JSON format) or asking a question. - -## The "Directory Service": Routing - -Once your request arrives at the `scheduler`'s main entrance (its network address), **Routing** is the process that looks at the specific endpoint you targeted (e.g., `/api/schedule` or `/api/status`) and the type of request you made (e.g., "create," "delete," "get information") and directs it to the correct "internal department" – a specific piece of code called a **handler function**. - -In our `scheduler` project, a function named `routes.GetRouter` is responsible for setting up all these "reception desks" and the "directory service." It acts like the building manager who decides where each desk goes and trains the receptionists (the router itself) to send visitors to the right place. - -## The "Office Workers": Handler Functions - -A **handler function** is a piece of Go code that actually does the work for a specific request. For example: -* If you send a request to create a new schedule, the router will pass it to a handler like `schedules.Save`. This handler takes the schedule details you sent, validates them, and then probably uses the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md) to store it. -* If you ask for the system status, the router directs your request to a handler like `status.Get`. This handler checks the system's health, perhaps using the [Application Health Monitor (`api.InstrumentationService`)](10_application_health_monitor___api_instrumentationservice___.md), and sends back a response. - -These handlers often interact with a central coordinator called the [Core Services Aggregator (`api.Core`)](04_core_services_aggregator___api_core___.md) to get access to various services they need to fulfill the request. - -## Our Main Use Case: Submitting a Daily Report Schedule - -Let's say you've crafted a `schedule.Schedule` blueprint for a daily report, as we discussed in Chapter 1. Now, you want to submit it to the `scheduler`. You'll do this by sending an HTTP request to a specific endpoint. - -**HTTP Methods: The Verbs of Your Request** - -When you make an HTTP request, you also specify an "HTTP method." These are like verbs telling the server what kind of action you want to perform on the resource identified by the endpoint. Common ones include: -* `POST`: Typically used to create a new resource (e.g., submit a new schedule). -* `GET`: Used to retrieve information (e.g., get system status, get a list of schedules). -* `DELETE`: Used to remove a resource (e.g., delete an existing schedule). -* `PUT`: Often used to update an existing resource entirely. - -**Submitting Our Schedule (POST Request)** - -To submit your new daily report schedule, you would typically send an HTTP `POST` request to an endpoint like `/api/schedule`. The body of this request would contain your `schedule.Schedule` blueprint, usually in JSON format. - -Let's imagine our schedule blueprint in JSON (simplified): -```json -// This is what we want to send -{ - "key": "daily-sales-report", - "localExecutionTime": "2024-08-01T09:00:00Z", - "timeLocation": "UTC", - "recurrence": { "scheme": "DAILY" }, - "targetTopic": "reports-generation", - "targetPayload": "eyJhY3Rpb24iOiAiZ2VuZXJhdGVfc2FsZXNfcmVwb3J0In0=" // Base64 for: {"action": "generate_sales_report"} -} -``` - -You could use a command-line tool like `curl` to send this (don't worry if `curl` is new to you, it's just a way to send HTTP requests): -```bash -# Example using curl (conceptual) -curl -X POST \ - http://your-scheduler-address/api/schedule \ - -H "Content-Type: application/json" \ - -d '{ - "key": "daily-sales-report", - "localExecutionTime": "2024-08-01T09:00:00Z", - "timeLocation": "UTC", - "recurrence": { "scheme": "DAILY" }, - "targetTopic": "reports-generation", - "targetPayload": "eyJhY3Rpb24iOiAiZ2VuZXJhdGVfc2FsZXNfcmVwb3J0In0=" - }' -``` -* `-X POST`: Specifies the HTTP POST method. -* `http://your-scheduler-address/api/schedule`: The endpoint URL. -* `-H "Content-Type: application/json"`: Tells the server we're sending JSON data. -* `-d '{...}'`: The actual JSON data (our schedule blueprint). - -**What Happens Next?** -1. The `scheduler`'s web server receives this `POST` request to `/api/schedule`. -2. The router sees `/api/schedule` and `POST`, and knows to send it to the `schedules.Save` handler function. -3. The `schedules.Save` handler (which we'll see more of later) reads the JSON data, converts it into a `schedule.Schedule` Go object, and then uses the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md) to save it. -4. If successful, the `scheduler` sends back an HTTP response, typically `201 Created`. - -**Other Common Interactions:** - -* **Deleting a Schedule (DELETE Request):** - To delete the "daily-sales-report" schedule, you might send: - ```bash - curl -X DELETE http://your-scheduler-address/api/schedule/daily-sales-report - ``` - The router would direct this to the `schedules.Delete` handler. If successful, you'd get a `200 OK` response. - -* **Checking System Status (GET Request):** - To check if the scheduler is running and its version: - ```bash - curl -X GET http://your-scheduler-address/api/status - ``` - The router would send this to the `status.Get` handler. You might get back a JSON response like: - ```json - { - "status": 200, - "time": "2024-07-15T12:30:00Z", - "version": "v1.2.3" - } - ``` - -## Under the Hood: How Routing Works in `scheduler` - -Let's peek into how the `scheduler` sets up these routes. The main file to look at is `api/cmd/scheduler-api/routes/routes.go`. - -**1. The Main Router Setup (`GetRouter`)** - -The `GetRouter` function is the starting point. It creates a new router object (using a popular Go library called `chi`) and configures it. - -```go -// Simplified from: api/cmd/scheduler-api/routes/routes.go -package routes - -import ( - // ... other imports - "github.com/go-chi/chi" - "github.com/nestoca/scheduler/api" - // ... handlers like status, schedules -) - -func GetRouter(c *api.Core) chi.Router { - r := chi.NewRouter() // Creates a new router instance - - // r.Use(...) // Applies some standard "middleware" (explained below) - r.Mount("/api", getAPIRoutes(c)) // All our API routes will start with /api - - return r -} -``` -* `chi.NewRouter()`: This creates the main router object. -* `r.Mount("/api", getAPIRoutes(c))`: This is key! It says that any URL starting with `/api` should be handled by another set of routes defined in the `getAPIRoutes` function. The `c *api.Core` is an instance of the [Core Services Aggregator (`api.Core`)](04_core_services_aggregator___api_core___.md), which provides access to all necessary services for the handlers. - -The `r.Use(...)` line (commented for brevity above, but present in the actual code) sets up "middleware." Middleware are like checkpoints that every request passes through before reaching the final handler. They can do things like log the request, check for authentication, or add information to the request. - -**2. Defining API-Specific Routes (`getAPIRoutes`)** - -The `getAPIRoutes` function defines the actual endpoints like `/status`, `/schedule`, etc. - -```go -// Simplified from: api/cmd/scheduler-api/routes/routes.go -func getAPIRoutes(core *api.Core) http.Handler { - r := chi.NewRouter() // A sub-router for /api paths - - // Status endpoint - r.Get("/status", status.Get(core.HealthService)) - - // Schedule endpoints - r.Route("/schedule", func(r chi.Router) { - // POST /api/schedule -> creates a new schedule - r.Post("/", schedules.Save(core.SchedulesService)) - // DELETE /api/schedule/{key} -> deletes a schedule by its key - r.Delete("/{key}", schedules.Delete(core.SchedulesService)) - // ... other schedule-related routes - }) - - // ... routes for instrumentation/metrics ... - - return r -} -``` -Let's break this down: -* `r.Get("/status", status.Get(core.HealthService))`: - * This says: If a `GET` request comes to `/api/status` (remember the `/api` prefix from `Mount`), then call the handler function returned by `status.Get(...)`. - * `status.Get` is a function (from `api/cmd/scheduler-api/status/get.go`) that takes the `core.HealthService` (an instance of [Application Health Monitor (`api.InstrumentationService`)](10_application_health_monitor___api_instrumentationservice___.md)) and returns the actual handler function. -* `r.Route("/schedule", func(r chi.Router) { ... })`: - * This groups all routes that start with `/api/schedule`. - * `r.Post("/", schedules.Save(core.SchedulesService))`: For a `POST` request to `/api/schedule/` (the `/` here means the base of the `/schedule` route), use the handler from `schedules.Save(...)`. This handler gets the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md) from the `core`. - * `r.Delete("/{key}", schedules.Delete(core.SchedulesService))`: For a `DELETE` request to `/api/schedule/some-schedule-key`, use the `schedules.Delete(...)` handler. The `{key}` part is a placeholder for the actual schedule key (e.g., "daily-sales-report"). The handler can extract this key. - -**A Visual Flow of a Request:** - -Here's how a request to create a schedule flows through the system: - -```mermaid -sequenceDiagram - participant Client as External Client - participant WebServer as Scheduler Web Server (Chi) - participant Router as Main Router (from GetRouter) - participant APIRouter as /api Sub-Router (from getAPIRoutes) - participant Handler as schedules.Save Handler - participant Svc as api.SchedulesService - - Client->>WebServer: POST /api/schedule (JSON data) - WebServer->>Router: Request for /api/schedule - Router->>APIRouter: Mounts to /api, passes to APIRouter for /schedule - APIRouter->>Handler: Matches POST / and calls schedules.Save - Handler->>Svc: Processes JSON, calls SaveSchedule() - Svc-->>Handler: Returns success/error - Handler-->>WebServer: Sends HTTP Response (e.g., 201 Created) - WebServer-->>Client: Delivers HTTP Response -``` - -**3. Inside a Handler Function (e.g., `schedules.Save`)** - -Let's look at a simplified piece of a handler function, like `schedules.Save` from `api/cmd/scheduler-api/schedules/save.go`: - -```go -// Simplified from: api/cmd/scheduler-api/schedules/save.go -package schedules - -// ... imports ... -import ( - "github.com/nestoca/scheduler/api" - "github.com/nestoca/scheduler/api/pkg/schedule" -) - -func Save(s api.SchedulesService) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - // 1. Get the schedule data from the request body - scheduleBlueprint, err := getSchedule(r) - if err != nil { - // ... send an error response ... - return - } - - // 2. Use the SchedulesService to save it - err = s.SaveSchedule(r.Context(), scheduleBlueprint) - if err != nil { - // ... send an error response ... - return - } - - // 3. Send a success response - w.WriteHeader(http.StatusCreated) // 201 Created - } -} -``` -* `Save(s api.SchedulesService)`: This function takes the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md) as an argument (provided by `core.SchedulesService` when setting up the route) and returns the actual handler function `func(w http.ResponseWriter, r *http.Request)`. -* `getSchedule(r)`: This helper function (also in `save.go`, shown in Chapter 1 context) reads the JSON data from the request body (`r.Body`) and decodes it into a `*schedule.Schedule` object. -* `s.SaveSchedule(...)`: This is where the handler calls the actual service responsible for saving the schedule. -* `w.WriteHeader(http.StatusCreated)`: If everything is successful, it sends back an HTTP status code `201 Created` to the client. - -The other handlers like `schedules.Delete` or `status.Get` follow similar patterns: they receive the request, potentially extract information from it (like a URL parameter or request body), call the appropriate service from the `api.Core`, and then formulate an HTTP response. - -## Conclusion - -You've now seen how the `scheduler` opens its doors to the outside world! **HTTP API Endpoints** are the specific "reception desks" for different types of requests (create schedule, delete schedule, get status). **Routing** is the "directory service" (`routes.GetRouter` and `chi` router) that ensures your request reaches the correct **handler function** (like `schedules.Save` or `status.Get`). These handlers then do the actual work, often by using services provided by the [Core Services Aggregator (`api.Core`)](04_core_services_aggregator___api_core___.md). - -So, we can create a `schedule.Schedule` blueprint (Chapter 1) and we know how to send it to the scheduler via an HTTP API (this chapter). But what exactly happens when `schedules.Save` calls `s.SaveSchedule()`? What does the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md) do with our blueprint? That's what we'll explore in the next chapter! - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/03_schedule_management_service___api_schedulesservice___.md b/documentation/03_schedule_management_service___api_schedulesservice___.md deleted file mode 100644 index 6176106c..00000000 --- a/documentation/03_schedule_management_service___api_schedulesservice___.md +++ /dev/null @@ -1,265 +0,0 @@ -# Chapter 3: Schedule Management Service (`api.SchedulesService`) - -In [Chapter 2: HTTP API Endpoints & Routing](02_http_api_endpoints___routing_.md), we saw how our `scheduler` system receives requests from the outside world, like a request to create a new schedule. We learned that an API handler function, such as `schedules.Save`, takes the schedule details and calls a service to do the actual work of saving it. That service is the star of this chapter: the **`api.SchedulesService`**. - -So, you've handed your `schedule.Schedule` blueprint (your "recipe card" from [Chapter 1: Schedule Blueprint (`schedule.Schedule`)](01_schedule_blueprint___schedule_schedule___.md)) to the API. What happens next? How does the system ensure this blueprint is valid and store it safely? - -## The Problem: Managing the "Recipe Card Collection" - -Imagine our `scheduler` system is a grand library for task recipes (our `schedule.Schedule` blueprints). We can't just throw new recipes onto a pile! We need a librarian or an administrator to: -1. **Check the recipe:** Is it complete? Does it make sense (e.g., is the cooking time in a valid timezone)? -2. **File it properly:** Store it in a way that it can be found and used later. -3. **Update it if needed:** If we change a recipe, the old one needs to be updated. -4. **Remove it when done:** If a recipe is no longer needed, it should be discarded. -5. **Help find future cooking times:** If it's a recipe that's made regularly (a recurring schedule), help figure out when it's next due. - -The `api.SchedulesService` is this librarian or administrator for all our "schedule blueprints." It's the central point for managing the lifecycle of these `schedule.Schedule` objects. - -## Meet the `api.SchedulesService`: Your Schedule Librarian - -The `api.SchedulesService` is a core component in our `scheduler`. It's an interface (a contract defining what it can do) and has a default implementation (`SchedulesServiceDefault`). Its main job is to provide a clean way for other parts of the system (especially the API handlers) to manage schedule definitions. - -Here are its key responsibilities: - -* **Creating New Schedules:** When you want to add a new task, you give the `schedule.Schedule` blueprint to this service. -* **Validating Schedule Details:** Before saving, it checks if the blueprint is valid. For example: - * Is the `TimeLocation` (timezone) a real, recognized timezone? - * If it's a recurring schedule (e.g., "monthly"), are the recurrence rules sensible (e.g., does it specify which day of the month)? - It uses a helper called a `LocationProvider` to check timezones. -* **Saving Schedules Persistently:** Once validated, it saves the schedule. "Persistently" means it's stored somewhere durable, so it won't be lost if the program restarts. This is typically done via another helper called a `ScheduleWriter`, which often writes to a reliable system like Kafka (more on this in [Chapter 6: Kafka-based Schedule Persistence and Communication](06_kafka_based_schedule_persistence_and_communication_.md)). -* **Calculating Next Execution Times:** For a given schedule (especially a recurring one), it can calculate when it should run next. -* **Deleting Schedules:** When a task is no longer needed, this service handles its removal. - -Let's look at the `SchedulesService` interface defined in `api/schedules.go`: -```go -// Simplified from: api/schedules.go -package api - -import ( - "context" - "time" - "github.com/nestoca/scheduler/api/pkg/schedule" -) - -// SchedulesService abstract representation of a SchedulesService -type SchedulesService interface { - SaveSchedule(ctx context.Context, schedule *schedule.Schedule) error - CalculateNextLocalExecutionTime(ctx context.Context, currentSchedule *schedule.Schedule) (time.Time, error) - DeleteSchedule(ctx context.Context, scheduleKey string) error - // Shutdown() error // For gracefully stopping the service -} -``` -This tells us that any `SchedulesService` must know how to `SaveSchedule`, `CalculateNextLocalExecutionTime`, and `DeleteSchedule`. - -## How the API Uses the `SchedulesService` - -Let's revisit our use case from Chapter 2: submitting a new schedule for a daily report. The API handler `schedules.Save` receives the JSON blueprint, converts it into a `schedule.Schedule` object, and then calls `SchedulesService.SaveSchedule()`. - -Here's a simplified snippet of how `schedules.Save` (from `api/cmd/scheduler-api/schedules/save.go`) uses the service: -```go -// Simplified from api/cmd/scheduler-api/schedules/save.go -package schedules - -import ( - "net/http" - "github.com/nestoca/scheduler/api" // Where SchedulesService lives - "github.com/nestoca/scheduler/api/pkg/schedule" -) - -func Save(s api.SchedulesService) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - // 1. Get the schedule blueprint from the HTTP request - scheduleBlueprint, err := getSchedule(r) // Decodes JSON to schedule.Schedule - if err != nil { /* handle error */ return } - - // 2. Use the SchedulesService to save it! - err = s.SaveSchedule(r.Context(), scheduleBlueprint) - if err != nil { /* handle error */ return } - - // 3. Send a success response (e.g., HTTP 201 Created) - w.WriteHeader(http.StatusCreated) - } -} -``` -The `s` variable here is an instance of `api.SchedulesService`. The handler doesn't need to know *how* the schedule is validated or stored; it just trusts the `SchedulesService` to do it correctly. - -**Input to `SaveSchedule`:** -* `r.Context()`: A context object (common in Go for managing request lifecycles, timeouts, etc.). -* `scheduleBlueprint`: The `*schedule.Schedule` object we want to save. - -**What happens (at a high level) when `s.SaveSchedule()` is called:** -1. The `SchedulesService` validates the `scheduleBlueprint`. -2. If valid, it assigns a unique ID to this version of the schedule. -3. It then tells its `ScheduleWriter` to persist this blueprint. -4. If any step fails (e.g., validation error, storage error), it returns an error. Otherwise, it returns `nil` (success). - -## Under the Hood: A Look Inside `SchedulesServiceDefault` - -The `scheduler` project provides `SchedulesServiceDefault` as the standard implementation of the `SchedulesService` interface. It's defined in `api/schedules.go`. - -**1. What's Inside `SchedulesServiceDefault`?** - -It needs two helpers to do its job: -* `writer ScheduleWriter`: This is responsible for actually writing the schedule data to a persistent store (like Kafka). -* `locationProvider LocationProvider`: This helps validate timezones. - -```go -// Simplified from: api/schedules.go -package api - -// ... (ScheduleWriter and LocationProvider interfaces are also in this file) ... - -// SchedulesServiceDefault default implementation of SchedulesService -type SchedulesServiceDefault struct { - writer ScheduleWriter - locationProvider LocationProvider -} - -// NewSchedulesService creates a new SchedulesService -func NewSchedulesService(writer ScheduleWriter, locationProvider LocationProvider) *SchedulesServiceDefault { - // ... (nil check) ... - return &SchedulesServiceDefault{ - writer: writer, - locationProvider: locationProvider, - } -} -``` -When an instance of `SchedulesServiceDefault` is created (usually when the application starts, as part of setting up the [Core Services Aggregator (`api.Core`)](04_core_services_aggregator___api_core___.md)), it's given these two helpers. - -**2. The `SaveSchedule` Method in Detail** - -Let's follow the journey of a `schedule.Schedule` blueprint when `SaveSchedule` is called: - -```mermaid -sequenceDiagram - participant Handler as API Handler (schedules.Save) - participant Svc as SchedulesServiceDefault - participant Validator as s.isValid() method - participant LocProv as LocationProvider - participant Writer as ScheduleWriter - - Handler->>Svc: SaveSchedule(ctx, blueprint) - Svc->>Validator: isValid(blueprint)? - Validator->>LocProv: Valid(blueprint.TimeLocation)? - LocProv-->>Validator: true (e.g., "America/New_York" is valid) - Validator-->>Svc: validation result (ok) - Svc->>Svc: Generates unique blueprint.ID - Svc->>Writer: Save(ctx, blueprintWithID) - Writer-->>Svc: Save successful - Svc-->>Handler: Returns success (nil error) -``` - -Now, let's look at the simplified code for `SaveSchedule` in `api/schedules.go`: -```go -// Simplified from: api/schedules.go -func (s SchedulesServiceDefault) SaveSchedule(ctx context.Context, sch *schedule.Schedule) error { - if sch == nil { - return fmt.Errorf("supplied no schedule") - } - - // Step 1: Validate the schedule details - if err := s.isValid(sch); err != nil { - return err // If invalid, return the error - } - - // Step 2: Generate a unique ID for this schedule instance. - // This ID might change for each run of a recurring schedule. - sch.ID = uuid.NewString() - - // Step 3: Pass it to the writer to be saved. - return s.writer.Save(ctx, sch) -} -``` -* First, it checks if the schedule (`sch`) itself is provided. -* Then, it calls its own private method `s.isValid(sch)` to perform validation. -* If valid, it assigns a new unique `ID` (using `uuid.NewString()`) to the `sch.ID` field. This `ID` is an internal system identifier, different from the `Key` you provide. For recurring schedules, each future instance might get a new `ID`. -* Finally, it calls `s.writer.Save(ctx, sch)`, delegating the actual storage to the `ScheduleWriter`. - -**3. The Validation Logic: `isValid()`** - -The `isValid()` method (also in `SchedulesServiceDefault`) is crucial. It checks various parts of the `schedule.Schedule`. - -```go -// Simplified from: api/schedules.go -func (s SchedulesServiceDefault) isValid(sched *schedule.Schedule) error { - // Check 1: Is the TimeLocation valid? - if !s.locationProvider.Valid(sched.TimeLocation) { - return e.NewInvalidParameters("schedule time location") // Error if not valid - } - - if sched.Recurrence == nil { - return nil // No recurrence to validate - } - - // Check 2: Are recurrence rules valid? (Example for monthly) - switch sched.Recurrence.Scheme { - case schedule.RecurrenceMonthly: - dayStr, found := sched.Recurrence.Metadata[schedule.RecurrenceMetadataMonthlyPaymentDay] - if !found { - return e.NewInvalidParameters("missing day for monthly recurrence") - } - // ... (code to check if dayStr is a valid number 1-31) ... - // ... (other cases for RecurrenceDaily, RecurrenceWeekly, etc.) ... - case schedule.RecurrenceCustom: - // ... (check if custom delta is a valid duration string) ... - } - return nil // All good! -} -``` -* It uses the `s.locationProvider` to confirm that `sched.TimeLocation` (e.g., `"America/New_York"`) is a recognized timezone. -* If `sched.Recurrence` is set, it checks the `Scheme` and `Metadata`. For example, if `Scheme` is `schedule.RecurrenceMonthly`, it expects `Metadata` to contain `schedule.RecurrenceMetadataMonthlyPaymentDay` with a valid day number (1-31). - -If any check fails, `isValid()` returns an error, which then `SaveSchedule` returns to the API handler, and ultimately back to the client who tried to save the schedule. - -**4. Deleting a Schedule: `DeleteSchedule()`** - -Deleting is simpler. The API handler (`schedules.Delete`) gets the `scheduleKey` from the URL, and calls `SchedulesService.DeleteSchedule()`: - -```go -// Simplified from: api/schedules.go -func (s SchedulesServiceDefault) DeleteSchedule(ctx context.Context, scheduleKey string) error { - if len(scheduleKey) == 0 { - return fmt.Errorf("supplied invalid key") - } - // Just pass the key to the writer to handle deletion. - return s.writer.Delete(ctx, scheduleKey) -} -``` -The `SchedulesService` mostly just delegates this to its `ScheduleWriter`. The `ScheduleWriter` would then be responsible for telling the persistence layer (e.g., Kafka) to mark this schedule (identified by its `scheduleKey`) for deletion. - -**5. Calculating Next Execution Time: `CalculateNextLocalExecutionTime()`** - -This method helps figure out when a schedule should run next, based on its current definition and recurrence rules. - -```go -// Simplified from: api/schedules.go -func (s SchedulesServiceDefault) CalculateNextLocalExecutionTime( - ctx context.Context, - currentSchedule *schedule.Schedule, -) (time.Time, error) { - if currentSchedule == nil { /* error */ } - if err := s.isValid(currentSchedule); err != nil { /* error */ } // Must be valid - - // Delegates to a dedicated calculator function - return scheduler.CalculateNextLocalExecutionTime(ctx, currentSchedule, s.locationProvider) -} -``` -* It first validates the provided `currentSchedule`. -* Then, it calls a specialized function `scheduler.CalculateNextLocalExecutionTime` (from `api/pkg/scheduler/calculator.go`, not shown here). This function contains the logic to interpret `LocalExecutionTime`, `TimeLocation`, and `Recurrence` rules, using the `locationProvider` for timezone calculations, to determine the next run time. This is very useful for recurring tasks. - -## Why is the `SchedulesService` So Important? - -* **Centralized Logic:** It keeps all the rules about what makes a schedule valid and how to manage its lifecycle in one place. -* **Abstraction:** API Handlers (and other potential clients of this service) don't need to worry about the nitty-gritty details of validation or how schedules are stored. They just talk to the `SchedulesService`. -* **Testability:** Because it's a well-defined interface, it's easier to test this component in isolation. -* **Flexibility:** If we wanted to change *how* schedules are stored (e.g., move from Kafka to a database), we'd primarily change the `ScheduleWriter` implementation. The `SchedulesService` interface and its interaction with API handlers could remain largely the same. - -## Conclusion - -The `api.SchedulesService` acts as the diligent administrator for all our `schedule.Schedule` blueprints. It's responsible for receiving new schedule definitions, validating them thoroughly (using helpers like a `LocationProvider`), ensuring they are saved persistently (via a `ScheduleWriter`), calculating their future execution times, and handling their deletion. This service ensures that our "library of recipes" is well-organized, correct, and ready for the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) to use. - -We've seen that the `SchedulesService` relies on helpers like `ScheduleWriter` and `LocationProvider`. But how does it get these helpers? And how do the API handlers from Chapter 2 get an instance of `SchedulesService`? This is where the [Core Services Aggregator (`api.Core`)](04_core_services_aggregator___api_core___.md) comes in, which we'll explore in the next chapter! - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/04_core_services_aggregator___api_core___.md b/documentation/04_core_services_aggregator___api_core___.md deleted file mode 100644 index 9d107444..00000000 --- a/documentation/04_core_services_aggregator___api_core___.md +++ /dev/null @@ -1,205 +0,0 @@ -# Chapter 4: Core Services Aggregator (`api.Core`) - -In [Chapter 3: Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md), we learned how the `SchedulesService` acts as a librarian for our schedule blueprints. We saw it validates and saves schedules, often relying on other helpers. You might have wondered: how do different parts of our `scheduler` application, like the API handlers from [Chapter 2: HTTP API Endpoints & Routing](02_http_api_endpoints___routing_.md), get access to the `SchedulesService` or other essential tools? This chapter introduces the component that makes this possible: the **`api.Core`**. - -## The Problem: Accessing Specialized Tools - -Imagine you're building a complex machine, like a car. The car has many specialized systems: an engine, brakes, a steering system, and an electronics system for the radio and dashboard. For the car to work, these systems need to be connected and accessible. The driver (representing a user or another part of our application) needs a way to use these systems without knowing every single detail of how they are built. They just need a steering wheel, an accelerator pedal, and a brake pedal. - -In our `scheduler` application, we have several specialized "tools" or "services": -* The `SchedulesService` to manage schedule definitions. -* A `HealthService` to check if the application is running correctly. -* An `InstrumentationService` to gather performance metrics. -* The `Scheduler` engine itself, which actually runs the tasks. - -How can components like our API handlers easily get a reference to the specific service they need? If every handler had to create these services itself, it would be messy and inefficient. We need a central place that holds all these tools and makes them available. - -## Meet `api.Core`: The Scheduler's Main Toolbox - -Think of `api.Core` as the main **toolbox** or **control panel** for our `scheduler` application. It's a Go struct that doesn't perform tasks directly. Instead, its primary job is to **hold and provide access to all the essential, specialized services** needed for the application to operate. - -It's the central hub that connects various parts of the application. Components like the web API handlers (which we met in Chapter 2) can ask `api.Core` for the specific service they need to perform their job. - -Here's a look at the structure of `api.Core` from `api/core.go`: -```go -// Simplified from: api/core.go -package api - -// Core represents the entrypoint to call the business logic. -type Core struct { - HealthService HealthService - InstrumentationService InstrumentationService - Scheduler Scheduler - SchedulesService SchedulesService -} -``` -As you can see, an `api.Core` object simply contains fields for each of the major services: -* `HealthService`: Used for health checkups (e.g., for the `/api/status` endpoint). -* `InstrumentationService`: Used for collecting performance metrics ([Application Health Monitor (`api.InstrumentationService`)](10_application_health_monitor___api_instrumentationservice___.md)). -* `Scheduler`: This is the actual [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) responsible for running scheduled tasks. -* `SchedulesService`: The [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md) we learned about in the previous chapter. - -## How `api.Core` is Used: Connecting Handlers to Services - -Let's revisit how our API routes and handlers are set up. In [Chapter 2: HTTP API Endpoints & Routing](02_http_api_endpoints___routing_.md), we saw a function `routes.GetRouter` that configures all the API endpoints. This function receives an `*api.Core` instance. - -Here's a simplified snippet from `api/cmd/scheduler-api/routes/routes.go` showing how `api.Core` is used: -```go -// Simplified from: api/cmd/scheduler-api/routes/routes.go -package routes - -import ( - "github.com/go-chi/chi" - "github.com/nestoca/scheduler/api" // Where api.Core is defined - // ... imports for status, schedules handlers -) - -// GetRouter configures routes using a core. -func GetRouter(c *api.Core) chi.Router { - r := chi.NewRouter() - // ... - r.Mount("/api", getAPIRoutes(c)) // Pass core to API route setup - return r -} -``` -The `GetRouter` function takes `c *api.Core` as an argument. This `c` is our "toolbox." It's then passed to `getAPIRoutes`. - -Inside `getAPIRoutes`, the individual services from the `api.Core` instance are given to the specific handlers that need them: -```go -// Simplified from: api/cmd/scheduler-api/routes/routes.go -func getAPIRoutes(core *api.Core) http.Handler { - r := chi.NewRouter() - - // Status endpoint needs the HealthService - r.Get("/status", status.Get(core.HealthService)) - - // Schedule endpoints need the SchedulesService - r.Route("/schedule", func(r chi.Router) { - r.Post("/", schedules.Save(core.SchedulesService)) - r.Delete("/{key}", schedules.Delete(core.SchedulesService)) - // ... other schedule routes ... - }) - - return r -} -``` -Look closely: -* `status.Get(core.HealthService)`: The `status.Get` handler function is created and given `core.HealthService` (the `HealthService` tool from our `api.Core` toolbox). -* `schedules.Save(core.SchedulesService)`: The `schedules.Save` handler function is given `core.SchedulesService` (the `SchedulesService` tool from the toolbox). - -This way, the `schedules.Save` handler doesn't need to know how to create or find a `SchedulesService`; it's simply handed one. The `api.Core` acts as a central provider. - -## Under the Hood: Creating and Populating `api.Core` - -So, where does this `api.Core` instance come from? How is the toolbox filled with its tools? - -Typically, the `api.Core` instance is created once when the application starts up. There's a special function, `FromEnv()`, located in `api/pkg/core/core.go`, that is responsible for building all the necessary services and then packaging them into an `api.Core` struct. - -**A Visual Analogy:** - -Imagine a workshop preparing for a big project. -1. First, specialized tools are forged or assembled (this is like `FromEnv()` creating each service like `HealthService`, `SchedulesService`, etc.). -2. Then, all these tools are placed into a central toolbox (`api.Core`). -3. When a worker (an API handler) needs a tool, they get it from this central toolbox. - -```mermaid -graph TD - subgraph "Initialization (e.g., in api/pkg/core/core.go's FromEnv)" - S1[HealthService instance] - S2[InstrumentationService instance] - S3[Scheduler Engine instance] - S4[SchedulesService instance] - end - - Toolbox["api.Core (The Main Toolbox)"] - style Toolbox fill:#ccf,stroke:#333,stroke-width:2px - - S1 --> Toolbox - S2 --> Toolbox - S3 --> Toolbox - S4 --> Toolbox - - subgraph "Usage by API Handlers" - SaveHandler["schedules.Save Handler"] - StatusHandler["status.Get Handler"] - end - - Toolbox -->|Provides SchedulesService| SaveHandler - Toolbox -->|Provides HealthService| StatusHandler -``` - -**The `FromEnv()` Function (Highly Simplified)** - -The `FromEnv()` function in `api/pkg/core/core.go` does a lot of work to initialize each service. This often involves reading configuration from environment variables (hence "FromEnv"), setting up connections to things like Kafka (which we'll see in [Chapter 6: Kafka-based Schedule Persistence and Communication](06_kafka_based_schedule_persistence_and_communication_.md)), and more. - -For our purposes, the most important part is the end, where it assembles the `api.Core` struct: - -```go -// Highly simplified concept from: api/pkg/core/core.go -package core - -import ( - "github.com/nestoca/scheduler/api" - // ... many other imports for creating actual services ... -) - -// FromEnv creates the default instances of all services. -func FromEnv() *api.Core { - // 1. Create HealthService instance (details omitted) - healthSvc := api.NewHealthService() - - // 2. Create InstrumentationService instance (details omitted) - metricsStore := instrumentation.NewMetricsDataStore() // Example helper - instrumentationSvc := api.NewInstrumentationService(metricsStore) - - // 3. Create SchedulesService instance (details omitted) - // This involves creating a ScheduleWriter, LocationProvider, etc. - // scheduleProducer := kafkaschedules.NewScheduleProducer(...) - // locationProvider := location.NewCachedProvider() - // schedulesSvc := api.NewSchedulesService(scheduleProducer, locationProvider) - var schedulesSvc api.SchedulesService // Placeholder for actual creation - // ... (actual complex creation of schedulesSvc here) ... - - - // 4. Create Scheduler engine instance (details omitted) - // This involves creating stores, post-processors, DLQs, etc. - // store := scheduler.NewDailyScheduleConsumer(...) - // postProcessor := scheduler.NewSchedulePostProcessor(...) - // dlq := scheduler.NewDeadLetterQueue(...) - // actor := scheduler.NewConcurrentScheduleActor(...) - // schedulerEngine := scheduler.NewScheduler(store, postProcessor, dlq, actor) - var schedulerEngine api.Scheduler // Placeholder for actual creation - // ... (actual complex creation of schedulerEngine here) ... - - - // 5. Assemble and return the api.Core struct - return &api.Core{ - HealthService: healthSvc, - InstrumentationService: instrumentationSvc, - Scheduler: schedulerEngine, // The main scheduling engine - SchedulesService: schedulesSvc, // Our schedule manager - } -} -``` -**Important:** The snippet above is *heavily* simplified. The actual `FromEnv()` function is much more complex because creating each service (like `SchedulesService` or `Scheduler`) involves setting up its own dependencies. For example, creating `SchedulesService` requires providing it with a `ScheduleWriter` and a `LocationProvider`, as we saw in Chapter 3. - -The key takeaway is that `FromEnv()` acts as a "factory" that produces a fully populated `api.Core` instance. This instance is then typically passed to the parts of the application that need to set up routes or initialize other components. In our `scheduler-api` command, `main.go` would call `core.FromEnv()` and then pass the result to `routes.GetRouter()`. - -## Why is `api.Core` Important? - -The `api.Core` aggregator plays a vital role: - -1. **Centralization:** It provides a single, well-defined place to get access to all core application services. If you need a service, you know where to look. -2. **Dependency Management (Simplified):** It helps manage dependencies. Components like API handlers don't need to create services themselves; they receive them. This is a form of "dependency injection." -3. **Organization:** It keeps the application startup and wiring of components cleaner. The `main` function (or equivalent startup code) creates the `api.Core`, and then passes this single "toolbox" around, rather than passing many individual services. -4. **Testability:** When testing a component that uses services (e.g., an API handler), it's easier to provide "mock" or "fake" versions of these services if they are accessed through a central point like `api.Core`. You could create an `api.Core` with mock services for your tests. - -## Conclusion - -You've now learned about `api.Core`, the central toolbox or control panel of our `scheduler` application. It doesn't do the heavy lifting itself, but it holds and provides convenient access to all the specialized services that do, such as `SchedulesService`, `HealthService`, `InstrumentationService`, and the `Scheduler` engine. This makes it easy for components like API handlers to get the tools they need and helps keep our application organized. - -One of the most important tools held within `api.Core` is the `Scheduler` itself – the engine that actually picks up schedules and makes sure tasks get triggered. In the next chapter, we'll dive into this crucial component: [Chapter 5: Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md). - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md b/documentation/05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md deleted file mode 100644 index 21e6c3fb..00000000 --- a/documentation/05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md +++ /dev/null @@ -1,288 +0,0 @@ -# Chapter 5: Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`) - -In [Chapter 4: Core Services Aggregator (`api.Core`)](04_core_services_aggregator___api_core___.md), we saw how `api.Core` acts as a central toolbox, holding all the essential services our `scheduler` application needs. One of the most critical tools in that toolbox is the `api.Scheduler` itself. This chapter focuses on this "engine" of our system, specifically its main implementation, `scheduler.DefaultScheduler`. - -So far, we've learned how to: -1. Define a task with its timing: a `schedule.Schedule` blueprint ([Chapter 1: Schedule Blueprint (`schedule.Schedule`)](01_schedule_blueprint___schedule_schedule___.md)). -2. Send this blueprint to our system: via [HTTP API Endpoints & Routing](02_http_api_endpoints___routing_.md). -3. Have the blueprint validated and prepared for storage: by the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md). - -But how does a stored blueprint actually cause something to happen at the scheduled time? Who is constantly watching the clock and saying, "Aha! Time to run this task!"? That's the job of the Task Execution Engine. - -## The Problem: Making Scheduled Tasks Actually Happen - -Imagine you've written down a list of chores on a to-do list with specific times for each: -* "Take out trash - Tuesday 8:00 AM" -* "Water plants - Wednesday 6:00 PM" -* "Send weekly email summary - Friday 9:00 AM, repeat weekly" - -Having this list is great, but it doesn't do the chores. You need someone (or something) to: -1. Constantly check the list and the current time. -2. When a chore's time arrives, actually *start* doing it. -3. If it's a repeating chore, remember to schedule it for the next time. -4. If something goes wrong while doing a chore, decide what to do (e.g., try again later). - -In our `scheduler` project, the **Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)** is this diligent worker. It's like an automated conductor for an orchestra, ensuring each instrument plays at the right time, or the engine of a car, providing the power to move forward. - -## Meet the `api.Scheduler` and `scheduler.DefaultScheduler` - -The `api.Scheduler` is an interface (a contract) defining what a task execution engine should be able to do. It's very simple: - -```go -// Simplified from: api/scheduler.go -package api - -import "context" - -// Scheduler Abstract representation of a scheduler -type Scheduler interface { - StartBlocking(ctx context.Context) // Starts the engine's main work loop - // ForceGenerationChange(ctx context.Context) // A way to signal internal changes -} -``` -* `StartBlocking(ctx context.Context)`: This is the main command to tell the engine to begin its work of monitoring and executing tasks. It's "blocking" because it will typically run continuously until the application is told to stop. - -The primary implementation of this interface is `scheduler.DefaultScheduler` (found in `api/pkg/scheduler/scheduler.go`). This is the component that does the actual work. - -**What `scheduler.DefaultScheduler` Does:** - -Think of `scheduler.DefaultScheduler` as the main coordinator on the factory floor. It doesn't build everything itself, but it directs other specialized workers and machines: - -1. **Monitors for Due Tasks:** It continuously asks a component called the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) (let's call it the `ScheduleStore` for short): "Are there any tasks due *right now*?" -2. **Triggers Task Execution:** When the `ScheduleStore` says, "Yes, this task is due!", the `DefaultScheduler` tells another component (an `ScheduleActor`) to actually perform the task. "Performing the task" in our system usually means sending a message (the `TargetPayload` from our `schedule.Schedule` blueprint) to a specific Kafka topic (the `TargetTopic`). -3. **Handles Post-Execution:** After a task is attempted: - * If it was successful, the `DefaultScheduler` passes the task to the [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md). This handler figures out if the task needs to be rescheduled (if it's recurring) or if it's done for good. - * If it failed, the `DefaultScheduler` sends it to the [Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md). This queue will try to run the task again later. -4. **Keeps Time:** It also manages a daily timer to know when a new day starts, which is important for the `ScheduleStore` to load tasks for the current day. - -It ensures tasks are picked up and processed in a timely and orderly manner. - -## How the Engine Starts and Works - -When our `scheduler` application starts up (as seen in `api/cmd/scheduler-api/api.go`), it gets an instance of the `api.Scheduler` from the `api.Core` toolbox and tells it to start: - -```go -// Simplified from: api/cmd/scheduler-api/api.go -func main() { - // ... setup code ... - c := core.FromEnv() // Gets the api.Core toolbox - - // ... - // Run Scheduler async - log.L.Infof("[START] starting scheduler") - startScheduler(ctx, c.Scheduler, &wg, schedulerError) // c.Scheduler is our engine - // ... -} - -func startScheduler(ctx context.Context, s api.Scheduler, /*...*/) { - // ... - go func() { // Runs in a separate goroutine (like a parallel thread) - // ... - s.StartBlocking(ctx) // This kicks off the engine! - }() -} -``` -The `s.StartBlocking(ctx)` call is what makes the `scheduler.DefaultScheduler` spring to life. - -**The Main Work Loop (Conceptual):** - -Once started, `scheduler.DefaultScheduler` enters a continuous loop. Here's a simplified idea of what it's doing: - -```mermaid -graph TD - A[Start Engine] --> B{Anything to do?}; - - subgraph "Engine's Continuous Loop" - B -- "Daily Timer Ticked (Midnight!)" --> C[Notify ScheduleStore: New Day!]; - C --> B; - - B -- "ScheduleStore: Task X is Due!" --> D[Get Task X]; - D --> E{Attempt Task X via ScheduleActor}; - E -- "Success!" --> F[Send Task X to PostProcessor]; - F --> B; - E -- "Failure!" --> G[Send Task X to DeadLetterQueue]; - G --> B; - - B -- "DeadLetterQueue: Task Y (retried) is Done" --> H[Send Task Y to PostProcessor]; - H --> B; - - B -- "Application Shutting Down" --> I[Stop]; - end -``` -This loop continues, making the scheduler responsive to due tasks, daily rollovers, and retry completions. - -## Under the Hood: The `scheduler.DefaultScheduler` and its Helpers - -The `scheduler.DefaultScheduler` doesn't work in isolation. It relies on several helper components, which are given to it when it's created. This creation happens inside `api/pkg/core/core.go` as part of the `FromEnv()` function we saw in [Chapter 4: Core Services Aggregator (`api.Core`)](04_core_services_aggregator___api_core___.md). - -```go -// Simplified from: api/pkg/core/core.go -// Inside FromEnv() function: - -// ... (store, postProcessor, dlq, actor are created first) ... -// store is an instance of scheduler.ScheduleStore -// postProcessor is an instance of scheduler.Queue (for post-processing) -// dlq is an instance of scheduler.Queue (for dead letters) -// actor is an instance of scheduler.ScheduleActor - -schedulerEngine := scheduler.NewScheduler(store, postProcessor, dlq, actor) - -// This schedulerEngine is then put into api.Core -// return &api.Core{ -// Scheduler: schedulerEngine, -// // ... other services -// } -``` - -The `scheduler.NewScheduler` function (which creates our `DefaultScheduler`) takes these key helpers: -* `store ScheduleStore`: This is the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md). The engine asks it for the next task that's due. -* `schedulePostProcessor Queue`: This is the [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md). Tasks go here after successful execution (or after exhausting retries) for final processing like rescheduling or deletion. -* `deadLetterQueue Queue`: This is the [Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md). Tasks that fail execution are sent here to be retried later. -* `actor ScheduleActor`: This component is responsible for the actual "execution" of the task – typically, sending a Kafka message. - -Let's look at a very simplified structure of `DefaultScheduler` and its `StartBlocking` method from `api/pkg/scheduler/scheduler.go`: - -```go -// Simplified from: api/pkg/scheduler/scheduler.go -package scheduler - -// ... (imports for context, sync, time, schedule package) ... - -type DefaultScheduler struct { - store ScheduleStore // Gets due tasks - deadLetterQueue Queue // Handles failed tasks for retry - schedulePostProcessor Queue // Handles tasks after execution/retry - actor ScheduleActor // Performs the task action - - dailyTimer *time.Timer // For midnight signal - // ... (mutex for synchronization) ... -} - -func NewScheduler(store ScheduleStore, schedulePostProcessor Queue, deadLetterQueue Queue, actor ScheduleActor) *DefaultScheduler { - // ... (nil checks) ... - return &DefaultScheduler{ - store: store, - deadLetterQueue: deadLetterQueue, - schedulePostProcessor: schedulePostProcessor, - actor: actor, - dailyTimer: time.NewTimer(calculateTimeToMidnight()), // Sets timer for next midnight - } -} -``` -This shows how the `DefaultScheduler` holds onto its helpers. The `dailyTimer` is set to fire at the next UTC midnight. - -Now, a simplified look at the `StartBlocking` method: -```go -// Simplified from: api/pkg/scheduler/scheduler.go -func (s *DefaultScheduler) StartBlocking(ctx context.Context) { - // ... (logging, waitgroup setup for graceful shutdown) ... - - // Start helper components (they run their own loops) - // s.startLongLastingProcess(ctx, &wg, s.store.RunBlocking) - // s.startLongLastingProcess(ctx, &wg, s.deadLetterQueue.RunBlocking) - // s.startLongLastingProcess(ctx, &wg, s.schedulePostProcessor.RunBlocking) - -readLoop: - for { - select { // Waits for any of these events - case <-s.dailyTimer.C: - // Midnight reached! - s.store.MidnightReachedChannel() <- struct{}{} // Signal the store - s.dailyTimer.Reset(calculateTimeToMidnight()) // Reset timer for next midnight - - case sched := <-s.store.ProduceChannel(): // A task is due from the store! - log.G(ctx).Tracef("[SCHED] Task %s ready", sched.Key) - err := s.actor.IssueSchedule(ctx, sched) // Tell actor to run it - if err == nil { - // Success! Send to post-processor for cleanup/rescheduling - s.schedulePostProcessor.PushChannel() <- sched - } else { - // Failure! Send to dead-letter queue for retry - s.deadLetterQueue.PushChannel() <- sched - } - - case sched := <-s.deadLetterQueue.ExitChannel(): // Task finished retries (success or fail) - // Send to post-processor for final handling - s.schedulePostProcessor.PushChannel() <- sched - - case err := <-s.store.ConsumeErrorChannel(): // An error from the store - log.G(ctx).Errorf("[SCHED] Store error: %s", err.Error()) - - case <-ctx.Done(): // Application is shutting down - break readLoop // Exit the loop - } - } -} -``` - -**Let's break down the `select` block, the heart of the engine:** - -1. `case <-s.dailyTimer.C:` - * The `dailyTimer` (which was set to fire at midnight) has gone off. - * The engine sends a signal to the `s.store.MidnightReachedChannel()`. This tells the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) that it's a new day, so it might need to load schedules for this new day. - * The `dailyTimer` is reset for the *next* midnight. - -2. `case sched := <-s.store.ProduceChannel():` - * The engine receives a `schedule.Schedule` (named `sched`) from the `s.store.ProduceChannel()`. This means the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) has determined this task is due *right now*. - * It calls `s.actor.IssueSchedule(ctx, sched)`. The `ScheduleActor` attempts to "issue" the task (e.g., send its payload to a Kafka topic). - * If `IssueSchedule` returns no error (`err == nil`), the task was successfully issued. It's then sent to the `s.schedulePostProcessor.PushChannel()`. The [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md) will take care of it (e.g., if it's a daily task, it will calculate the next run time and save a new schedule instance for tomorrow). - * If `IssueSchedule` returns an error, the task failed. It's sent to the `s.deadLetterQueue.PushChannel()`. The [Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md) will try to run it again later. - -3. `case sched := <-s.deadLetterQueue.ExitChannel():` - * A task has "exited" the [Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md). This means the DLQ has finished its retry attempts for this task (either it eventually succeeded, or it reached the maximum number of retries). - * This task is then sent to the `s.schedulePostProcessor.PushChannel()` for final processing, just like a task that succeeded on its first try. The post-processor will handle its removal or logging as appropriate. - -4. `case err := <-s.store.ConsumeErrorChannel():` - * If the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) has a problem (e.g., trouble reading from its data source like Kafka), it can send an error here. The engine logs this error. - -5. `case <-ctx.Done():` - * The `context` (`ctx`) has been "canceled." This is the signal for the application to shut down gracefully. - * The `break readLoop` statement exits the `for` loop, and the `StartBlocking` method will eventually return, stopping the engine. - -**A Visual Summary of Interactions:** - -Here's how `DefaultScheduler` coordinates with its main helpers for a typical task lifecycle: - -```mermaid -sequenceDiagram - participant Store as ScheduleStore - participant Engine as DefaultScheduler - participant Actor as ScheduleActor - participant PostProc as SchedulePostProcessor - participant DLQ as DeadLetterQueue - - Store->>Engine: Task `S` is due! (via ProduceChannel) - Engine->>Actor: Issue Task `S` - alt Task S succeeds - Actor-->>Engine: Success! - Engine->>PostProc: Handle Task `S` (via PushChannel) - else Task S fails - Actor-->>Engine: Failed! - Engine->>DLQ: Retry Task `S` (via PushChannel) - DLQ-->>Engine: Task `S` finished retries (via ExitChannel) - Engine->>PostProc: Handle Task `S` (via PushChannel) - end -``` - -## Why is the Task Execution Engine So Important? - -* **The Heartbeat:** It's the component that makes the scheduler "live." Without it, schedules would just be data sitting in storage. -* **Orchestration:** It masterfully coordinates several other specialized components (`ScheduleStore`, `ScheduleActor`, `SchedulePostProcessor`, `DeadLetterQueue`) to ensure the entire lifecycle of a task is managed. -* **Reliability:** By integrating with a dead-letter queue, it provides a mechanism for retrying failed tasks, increasing the overall reliability of the system. -* **Timeliness:** Its core loop, driven by signals from the `ScheduleStore` and internal timers, ensures tasks are picked up when they are due. - -## Conclusion - -The Task Execution Engine, primarily `scheduler.DefaultScheduler`, is the automated conductor at the heart of our `scheduler` project. It doesn't do everything itself but intelligently uses other components: -* It gets due tasks from the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md). -* It uses a `ScheduleActor` to trigger the task's action (like sending a message). -* It sends successfully completed (or fully retried) tasks to the [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md) for cleanup and rescheduling. -* It sends failed tasks to the [Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md). - -This orchestration ensures that the "recipe cards" (`schedule.Schedule` blueprints) we create are actually acted upon at the right times. - -Now, we've seen tasks being "issued" or "produced," often involving Kafka. But how exactly are schedules stored in Kafka, and how do components like the `ScheduleStore` or `ScheduleActor` use Kafka to communicate? That's what we'll explore in the next chapter: [Chapter 6: Kafka-based Schedule Persistence and Communication](06_kafka_based_schedule_persistence_and_communication_.md). - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/06_kafka_based_schedule_persistence_and_communication_.md b/documentation/06_kafka_based_schedule_persistence_and_communication_.md deleted file mode 100644 index 6f30de23..00000000 --- a/documentation/06_kafka_based_schedule_persistence_and_communication_.md +++ /dev/null @@ -1,314 +0,0 @@ -# Chapter 6: Kafka-based Schedule Persistence and Communication - -Welcome to Chapter 6! In [Chapter 5: Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md), we saw how the scheduler engine orchestrates task execution. We learned that it gets tasks from a `ScheduleStore` and uses an `ScheduleActor` to issue tasks. But where does the `ScheduleStore` get its information? And how do different parts of our `scheduler` system stay in sync about all the schedule definitions, especially if components restart or if we have multiple instances running? - -This chapter dives into **Apache Kafka**, the technology our `scheduler` uses as a robust "event bus" (a way to send messages) and a "persistence layer" (a way to store data reliably) for schedules. - -## The Problem: Reliable Schedule Storage and Announcements - -Imagine our `scheduler` system is a busy train station. -* **New Schedules are like New Train Routes:** When someone wants to schedule a new task (like our daily report from [Chapter 1: Schedule Blueprint (`schedule.Schedule`)](01_schedule_blueprint___schedule_schedule___.md)), it's like announcing a new train route. -* **Updates are Timetable Changes:** If a schedule changes, it's like a train timetable update. -* **Deletions are Route Cancellations:** If a schedule is removed, the route is cancelled. - -We need a central, super-reliable announcement system and archive for all these train routes and timetable changes. -1. **Durability:** Announcements must not be lost, even if the station master (one part of our program) goes home and a new one comes on duty (the program restarts). -2. **Broadcasting:** All relevant departments in the station (different parts of our `scheduler` system) need to hear these announcements. For example, the department that displays "upcoming trains" (our [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md)) needs to know about new routes and changes. -3. **Order:** It's helpful if announcements are kept in order. - -This is where Kafka comes in. - -## Meet Kafka: Our Reliable, Distributed Postal Service - -Think of **Kafka** as a highly reliable, distributed postal service combined with a public archive of all sent mail. -* **Topics:** A Kafka **topic** is like a specific mailbox or a public bulletin board dedicated to a certain kind of information. For our schedules, we'll have a topic, let's call it `schedules-log`. -* **Messages:** Each piece of information (a new schedule, an update, a deletion) is sent as a **message** (a "letter" or a "bulletin post") to this topic. -* **Producers:** Components that *write* or *send* messages to a Kafka topic are called **Producers**. They are like people dropping letters into the mailbox. -* **Consumers:** Components that *read* or *receive* messages from a Kafka topic are called **Consumers**. They are like people checking the mailbox or reading the bulletin board. - -Kafka ensures that once a message is successfully sent to a topic, it's stored durably (it won't get lost easily) and can be read by any authorized consumer, even much later. - -## Key Players in Our Kafka-Powered Scheduler - -In our `scheduler` project, we have specific components that interact with Kafka: - -1. **`ScheduleProducer` (The Mail Sender for Schedules)** - * Found in: `api/pkg/schedule/kafka/producer.go` - * This component is responsible for writing schedule information to our Kafka `schedules-log` topic. - * When you create a new schedule (via the API and the [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md)), the `SchedulesService` uses the `ScheduleProducer` to send a "new schedule" message to Kafka. - * It also sends messages for updates or deletions. - -2. **`ScheduleConsumer` (The Mail Reader for Schedules)** - * Found in: `api/pkg/schedule/kafka/consumer.go` - * This component reads the schedule messages from the Kafka `schedules-log` topic. - * Other parts of the system, like the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) (which we'll see in the next chapter), use a `ScheduleConsumer` to learn about all active schedules. - -3. **Kafka Messages: The "Letters"** - * A message typically contains the details of a `schedule.Schedule` blueprint, packaged in a standard format. - * Our system converts the `schedule.Schedule` object into a byte array (a sequence of 0s and 1s) to send it over Kafka, and then converts it back when reading. This is called **serialization** and **deserialization**. - -4. **"Tombstones": Marking Schedules for Deletion** - * How do you delete something from Kafka? You can't just erase a message that's already written. - * Instead, Kafka uses a special kind of message called a **tombstone**. If we want to delete a schedule with `Key: "daily-report"`, the `ScheduleProducer` sends a message to Kafka with `Key: "daily-report"` but with an empty (or `nil`) body. - * Consumers see this "tombstone" message and understand that the schedule identified by `"daily-report"` should be considered deleted. - -5. **`ConsumerGroup`: A Team of Mail Sorters** - * Kafka allows multiple `ScheduleConsumer` instances to work together as a **Consumer Group**. - * Imagine having several mail sorters for the same `schedules-log` mailbox. Kafka cleverly divides the work among them. - * **Benefits:** - * **Scalability:** If there are many messages, more consumers can share the load. - * **Fault Tolerance:** If one consumer in the group crashes, another one can take over its work, ensuring no messages are missed. - * The `api/pkg/schedule/kafka/client/consumer_group.go` file provides the underlying mechanism for this, ensuring our `ScheduleConsumer` can be part of such a resilient team. - -## How It Works: Storing and Communicating Schedules - -Let's trace how a schedule makes its way into Kafka and how it's read. - -### 1. Saving a New Schedule (or Updating an Existing One) - -When you submit a new `schedule.Schedule` blueprint (or update an existing one): -1. The [Schedule Management Service (`api.SchedulesService`)](03_schedule_management_service___api_schedulesservice___.md) validates it. -2. It then calls `Save()` on our `ScheduleProducer`. - -Here's a simplified look at what `ScheduleProducer.Save()` does: -```go -// Simplified from: api/pkg/schedule/kafka/producer.go -func (p *ScheduleProducer) Save(ctx context.Context, s *schedule.Schedule) error { - // 1. Convert our schedule (s) into an "UpsertedEvent" format. - // "Upserted" means it's for an insert (new) or update. - upsertData := toUpsertScheduleEvent(s) // From producer.go - - // 2. Package this data into a standard "event" envelope. - // This helps track changes systematically. - event := &es.Event{ - ID: uuid.New(), // A unique ID for this event itself - Type: /* type like "schedule.UpsertedEvent" */, - Data: upsertData, // Our schedule details - Created: time.Now().UTC(), - } - - // 3. Create the actual Kafka "letter" (message). - // s.Key (e.g., "daily-report") is the address label for this schedule. - // p.dataStoreTopic is the main mailbox for all schedule definitions. - // p.eventHandler helps turn the `event` into bytes. - kafkaMessage, err := ToDataStoreMessage(s.Key, event, p.eventHandler, p.dataStoreTopic) // From converter.go - if err != nil { return err } - - // 4. Send the letter! The `p.producer` is a generic Kafka message writer. - return p.producer.WriteMessage(ctx, *kafkaMessage) -} -``` -* `toUpsertScheduleEvent(s)`: This helper (also in `producer.go`) takes your `schedule.Schedule` object and converts some fields (like `LocalExecutionTime`) into strings, preparing it for storage. - ```go - // Simplified from: api/pkg/schedule/kafka/producer.go - func toUpsertScheduleEvent(s *schedule.Schedule) *schedule.UpsertedEvent { - return &schedule.UpsertedEvent{ // This is defined in schedule/events.go - ID: s.ID, // The schedule's system-generated ID - LocalExecutionTime: s.LocalExecutionTime.Format("2006-01-02 15:04"), // Time as string - TimeLocation: s.TimeLocation, - Recurrence: s.Recurrence, - TargetTopic: s.TargetTopic, - TargetKey: s.TargetKey, - TargetPayload: s.TargetPayload, - } - } - ``` -* `ToDataStoreMessage(...)`: This helper from `converter.go` actually creates the Kafka message structure. The `eventHandler` turns the `event` struct into bytes (`[]byte`). - ```go - // Simplified from: api/pkg/schedule/kafka/converter.go - func ToDataStoreMessage(key string, event *es.Event, eventHandler es.EventHandler, dataStoreTopic string) (*kafkaclient.Message, error) { - // Convert the 'event' (which contains our schedule data) into bytes - eventBytes, /* schemaID */ _, err := eventHandler.Marshal(event) - if err != nil { return nil, err } - // ... (schema ID might be added to eventBytes for versioning) ... - - return &kafkaclient.Message{ - Topic: dataStoreTopic, // The mailbox name - Key: key, // The schedule's unique name (e.g., "daily-report") - Value: eventBytes, // The schedule data, as bytes - }, nil - } - ``` - -**Visualized: Saving a Schedule** -```mermaid -sequenceDiagram - participant SMService as Schedule Management Service - participant SProducer as ScheduleProducer - participant KafkaTopic as Kafka Topic (schedules-log) - - SMService->>SProducer: Save(scheduleBlueprint) - SProducer->>SProducer: scheduleBlueprint -> UpsertedEvent - SProducer->>SProducer: UpsertedEvent -> es.Event - SProducer->>SProducer: es.Event -> Kafka Message (Key, Value bytes) - SProducer->>KafkaTopic: Write Message -``` -Now, the schedule blueprint is safely stored in Kafka! - -### 2. Deleting a Schedule - -When `ScheduleProducer.Delete()` is called: -```go -// Simplified from: api/pkg/schedule/kafka/producer.go -func (p *ScheduleProducer) Delete(ctx context.Context, scheduleKey string) error { - // 1. Create a special "tombstone" message. - // It has the scheduleKey, but its Value (body) is nil. - tombstoneMessage, err := ToDataStoreTombstoneMessage(scheduleKey, p.dataStoreTopic) // From converter.go - if err != nil { return err } - - // 2. Send this tombstone message to Kafka. - return p.producer.WriteMessage(ctx, *tombstoneMessage) -} -``` -* `ToDataStoreTombstoneMessage(...)` from `converter.go`: - ```go - // Simplified from: api/pkg/schedule/kafka/converter.go - func ToDataStoreTombstoneMessage(key string, dataStoreTopic string) (*kafkaclient.Message, error) { - return &kafkaclient.Message{ - Topic: dataStoreTopic, - Key: key, // The schedule's unique name - Value: nil, // Empty body signifies a tombstone! - }, nil - } - ``` -Kafka and its consumers understand this `nil` value as a "delete" instruction for the given `key`. - -### 3. Reading Schedules (The `ScheduleConsumer` in Action) - -Components like the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) need to get the list of schedules. They use the `ScheduleConsumer`. -```go -// Simplified from: api/pkg/schedule/kafka/consumer.go -func (c *ScheduleConsumer) GetNextSchedule(ctx context.Context) (*schedule.ConsumedSchedule, error) { - // 1. Read the next raw message from the Kafka topic. - // This `c.consumer` is a generic Kafka message reader. - kafkaMessage, err := c.consumer.ReadFirstAvailableMessageNonBlocking(ctx) - if err != nil { return nil, err } // e.g., no new messages right now - - // 2. Convert the raw Kafka message back into our schedule event format. - // `c.eventHandler` helps decode the message bytes. - scheduleKey, event, err := FromDataStoreKafkaMessage(kafkaMessage, c.eventHandler) // From converter.go - if err != nil { return nil, err } - - if event == nil { - // It's a tombstone! This schedule (identified by scheduleKey) was deleted. - // Return a special marker for deletion. - return schedule.NewConsumedSchedule( /* metadata */, &schedule.Schedule{Key: scheduleKey, TargetTopic: schedule.TombstoneIdentifier}), nil - } - - // 3. If it's not a tombstone, it's an event (e.g., UpsertedEvent). - // Extract the schedule data from the event. - s, err := fromUpsertScheduleEvent(scheduleKey, event.Data.(*schedule.UpsertedEvent)) // From consumer.go - if err != nil { return nil, err } - - return schedule.NewConsumedSchedule( /* metadata */, s), nil -} -``` -* `FromDataStoreKafkaMessage(...)` from `converter.go`: - ```go - // Simplified from: api/pkg/schedule/kafka/converter.go - func FromDataStoreKafkaMessage(message kafkaclient.Message, eventHandler es.EventHandler) (string, *es.Event, error) { - if message.Value == nil { - // This is a tombstone message (deletion)! - return message.Key, nil, nil - } - - // ... (schema ID might be removed from message.Value before unmarshalling) ... - // Convert the bytes back into an `es.Event` struct - event, err := eventHandler.Unmarshal(message.Value, /* schemaID */) - if err != nil { return "", nil, err } - - return message.Key, event, nil - } - ``` -* `fromUpsertScheduleEvent(...)` from `consumer.go`: - ```go - // Simplified from: api/pkg/schedule/kafka/consumer.go - func fromUpsertScheduleEvent(key string, eventData *schedule.UpsertedEvent) (*schedule.Schedule, error) { - // Convert string time back to time.Time object - execTime, _ := time.Parse("2006-01-02 15:04", eventData.LocalExecutionTime) - - return &schedule.Schedule{ - ID: eventData.ID, - LocalExecutionTime: execTime, - TimeLocation: eventData.TimeLocation, - Recurrence: eventData.Recurrence, - Key: key, // The key comes from the Kafka message itself - TargetTopic: eventData.TargetTopic, - TargetKey: eventData.TargetKey, - TargetPayload: eventData.TargetPayload, - }, nil - } - ``` - -**Visualized: Reading a Schedule** -```mermaid -sequenceDiagram - participant SConsumer as ScheduleConsumer - participant KafkaTopic as Kafka Topic (schedules-log) - participant DailyStore as Daily Schedule Store - - DailyStore->>SConsumer: GetNextSchedule() - SConsumer->>KafkaTopic: Read Message - KafkaTopic-->>SConsumer: Kafka Message (Key, Value bytes) - SConsumer->>SConsumer: Kafka Message -> es.Event (or Tombstone) - alt Tombstone - SConsumer->>SConsumer: Mark schedule as deleted - else UpsertedEvent - SConsumer->>SConsumer: es.Event -> schedule.Schedule - end - SConsumer-->>DailyStore: ConsumedSchedule (or deletion signal) -``` - -### 4. Communicating Task Execution (Bonus Kafka Use!) - -Kafka isn't just for storing schedule *definitions*. It's also used to send the actual task *payloads* when it's time for a task to run. -The [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md), through its `ScheduleActor`, uses `ScheduleProducer.IssueOnTargetTopic()`: - -```go -// Simplified from: api/pkg/schedule/kafka/producer.go -func (p *ScheduleProducer) IssueOnTargetTopic(ctx context.Context, s *schedule.Schedule) error { - // 1. Create a Kafka message destined for the schedule's `TargetTopic`. - // The body of this message is the schedule's `TargetPayload`. - targetMessage, err := ToTargetMessage(s) // From converter.go - if err != nil { return err } - - // 2. Send this "task payload" message. - // Another system (or another part of this system) will be listening - // on `s.TargetTopic` to actually perform the work. - return p.producer.WriteMessage(ctx, *targetMessage) -} -``` -* `ToTargetMessage(...)` from `converter.go`: - ```go - // Simplified from: api/pkg/schedule/kafka/converter.go - func ToTargetMessage(s *schedule.Schedule) (*kafkaclient.Message, error) { - return &kafkaclient.Message{ - Topic: s.TargetTopic, // The topic specified in the schedule blueprint - Key: s.TargetKey, // The key specified in the schedule blueprint - Value: s.TargetPayload, // The actual work data! - Headers: { /* schedule ID, timestamp, etc. */ }, - }, nil - } - ``` -This decouples the scheduler (which decides *when* to run) from the worker that actually *does* the task. - -## Why Kafka? The Advantages for Our Scheduler - -Using Kafka as the backbone for schedule persistence and communication offers several benefits: - -* **Durability & Reliability:** Schedule definitions are safely stored and won't be lost if a component restarts. -* **Decoupling:** The `ScheduleProducer` doesn't need to know who the `ScheduleConsumers` are, and vice-versa. They only need to agree on the Kafka topic and message format. This makes the system more flexible. -* **Scalability:** Kafka is designed to handle a high volume of messages and many consumers. If our scheduler gets very busy, Kafka can keep up. -* **Fault Tolerance:** With consumer groups, if one instance of a `ScheduleConsumer` fails, others can take over, ensuring continuous processing. -* **Central Log:** The Kafka topic acts as a chronological log of all changes to schedules. This can be useful for auditing or debugging. -* **Asynchronous Communication:** Components can send messages without waiting for an immediate response, which can improve performance and responsiveness. - -## Conclusion - -You've now seen how Apache Kafka serves as the reliable "postal service" and "archive" for our `scheduler` system. The `ScheduleProducer` writes schedule information (new tasks, updates, and "tombstone" deletions) as messages to a Kafka topic. The `ScheduleConsumer` (often as part of a resilient `ConsumerGroup`) reads these messages. This ensures that schedule definitions are durably stored and can be processed by different parts of the scheduler system, keeping everything in sync even if components restart or scale out. Kafka is also used to send the actual task payloads to their designated target topics when it's time for execution. - -But how does a component like the `ScheduleConsumer` intelligently manage all these schedule messages from Kafka to figure out which tasks are due *today* or *right now*? That's where the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) comes in, and we'll explore it in the next chapter! - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md b/documentation/07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md deleted file mode 100644 index 564c57bd..00000000 --- a/documentation/07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md +++ /dev/null @@ -1,307 +0,0 @@ -# Chapter 7: Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`) - -In [Chapter 6: Kafka-based Schedule Persistence and Communication](06_kafka_based_schedule_persistence_and_communication_.md), we learned how our `scheduler` system uses Kafka as a reliable way to store all the "recipe cards" for tasks (`schedule.Schedule` blueprints). Kafka acts like a giant, permanent library of every schedule we've ever defined. But if the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) had to sift through this entire library every second to see what's due, it would be very inefficient! - -This chapter introduces the **`scheduler.DailyScheduleStoreDefault`**, our "Daily Task Organizer." It's like a smart personal assistant who focuses only on what needs to be done *today*. - -## The Problem: Focusing on Today's Tasks - -Imagine you have a master calendar with all your appointments for the entire year (that's like our Kafka topic). Every morning, you don't want to re-read the whole year's calendar. Instead, you'd probably: -1. Look at the master calendar once in the morning. -2. Write down *only today's* appointments on a small notepad, in order of time. -3. Throughout the day, you just glance at this notepad to see what's next. -4. At the end of the day (or at midnight), you throw away the notepad and prepare a new one for tomorrow. - -The `DailyScheduleStoreDefault` does exactly this for our `scheduler`. It needs to: -* Efficiently find out which tasks are relevant for the current day. -* Keep them sorted by time. -* Make a task available precisely when its scheduled time arrives. -* Clear out old tasks and prepare for the next day (the "midnight reset"). - -## Meet the `scheduler.DailyScheduleStoreDefault`: Your Daily Planner - -The `scheduler.DailyScheduleStoreDefault` acts like a meticulous daily planner or a short-term memory for the scheduler. Its main job is to take all the schedule definitions it reads from Kafka (via a `ScheduleReader`) and organize a focused list of tasks specifically for the current day. - -Here's what it does: - -1. **Reads from Kafka:** It continuously reads schedule messages (new schedules, updates, deletions) from the Kafka topic where all schedules are stored. It uses a helper called a `ScheduleReader` for this. -2. **Filters for "Today":** It looks at each schedule's execution time and decides if it falls within the current day's active window. This "window" is typically from a little before "now" (e.g., 5 minutes in the past, to catch anything slightly delayed) up to the next midnight. -3. **Maintains an Ordered List:** It keeps an internal, sorted list of all tasks scheduled for this "today" window, ordered by their execution time. This way, it always knows which task is next. -4. **Provides Due Tasks:** When a task's scheduled time arrives, this organizer makes it available to the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) for execution. It does this by sending the task on a Go channel (a way for different parts of a Go program to communicate). -5. **Handles Midnight Reset:** When it's notified that midnight has passed (usually by the Task Execution Engine), it clears its internal list of "today's" tasks and starts building a new list for the new day by re-reading from Kafka. - -Think of it as the scheduler's highly efficient secretary, ensuring the boss (the Task Execution Engine) only sees the tasks that are immediately relevant. - -## How It Works: A Day in the Life of the Organizer - -Let's follow the `DailyScheduleStoreDefault` through its daily routine: - -1. **Morning Prep (Initialization or after Midnight):** - * The organizer starts with a blank slate. - * It begins reading *all* schedule definitions from the Kafka topic using its `ScheduleReader`. This is like getting the master list of all possible tasks. - * For each schedule, it checks if its `LocalExecutionTime` falls within the current day's active window. - * If a schedule is for "today," it's added to an internal, sorted list. If it's a deletion message (a "tombstone" from Kafka), the corresponding task is removed. - -2. **During the Day (Continuous Operation):** - * The organizer has a timer set for the *very next* task in its sorted list. - * When this timer goes off, it means that task is now due! - * It takes this due task and sends it to the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) through a special communication pipe called a Go channel (`ProduceChannel`). - * The task is then removed from its internal list of "today's" tasks. - * The organizer then looks at the *new* next task in its list and resets its timer for that task's execution time. - * It also continues to listen for new schedule messages from Kafka. If a new task for "today" is added or an existing one is updated/deleted, the internal list and the timer are adjusted accordingly. - -3. **Midnight! (The Reset):** - * The [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) sends a signal to the organizer: "It's midnight!" - * The organizer stops its current timer, throws away its entire internal list of "today's" tasks. - * It then tells its `ScheduleReader` to "reset," meaning it should start reading all schedules from the beginning of the Kafka topic again. - * The process then repeats from "Morning Prep" for the new day. - -This cycle ensures that the scheduler is always working with an up-to-date, focused list of tasks. - -## Under the Hood: A Peek Inside `DailyScheduleStoreDefault` - -The `DailyScheduleStoreDefault` is defined in `api/pkg/scheduler/daily_schedule_store.go`. Let's look at its key parts and how it's created. - -**1. Key Components Held by `DailyScheduleStoreDefault`:** - -When a `DailyScheduleStoreDefault` is created (usually by the `NewDailyScheduleConsumer` function), it's given several helpers: - -```go -// Simplified from: api/pkg/scheduler/daily_schedule_store.go -package scheduler - -// ... (other imports) ... -import ( - "github.com/nestoca/scheduler/api/pkg/schedule" -) - -type DailyScheduleStoreDefault struct { - executionTimer *storeTimer // A timer for the next due schedule - scheduleReader ScheduleReader // Reads schedules from Kafka - - // Channel to send due schedules to the Task Execution Engine - produceChannel chan *schedule.Schedule - // Channel to receive the "midnight has arrived" signal - midnightReachedChannel chan struct{} - // Channel to send very old/problematic schedules to the Dead Letter Queue - deadLetterChannel chan<- *schedule.Schedule - - // Internal storage for today's schedules, conceptually a sorted list. - // (Actual implementation uses 'dailySchedules' map for Kafka partitions) - dailySchedules dailyScheduleStore - - locationProvider LocationProvider // Helps with timezone calculations - // ... (other fields for metrics, retry logic, etc.) ... -} - -// scheduleEntry stores a schedule and its specific execution time. -type scheduleEntry struct { - schedule *schedule.Schedule - localExecutionTime time.Time // The actual time.Time object for execution - destination string // e.g., "OUTPUT" or "DEAD_LETTER_QUEUE" -} - -// dailyScheduleStore is a map where keys are Kafka partition IDs and values -// are partitions, each holding a list of scheduleEntry. -// For simplicity, think of this as managing one combined sorted list. -type dailyScheduleStore map[int]*dailyScheduleStorePartition -type dailyScheduleStorePartition struct { - entries []scheduleEntry // Sorted list of tasks for this partition for today - // ... (fields for readiness tracking) ... -} -``` -* `executionTimer`: A smart timer that's always set to fire when the *next* schedule in its internal list is due. -* `scheduleReader`: An instance of the `ScheduleReader` interface (which we learned about in [Chapter 6: Kafka-based Schedule Persistence and Communication](06_kafka_based_schedule_persistence_and_communication_.md)) to fetch schedule definitions. -* `produceChannel`: This is how it "produces" or "outputs" due schedules to the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md). -* `midnightReachedChannel`: It listens on this channel for a signal indicating a new day has begun. -* `deadLetterChannel`: If it encounters a schedule that is too old (e.g., its execution time was hours ago and missed), it sends it here. -* `dailySchedules`: This is its internal "notepad" for today's tasks. Although the actual structure is a bit more complex to handle Kafka partitions efficiently, you can think of it as maintaining one big, sorted list of `scheduleEntry` items for the current day. Each `scheduleEntry` holds a `schedule.Schedule` and its precisely calculated `localExecutionTime`. -* `locationProvider`: Essential for correctly interpreting timezones from `schedule.Schedule` blueprints. - -**2. The Main Loop (`RunBlocking`)** - -The `RunBlocking` method is the heart of the `DailyScheduleStoreDefault`. It runs in a continuous loop, waiting for different events: - -```go -// Simplified from: api/pkg/scheduler/daily_schedule_store.go -func (s *DailyScheduleStoreDefault) RunBlocking(ctx context.Context) { - // ... (setup) ... - for { // Infinite loop - select { - case <-s.executionTimer.C: // Timer for the next schedule fired! - log.G(ctx).Tracef("[Store] Execution time of schedule reached") - s.sendNextScheduleForConsumption() // Send it out - - case <-s.midnightReachedChannel: // Midnight signal received! - log.G(ctx).Tracef("[Store] Midnight boundary reached, we must reset") - s.reset(s.consumeErrorChannel, true) // Clear list, prepare for new day - - case <-s.readRetryTimer.C: // Time to try reading a new schedule from Kafka - s.readRetryTimer.Stop() // Stop this timer first - sched, err := s.scheduleReader.GetNextSchedule(ctx) // Get a schedule - if err == nil && sched != nil { - s.manageScheduleEntry(sched, s.consumeErrorChannel) // Process it - } - // ... (handle errors, reset readRetryTimer for next read attempt) ... - - case <-ctx.Done(): // Application is shutting down - return // Exit the loop - // ... (other cases like new generation events) ... - } - } -} -``` -* **`<-s.executionTimer.C`**: The `executionTimer` has fired. This means the task at the top of its internal "today's list" is due. The `sendNextScheduleForConsumption()` method is called. -* **`<-s.midnightReachedChannel`**: A signal arrives telling the store it's midnight. The `reset()` method is called to clear everything for the new day. -* **`<-s.readRetryTimer.C`**: It's time to attempt to read another message from the `scheduleReader` (which gets data from Kafka). If a schedule is read, `manageScheduleEntry()` is called. This timer ensures the store periodically checks Kafka for new or updated schedules. -* **`<-ctx.Done()`**: The application is shutting down, so the loop exits. - -**3. Processing a Schedule (`manageScheduleEntry`)** - -When a new schedule (`sched`) is read from Kafka, `manageScheduleEntry` decides what to do: - -```go -// Simplified logic of manageScheduleEntry from api/pkg/scheduler/daily_schedule_store.go -func (s *DailyScheduleStoreDefault) manageScheduleEntry( - consumedSched *schedule.ConsumedSchedule, - errChan chan<- error) { - - actualSchedule := consumedSched.Schedule() // The schedule.Schedule blueprint - - // If this schedule already exists in our list, remove the old one first. - idx, found := s.findKeyMatch(consumedSched) - if found { - s.deleteEntry(consumedSched.Partition(), idx, actualSchedule) - } - - // If it's a "tombstone" (delete message), we're done with it. - if isTombstone(actualSchedule) { - s.updateExecTimerAfterAction("deletion") // May need to update timer - return - } - - // Add the schedule to our internal "today's list" if it's relevant. - // This involves checking if it's within time boundaries for today. - // addEntry returns true if the new entry is now the *very next* one. - if s.addEntry(consumedSched.Partition(), actualSchedule, s.lowBoundaryFunc, s.highBoundaryFunc) { - s.updateExecTimerAfterAction("insertion") // Update timer if next task changed - } -} -``` -* It first checks if this schedule (by its `Key`) is already in the internal list. If so, the old version is removed (`deleteEntry`). -* If the message from Kafka is a "tombstone" (meaning the schedule was deleted), then nothing more is done for this key. -* Otherwise (`addEntry`), it checks if the schedule's `LocalExecutionTime` is within the "today" window (using `lowBoundaryFunc` and `highBoundaryFunc`). - * `lowBoundaryFunc`: Checks if the task is not too old (e.g., not older than 5 minutes ago). If it is, it might be sent to a dead-letter queue. - * `highBoundaryFunc`: Checks if the task is before the next midnight. - * If it's for today, it's inserted into the internal sorted list. -* `updateExecTimerAfterAction` is called to potentially reset the `executionTimer` if the newly added/deleted schedule affects which task is next. - -**4. Sending a Due Schedule (`sendNextScheduleForConsumption`)** - -When `executionTimer` fires: - -```go -// Simplified logic of sendNextScheduleForConsumption from api/pkg/scheduler/daily_schedule_store.go -func (s *DailyScheduleStoreDefault) sendNextScheduleForConsumption() { - // Find the very next schedule from our internal sorted list. - entry, partition := s.findNextScheduleToExecute() - if entry == nil { // No tasks left for today or store not ready - s.executionTimer.Stop() // Stop timer - return - } - - // Check if all parts of the store are ready (relevant for Kafka partitions). - // For simplicity, let's assume it's ready. - - // Send the schedule to the correct channel. - if entry.destination == scheduleDestinationDLQ { // Too old - s.deadLetterChannel <- entry.schedule - } else { // It's due now! - s.produceChannel <- entry.schedule // Send to Task Execution Engine - } - - // Remove this task from our internal list. - s.deleteEntry(partition, 0, entry.schedule) // 0 is the index of the first item - - // Set the timer for the NEW next schedule. - s.updateExecTimerAfterAction("execution") -} -``` -* It gets the task at the top of its sorted list (`findNextScheduleToExecute`). -* It sends this `schedule.Schedule` object to the `produceChannel` (if it's not too old). The [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) is listening on this channel. -* It removes the task from its list and then calls `updateExecTimerAfterAction` to set the `executionTimer` for the next task in the list. - -**5. Midnight Reset (`reset`)** - -When the midnight signal arrives: - -```go -// Simplified logic of reset from api/pkg/scheduler/daily_schedule_store.go -func (s *DailyScheduleStoreDefault) reset(errChan chan<- error, forceEndOfGeneration bool) { - s.executionTimer.Stop() // Stop the main timer - s.readRetryTimer.Stop() // Stop trying to read from Kafka for a moment - - // Clear the internal list of "today's" schedules. - clear(s.dailySchedules) // 'clear' is a Go built-in for maps - - // Tell the ScheduleReader to reset its position in Kafka, - // so it starts reading all schedules from the beginning for the new day. - if err := s.scheduleReader.Reset(); err != nil { - errChan <- err // Report error if reset fails - } - - // ... (reset metrics) ... - - // Start the Kafka reading process again for the new day. - s.readRetryTimer.Reset(s.readRetryStrategy.ResetSequence()) -} -``` -* It stops all its timers. -* It completely clears its `dailySchedules` internal storage (the "notepad" is wiped clean). -* Crucially, it calls `s.scheduleReader.Reset()`. This tells the component reading from Kafka to go back to the beginning of the topic and re-process all schedule definitions, so the store can build a fresh list for the new day. -* It then restarts its `readRetryTimer` to begin populating its list for the new day. - -**Visualizing the Flow:** - -Here's a simplified diagram of how the `DailyScheduleStoreDefault` interacts: - -```mermaid -sequenceDiagram - participant Reader as ScheduleReader (from Kafka) - participant Store as DailyScheduleStoreDefault - participant Engine as Task Execution Engine - - Store->>Reader: GetNextSchedule() - Reader-->>Store: ConsumedSchedule (e.g., new_task_for_today) - Store->>Store: manageScheduleEntry(new_task_for_today)
(Adds to internal sorted list) - Store->>Store: updateExecTimerAfterAction()
(Sets executionTimer for new_task_for_today) - - Note over Store,Engine: Time passes... executionTimer for new_task_for_today fires! - - Store->>Store: sendNextScheduleForConsumption() - Store->>Engine: produceChannel <- new_task_for_today - Store->>Store: (Removes new_task_for_today from list) - Store->>Store: updateExecTimerAfterAction()
(Sets timer for the *next* task in list) - - Note over Store,Engine: Midnight signal arrives! - Engine->>Store: midnightReachedChannel <- signal - Store->>Store: reset() (Clears internal list) - Store->>Reader: Reset() (Start reading Kafka from beginning) -``` - -## Why is the Daily Task Organizer So Important? - -* **Efficiency:** It prevents the core [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) from having to constantly scan *all* schedules in Kafka. It only deals with a small, relevant subset. -* **Focus:** It maintains a clear, short-term view of tasks due "today," making the system responsive. -* **Timeliness:** By keeping an ordered list and using a precise timer, it ensures tasks are made available for execution exactly when they are scheduled. -* **Cleanliness:** The "midnight reset" provides a clean way to roll over to the next day, discarding completed or outdated tasks from the active set and rebuilding with fresh data. - -## Conclusion - -The `scheduler.DailyScheduleStoreDefault` is like the scheduler's dedicated daily assistant. It diligently reads all potential tasks from Kafka (via `ScheduleReader`), filters them to create an ordered to-do list for just the current day, and hands over tasks to the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) precisely when they are due. Its "midnight reset" ensures each day starts fresh. This component is crucial for keeping the scheduler efficient and focused on what needs to happen *now*. - -Now that a task has been picked up by the Daily Task Organizer and sent for execution, what happens *after* the Task Execution Engine attempts to run it? How are recurring tasks rescheduled for their next run? What if a task needs special handling after it's done? That's what we'll explore in the next chapter: [Chapter 8: Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md). - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/08_post_execution_handler___scheduler_schedulepostprocessordefault___.md b/documentation/08_post_execution_handler___scheduler_schedulepostprocessordefault___.md deleted file mode 100644 index 5276ca3d..00000000 --- a/documentation/08_post_execution_handler___scheduler_schedulepostprocessordefault___.md +++ /dev/null @@ -1,248 +0,0 @@ -# Chapter 8: Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`) - -In [Chapter 7: Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md), we saw how our scheduler efficiently manages and provides tasks that are due for execution "today." The [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) then picks up these due tasks and attempts to run them. - -But what happens *after* a task has been successfully executed (or has completed its retries)? Is it just forgotten? What if it's a task that needs to run every day? This is where our **Post-Execution Handler**, the `scheduler.SchedulePostProcessorDefault`, steps in. - -## The Problem: What To Do After the Work is Done? - -Imagine you have a personal assistant who helps you with your daily tasks. -* If you had a one-time task like "Mail a birthday card to Alex," once it's done, your assistant should probably cross it off your list for good. -* But if you have a recurring task like "Water the plants every Monday," after your assistant does it this Monday, they need to remember to schedule it again for *next* Monday. - -The `scheduler.SchedulePostProcessorDefault` acts like this diligent administrative assistant or a cleanup crew for our tasks. After a task finishes its journey through the execution engine, this handler takes charge of the follow-up actions. - -## Meet `scheduler.SchedulePostProcessorDefault`: The After-Task Admin - -The `scheduler.SchedulePostProcessorDefault` (let's call it "PostProcessor" for short) is a crucial component that ensures tasks are handled correctly *after* they've been processed by the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md). - -Its main responsibilities are: - -1. **Handling One-Time Tasks:** If a task was a one-off event (meaning its `Recurrence` from the [Schedule Blueprint (`schedule.Schedule`)](01_schedule_blueprint___schedule_schedule___.md) is set to `RecurrenceNone` or its recurrence period has ended), the PostProcessor ensures this task is marked for deletion from the system. We don't want to keep old, completed one-time tasks cluttering our schedule list. -2. **Rescheduling Recurring Tasks:** If a task is meant to repeat (e.g., daily, weekly, hourly), the PostProcessor calculates the *next* time this task should run based on its `Recurrence` rules. It then updates the schedule definition, effectively re-scheduling it for the future. This "update" usually means creating a new `schedule.Schedule` instance with the new execution time and saving it (often back to Kafka, as we saw in [Chapter 6: Kafka-based Schedule Persistence and Communication](06_kafka_based_schedule_persistence_and_communication_.md)). - -It uses a helper component called a `SchedulePostProcessUpdater` to perform these deletions or updates. - -## How the PostProcessor Gets Involved - -As we saw in [Chapter 5: Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md), when the Task Execution Engine successfully issues a task, or when a task finishes its retries from the [Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md), it sends the `schedule.Schedule` object to the PostProcessor. - -Here's a simplified view of that interaction from the `DefaultScheduler`'s `StartBlocking` method: -```go -// Simplified from: api/pkg/scheduler/scheduler.go -// Inside DefaultScheduler's main loop: - -// ... (when a task `sched` is successfully issued by `s.actor`) ... -s.schedulePostProcessor.PushChannel() <- sched - -// ... (or when a task `sched` exits the deadLetterQueue) ... -s.schedulePostProcessor.PushChannel() <- sched -``` -The `s.schedulePostProcessor` is our `SchedulePostProcessorDefault`. It receives the just-processed schedule on its `PushChannel()`. - -## A Day in the Life of the PostProcessor - -Let's consider two scenarios: - -**Scenario 1: A One-Time Abandoned Cart Reminder** -1. An "Abandoned Cart Reminder" task (which is a one-time task) for `user123` is successfully executed. -2. The Task Execution Engine sends this `schedule.Schedule` object to the PostProcessor. -3. The PostProcessor looks at the schedule's `Recurrence` settings. It sees it's `RecurrenceNone`. -4. It then instructs its `SchedulePostProcessUpdater` to "delete" this schedule (e.g., `user123-cart-reminder`). The updater will typically send a "tombstone" message to Kafka. - -**Scenario 2: A Daily Sales Report Task** -1. The "Daily Sales Report" task (which runs daily at 9 AM) is successfully executed today. -2. The Task Execution Engine sends this `schedule.Schedule` to the PostProcessor. -3. The PostProcessor sees the `Recurrence` is `RecurrenceDaily`. -4. It calculates the next execution time: tomorrow at 9 AM. -5. It creates a new `schedule.Schedule` instance (or modifies the existing one) with this new `LocalExecutionTime` and a new unique internal `ID`. -6. It then tells its `SchedulePostProcessUpdater` to "update" (effectively, save) this new schedule instance. The updater will typically save this new blueprint to Kafka. - -## Under the Hood: Inside `SchedulePostProcessorDefault` - -The `SchedulePostProcessorDefault` is defined in `api/pkg/scheduler/schedule_postprocessor.go`. Let's explore its inner workings. - -**1. Key Components:** -The PostProcessor needs a few helpers: -```go -// Simplified from: api/pkg/scheduler/schedule_postprocessor.go -package scheduler - -type SchedulePostProcessorDefault struct { - postProcessUpdater SchedulePostProcessUpdater // To save or delete schedules - locationProvider LocationProvider // To help with time calculations - - postProcessInputChannel chan *schedule.Schedule // Receives schedules to process - scheduleQueue *schedule.Queue // Internal queue for schedules - // ... (timers, metrics, retry logic) ... -} -``` -* `postProcessUpdater`: This is an instance of `SchedulePostProcessUpdater`. It's the component that actually performs the save (for rescheduling) or delete operations. Often, this is an instance of `ConcurrentScheduleActor` which writes to Kafka. -* `locationProvider`: Used to correctly interpret timezones when calculating next execution times. -* `postProcessInputChannel`: The channel where it receives schedules from the Task Execution Engine. -* `scheduleQueue`: An internal queue to hold schedules waiting for post-processing. - -**2. The Main Loop (`RunBlocking`)** -The PostProcessor runs its own loop, much like other core components: -```go -// Simplified from: api/pkg/scheduler/schedule_postprocessor.go -func (s SchedulePostProcessorDefault) RunBlocking(ctx context.Context) { - for { - select { - case schedule := <-s.postProcessInputChannel: // New schedule arrived! - s.scheduleQueue.Enqueue(schedule) // Add to internal queue - // ... (logic to start/reset a processing timer if needed) ... - - case <-s.processTimer.timer.C: // Internal timer fired, time to process - // Calls a helper that uses s.selectActionAndPrepareQueue() - processQueueEntry(ctx, s.scheduleQueue, /*...other args...*/, s.selectActionAndPrepareQueue(ctx)) - - case <-ctx.Done(): // Application shutting down - return // Exit loop - // ... (other cases like generation changes) ... - } - } -} -``` -* When a schedule arrives on `postProcessInputChannel`, it's added to an internal `scheduleQueue`. -* A timer (`s.processTimer`) ensures that schedules in the queue are processed. When this timer fires, `processQueueEntry` is called, which eventually uses the logic from `selectActionAndPrepareQueue`. - -**3. The Decision Maker: `selectActionAndPrepareQueue`** -This is where the core logic resides for deciding what to do with a schedule: -```go -// Simplified from: api/pkg/scheduler/schedule_postprocessor.go -func (s *SchedulePostProcessorDefault) selectActionAndPrepareQueue(ctx context.Context) actionFunc { - sched := s.scheduleQueue.Dequeue() // Get next schedule from internal queue - if sched == nil { - return noOpAction // Nothing to do - } - defer s.scheduleQueue.Requeue(sched) // Put it back if processing fails & needs retry - - // Calculate when this schedule should run next. - nextLocalExecutionTime, err := CalculateNextLocalExecutionTime(ctx, sched, s.locationProvider) - if err != nil { - // Problem calculating next time, probably delete it. - return s.deleteScheduleProxy() - } - - if nextLocalExecutionTime.IsZero() { - // If next time is zero, it means no more recurrences. Delete it. - return s.deleteScheduleProxy() - } - - // Otherwise, it needs to be rescheduled (updated). - return s.updateScheduleProxy() -} -``` -* It takes a schedule (`sched`) from its internal queue. -* It calls `CalculateNextLocalExecutionTime` (a helper function also in this file) to determine the next run time. -* If `CalculateNextLocalExecutionTime` returns an error or a "zero" time (meaning no further runs), it decides to delete the schedule (`s.deleteScheduleProxy()`). -* Otherwise, it decides to update (reschedule) it (`s.updateScheduleProxy()`). - -**4. Calculating the Next Run: `CalculateNextLocalExecutionTime`** -This helper function checks the `sched.Recurrence` rules and current `sched.LocalExecutionTime` to figure out the next one. -```go -// Simplified from: api/pkg/scheduler/schedule_postprocessor.go -func CalculateNextLocalExecutionTime(ctx context.Context, sched *schedule.Schedule, provider LocationProvider) (time.Time, error) { - recurrenceScheme := schedule.RecurrenceNone - if sched.Recurrence != nil { - recurrenceScheme = sched.Recurrence.Scheme - } - - nextTime := time.Time{} // A "zero" time.Time - switch recurrenceScheme { - case schedule.RecurrenceDaily: - nextTime = sched.LocalExecutionTime.AddDate(0, 0, 1) // Add 1 day - case schedule.RecurrenceHourly: - nextTime = sched.LocalExecutionTime.Add(1 * time.Hour) // Add 1 hour - // ... (cases for RecurrenceWeekly, RecurrenceMonthly, RecurrenceCustom, etc.) ... - case schedule.RecurrenceNone: - // No recurrence, nextTime remains zero. - } - return nextTime, nil // Error handling omitted for simplicity -} -``` -* For `RecurrenceDaily`, it adds 1 day. For `RecurrenceHourly`, 1 hour, and so on. -* If `RecurrenceNone`, `nextTime` remains zero, signaling it's a one-time task. - -**5. Taking Action: `updateScheduleProxy` and `deleteScheduleProxy`** -These functions return the actual "action" to be performed on the schedule, using the `s.postProcessUpdater`. -```go -// Simplified from: api/pkg/scheduler/schedule_postprocessor.go -func (s *SchedulePostProcessorDefault) updateScheduleProxy() actionFunc { - return func(ctx context.Context, sched *schedule.Schedule) (*schedule.Schedule, error) { - // Recalculate next time (important for retries) - nextLocalExecutionTime, _ := CalculateNextLocalExecutionTime(ctx, sched, s.locationProvider) - - updateSchedule := sched.Clone() // Work on a copy - updateSchedule.LocalExecutionTime = nextLocalExecutionTime - updateSchedule.ID = uuid.NewString() // Give new instance a new system ID - - // Use the updater to save this new version - err := s.postProcessUpdater.UpdateSchedule(ctx, updateSchedule) - return updateSchedule, err - } -} - -func (s *SchedulePostProcessorDefault) deleteScheduleProxy() actionFunc { - return func(ctx context.Context, sched *schedule.Schedule) (*schedule.Schedule, error) { - // Use the updater to delete this schedule - err := s.postProcessUpdater.DeleteSchedule(ctx, sched) - return sched, err - } -} -``` -* `updateScheduleProxy`: Clones the original schedule, sets its `LocalExecutionTime` to the newly calculated next run time, assigns a fresh `ID`, and then calls `s.postProcessUpdater.UpdateSchedule()`. -* `deleteScheduleProxy`: Simply calls `s.postProcessUpdater.DeleteSchedule()`. - -**6. The `SchedulePostProcessUpdater` Interface** -This interface defines what the actual updater component must do: -```go -// From: api/pkg/scheduler/schedule_postprocessor.go -type SchedulePostProcessUpdater interface { - UpdateSchedule(ctx context.Context, schedule *schedule.Schedule) error - DeleteSchedule(ctx context.Context, schedule *schedule.Schedule) error -} -``` -As mentioned, the `ConcurrentScheduleActor` (from `api/pkg/scheduler/concurrent_schedule_actor.go`) often implements this. Its `UpdateSchedule` method calls `writer.Save()` and `DeleteSchedule` calls `writer.Delete()`. The `writer` is usually a `ScheduleProducer` that writes these changes to Kafka. - -**Visualizing the Post-Execution Flow:** -```mermaid -sequenceDiagram - participant Engine as Task Execution Engine - participant PostProc as SchedulePostProcessorDefault - participant Calculator as CalculateNextLocalExecutionTime - participant Updater as SchedulePostProcessUpdater (e.g., ConcurrentScheduleActor) - participant Kafka as Kafka - - Engine->>PostProc: PushChannel <- executedSchedule - PostProc->>PostProc: Enqueues executedSchedule - Note over PostProc: Timer fires for processing... - PostProc->>Calculator: CalculateNextTime(executedSchedule) - alt Recurring Task - Calculator-->>PostProc: nextRunTime (e.g., tomorrow 9 AM) - PostProc->>Updater: UpdateSchedule(newScheduleInstance for nextRunTime) - Updater->>Kafka: Save newScheduleInstance - else One-Time Task or Error - Calculator-->>PostProc: zeroTime or error - PostProc->>Updater: DeleteSchedule(executedSchedule) - Updater->>Kafka: Send Tombstone for executedSchedule - end -``` - -## Why is the Post-Execution Handler So Important? - -* **Automates Task Lifecycle:** It ensures that recurring tasks are automatically rescheduled, keeping the system running smoothly without manual intervention. -* **Keeps System Tidy:** It cleans up one-time tasks after they are done, preventing the accumulation of obsolete schedule definitions. -* **Centralizes Post-Execution Logic:** All the rules about what to do after a task runs are concentrated in this component, making the system easier to understand and maintain. -* **Enables Recurrence:** Without it, recurring schedules would just run once and then stop. This handler is the engine that drives continuous, repeated execution. - -## Conclusion - -The `scheduler.SchedulePostProcessorDefault` is the vital "after-action report" handler for our scheduler. Once a task has been executed by the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md), this PostProcessor steps in. It intelligently decides whether to mark the task for deletion (if it was a one-time event) or to calculate its next run time and re-submit it for future execution (if it's a recurring task). This is done using a `SchedulePostProcessUpdater` that typically writes these changes back to Kafka. This ensures the scheduler can handle both one-off jobs and continuously repeating tasks effectively. - -We've seen tasks get executed successfully and then handled by the PostProcessor. But what if a task *fails* during its execution attempt? How does our scheduler handle such failures and try again? That's the topic of our next chapter: [Chapter 9: Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md). - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md b/documentation/09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md deleted file mode 100644 index 2216ed9e..00000000 --- a/documentation/09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md +++ /dev/null @@ -1,232 +0,0 @@ -# Chapter 9: Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`) - -In [Chapter 8: Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md), we learned how tasks are managed *after* they've been successfully executed or have finished their retry attempts. Recurring tasks get rescheduled, and one-time tasks get cleaned up. - -But what if a task *fails* during its initial execution attempt? Imagine our scheduler tries to send a daily report, but the email server is temporarily down. Do we just give up? Of course not! This is where the **Failed Task Retry Queue**, specifically `scheduler.DeadLetterQueueDefault`, comes into play. - -## The Problem: Dealing With Temporary Glitches - -Life isn't always perfect, and sometimes tasks fail. -* A network connection might blip for a few seconds. -* A service the task depends on might be briefly unavailable. -* There might be a temporary hiccup that prevents the task from completing right away. - -If our scheduler tried a task once and gave up immediately upon any error, many tasks would never get done. We need a way to handle these temporary failures gracefully and give tasks a second (or third, or fourth...) chance. - -Think of it like trying to call a friend. If the line is busy, you don't just give up forever. You wait a bit and try again. If it's still busy, you might wait a little longer before trying again. - -## Meet `scheduler.DeadLetterQueueDefault`: The Task ICU - -When a scheduled task encounters an error during its execution attempt by the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md), it's not immediately discarded. Instead, it's sent to this "Dead Letter Queue" (DLQ), specifically our `scheduler.DeadLetterQueueDefault`. - -Think of the DLQ as an **intensive care unit (ICU)** or a **special retry inbox** for tasks. -* It holds these failed tasks. -* It attempts to re-issue them later, often using a `RetryStrategy` with increasing delays (this is called "backoff"). -* This provides resilience against temporary issues that might have caused the initial failure, increasing the chances that tasks eventually get processed. - -If a task in the DLQ finally succeeds on a retry attempt, great! It then goes to the [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md) just like a task that succeeded on its first try. If, after several retries, the task still fails, it also exits the DLQ and goes to the Post-Execution Handler, which will then decide on its final fate (e.g., log it as permanently failed, or if recurring, schedule the *next* instance). - -## How a Task Lands in the DLQ - -Let's look back at the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md). When it tries to issue a schedule using its `ScheduleActor` (which actually performs the task, like sending a Kafka message): - -```go -// Simplified from: api/pkg/scheduler/scheduler.go -// Inside DefaultScheduler's main loop (StartBlocking method) - -// sched is the task to be processed -err := s.actor.IssueSchedule(ctx, sched) // Attempt to run the task -if err == nil { - // SUCCESS! Send to post-processor - s.schedulePostProcessor.PushChannel() <- sched -} else { - // FAILURE! Send to dead letter queue - log.G(ctx).Tracef("[SCHED] Process failure for key %s, sending schedule to be retried", sched.Key) - s.deadLetterQueue.PushChannel() <- sched // <--- This sends it to the DLQ! -} -``` -If `s.actor.IssueSchedule()` (the attempt to run the task) returns an error, the schedule `sched` is sent to `s.deadLetterQueue.PushChannel()`. This `s.deadLetterQueue` is our `scheduler.DeadLetterQueueDefault`. - -## A Day in the Life of a Failed Task (DLQ Journey) - -Here's what happens when a task enters the `DeadLetterQueueDefault`: - -1. **Admission to ICU:** The task arrives at the DLQ. -2. **Waiting Period:** The DLQ doesn't try to re-run it immediately. It uses a `RetryStrategy` to decide how long to wait. The first wait might be short (e.g., a few seconds). -3. **Retry Attempt:** After the wait, the DLQ attempts to "issue" the task again. It uses a `ScheduleIssuer` (often the same kind of actor that the main engine uses) to do this. -4. **Outcome:** - * **Success!** If the task runs successfully this time, it "exits" the DLQ and is sent to the [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md) for normal follow-up (like rescheduling if it's a recurring task). - * **Still Fails!** If the task fails again, the `RetryStrategy` calculates a *new, longer* waiting period (this is the "backoff"). The task goes back to step 2. -5. **Max Retries Reached:** The `RetryStrategy` usually has a limit on how many times a task can be retried. If this limit is reached and the task still hasn't succeeded, it's considered "terminally failed" *for this specific run*. It then "exits" the DLQ and is sent to the [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md) for final processing (e.g., logging the failure). - -**Visualizing the DLQ Flow:** - -```mermaid -sequenceDiagram - participant Engine as Task Execution Engine - participant DLQ as DeadLetterQueueDefault - participant Issuer as DLQ's ScheduleIssuer - participant PostProc as Post-Execution Handler - - Engine->>DLQ: Failed Task (via PushChannel) - DLQ->>DLQ: Add to internal queue, set timer (RetryStrategy) - - loop Retry Attempts - Note over DLQ: Timer fires... - DLQ->>Issuer: Issue Task (Retry Attempt) - alt Retry Successful - Issuer-->>DLQ: Success! - DLQ->>PostProc: Task (via ExitChannel) - break - else Retry Fails - Issuer-->>DLQ: Failed! - DLQ->>DLQ: Update retry count, set longer timer (RetryStrategy) - Note over DLQ: If max retries, then... - DLQ->>PostProc: Task (failed) (via ExitChannel) - break - end - end -``` - -## Under the Hood: Inside `scheduler.DeadLetterQueueDefault` - -The `scheduler.DeadLetterQueueDefault` is defined in `api/pkg/scheduler/dead_letter_queue.go`. Let's look at its key parts. - -**1. Key Components Held by `DeadLetterQueueDefault`:** -When a `DeadLetterQueueDefault` is created (usually by `NewDeadLetterQueue`), it's given: -```go -// Simplified from: api/pkg/scheduler/dead_letter_queue.go -package scheduler - -type DeadLetterQueueDefault struct { - issuer ScheduleIssuer // To re-attempt issuing the schedule - - deadLetterEnterChannel chan *schedule.Schedule // Receives failed tasks - deadLetterExitChannel chan *schedule.Schedule // Sends out tasks after retries - - processTimer *queryableTimer // Timer for the next retry attempt - retryStrategy RetryStrategy // Decides wait times and max retries - scheduleQueue *schedule.Queue // Internal list of tasks waiting for retry - // ... (metrics, newGenEventHandler for Kafka rebalancing) ... -} -``` -* `issuer ScheduleIssuer`: This is crucial. It's the component the DLQ uses to try and re-run the task. It's an interface, often implemented by `ConcurrentScheduleActor` (from `api/pkg/scheduler/concurrent_schedule_actor.go`), which can issue tasks (e.g., send a Kafka message). -* `deadLetterEnterChannel`: Failed tasks from the [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`)](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) arrive here. -* `deadLetterExitChannel`: Tasks leave the DLQ through this channel after they either succeed on a retry or exhaust all retry attempts. The Task Execution Engine listens to this and sends them to the [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`)](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md). -* `processTimer`: A timer that's set according to the `retryStrategy` to trigger the next retry attempt for the task at the head of the `scheduleQueue`. -* `retryStrategy RetryStrategy`: An interface (defined in the same file) that dictates the retry logic. It provides methods like `GetNewDuration()` to get the next wait time and can signal if `ErrMaxRetriesReached`. An example implementation could be an exponential backoff strategy (wait longer after each failure). -* `scheduleQueue *schedule.Queue`: An internal queue holding the schedules that are currently in the DLQ, waiting for their next retry attempt. - -**2. Creation (`NewDeadLetterQueue`)** -```go -// Simplified from: api/pkg/scheduler/dead_letter_queue.go -func NewDeadLetterQueue( - issuer ScheduleIssuer, - newGenEventHandler EntityGenerationChangeHandler, // For Kafka changes - retryStrategy RetryStrategy, - aggregator MetricAggregator, // For metrics -) *DeadLetterQueueDefault { - if issuer == nil || retryStrategy == nil { /* ... panic ... */ } - - // ... (timer and metrics setup) ... - - return &DeadLetterQueueDefault{ - issuer: issuer, - retryStrategy: retryStrategy, - deadLetterEnterChannel: make(chan *schedule.Schedule, 10), - deadLetterExitChannel: make(chan *schedule.Schedule, 10), - scheduleQueue: schedule.NewQueue(), - processTimer: newQueryableTimer( /* ... */ ), - // ... (other fields) ... - } -} -``` -This function sets up the DLQ with its necessary helpers, most importantly the `issuer` to re-run tasks and the `retryStrategy` to manage the retry timings. - -**3. The Main Loop (`RunBlocking`)** -The `RunBlocking` method is the DLQ's engine room. It listens for incoming tasks and manages the retry process: -```go -// Simplified from: api/pkg/scheduler/dead_letter_queue.go -func (d *DeadLetterQueueDefault) RunBlocking(ctx context.Context) { - for { - select { - case schedule := <-d.deadLetterEnterChannel: // A failed task arrives! - log.G(ctx).Tracef("[DLQ] Received new schedule to enqueue: %s", schedule.Key) - d.scheduleQueue.Enqueue(schedule) // Add to internal queue - // If timer wasn't running, start it for the new task. - if !d.processTimer.running { - // updateTimerAfter sets timer based on retryStrategy for head of queue - updateTimerAfter(processEventNone, d.scheduleQueue, d.processTimer, d.retryStrategy) - } - - case <-d.processTimer.timer.C: // Timer for a retry attempt fired! - // processQueueEntry tries to run the task at head of queue - // It uses d.issueScheduleProxy() as the action. - // It returns the schedule if it succeeded OR if max retries hit. - if exitSched := processQueueEntry(ctx, d.scheduleQueue, d.processTimer, d.retryStrategy, d.issueScheduleProxy()); exitSched != nil { - // Task succeeded or max retries reached. Send to exit channel. - d.deadLetterExitChannel <- exitSched - } - // If exitSched is nil, it means retry failed but more retries are pending. - // processQueueEntry has already reset the timer for the next attempt. - - case <-ctx.Done(): // Application is shutting down - d.reset() // Clear queue, stop timer - return // Exit loop - // ... (case for newGenEventHandler for Kafka rebalancing - more advanced) ... - } - } -} -``` -* **`schedule := <-d.deadLetterEnterChannel`**: A task that failed its initial execution attempt arrives. It's added to the `d.scheduleQueue`. The `updateTimerAfter` helper function (not shown in detail, but part of `queue_processor.go` utilities) is called to ensure the `d.processTimer` is set to fire for this task's first retry attempt, according to the `d.retryStrategy`. -* **`<-d.processTimer.timer.C`**: The timer for a retry has elapsed. The `processQueueEntry` function (a shared helper, also used by the PostProcessor) is called. This function: - 1. Takes the task from the front of `d.scheduleQueue`. - 2. Calls the action function provided, which for the DLQ is `d.issueScheduleProxy()`. - 3. If `d.issueScheduleProxy()` (see below) succeeds, `processQueueEntry` returns the schedule. - 4. If `d.issueScheduleProxy()` fails, `processQueueEntry` consults the `d.retryStrategy`: - * If more retries are allowed, `processQueueEntry` resets the `d.processTimer` for the next attempt (with backoff) and returns `nil` (meaning the task stays in the DLQ). - * If max retries are reached, `processQueueEntry` returns the schedule (even though it failed). - 5. Back in `RunBlocking`, if `exitSched` is not `nil` (meaning success or max retries), the schedule is sent to `d.deadLetterExitChannel`. - -**4. Attempting the Retry (`issueScheduleProxy`)** -This small but vital function defines *what action* the `processQueueEntry` should try: -```go -// From: api/pkg/scheduler/dead_letter_queue.go -func (d *DeadLetterQueueDefault) issueScheduleProxy() actionFunc { - // actionFunc is: func(ctx context.Context, sched *schedule.Schedule) (*schedule.Schedule, error) - return func(ctx context.Context, sched *schedule.Schedule) (*schedule.Schedule, error) { - // Use the DLQ's issuer to try and run the schedule again. - return sched, d.issuer.IssueSchedule(ctx, sched) - } -} -``` -It simply calls `d.issuer.IssueSchedule(ctx, sched)`. If this returns `nil` (no error), the retry was successful! If it returns an error, the retry failed. - -**5. The `RetryStrategy` Interface** -This interface (defined in `dead_letter_queue.go`) is key to the "wait and try again, maybe longer" logic. -```go -// From: api/pkg/scheduler/dead_letter_queue.go -type RetryStrategy interface { - GetNewDuration() (time.Duration, error) // Gets next wait duration. Returns ErrMaxRetriesReached if done. - MustGetNewDuration() time.Duration // Similar, but panics on max retries. - ResetSequence() time.Duration // Resets retry count, returns first duration. -} -``` -The `scheduler` project might use an implementation like "exponential backoff" (e.g., wait 2s, then 4s, then 8s, etc., up to a maximum number of tries). This strategy is passed to `processQueueEntry` which uses it to manage the retry loop for each task. - -## Why is the Failed Task Retry Queue So Important? - -* **Resilience:** It makes the scheduler robust against temporary, transient failures. Many tasks that would otherwise be lost can succeed on a retry. -* **Automatic Recovery:** It automates the retry process. No manual intervention is needed for common temporary issues. -* **Controlled Retries:** The `RetryStrategy` ensures that retries don't happen too aggressively (which could overload a struggling downstream system) and that they don't go on forever. -* **Improved Task Completion Rates:** By giving tasks multiple chances, the overall success rate of task processing increases significantly. - -## Conclusion - -The `scheduler.DeadLetterQueueDefault` is our scheduler's "intensive care unit" for tasks that fail their initial execution. Instead of discarding them, it holds them and intelligently retries them using a `RetryStrategy` that often involves increasing delays (backoff). If a retry succeeds, the task moves on to post-processing. If all retries are exhausted, the task also moves to post-processing, but marked as failed. This mechanism is crucial for building a resilient and reliable scheduling system that can handle the inevitable hiccups of distributed environments. - -We've now covered how schedules are defined, submitted, managed, executed, handled post-execution, and retried upon failure. But how do we keep an eye on the overall health and performance of our scheduler application itself? That's what we'll explore in the final chapter: [Chapter 10: Application Health Monitor (`api.InstrumentationService`)](10_application_health_monitor___api_instrumentationservice___.md). - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/10_application_health_monitor___api_instrumentationservice___.md b/documentation/10_application_health_monitor___api_instrumentationservice___.md deleted file mode 100644 index 90a08a68..00000000 --- a/documentation/10_application_health_monitor___api_instrumentationservice___.md +++ /dev/null @@ -1,335 +0,0 @@ -# Chapter 10: Application Health Monitor (`api.InstrumentationService`) - -Welcome to the final chapter of our `scheduler` project tutorial! In [Chapter 9: Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md), we explored how our system handles tasks that fail, giving them a second chance through a retry mechanism. We've covered a lot, from defining schedules to executing them and handling various outcomes. Now, let's look at how we can keep an eye on the overall health and performance of our scheduler. - -## The Problem: Knowing How Your Scheduler is Doing - -Imagine you've built a complex machine, like our `scheduler`. It's running, processing tasks, handling retries, and more. But how do you know if it's running smoothly? -* Is it processing tasks quickly, or is there a growing backlog? -* How many tasks are currently waiting in the retry queue? -* Are tasks being successfully executed, or are many ending up deleted after post-processing? - -Without a way to see these "vital signs," troubleshooting problems or understanding the system's load becomes very difficult. It's like driving a car without a dashboard – you wouldn't know your speed, fuel level, or if the engine is overheating until it's too late! - -We need a system that acts like this dashboard, or like a doctor performing a checkup, providing real-time information about how the scheduler is performing. This is crucial for: -* **Observability:** Understanding what's happening inside the system. -* **Monitoring Health:** Quickly identifying if something is wrong. -* **Debugging:** Pinpointing bottlenecks or areas of high error rates. -* **Capacity Planning:** Knowing if the system is handling its current load or if it needs more resources. - -## Meet the `api.InstrumentationService`: Your Scheduler's Dashboard - -The `api.InstrumentationService` is our scheduler's built-in health and performance monitoring system. It provides a way to collect and expose various **metrics** – numerical data points that represent some aspect of the scheduler's operation. - -Think of it like this: -* Different parts of the scheduler (like the [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`)](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) or the [Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`)](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md)) are like sensors in your car. -* These "sensors" report data (e.g., "one task entered the queue," "one task was processed"). -* This data is collected and stored by a component called `instrumentation.MetricsDataStore`. -* The `api.InstrumentationService` then acts as the interface to read this collected data, much like how your car's dashboard displays information from the sensors. - -Other systems, or even human operators, can then query this service (usually via API endpoints) to get a snapshot of the scheduler's current status. - -## Key Concepts of the Health Monitor - -Let's break down the main parts involved: - -1. **Metrics (The "Vital Signs"):** - These are specific counts or values that tell us something about the system. Our `scheduler` tracks various metrics, defined as constants in `api/pkg/instrumentation/supported_metrics.go`: - ```go - // From: api/pkg/instrumentation/supported_metrics.go - package instrumentation - - const ( - MetricDataStoreReadFromTopic = "metrics.dataStore.read" // Schedules read by DailyStore - MetricDataStoreStored = "metrics.dataStore.stored" // Schedules sent to DailyStore's output - MetricDataStoreExited = "metrics.dataStore.exited" // Schedules too old for DailyStore - - MetricDeadLetterQueueEntered = "metrics.dlq.entered" // Tasks entering DLQ - MetricDeadLetterQueueExited = "metrics.dlq.exited" // Tasks leaving DLQ (succeeded/failed) - - MetricPostProcessorEntered = "metrics.postProcessor.entered" // Tasks entering PostProcessor - MetricPostProcessorRescheduled = "metrics.postProcessor.rescheduled" // Tasks rescheduled - MetricPostProcessorDeleted = "metrics.postProcessor.deleted" // Tasks deleted by PostProcessor - ) - ``` - These are just examples. A real system might have many more, like "tasks_successfully_issued," "tasks_failed_permanently," etc. Each metric is a simple counter. - -2. **`instrumentation.MetricsDataStore` (The "Data Collector and Storage"):** - * Found in: `api/pkg/instrumentation/metrics_data_store.go` - * This component is responsible for actually storing the current values of all metrics. - * Other parts of the system (like the `DailyScheduleStoreDefault` or `DeadLetterQueueDefault`) "register" the metrics they care about with the `MetricsDataStore`. - * When an event happens (e.g., a task enters the DLQ), the relevant component tells the `MetricsDataStore` to `Increment()` the corresponding metric (e.g., `MetricDeadLetterQueueEntered`). - * The `MetricsDataStore` keeps all these counts in memory, protected by a mutex for safe access from multiple parts of the program. - -3. **`api.MetricExtractor` (The "Data Retriever Contract"):** - * Found in: `api/instrumentation.go` - * This is an interface that defines how metrics can be extracted (retrieved). - ```go - // From: api/instrumentation.go - package api - // ... - type MetricExtractor interface { - ListSupported(ctx context.Context) []string - Extract(ctx context.Context, metric string) (int, error) - ExtractAll(ctx context.Context) map[string]int - } - ``` - * The `instrumentation.MetricsDataStore` implements this interface, meaning it knows how to list available metrics, get a specific one, or get all of them. - -4. **`api.InstrumentationService` (The "Dashboard Interface"):** - * Found in: `api/instrumentation.go` - * This is the service that other parts of the application (like API handlers) will use to get metric data. - * It's a very simple service; its main job is to delegate requests to a `MetricExtractor` (which is typically our `MetricsDataStore`). - ```go - // From: api/instrumentation.go - package api - // ... - type InstrumentationService interface { - ListSupportedMetrics(ctx context.Context) []string - ExtractMetric(ctx context.Context, metric string) (int, error) - ExtractAllMetrics(ctx context.Context) map[string]int - } - - type InstrumentationServiceDefault struct { // The implementation - extractor MetricExtractor - } - // ... (NewInstrumentationService and method implementations) - ``` - -## How to Use the Health Monitor: Checking the Scheduler's Pulse - -You typically interact with the `api.InstrumentationService` through HTTP API endpoints, which we learned about in [Chapter 2: HTTP API Endpoints & Routing](02_http_api_endpoints___routing_.md). The `scheduler` provides a few endpoints specifically for metrics, usually under a path like `/api/instrumentation/metrics`. - -These endpoints are defined in `api/cmd/scheduler-api/routes/routes.go` and use handler functions from `api/cmd/scheduler-api/instrumentation/metrics.go`. - -Let's see how you might use them with a tool like `curl` (a command-line tool for making HTTP requests): - -**1. Listing Supported Metrics:** -You want to know what metrics are even available. -* **Request:** Send a `GET` request to `/api/instrumentation/metrics` (or a similar path depending on router setup). - ```bash - curl http://your-scheduler-address/api/instrumentation/metrics - ``` -* **Handler:** The `instrumentation.ListSupportedMetrics` handler (from `api/cmd/scheduler-api/instrumentation/metrics.go`) is called. It uses the `InstrumentationService`'s `ListSupportedMetrics` method. - ```go - // From: api/cmd/scheduler-api/instrumentation/metrics.go - func ListSupportedMetrics(s api.InstrumentationService) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - metrics := s.ListSupportedMetrics(r.Context()) // Calls service - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(metrics) // Sends response - } - } - ``` -* **Output (Example JSON):** - ```json - [ - "metrics.dataStore.read", - "metrics.dataStore.stored", - "metrics.dlq.entered", - "metrics.dlq.exited", - "metrics.postProcessor.rescheduled" - ] - ``` - This tells you the names of metrics you can query. - -**2. Getting a Specific Metric:** -You want to know the current value of `metrics.dlq.entered`. -* **Request:** Send a `GET` request to `/api/instrumentation/metrics/{metric_name}`. - ```bash - curl http://your-scheduler-address/api/instrumentation/metrics/metrics.dlq.entered - ``` -* **Handler:** The `instrumentation.GetMetric` handler is called. - ```go - // From: api/cmd/scheduler-api/instrumentation/metrics.go - func GetMetric(s api.InstrumentationService) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - metricName, _ := getMetricName(r) // Extracts "metrics.dlq.entered" - val, err := s.ExtractMetric(r.Context(), metricName) // Calls service - // ... (error handling) ... - json.NewEncoder(w).Encode(val) // Sends value - } - } - ``` -* **Output (Example JSON):** - ```json - 15 - ``` - This means 15 tasks have entered the Dead Letter Queue since the counter was last reset (or since the application started). - -**3. Getting All Metrics:** -You want a snapshot of all current metric values. -* **Request:** Send a `GET` request to `/api/instrumentation/metrics/all` (or similar). - ```bash - curl http://your-scheduler-address/api/instrumentation/metrics/all - ``` -* **Handler:** The `instrumentation.GetAllMetrics` handler is called. - ```go - // From: api/cmd/scheduler-api/instrumentation/metrics.go - func GetAllMetrics(s api.InstrumentationService) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - metricsMap := s.ExtractAllMetrics(r.Context()) // Calls service - // ... - json.NewEncoder(w).Encode(toArray(metricsMap)) // Sends array - } - } - ``` - The `toArray` helper converts the map of metrics into an array of objects for the JSON response. -* **Output (Example JSON):** - ```json - [ - { "name": "metrics.dataStore.read", "value": 1052 }, - { "name": "metrics.dataStore.stored", "value": 1000 }, - { "name": "metrics.dlq.entered", "value": 15 }, - { "name": "metrics.dlq.exited", "value": 12 }, - { "name": "metrics.postProcessor.rescheduled", "value": 980 } - ] - ``` - -This data gives you a rich overview of the scheduler's activity! - -## Under the Hood: How Metrics are Collected and Served - -Let's trace the journey of a metric from its origin to being displayed. - -**Step-by-Step Flow:** - -1. **Registration:** When a component (e.g., `DeadLetterQueueDefault`) starts, it tells the `MetricsDataStore` about the metrics it will manage. - * Example: `DeadLetterQueueDefault` calls `MetricsDataStore.RegisterForManagement("metrics.dlq.entered")`. - * The `MetricsDataStore` initializes this metric to `0`. - * It returns an `MetricIncrementor` and `MetricResetter` to the `DeadLetterQueueDefault`. - -2. **Incrementing:** When an event occurs (e.g., a task enters the `DeadLetterQueueDefault`), it uses its `MetricIncrementor` to update the count. - * Example: `DeadLetterQueueDefault` calls `dlqEnteredIncrementor.Increment()`. - * The `MetricsDataStore` (behind the scenes) safely increases the value of `"metrics.dlq.entered"`. - -3. **Extraction Request:** A user or monitoring tool makes an API call (e.g., `GET /api/instrumentation/metrics/all`). - -4. **Service Call:** The API handler calls the `api.InstrumentationService` (e.g., `ExtractAllMetrics()`). - -5. **Extractor Call:** The `InstrumentationServiceDefault` implementation simply calls its `MetricExtractor` (which is our `MetricsDataStore`). - ```go - // From: api/instrumentation.go - func (i *InstrumentationServiceDefault) ExtractAllMetrics(ctx context.Context) map[string]int { - return i.extractor.ExtractAll(ctx) // Delegates to extractor - } - ``` - -6. **Data Retrieval:** The `MetricsDataStore` retrieves the current values of all its stored metrics. - ```go - // From: api/pkg/instrumentation/metrics_data_store.go - func (s *MetricsDataStore) ExtractAll(ctx context.Context) map[string]int { - s.mutex.RLock() // Read lock for safety - defer s.mutex.RUnlock() - return s.metrics // Returns the internal map - } - ``` - -7. **Response:** The data flows back through the service and API handler to the client. - -**Visualizing the Metric Retrieval Flow:** - -```mermaid -sequenceDiagram - participant Client as External Client/Tool - participant APIEndpoint as /api/instrumentation/metrics/... - participant InstrService as InstrumentationService - participant MDS as MetricsDataStore (as MetricExtractor) - - Client->>APIEndpoint: GET /api/instrumentation/metrics/all - APIEndpoint->>InstrService: ExtractAllMetrics() - InstrService->>MDS: ExtractAll() - MDS-->>InstrService: map[string]int {metric_name: value, ...} - InstrService-->>APIEndpoint: map[string]int {...} - APIEndpoint-->>Client: JSON Response -``` - -### Diving into `MetricsDataStore` - -The `instrumentation.MetricsDataStore` is the heart of metric collection. -File: `api/pkg/instrumentation/metrics_data_store.go` - -1. **Structure:** - ```go - // From: api/pkg/instrumentation/metrics_data_store.go - type MetricsDataStore struct { - metrics map[string]int // Where the counts are stored - mutex sync.RWMutex // For thread-safe access - } - ``` - It's a simple map to hold metric names and their integer values, with a `sync.RWMutex` to prevent issues if multiple parts of the program try to read or write metrics at the same time. - -2. **Registering a Metric:** - ```go - // From: api/pkg/instrumentation/metrics_data_store.go - func (s *MetricsDataStore) RegisterForManagement(metric string) (MetricIncrementor, MetricResetter, error) { - s.mutex.Lock() // Exclusive lock for modification - defer s.mutex.Unlock() - - if _, found := s.metrics[metric]; found { - return nil, nil, fmt.Errorf("metric is already under management") - } - s.metrics[metric] = 0 // Initialize to zero - - // Return functions that can increment/reset this specific metric - inc := func() { s.increment(metric) } - res := func() { s.reset(metric) } - return metricIncrementAction(inc), metricResetAction(res), nil - } - ``` - When a component registers a metric (e.g., `MetricDeadLetterQueueEntered`), the `MetricsDataStore` adds it to its internal `metrics` map (initialized to 0). It then returns two small functions (wrapped as `MetricIncrementor` and `MetricResetter`) that are tied *specifically* to that metric name. - -3. **Incrementing a Metric:** - The `MetricIncrementor` returned by `RegisterForManagement` calls this internal `increment` method: - ```go - // From: api/pkg/instrumentation/metrics_data_store.go - func (s *MetricsDataStore) increment(metric string) { - s.mutex.Lock() // Exclusive lock - defer s.mutex.Unlock() - - val, found := s.metrics[metric] - if !found { return } // Should not happen if registered - s.metrics[metric] = val + 1 - } - ``` - This safely finds the metric in the map and adds 1 to its value. - -### How Other Components Update Metrics - -Components like `DeadLetterQueueDefault` or `DailyScheduleStoreDefault` integrate with this system. -The file `api/pkg/scheduler/metrics.go` shows a helper struct: -```go -// From: api/pkg/scheduler/metrics.go -package scheduler -import "github.com/nestoca/scheduler/api/pkg/instrumentation" - -type metricModifier struct { - instrumentation.MetricIncrementor - instrumentation.MetricResetter -} -``` -A component (e.g., `DeadLetterQueueDefault`) would typically: -1. Have fields for each `MetricIncrementor` it needs (e.g., `dlqEnteredCounter`). -2. During its initialization, it calls `MetricsDataStore.RegisterForManagement()` for each metric it's responsible for and stores the returned `MetricIncrementor`. -3. When a relevant event occurs (e.g., a schedule enters the DLQ), it calls `dlqEnteredCounter.Increment()`. - -This design keeps the `MetricsDataStore` as a central, simple collector, and lets individual components manage their own metric updates cleanly. - -## Why is the Application Health Monitor So Important? - -* **Visibility:** It shines a light into the "black box" of the scheduler, letting you see what it's doing. -* **Proactive Problem Detection:** By monitoring key metrics (e.g., queue sizes, error rates), you can often detect problems before they become critical. -* **Performance Tuning:** Metrics can help identify bottlenecks. If `metrics.dataStore.read` is very high but `metrics.dataStore.stored` is low, it might indicate a problem in the `DailyScheduleStoreDefault`'s filtering or processing. -* **Foundation for Alerting:** Monitoring systems can use these metrics to trigger alerts if values go outside expected ranges (e.g., "DLQ size > 100 for 5 minutes"). - -## Conclusion - -The `api.InstrumentationService`, powered by the `instrumentation.MetricsDataStore` and various `MetricIncrementor`s throughout the application, provides a vital "dashboard" for our `scheduler`. It collects key performance indicators (metrics) like queue lengths, items processed, and errors encountered. This data is then exposed via API endpoints, allowing operators and monitoring tools to observe the system's health, diagnose issues, and understand its performance characteristics. This capability is essential for running any robust application in a production environment. - -Congratulations on completing this tutorial series on the `scheduler` project! You've journeyed through defining tasks with `schedule.Schedule`, exposing functionality via [HTTP API Endpoints & Routing](02_http_api_endpoints___routing_.md), managing schedules with `api.SchedulesService`, understanding the `api.Core` aggregator, seeing the `api.Scheduler` execution engine in action, learning about [Kafka-based Schedule Persistence and Communication](06_kafka_based_schedule_persistence_and_communication_.md), how tasks are organized daily by `scheduler.DailyScheduleStoreDefault`, handled post-execution by `scheduler.SchedulePostProcessorDefault`, retried with `scheduler.DeadLetterQueueDefault`, and finally, how the system's health is monitored. We hope this gives you a solid foundation for understanding and working with the `scheduler`! - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/documentation/index.md b/documentation/index.md deleted file mode 100644 index f77583a5..00000000 --- a/documentation/index.md +++ /dev/null @@ -1,81 +0,0 @@ -# Tutorial: scheduler - -This project implements a **task scheduling system**. It allows users to define *tasks* (what to do, like sending a Kafka message) and *schedules* (when and how often to do it). These schedules are reliably stored and communicated using **Kafka**. The system's *engine* then picks up due tasks, *executes* them by, for example, sending a message to a target Kafka topic, handles *recurring tasks* by rescheduling them, and manages *failed tasks* through a retry mechanism. It also offers *monitoring* of its operations through metrics. - - -**Source Repository:** [None](None) - -```mermaid -flowchart TD - A0["Core Services Aggregator (`api.Core`) -"] - A1["Schedule Blueprint (`schedule.Schedule`) -"] - A2["Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`) -"] - A3["Schedule Management Service (`api.SchedulesService`) -"] - A4["Kafka-based Schedule Persistence and Communication -"] - A5["Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`) -"] - A6["Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`) -"] - A7["Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`) -"] - A8["Application Health Monitor (`api.InstrumentationService`) -"] - A9["HTTP API Endpoints & Routing -"] - A9 -- "Accesses core services via" --> A0 - A9 -- "Manages schedules via" --> A3 - A9 -- "Exposes metrics from" --> A8 - A9 -- "Controls engine via" --> A2 - A0 -- "Holds instance of" --> A2 - A0 -- "Holds instance of" --> A3 - A0 -- "Holds instance of" --> A8 - A3 -- "Defines/Validates" --> A1 - A3 -- "Writes schedule definitions to" --> A4 - A5 -- "Reads schedule definitions ..." --> A4 - A5 -- "Organizes for daily execution" --> A1 - A2 -- "Gets due tasks from" --> A5 - A2 -- "Triggers execution of" --> A1 - A2 -- "Issues tasks to target topi..." --> A4 - A2 -- "Delegates post-execution to" --> A6 - A6 -- "Processes executed" --> A1 - A6 -- "Updates/Deletes schedule de..." --> A4 - A2 -- "Sends failed tasks to" --> A7 - A7 -- "Queues failed" --> A1 - A7 -- "Re-issues tasks via" --> A4 - A5 -- "Reports metrics to" --> A8 - A6 -- "Reports metrics to" --> A8 - A7 -- "Reports metrics to" --> A8 -``` - -## Chapters - -1. [Schedule Blueprint (`schedule.Schedule`) -](01_schedule_blueprint___schedule_schedule___.md) -2. [HTTP API Endpoints & Routing -](02_http_api_endpoints___routing_.md) -3. [Schedule Management Service (`api.SchedulesService`) -](03_schedule_management_service___api_schedulesservice___.md) -4. [Core Services Aggregator (`api.Core`) -](04_core_services_aggregator___api_core___.md) -5. [Task Execution Engine (`api.Scheduler` / `scheduler.DefaultScheduler`) -](05_task_execution_engine___api_scheduler_____scheduler_defaultscheduler___.md) -6. [Kafka-based Schedule Persistence and Communication -](06_kafka_based_schedule_persistence_and_communication_.md) -7. [Daily Task Organizer (`scheduler.DailyScheduleStoreDefault`) -](07_daily_task_organizer___scheduler_dailyschedulestoredefault___.md) -8. [Post-Execution Handler (`scheduler.SchedulePostProcessorDefault`) -](08_post_execution_handler___scheduler_schedulepostprocessordefault___.md) -9. [Failed Task Retry Queue (`scheduler.DeadLetterQueueDefault`) -](09_failed_task_retry_queue___scheduler_deadletterqueuedefault___.md) -10. [Application Health Monitor (`api.InstrumentationService`) -](10_application_health_monitor___api_instrumentationservice___.md) - - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/01_event_.md b/nesto/corebanking/01_event_.md new file mode 100644 index 00000000..8d1f7c5c --- /dev/null +++ b/nesto/corebanking/01_event_.md @@ -0,0 +1,179 @@ +# Chapter 1: Event + +Welcome to the `corebanking` project tutorial! We're thrilled to have you on board. In this first chapter, we'll dive into one of the most fundamental concepts in our system: the **Event**. + +## What's the Big Deal with Events? + +Imagine you're building a brand new digital bank. A customer, let's call her Alice, wants to open a new savings account. She fills out a form, clicks "Submit," and voilà, her account is created. + +But how does the bank *remember* that this account was created? How does it keep a permanent, trustworthy record of this action? What if we need to look back and see exactly when Alice's account was opened or what its initial details were? + +This is where **Events** come into play. + +In our `corebanking` system, an **Event** is an **immutable record of something significant that has already happened**. Think of it like an official entry in a historical logbook or a diary. Once an entry is written, it's not changed. + +When something important occurs in the system, like Alice's account being created, we record this fact as an Event. For example, after successfully processing Alice's request to create an account (which we'll later learn is a [Command](02_command_.md)), an `AccountCreatedEvent` is generated. + +## Key Characteristics of an Event + +Events have a few defining features: + +1. **Immutable Record:** Once an Event is recorded, it **cannot be changed**. It's a statement of fact about something that happened in the past. Like carving something in stone. +2. **Significant Happenings:** We don't create Events for every tiny operation. They are reserved for actions that have a meaningful impact on the system's state. Examples: `AccountCreatedEvent`, `FundsDepositedEvent`, `CustomerAddressUpdatedEvent`. +3. **Past Tense:** Event names usually reflect that the action has already completed. Notice the "ed" or "d" at the end: `AccountCreatedEvent`, `PaymentDeviceLinkedEvent`. +4. **Source of Truth:** Events are the ultimate, undeniable truth for what has occurred. If we ever need to understand the current state of something (like an account's balance or status), we can, in theory, replay all the Events related to it from the very beginning. It's like re-reading a ship's log to know its entire journey. +5. **Result of a [Command](02_command_.md):** Typically, an Event is produced after the system processes a [Command](02_command_.md). A [Command](02_command_.md) is a request to do something (e.g., "Create Account"). If the command is successful and changes the system's state, one or more Events are generated as a result. + +## Events in Action: Creating Alice's Account + +Let's revisit Alice creating her savings account: + +1. Alice submits her request (this is a [Command](02_command_.md)). +2. The `corebanking` system processes this request. It checks if Alice can open an account, if she provided all necessary details, etc. +3. If everything is okay, the system changes its state: a new account for Alice now exists! +4. To record this fact, the system generates an `AccountCreatedEvent`. This Event will contain all the crucial information: Alice's customer ID, the new account ID, the type of account, the currency, the date and time of creation, etc. +5. This `AccountCreatedEvent` is then durably stored, perhaps in a special database. It becomes part of the bank's permanent history. + +##What Does an Event Look Like? (A Peek at the Code) + +In Go, an Event is often represented as a struct. There's a general structure for all events, and then specific data for each type of event. + +Here's a simplified general `Event` structure from our system: + +```go +// From: api/events.go + +// Event represents an event +type Event struct { + ID uuid.UUID // Unique ID for this specific event instance + AggregateID uuid.UUID // ID of the entity this event pertains to (e.g., Account ID) + AggregateType string // Type of the entity (e.g., "Account") + Type string // Specific type of event (e.g., "AccountCreatedEvent") + Data interface{} // The actual data specific to this event type + Created time.Time // When the event was created + // ... other general fields like Actor, TenantID ... +} +``` + +* `ID`: Every single event occurrence gets its own unique identifier. +* `AggregateID`: This tells us which specific entity the event is about. For an `AccountCreatedEvent`, this would be the ID of the newly created account. We'll learn more about [Aggregates](03_aggregate_.md) later. +* `AggregateType`: The kind of entity, like "Account" or "Customer". +* `Type`: A string that clearly states what kind of event this is, e.g., `com.nesto.corebanking.accounts.events.AccountCreatedEvent`. +* `Data`: This is the payload, containing the specific details of *what happened*. For an `AccountCreatedEvent`, this would hold the account category, currency, etc. +* `Created`: A timestamp indicating when the event occurred. + +The `Data` field itself would be another struct, specific to the event type. For example, an `AccountCreatedEvent` might have data like this (simplified): + +```go +// From: api/account_aggregate.go + +// AccountCreatedEvent represents a created account. +type AccountCreatedEvent struct { + Category string `avro:"category"` + Product *AccountProduct `avro:"product"` + AvailableCurrencies []string `avro:"availableCurrencies"` + CustomerIDs []uuid.UUID `avro:"customerIds"` + // ... other fields like Parameters, Metadata ... +} +``` +This `AccountCreatedEvent` struct holds all the specific details that are relevant when a new account is made. + +## How an Event is "Born" - The Internals + +Let's look under the hood to see how an `AccountCreatedEvent` comes to life. + +1. **A Request Arrives:** Alice's request to create an account comes in as a [Command](02_command_.md) (e.g., `CreateAccountCommand`). +2. **Processing by an [Aggregate](03_aggregate_.md):** An [Aggregate](03_aggregate_.md) is responsible for handling commands related to a specific entity (like an `AccountAggregate` for accounts). It takes the `CreateAccountCommand`. +3. **Validation and State Change:** The `AccountAggregate` checks if the command is valid (e.g., are all required fields present?). If valid, it determines what state changes are needed. For a new account, this means setting its initial properties. +4. **Event Generation:** Because the state changed, the `AccountAggregate` now creates an `AccountCreatedEvent`, filling it with details from the command and any system-generated data (like the creation timestamp). +5. **Event Storage:** This newly minted event is then passed to an "Event Store" (a specialized database or system component) to be saved permanently. + +Here's a simplified sequence diagram: + +```mermaid +sequenceDiagram + participant Alice + participant CoreBankingAPI as API + participant AccountAgg as Account Aggregate + participant EventStore + + Alice->>API: Submit Create Account Form + API->>AccountAgg: Process CreateAccountCommand + AccountAgg-->>AccountAgg: Validate, decide to create account + AccountAgg->>EventStore: New AccountCreatedEvent(details...) + EventStore-->>AccountAgg: Event Saved! + AccountAgg-->>API: Account Creation Successful + API-->>Alice: Your account is created! +``` + +Let's look at a snippet from `api/account_aggregate.go` where an event is created within the `HandleCommand` method. This method is part of the [Aggregate](03_aggregate_.md) (which we'll cover in detail later). + +```go +// Simplified from api/account_aggregate.go +// Inside AccountAggregate's HandleCommand method: + +case *CreateAccountCommand: // This is the request to create an account + // ... (some validation and setup logic) ... + + // Prepare the specific data for our AccountCreatedEvent + data := &AccountCreatedEvent{ + Category: c.Category, + Product: c.Product, + AvailableCurrencies: c.AvailableCurrencies, + CustomerIDs: c.CustomerIDs, + // ... other relevant details from the command ... + } + + // This is the magic moment! An Event is created. + // a.NewEvent() wraps 'data' with general event info (ID, timestamp, etc.) + event = a.NewEvent(command, data) + + // ... (logic to apply this event to the aggregate's state and save it) ... + a.ApplyChangeHelper(a, event, true) +``` +In this code: +* We receive a `CreateAccountCommand` (aliased as `c`). +* We gather the necessary details into an `AccountCreatedEvent` struct (`data`). +* `a.NewEvent(command, data)` is a helper method that takes the original command and the event-specific `data` to construct the full `Event` object, including its unique ID, timestamp, and type. +* `a.ApplyChangeHelper` is another crucial step. After an event is "born," the [Aggregate](03_aggregate_.md) itself uses this event to update its own internal state. This ensures the [Aggregate's](03_aggregate_.md) in-memory representation reflects the new reality. + +The `ApplyChange` method (or a helper it calls) looks at the type of event and updates the [Aggregate's](03_aggregate_.md) fields accordingly: + +```go +// Simplified from api/account_aggregate.go +// Inside AccountAggregate's ApplyChange method: + +func (a *AccountAggregate) ApplyChange(event es.Event) { + switch e := event.Data.(type) { // e is the specific event data + case *AccountCreatedEvent: + a.ID = event.AggregateID // The aggregate now knows its ID + a.Category = e.Category + a.Status = AccountStatusCreated // Set initial status + a.Created = event.Created // Record creation time + // ... update other fields based on AccountCreatedEvent ... + case *AccountActivatedEvent: + a.Status = AccountStatusActivated + // ... + // ... other event types ... + } +} +``` +This ensures that the `AccountAggregate`'s state is consistent with the events it has produced and processed. + +## Why Immutability and "Facts" Matter So Much + +The fact that Events are immutable historical records is incredibly powerful: + +* **Reliability & Auditability:** You have a perfect, trustworthy audit trail of everything significant that has happened. Need to know why an account is in a particular state? Replay its events! This is invaluable for debugging, compliance, and understanding system behavior. +* **State Reconstruction:** If, for some reason, the current "snapshot" of an account's data gets corrupted, you can rebuild it by replaying all its historical Events in order. +* **Decoupling:** Other parts of the system can subscribe to events and react to them independently, without needing to know the intricate details of how the event was produced. For example, a notification service could listen for `AccountCreatedEvent` and send a welcome email to Alice. + +## Conclusion + +Events are the bedrock of our `corebanking` system's memory. They are **immutable facts** representing **significant past occurrences**. Each Event tells a small part of a larger story, like the story of Alice's bank account. By recording and storing these Events, we build a reliable, auditable, and resilient system. + +We've seen that Events are often born from processing requests. In the next chapter, we'll take a closer look at these requests themselves. Get ready to learn about the [Command](02_command_.md)! + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/02_command_.md b/nesto/corebanking/02_command_.md new file mode 100644 index 00000000..d8113c5b --- /dev/null +++ b/nesto/corebanking/02_command_.md @@ -0,0 +1,179 @@ +# Chapter 2: Command + +In our [previous chapter](01_event_.md), we learned about [Events](01_event_.md) – the immutable records of things that have already happened in our `corebanking` system. For example, when Alice successfully created her bank account, an `AccountCreatedEvent` was generated and stored. + +But how did Alice *tell* the bank she wanted to create an account in the first place? How does she ask the bank to do something, like deposit money or change her address? This is where **Commands** come in. + +## What's a Command? The Bank's Instruction Slip + +Imagine you walk into a physical bank. To make a deposit, you fill out a deposit slip. To open an account, you fill out an application form. These forms are your formal instructions to the bank. + +In the `corebanking` system, a **Command** is very similar. It's like an **instruction or a formal request** you submit to the system. It represents an **intent** to change something. + +Think of it this way: +* An [Event](01_event_.md) says: "This happened!" (e.g., `AccountCreatedEvent`) +* A **Command** says: "I want this to happen!" (e.g., `CreateAccountCommand`) + +Commands are specific, named operations that carry all the necessary data to perform an action. They are the primary way external users or other systems interact with the core logic of our bank to request changes. + +## Key Characteristics of a Command + +1. **Intent to Act:** A Command expresses a desire for the system to perform an action. It's a request, not a statement of fact. +2. **Specific Operation:** Each Command has a clear name that describes the intended action, like `CreateAccountCommand`, `DepositFundsCommand`, or `UpdateCustomerAddressCommand`. +3. **Carries Data:** A Command bundles all the information needed to execute the desired action. For instance, a `CreateAccountCommand` would include details like who the customer is, what type of account they want, and in what currency. +4. **Sent for Processing:** Commands are typically sent to a specific part of our system called an [Aggregate](03_aggregate_.md) (we'll learn about Aggregates in the next chapter!). The [Aggregate](03_aggregate_.md) is responsible for deciding if the Command is valid and then carrying out the action. +5. **May Result in [Events](01_event_.md):** If a Command is successfully processed and leads to a change in the system's state, one or more [Events](01_event_.md) are usually generated to record that change. If the Command is invalid (e.g., trying to deposit into a non-existent account), it might be rejected, and no [Events](01_event_.md) related to the intended change would be created. + +## Commands in Action: Alice's Account Creation (The "How") + +Let's go back to Alice wanting to open her savings account: + +1. **Alice Expresses Her Intent:** Alice, through a web form or a mobile app, indicates she wants a new account. She provides her details (customer ID, desired currency, etc.). +2. **The `CreateAccountCommand` is Born:** The application she's using takes this information and packages it into a `CreateAccountCommand`. This command now holds all the data needed to open the account. +3. **Command is Sent:** This `CreateAccountCommand` is sent to the `corebanking` system. +4. **Processing by an [Aggregate](03_aggregate_.md):** An `AccountAggregate` (which is responsible for managing account-related operations) receives the `CreateAccountCommand`. +5. **Validation & Logic:** The `AccountAggregate` checks: + * Is the customer ID valid? + * Is the currency supported? + * Are all required fields present? + * Any other business rules? +6. **Success and [Event](01_event_.md) Generation:** If everything is valid, the `AccountAggregate` proceeds to create the account. As a result of this successful operation, it generates an `AccountCreatedEvent` (which we saw in Chapter 1!). +7. **Failure:** If the command is invalid (e.g., Alice provides an unsupported currency), the `AccountAggregate` rejects the command, and no `AccountCreatedEvent` is generated. An error message would typically be returned to Alice. + +## What Does a Command Look Like? (A Peek at the Code) + +In Go, just like [Events](01_event_.md), Commands are often represented as structs. They carry the data necessary for the operation. + +Here's a simplified look at the `CreateAccountCommand` from our `corebanking` project (`api/account_aggregate.go`): + +```go +// From: api/account_aggregate.go + +// CreateAccountCommand represents the creation of an account. +type CreateAccountCommand struct { + es.BaseCommand // Provides common fields for all commands + Category string `json:"category"` + Product *AccountProduct `json:"product"` // Details about the specific account product + AvailableCurrencies []string `json:"availableCurrencies"` + CustomerIDs []uuid.UUID `json:"customerIds"` + // ... other fields like Parameters, Metadata ... +} +``` + +Let's break this down: +* `es.BaseCommand`: This is a standard part of many commands in our system. It usually includes an `AggregateID`. For `CreateAccountCommand`, this `AggregateID` will be the ID of the *new* account we want to create. For a command like `DepositFundsCommand`, the `AggregateID` would be the ID of the *existing* account to deposit into. +* `Category`: What kind of account is it (e.g., "ASSET", "LIABILITY"). +* `Product`: Specifies the type of banking product (e.g., "Savings Account Basic", "Current Account Premium"). +* `AvailableCurrencies`: Which currencies can this account hold (e.g., `["USD", "EUR"]`). +* `CustomerIDs`: A list of customer IDs associated with this account. + +Each field in the command provides a piece of information that the system needs to fulfill the request. + +Another example, a `CloseAccountCommand`, would be simpler: + +```go +// From: api/account_aggregate.go + +// CloseAccountCommand represents the closing of an account. +type CloseAccountCommand struct { + es.BaseCommand // Contains the ID of the account to close + Reason string `json:"reason"` // Why is the account being closed? +} +``` +This command needs to know *which* account to close (via `BaseCommand`'s `AggregateID`) and optionally, a reason for closing it. + +## How a Command is Processed: Under the Hood + +When a command arrives, how does the system handle it? + +1. **Dispatch:** The command is typically sent from an API layer (like a web server handling Alice's request) to the core business logic. +2. **Targeting an [Aggregate](03_aggregate_.md):** The command is routed to the correct [Aggregate](03_aggregate_.md). An [Aggregate](03_aggregate_.md) is like a guardian for a specific piece of data (e.g., an `AccountAggregate` guards a specific bank account). If it's a `CreateAccountCommand`, a new `AccountAggregate` instance might be effectively created to handle it. If it's `DepositFundsCommand`, it's sent to the existing `AccountAggregate` for that account. +3. **Validation:** The [Aggregate](03_aggregate_.md) first validates the command. Can this action be performed? Is all the data correct and complete? +4. **Business Logic:** If valid, the [Aggregate](03_aggregate_.md) executes the business rules associated with the command. +5. **[Event](01_event_.md) Generation:** If the business logic results in a state change, the [Aggregate](03_aggregate_.md) creates one or more [Events](01_event_.md) to describe what happened. + +Here's a simplified sequence diagram illustrating the flow for creating an account: + +```mermaid +sequenceDiagram + participant AliceApp as Alice's App + participant CoreAPI as Core Banking API + participant AccountAgg as Account Aggregate + participant EventStore + + AliceApp->>CoreAPI: I want to create an account (details...) + CoreAPI->>AccountAgg: CreateAccountCommand{details...} + AccountAgg-->>AccountAgg: Validate command data (e.g., currency OK?) + AccountAgg-->>AccountAgg: Business logic: Okay, let's create it! + AccountAgg->>EventStore: Store this new AccountCreatedEvent(...) + EventStore-->>AccountAgg: Event saved! + AccountAgg-->>CoreAPI: Success! Account created. + CoreAPI-->>AliceApp: Your account is ready! +``` + +Let's look at a snippet from `api/account_aggregate.go` where an `AccountAggregate` handles a `CreateAccountCommand`. This happens inside a method called `HandleCommand`: + +```go +// Simplified from AccountAggregate.HandleCommand in api/account_aggregate.go + +// ... (inside HandleCommand method) ... +switch c := command.(type) { // 'c' is the incoming command +case *CreateAccountCommand: + // 1. Check if a product is specified and valid (simplified) + if c.Product != nil { + // ... (logic to validate product exists) ... + } + + // 2. The command carries the intended ID for the new account + a.ID = command.GetAggregateID() // 'a' is the AccountAggregate + + // 3. Prepare the data for the Event that will be created + eventData := &AccountCreatedEvent{ + Category: c.Category, + Product: c.Product, + CustomerIDs: c.CustomerIDs, + AvailableCurrencies: c.AvailableCurrencies, + // ... other details from the command 'c' ... + } + + // 4. Create the actual Event (as we saw in Chapter 1) + event := a.NewEvent(command, eventData) + + // 5. Apply the event to change the aggregate's state and record it + a.ApplyChangeHelper(a, event, true) // This stores the event for saving +``` + +In this snippet: +1. The `HandleCommand` method receives the `CreateAccountCommand` (aliased as `c`). +2. It performs some initial validation (like checking the product). +3. It uses the data *from the command* (`c.Category`, `c.Product`, etc.) to populate an `AccountCreatedEvent`. +4. The `a.NewEvent(...)` function (which we touched on in Chapter 1) creates the full [Event](01_event_.md) object, adding things like a unique event ID and timestamp. +5. `a.ApplyChangeHelper(...)` is a crucial step where the [Aggregate](03_aggregate_.md) updates its own state based on this new event and adds the event to a list of changes to be saved. + +So, the **Command** provides the *intent* and the *data*, and the [Aggregate's](03_aggregate_.md) `HandleCommand` method uses that to *do the work* and produce [Events](01_event_.md). + +## Commands vs. [Events](01_event_.md): A Quick Reminder + +It's vital to distinguish between Commands and [Events](01_event_.md): + +| Feature | Command | [Event](01_event_.md) | +| :-------------- | :-------------------------------------------- | :------------------------------------------------------ | +| **Purpose** | Request an action (intent) | Record a fact (something that happened) | +| **Tense** | Present/Future (e.g., "Create Account") | Past (e.g., "Account Created") | +| **Outcome** | May succeed or fail | Is a record of a successful state change | +| **Result of** | User input, system process | Successful command processing | +| **Analogy** | Filling out a request form | An entry in a historical logbook | + +You send a `CreateAccountCommand` (your request). If it's successful, an `AccountCreatedEvent` (the historical record) is generated. + +## Conclusion + +Commands are the way we tell our `corebanking` system what we want to do. They are formal, data-rich instructions representing an **intent to change the system's state**. They are named operations like `CreateAccountCommand` or `DepositFundsCommand`. + +When a Command is received, it's typically handled by an [Aggregate](03_aggregate_.md), which validates it, executes business logic, and, if successful, produces [Events](01_event_.md) to record the changes. + +Now that we understand how we *ask* the system to do things (Commands) and how the system *remembers* what happened ([Events](01_event_.md)), it's time to meet the component responsible for processing these Commands and generating those Events. Let's dive into the world of the [Aggregate](03_aggregate_.md)! + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/03_aggregate_.md b/nesto/corebanking/03_aggregate_.md new file mode 100644 index 00000000..4d605b58 --- /dev/null +++ b/nesto/corebanking/03_aggregate_.md @@ -0,0 +1,241 @@ +# Chapter 3: Aggregate + +Welcome to Chapter 3! In our [previous chapter](02_command_.md), we learned about [Commands](02_command_.md) – how we tell the `corebanking` system what we want to do, like "Create an Account" or "Deposit Funds." We also know from [Chapter 1](01_event_.md) that when these [Commands](02_command_.md) are successful, [Events](01_event_.md) are generated to record what happened. + +But who is in charge of receiving a [Command](02_command_.md), deciding if it's okay, and then creating the [Event](01_event_.md)? If Alice wants to deposit money into *her specific account*, how does the system make sure the change happens only to *her* account and follows all the rules? This is where the **Aggregate** steps in. + +## Meet the Main Character: The Aggregate + +Think of an **Aggregate** as the main character in a story, like a specific bank account (Account #12345) or a particular customer (Alice Wonderland). It's a self-contained unit that looks after its own information (its "state") and its own rules. + +When a [Command](02_command_.md) arrives—like a request to deposit money into Account #12345—it's the `AccountAggregate` for Account #12345 that handles it. This Aggregate will: +1. Check if the command is valid (e.g., "Is the account open?", "Is the deposit amount positive?"). +2. If everything is okay, it will change its state (e.g., increase its balance). +3. Then, it will record this change as a fact by producing an [Event](01_event_.md) (e.g., `FundsDepositedEvent`). + +This way, the Aggregate ensures that all changes to *its* data are consistent and follow the rules. It also provides a historical record of everything that has happened to it through the [Events](01_event_.md) it produces. Aggregates are a cornerstone of the Event Sourcing pattern used in our `corebanking` system. + +## Key Ideas About Aggregates + +1. **A Guardian of Data:** An Aggregate is responsible for a specific piece of data or a small group of related data. For example, an `AccountAggregate` is responsible for *one specific bank account*. It manages the account's balance, status, customer links, etc. +2. **Unique Identity:** Each Aggregate instance has a unique ID. So, `AccountAggregate` with ID `acc-123` is different from `AccountAggregate` with ID `acc-456`. +3. **Manages Its Own State:** The "state" is the current information about the entity. For an account, this includes its balance, status (CREATED, ACTIVATED, CLOSED), currency, etc. +4. **Enforces Business Rules:** Aggregates contain the logic to check if a requested action (a [Command](02_command_.md)) is allowed. Can you withdraw more money than you have? Can you close an account that has a pending transaction? The Aggregate knows! +5. **Processes [Commands](02_command_.md):** It takes a [Command](02_command_.md) as input. +6. **Produces [Events](01_event_.md):** If a [Command](02_command_.md) is valid and changes the state, the Aggregate creates one or more [Events](01_event_.md) to describe what happened. +7. **Consistency Boundary:** All changes within a single Aggregate happen together (atomically). Either all changes are applied, or none are. This keeps the data for that specific entity consistent. + +## Example: Alice Deposits Money + +Let's imagine Alice (customer `cust-001`) has an account (`acc-123`) and wants to deposit $50. + +1. **A [Command](02_command_.md) is Sent:** Alice's banking app creates a `DepositFundsCommand` with details: `AccountID: "acc-123"`, `Amount: 50.00`. +2. **The Right Aggregate is Found:** The system directs this command to the `AccountAggregate` responsible for account `acc-123`. +3. **The Aggregate Works:** + * The `AccountAggregate` for `acc-123` loads its current state (perhaps its current balance is $100 and its status is "ACTIVATED"). + * It checks business rules: Is the account "ACTIVATED"? Yes. Is $50 a valid amount? Yes. + * Since the command is valid, the Aggregate decides an `AccountCreditedEvent` (or similar) needs to happen. +4. **An [Event](01_event_.md) is Born:** It creates an `AccountCreditedEvent` with details: `AmountCredited: 50.00`, `NewBalance: 150.00` (or the event might just state the amount credited, and the balance is recalculated). +5. **State is Updated:** The Aggregate updates its own internal state. Its balance is now $150. +6. **[Event](01_event_.md) is Stored:** The `AccountCreditedEvent` is saved permanently. + +Now, account `acc-123` correctly reflects the deposit, and there's a permanent record of it. + +## What Does an Aggregate Look Like? (A Peek at the Code) + +In Go, an Aggregate is typically a struct. It often embeds a base struct that provides common functionalities and includes fields to hold its state. + +Here's a very simplified view of an `AccountAggregate` from `api/account_aggregate.go`: + +```go +// From: api/account_aggregate.go +// Simplified AccountAggregate structure +type AccountAggregate struct { + es.BaseAggregate // Provides common aggregate features (like ID, event helpers) + + // State of the account (the actual data for *this* account instance) + *AccountAggregateState + + // (Dependencies like repositories are omitted for simplicity here) +} +``` +* `es.BaseAggregate`: This is a helper struct from our event sourcing library (`es`). It provides common things an aggregate needs, like a place to store its ID, version, and methods for creating new [Events](01_event_.md). +* `*AccountAggregateState`: This pointer field is where the actual data specific to *this* account instance is stored. + +The `AccountAggregateState` itself is another struct: + +```go +// From: api/account_aggregate.go +// AccountAggregateState holds the data for an account +type AccountAggregateState struct { + Category string // e.g., "ASSET", "LIABILITY" + Status string // e.g., "CREATED", "ACTIVATED", "CLOSED" + CustomerIDs []uuid.UUID // Which customers own this account + // ... other fields like product info, currency, parameters ... + Created time.Time // When the account was created + // (The actual balance is often managed by postings and calculated, + // but for conceptual understanding, imagine it's part of the state) +} +``` +This `AccountAggregateState` holds all the important details about one specific account. When we talk about an Aggregate's "state," we're talking about the values in these fields. + +## How an Aggregate Handles a [Command](02_command_.md) + +There are two main methods in an Aggregate that work together: +1. `HandleCommand`: This is the entry point. It receives a [Command](02_command_.md), applies business rules, and if successful, creates [Event(s)](01_event_.md). +2. `ApplyChange`: This method takes an [Event](01_event_.md) and uses it to update the Aggregate's internal state. + +Let's look at simplified versions: + +### 1. `HandleCommand` + +When a [Command](02_command_.md) like `CreateAccountCommand` comes in: + +```go +// Simplified from AccountAggregate.HandleCommand in api/account_aggregate.go +// This method decides what to do when a command arrives. +func (a *AccountAggregate) HandleCommand(ctx context.Context, command es.Command) error { + var event es.Event // This will hold the event we generate + + switch c := command.(type) { // 'c' is the specific command (e.g., CreateAccountCommand) + case *CreateAccountCommand: + // 1. Validations (e.g., is product specified? are customers valid?) + // (Detailed validation logic omitted for brevity) + if c.Product == nil { + return errors.New("product is required") // Simplified error + } + + // 2. Set the ID for this new account aggregate from the command + // The command carries the intended ID for the new entity. + a.ID = command.GetAggregateID() + + // 3. Prepare data for the AccountCreatedEvent using info from the command + eventData := &AccountCreatedEvent{ + Category: c.Category, + Product: c.Product, // Details of the account type + CustomerIDs: c.CustomerIDs, + // ... other details from the CreateAccountCommand 'c' ... + } + + // 4. Create the actual Event object using a helper from BaseAggregate + event = a.NewEvent(command, eventData) // 'command' provides context + + // 5. Tell the aggregate to apply this change to itself + // and mark the event to be saved. + a.ApplyChangeHelper(a, event, true) + + // case *DepositFundsCommand: + // // ... logic to handle deposit, create FundsDepositedEvent ... + // case *ActivateAccountCommand: + // // ... logic to handle activation, create AccountActivatedEvent ... + } + return nil // If successful +} +``` +* The `switch` statement checks the type of [Command](02_command_.md). +* For `CreateAccountCommand`, it performs validations. +* It then prepares the data for an `AccountCreatedEvent` using information from the command. +* `a.NewEvent(...)` is a helper (from `es.BaseAggregate`) that wraps `eventData` with standard event fields like a unique Event ID, timestamp, Aggregate ID, etc. +* `a.ApplyChangeHelper(...)` is another crucial helper. It does two things: + 1. Calls `a.ApplyChange(event)` (see below) to update the aggregate's in-memory state. + 2. Adds the `event` to a list of "changes" that will be saved to the Event Store later. + +### 2. `ApplyChange` + +After an [Event](01_event_.md) is created by `HandleCommand`, the `ApplyChange` method is called to update the Aggregate's state. **The state of an Aggregate is *always* a result of applying its [Events](01_event_.md).** + +```go +// Simplified from AccountAggregate.ApplyChange in api/account_aggregate.go +// This method updates the aggregate's state based on an event. +func (a *AccountAggregate) ApplyChange(event es.Event) { + // 'e' is the specific event data (like AccountCreatedEvent details) + switch e := event.Data.(type) { + case *AccountCreatedEvent: + // The aggregate's ID is usually set by BaseAggregate from the event + a.ID = event.AggregateID + // Now, update the AccountAggregateState fields + a.Category = e.Category + a.Product = e.Product // Store product info + a.Status = AccountStatusCreated // Set initial status + a.CustomerIDs = e.CustomerIDs + a.Created = event.Created // Record creation time + // ... update other AccountAggregateState fields based on the event ... + + case *AccountActivatedEvent: // If an AccountActivatedEvent occurred + a.Status = AccountStatusActivated // Update the status + a.Updated = event.Created // Update modification time + + // case *FundsDepositedEvent: + // a.Balance = a.Balance + e.AmountDeposited // Hypothetical balance update + // a.Updated = event.Created + } +} +``` +* This method also uses a `switch` on the *type* of [Event](01_event_.md). +* For an `AccountCreatedEvent`, it sets the initial properties of the `AccountAggregateState` (like `Category`, `Status`, `Created` timestamp). +* For an `AccountActivatedEvent`, it would just update the `Status` field. +* This ensures that the Aggregate's in-memory state accurately reflects all the [Events](01_event_.md) that have happened to it. + +## The Life Cycle: Loading an Existing Aggregate + +What if Alice wants to deposit into an *existing* account? The system doesn't just create a new `AccountAggregate`. Instead: +1. It knows the `AccountID` (e.g., `acc-123`) from the `DepositFundsCommand`. +2. It goes to the Event Store (where all [Events](01_event_.md) are saved). +3. It fetches *all* the past [Events](01_event_.md) for `acc-123` in the order they happened. +4. It creates a fresh `AccountAggregate` instance. +5. It then "replays" each historical [Event](01_event_.md) by calling `ApplyChange` for each one. + * `ApplyChange(AccountCreatedEvent{...})` -> sets initial state. + * `ApplyChange(AccountActivatedEvent{...})` -> sets status to "ACTIVATED". + * `ApplyChange(SomeOtherEvent{...})` -> updates state further. +6. After replaying all its history, the `AccountAggregate` is now in its correct, current state, ready to handle the new `DepositFundsCommand`. + +This process of rebuilding state from [Events](01_event_.md) is fundamental to Event Sourcing. + +## Why Aggregates are So Important + +* **Encapsulation:** They bundle data (state) and behavior (rules) together, making the system easier to understand and manage. +* **Consistency:** They ensure that the data for a specific entity (like an account) is always valid and consistent according to business rules. +* **Clear Responsibilities:** It's clear which Aggregate is responsible for which [Commands](02_command_.md) and which part of the system's data. +* **Event Sourcing Enabler:** They are the primary producers of [Events](01_event_.md) in an Event Sourced system. The history of [Events](01_event_.md) an Aggregate produces *is* the history of that entity. + +## The Big Picture: Processing a Command + +Let's visualize how a command flows through an Aggregate to produce an Event. + +```mermaid +sequenceDiagram + participant ClientApp as Alice's App + participant CoreAPI as Core Banking API + participant AccountAgg as Account Aggregate (for acc-123) + participant EventStore + + ClientApp->>CoreAPI: Request: Deposit $50 into Account 123 + CoreAPI->>AccountAgg: Process DepositFundsCommand{AccountID: 123, Amount: 50} + Note over AccountAgg: 1. Load/Rehydrate Account 123's state
by replaying its past events from EventStore. + AccountAgg-->>AccountAgg: 2. HandleCommand: Validate Command (e.g., account active?) + AccountAgg-->>AccountAgg: 3. HandleCommand: Business Logic: Okay to deposit. + AccountAgg->>AccountAgg: 4. HandleCommand: Create FundsDepositedEvent{Amount: 50} + AccountAgg->>AccountAgg: 5. HandleCommand calls ApplyChange(FundsDepositedEvent)
--> Update internal state (e.g., balance) + Note over AccountAgg: 6. Event is marked for saving. + AccountAgg->>EventStore: 7. System saves FundsDepositedEvent + EventStore-->>AccountAgg: Event Saved! + AccountAgg-->>CoreAPI: Success! Deposit processed. + CoreAPI-->>ClientApp: Your deposit is successful! +``` +This diagram shows that the Aggregate is the central processor. It loads its state, validates the [Command](02_command_.md), applies logic, creates an [Event](01_event_.md), updates itself based on that [Event](01_event_.md), and then the [Event](01_event_.md) gets stored. + +## Conclusion + +Aggregates are like mini-managers for specific entities in our system, such as individual bank accounts or customers. They are responsible for: +* Guarding their own data (state). +* Enforcing business rules. +* Processing incoming [Commands](02_command_.md). +* Producing [Events](01_event_.md) as a record of successful changes. + +By doing this, Aggregates ensure data consistency and are the heart of how our `corebanking` system uses Event Sourcing. They take the "intent" from a [Command](02_command_.md) and, if valid, turn it into a historical "fact" as an [Event](01_event_.md). + +Now that we understand how Aggregates work and how their state can be built from [Events](01_event_.md), you might be wondering: how are these Aggregates (and their [Events](01_event_.md)) actually loaded from and saved to storage? That's where our next topic comes in. Get ready to learn about the [Repository](04_repository_.md)! + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/04_repository_.md b/nesto/corebanking/04_repository_.md new file mode 100644 index 00000000..1c6b2e27 --- /dev/null +++ b/nesto/corebanking/04_repository_.md @@ -0,0 +1,229 @@ +# Chapter 4: Repository + +Welcome to Chapter 4! In the [previous chapter](03_aggregate_.md), we learned about [Aggregates](03_aggregate_.md), like our `AccountAggregate`. We saw how they manage their own state (like an account's balance or status) by processing [Commands](02_command_.md) and then producing [Events](01_event_.md) to record what happened. + +Now, a couple of important questions arise: +1. When an `AccountAggregate` needs to process a [Command](02_command_.md) for an *existing* account, how does it get its current state? Remember, its state is built by replaying its past [Events](01_event_.md). Where do these [Events](01_event_.md) come from? +2. After an `AccountAggregate` processes a [Command](02_command_.md) and creates new [Events](01_event_.md), where do these new [Events](01_event_.md) get stored so they are not forgotten? +3. Beyond [Aggregates](03_aggregate_.md), what if other parts of our bank system simply need to *read* account information – say, to display a list of Alice's accounts on a web page? How do they get this data without needing to understand all the [Events](01_event_.md)? + +This is where the **Repository** pattern comes to our rescue! + +## What is a Repository? The Bank's Diligent Archivist + +Imagine a bank's large, old-fashioned archive room, filled with rows and rows of filing cabinets. This room has a team of dedicated archivists. + +* If a bank teller (let's say, an [Aggregate](03_aggregate_.md)) needs the history of a specific account to understand its current situation, they don't go rummaging through the cabinets themselves. They fill out a request slip and give it to an archivist. The archivist knows exactly where to find the files (the [Events](01_event_.md) or account data) and brings them back. +* If the teller completes a transaction and needs to file away the record (a new [Event](01_event_.md) or updated account information), they hand it to the archivist, who ensures it's stored in the right place, safely and correctly. + +A **Repository** in our `corebanking` system works just like this team of archivists. It's a component responsible for **persisting (saving) and retrieving data**. This data could be account details, customer information, transaction histories, or the [Events](01_event_.md) that make up an [Aggregate's](03_aggregate_.md) history. + +The crucial part is that the Repository provides an **abstraction** over the actual data storage mechanism. This means other parts of the system (like [Aggregates](03_aggregate_.md) or [Services](07_service_.md) which we'll see later) don't need to know if the data is stored in a MySQL database, a different type of database, or even text files. They just talk to the Repository using simple, clear methods. + +## Key Ideas About Repositories + +1. **Hides Storage Details:** The rest of the application doesn't care *how* or *where* data is stored. Is it SQL? Is it a NoSQL database? The Repository handles those details. This is like not needing to know the archivist's specific filing system, just that they can get you the file you need. +2. **Defined Contract (Interface):** A Repository offers a clear set of operations, like `GetByID()`, `Save()`, or `Search()`. In Go, this "contract" is usually defined by an `interface`. +3. **One Repository per Data Type (Usually):** You'll often have a specific repository for each main type of data you're managing. For example: + * An `AccountsRepository` to manage `Account` data. + * A `CustomersRepository` to manage `Customer` data. +4. **Different Repositories for Different Needs:** + * **For [Aggregates](03_aggregate_.md) (Event Sourcing):** In an Event Sourced system like ours, [Aggregates](03_aggregate_.md) (like `AccountAggregate`) are special. They are not stored directly as a single row in a database. Instead, their state is derived from their history of [Events](01_event_.md). A specialized type of repository (often called an `EventSourcedRepository` or similar) is used: + * To **load** an [Aggregate](03_aggregate_.md): It fetches all its past [Events](01_event_.md) from an "Event Store" (a database optimized for storing events). + * To **save** an [Aggregate](03_aggregate_.md): It takes any new [Events](01_event_.md) the [Aggregate](03_aggregate_.md) has produced and stores them in the Event Store. + * **For Read Data (Projections/Views):** Sometimes, we need to query data that's already nicely formatted for display or reporting – this is often called a "read model" or "projection." For example, when displaying account details on a screen, we want the current status, balance, etc., directly, not a list of [Events](01_event_.md). Repositories are also used to fetch this kind of data. Our `AccountsRepository` in `api/accounts.go` is an example of this type. + +## Using a Repository: Let's Look at Account Data + +Let's focus on how we might get information about an account that's easy to read and display, using the `AccountsRepository` from our project. Imagine a [Service](07_service_.md) (a component we'll discuss later) needs to fetch Alice's account details. + +### The Contract: The `AccountsRepository` Interface + +First, there's an interface that defines what operations can be performed for accounts. This is like the list of services the archivists offer. + +```go +// From: api/accounts.go + +// AccountsRepository defines methods for account data storage +type AccountsRepository interface { + Save(ctx context.Context, account *Account) error + // UpdateStatus updates the status of an account + UpdateStatus(ctx context.Context, accountID uuid.UUID, status string, updated time.Time) error + GetByID(ctx context.Context, accountID uuid.UUID) (*Account, error) + // ... other methods like Search, GetByIDs, SavePaymentDeviceLink etc. +} +``` +* This interface declares methods like `Save`, `UpdateStatus`, and `GetByID`. +* It uses `*Account`, which is a struct (`api.Account`) representing the "read model" of an account – a snapshot of its current, easily readable state. This is different from the `AccountAggregate` which is focused on processing [Commands](02_command_.md) and [Events](01_event_.md). + +### Getting Account Data + +If a part of our system needs to get details for account `acc-123`, it would use an implementation of `AccountsRepository`: + +```go +// Somewhere in our application (e.g., inside an Account Service) +var accountsRepo api.AccountsRepository // This would be a concrete implementation + +// ... +accountID := uuid.FromString("acc-123") // The ID of the account we want +account, err := accountsRepo.GetByID(context.Background(), accountID) +if err != nil { + // Handle error, maybe the account wasn't found + fmt.Println("Error fetching account:", err) + return +} + +fmt.Println("Fetched Account Category:", account.Category) +fmt.Println("Fetched Account Status:", account.Status) +``` +* **Input:** The `GetByID` method takes a `context` and the `accountID`. +* **Output:** It returns an `*api.Account` struct (containing the account's details like category, status, etc.) and an error (which will be `nil` if successful). + +The code calling `accountsRepo.GetByID` doesn't know or care if the data came from MySQL, a different database, or even a text file. That's the beauty of the repository abstraction! + +## Under the Hood: The `MySQLRepository` for Accounts + +Our project has a concrete implementation of the `AccountsRepository` interface that uses a MySQL database. It's located in `api/pkg/accounts/mysql_repository.go`. + +Let's see a simplified version of how its `GetByID` method might work: + +```go +// Simplified from: api/pkg/accounts/mysql_repository.go + +// MySQLRepository implements AccountsRepository using MySQL. +type MySQLRepository struct { + // (Internal details, like how it gets a database connection, are hidden here) +} + +// GetByID retrieves an account's projection (read model) by its ID from MySQL. +func (r *MySQLRepository) GetByID(ctx context.Context, accountID uuid.UUID) (*api.Account, error) { + // 1. Get a database connection (simplified) + conn, _ := repository.SQLConnection(ctx) + + // 2. This struct holds the raw data fetched from the 'account' table + var dbAccountData struct { // In real code, this is 'accountDbEntry' + ID uuid.UUID `db:"id"` + Category string `db:"category"` + Status string `db:"status"` + // ... other fields matching the database table columns ... + } + + // 3. Build the SQL query to select account data + sqlQuery := "SELECT id, category, status /*, ...other columns... */ FROM account WHERE id = ?" + + // 4. Execute the query against the MySQL database + err := conn.GetContext(ctx, &dbAccountData, sqlQuery, accountID) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, e.NewEntityIDNotFound("account", accountID.String()) // Specific error for "not found" + } + // Some other database error occurred + return nil, e.NewInternal(err) + } + + // 5. Convert the raw database data (dbAccountData) into the application's api.Account struct + apiAccount := &api.Account{ + ID: dbAccountData.ID, + Category: dbAccountData.Category, + Status: dbAccountData.Status, + // ... populate other fields ... + } + + // 6. In the real code, it also fetches related data like customer links, parameters, + // payment device links and populates them into apiAccount. + // Example (highly simplified): + // apiAccount.CustomerIDs, _ = r.getCustomersForAccount(ctx, accountID) + // apiAccount.Parameters, _ = r.getParametersForAccount(ctx, accountID) + + return apiAccount, nil +} +``` +Let's break this down step-by-step: +1. **Get Connection:** It first obtains a connection to the MySQL database. +2. **Data Holder:** `dbAccountData` is a temporary struct to hold the data exactly as it comes from the database table. +3. **SQL Query:** It defines the SQL `SELECT` statement to fetch data from the `account` table based on the `id`. +4. **Execute Query:** The query is executed. If no account is found, it returns a "not found" error. Other database errors are also handled. +5. **Map Data:** If data is found, the raw `dbAccountData` is converted into an `*api.Account` struct. This `api.Account` struct is what the rest of our application understands and expects. +6. **Fetch Related Data:** An account might have associated customers, specific parameters, or linked payment devices. The full repository method would also query other tables to fetch this related information and populate the `api.Account` struct completely. + +Here's a sequence diagram showing this: + +```mermaid +sequenceDiagram + participant AppCode as Application Code + participant AccRepo as AccountsRepository (MySQL) + participant MySQLDB as MySQL Database + + AppCode->>AccRepo: GetByID("acc-123") + AccRepo->>MySQLDB: SELECT id, category, status FROM account WHERE id = "acc-123" + MySQLDB-->>AccRepo: Row data (ID, Category, Status) + AccRepo-->>AccRepo: Map row to api.Account struct + AccRepo->>MySQLDB: SELECT customer_id FROM account_customer WHERE account_id = "acc-123" (etc. for other related data) + MySQLDB-->>AccRepo: Related data + AccRepo-->>AccRepo: Add related data to api.Account struct + AccRepo-->>AppCode: *api.Account (fully populated) +``` + +Similarly, a `Save` method in `MySQLRepository` would take an `*api.Account` struct, convert its fields into a format suitable for the database, and then execute an `INSERT` or `UPDATE` SQL statement. + +## What About [Aggregates](03_aggregate_.md) and Their [Events](01_event_.md)? + +As mentioned earlier, [Aggregates](03_aggregate_.md) like `AccountAggregate` are handled a bit differently in an Event Sourced system. They also use a Repository, but it's a specialized one. + +* **Loading an [Aggregate](03_aggregate_.md):** + When the system needs to load, say, `AccountAggregate` for `acc-123`: + 1. It asks an `EventSourcedRepository` to `Load("acc-123")`. + 2. This repository connects to an **Event Store** (a special database designed to store sequences of [Events](01_event_.md)). + 3. It fetches all [Events](01_event_.md) ever recorded for `acc-123`. + 4. It creates a new, empty `AccountAggregate` instance. + 5. It then "replays" each historical [Event](01_event_.md) on this instance by calling its `ApplyChange(event)` method. This rebuilds the [Aggregate's](03_aggregate_.md) current state. + 6. The fully rehydrated `AccountAggregate` is returned. + +* **Saving an [Aggregate](03_aggregate_.md):** + After `AccountAggregate` processes a [Command](02_command_.md) and generates new [Events](01_event_.md): + 1. The system calls `Save(accountAggregate)` on the `EventSourcedRepository`. + 2. The repository gets the list of newly generated (uncommitted) [Events](01_event_.md) from the `accountAggregate`. + 3. It appends these new [Events](01_event_.md) to the Event Store, associated with `acc-123`. + 4. It then clears the list of uncommitted [Events](01_event_.md) from the `accountAggregate`. + +Here's how loading an [Aggregate](03_aggregate_.md) via an EventSourced Repository looks: +```mermaid +sequenceDiagram + participant AppLogic as Application Logic + participant ESRepo as EventSourced Repository + participant EventStoreDB as Event Store Database + participant AccAgg as Account Aggregate + + AppLogic->>ESRepo: Load AccountAggregate (ID: "acc-123") + ESRepo->>EventStoreDB: GetEvents("acc-123") + EventStoreDB-->>ESRepo: [Event1, Event2, Event3...] + ESRepo->>AccAgg: New AccountAggregate() + loop For each Event in List + ESRepo->>AccAgg: ApplyChange(CurrentEvent) + end + ESRepo-->>AppLogic: AccountAggregate (state rehydrated) +``` +This shows that the Repository pattern is flexible. We can have different repository implementations for different kinds of data (read models like `api.Account`) and different storage strategies (like an Event Store for [Aggregates](03_aggregate_.md)). + +## Why is This Abstraction So Useful? + +Using the Repository pattern brings several benefits: + +1. **Testability:** When testing parts of your application that need data, you don't always want to connect to a real database. It can be slow and complicated to set up. With repositories, you can create a "mock" or "fake" repository for your tests. This fake repository can pretend to be a database, returning predefined data or checking if `Save` was called correctly, all without any actual database interaction. +2. **Flexibility (Change Your Database Later):** Imagine your bank starts with MySQL but later decides to switch to a different database, say PostgreSQL. If your application code directly uses MySQL-specific queries everywhere, changing the database would be a nightmare! With repositories, you only need to write a *new implementation* of the `AccountsRepository` interface (e.g., `PostgreSQLRepository`). The rest of your application code that uses the `AccountsRepository` interface doesn't need to change at all because it was only depending on the interface, not the specific MySQL details. +3. **Clear Separation of Concerns:** Business logic (what the bank *does*) is kept separate from data access logic (how data is saved and retrieved). This makes the code cleaner, easier to understand, and maintain. [Aggregates](03_aggregate_.md) and [Services](07_service_.md) focus on their tasks without being cluttered by SQL queries. + +## Conclusion + +Repositories are like the organized and efficient archivists of our `corebanking` system. They handle all the details of storing and retrieving data, whether it's: +* Fetching historical [Events](01_event_.md) to rebuild an [Aggregate's](03_aggregate_.md) state. +* Saving new [Events](01_event_.md) produced by an [Aggregate](03_aggregate_.md). +* Querying user-friendly "read models" (like an `api.Account`) from a database for display or reporting. + +By providing a clean abstraction layer over data storage, Repositories make our system more maintainable, testable, and flexible. They ensure that other parts of the system can request or save data using simple methods, without needing to know the nitty-gritty details of how or where that data is actually stored. + +Now that we know how data can be persisted and retrieved, how do external requests (like Alice trying to create an account through a mobile app) actually reach our core system and make use of these components? In the next chapter, we'll look at the entry points for such requests: the [API Handler](05_api_handler_.md). + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/05_api_handler_.md b/nesto/corebanking/05_api_handler_.md new file mode 100644 index 00000000..756e8914 --- /dev/null +++ b/nesto/corebanking/05_api_handler_.md @@ -0,0 +1,225 @@ +# Chapter 5: API Handler + +Welcome to Chapter 5! In the [previous chapter](04_repository_.md), we explored [Repositories](04_repository_.md) and how they help us save and retrieve data, like account information or the history of [Events](01_event_.md) for an [Aggregate](03_aggregate_.md). We now know how our system manages its data. + +But how does a request from the outside world, like Alice trying to open a new bank account using her mobile app, actually reach our `corebanking` system and start this whole process? How does Alice's app "talk" to our bank's brain? This is where the **API Handler** comes into play. + +## What's an API Handler? The Bank's Welcome Desk + +Imagine walking into a bank. You don't just wander into the vault or start talking to random employees. You usually go to a specific counter or desk: +* "Account Opening" desk. +* "Deposits & Withdrawals" window. +* A specific online form for "Loan Application." + +An **API Handler** in our `corebanking` system is just like that specific desk or form. It's the **official entry point** for any external request that wants to interact with our system. "API" stands for Application Programming Interface, which is a way for different software programs to communicate with each other. + +Each API Handler is responsible for: +1. A specific **URL path** (like a web address, e.g., `/api/accounts`). +2. A specific **HTTP method** (like the type of action, e.g., `POST` for creating something, `GET` for fetching something). + +When a request arrives matching its designated URL and method, the API Handler takes charge. Its job is to: +* Understand the incoming request (e.g., read the data Alice submitted). +* Translate this request into an instruction our system understands, like a [Command](02_command_.md) (e.g., "Please create an account with these details"). +* Or, it might ask a [Service](07_service_.md) to fetch some information. +* Finally, it sends back an HTTP response to tell the requester what happened (e.g., "Account created successfully!" or "Sorry, there was an error."). + +Essentially, API Handlers are the bridge connecting the "web world" (HTTP requests and responses) to the internal business logic of our bank. + +## Key Responsibilities of an API Handler + +Let's break down what an API Handler typically does: + +1. **Listens at a Specific Address:** Each handler is tied to a unique combination of a URL path and an HTTP method. For example: + * `POST /api/accounts`: Handler for creating new accounts. + * `GET /api/accounts/{accountId}`: Handler for fetching details of a specific account. + * `PUT /api/accounts/{accountId}/close`: Handler for closing an account. +2. **Parses the Incoming Request:** When Alice submits her account application form, her details (name, desired currency, etc.) are often sent as a JSON payload in the body of an HTTP request. The API Handler needs to read this data and convert it into a format our Go code can understand (like a Go struct). +3. **Basic Validation (Sometimes):** The handler might do some very basic checks, like "Is the JSON data correctly formatted?" or "Are essential fields like 'currency' present?". However, deep business rule validation (e.g., "Is this currency supported by our bank?") is usually done further inside the system by [Aggregates](03_aggregate_.md) or [Services](07_service_.md). +4. **Translates to Internal Actions:** This is a crucial step. + * **For actions that change data (Writes):** The handler often creates a [Command](02_command_.md) object (like `CreateAccountCommand`) and populates it with the data from the request. It then sends this [Command](02_command_.md) to a "Command Bus," which is like a central dispatcher that routes the [Command](02_command_.md) to the correct [Aggregate](03_aggregate_.md) (e.g., `AccountAggregate`) for processing. + * **For actions that only read data (Reads):** The handler might directly call a [Service](07_service_.md) (e.g., an `AccountsService`) to fetch the requested information (like account details from a [Repository](04_repository_.md)). +5. **Formats and Sends the Response:** After the [Command](02_command_.md) is processed or the [Service](07_service_.md) returns data, the API Handler takes the result and crafts an HTTP response. This includes: + * An **HTTP Status Code** (e.g., `200 OK` for success, `201 Created` if something new was made, `400 Bad Request` if the input was wrong, `500 Internal Server Error` if something went wrong on our side). + * A **Response Body** (often JSON data, like the details of the created account, or an error message). + +## Alice Creates an Account: The API Handler's Role + +Let's see how an API Handler helps Alice open her account: + +1. **Alice's App Sends a Request:** Alice fills out the "New Account" form on her banking app and taps "Submit." Her app sends an HTTP `POST` request to the URL `/api/accounts`. The body of this request contains her details in JSON format, like: + ```json + { + "category": "ASSET", + "product": { "id": "prod-savings-basic", "name": "Basic Savings" }, + "availableCurrencies": ["USD"], + "customerIds": ["cust-alice-123"] + } + ``` +2. **The Correct Handler is Activated:** Our `corebanking` system's web server receives this request. It looks at the method (`POST`) and the path (`/api/accounts`) and finds the specific API Handler responsible for this combination. +3. **Handler Parses the Request:** This API Handler (let's call it the `CreateAccountHandler`) takes the JSON data from the request's body. It uses a JSON parser to convert this data into a Go struct that matches the structure of a `CreateAccountCommand`. +4. **Handler Creates and Dispatches a [Command](02_command_.md):** The handler now creates an instance of `api.CreateAccountCommand` and fills it with the details parsed from Alice's request. It then passes this command to something called a `CommandBus` (part of `core.EventSourcingClient` in our project). + ```go + // Inside the handler (conceptual) + var cmd api.CreateAccountCommand + // ... (parse JSON from request into cmd) ... + err := commandBus.HandleCommand(context, &cmd) + ``` +5. **Handler Waits and Responds:** The Command Bus ensures the `CreateAccountCommand` is processed (by an `AccountAggregate`, which generates an `AccountCreatedEvent`, as we saw in earlier chapters). + * If the command is processed successfully, the handler sends back an HTTP response like `204 No Content` (meaning "I did what you asked, and there's no specific content to return") or `201 Created` (if it were to return details of the created account). + * If there's an error (e.g., Alice provided an invalid customer ID), the [Aggregate](03_aggregate_.md) would signal an error, and the handler would send back an HTTP response like `400 Bad Request` with a JSON body describing the error. + +## A Peek at the Code: An Account Creation Handler + +Our project has API handlers defined in packages under `api/cmd/corebanking-api/`. For creating accounts, the relevant file is `api/cmd/corebanking-api/accounts/create.go`. + +Let's look at a simplified version of the `Create` function, which acts as our API Handler: + +```go +// Simplified from: api/cmd/corebanking-api/accounts/create.go + +// 'cb' is a CommandBus, which knows how to send commands +// to the right part of our system. +func Create(cb es.CommandBus) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // 1. Parse the request to get command data + command, err := parseCreateAccountCommand(r) + if err != nil { + // If parsing fails, send an error response + e.MarshallHTTP(err, w) // Helper to send formatted error + return + } + + // 2. Send the command for processing + err = cb.HandleCommand(r.Context(), &command) + if err != nil { + // If command processing fails, send an error response + e.MarshallHTTP(err, w) + return + } + + // 3. If everything is OK, send a success response + w.WriteHeader(http.StatusNoContent) // "204 No Content" + } +} +``` +Let's break this down: +* `func Create(cb es.CommandBus) http.HandlerFunc`: This function `Create` takes a `CommandBus` (our `es.EventSourcingClient`) as an argument and *returns* another function. This returned function is the actual handler that Go's HTTP server will call for each incoming request. + * `w http.ResponseWriter`: Used to write the HTTP response back to Alice's app. + * `r *http.Request`: Represents the incoming HTTP request from Alice's app. + +Inside the returned function: +1. **Parse Request:** `command, err := parseCreateAccountCommand(r)` + This calls a helper function (we'll see it next) to read Alice's data from the request (`r`) and turn it into our `api.CreateAccountCommand` struct. If there's an error during parsing (e.g., malformed JSON), it sends an error response using `e.MarshallHTTP(err, w)`. +2. **Send Command:** `err = cb.HandleCommand(r.Context(), &command)` + This is where the magic happens! The parsed `command` is handed off to the `CommandBus` (`cb`). The `CommandBus` will make sure it gets to the right [Aggregate](03_aggregate_.md) for processing. If the [Aggregate](03_aggregate_.md) rejects the command (e.g., business rule violation), `HandleCommand` returns an error, which is then sent back as an HTTP error. +3. **Success Response:** `w.WriteHeader(http.StatusNoContent)` + If both parsing and command processing are successful, this line sends an HTTP status code `204 No Content` back to Alice's app, indicating success. + +### Parsing the Request Data + +The `parseCreateAccountCommand` helper function looks something like this (simplified): + +```go +// Simplified from: api/cmd/corebanking-api/accounts/create.go +func parseCreateAccountCommand(r *http.Request) (api.CreateAccountCommand, error) { + var command api.CreateAccountCommand // Prepare an empty command struct + + // Decode the JSON data from the request body into our 'command' struct + if err := json.NewDecoder(r.Body).Decode(&command); err != nil { + // If decoding fails, log it and return a "bad format" error + log.G(r.Context()).WithError(err).Error("could not decode json") + return command, e.NewBadFormat(err.Error()) + } + + return command, nil // Return the populated command and no error +} +``` +* `var command api.CreateAccountCommand`: An empty `CreateAccountCommand` struct is created. +* `json.NewDecoder(r.Body).Decode(&command)`: This is standard Go for reading JSON. It takes the request's body (`r.Body`), creates a JSON decoder, and tries to fill the `command` struct with the data from the JSON. +* If `Decode` fails (e.g., the JSON is broken or doesn't match the command structure), an error is logged and a specific "bad format" error is returned. + +So, the handler first uses this parsing logic to understand the request, then dispatches the [Command](02_command_.md). + +## How Does the System Know Which Handler to Call? Routing! + +You might be wondering: if Alice's app sends a `POST` request to `/api/accounts`, how does our `corebanking` application know to call the `accounts.Create` handler function we just looked at? + +This is done by a **router**. A router is like a traffic controller for web requests. It looks at the URL and HTTP method of an incoming request and directs it to the correct handler function. Our project uses a popular Go router called `chi`. + +The setup for these routes is typically done in a file like `api/cmd/corebanking-api/routes/routes.go`. Here's a tiny snippet showing how the account creation route is defined: + +```go +// Simplified from: api/cmd/corebanking-api/routes/routes.go + +// 'core' contains things like our CommandBus (EventSourcingClient) +// and Services. +func getAPIRoutes(core *api.Core) http.Handler { + r := chi.NewRouter() // Create a new router + + // ... other routes ... + + // This line tells the router: + // If a POST request comes to "/accounts", + // call the handler returned by accounts.Create(), + // passing it the EventSourcingClient from 'core'. + r.Post("/accounts", accounts.Create(core.EventSourcingClient)) + + // Example of a GET request handler: + // r.Get("/accounts/{id}", accounts.Get(core.AccountsService)) + + // ... many other routes ... + + return r +} +``` +* `r := chi.NewRouter()`: Initializes a new router. +* `r.Post("/accounts", accounts.Create(core.EventSourcingClient))`: This is the key line. + * `r.Post` means this rule applies to HTTP `POST` requests. + * `"/accounts"` is the URL path. + * `accounts.Create(core.EventSourcingClient)` is the handler function we want to execute. Notice that we *call* `accounts.Create(...)` here. This function, as we saw, *returns* the actual `http.HandlerFunc` that `chi` will use. We pass `core.EventSourcingClient` (which is our `CommandBus`) to it so the handler can dispatch commands. + +For requests that read data, like getting account details, you'd see something like `r.Get("/accounts/{id}", accounts.Get(core.AccountsService))`. Here, `accounts.Get` would be another handler function, likely taking an `AccountsService` to fetch data. + +## The Journey of a Request: From App to Response + +Let's visualize the entire flow for Alice creating an account: + +```mermaid +sequenceDiagram + participant AliceApp as Alice's App + participant WebRouter as API Router (Chi) + participant CreateAccHandler as accounts.Create Handler + participant CmdBus as CommandBus (EventSourcingClient) + participant AccAgg as AccountAggregate + + AliceApp->>WebRouter: POST /api/accounts (with JSON data) + WebRouter->>CreateAccHandler: Execute handler for this route + CreateAccHandler-->>CreateAccHandler: parseCreateAccountCommand(request) + CreateAccHandler->>CmdBus: HandleCommand(CreateAccountCommand) + CmdBus->>AccAgg: Process CreateAccountCommand + Note over AccAgg: Validates, creates AccountCreatedEvent,
updates state, saves event (via Repository). + AccAgg-->>CmdBus: Success (or Error) + CmdBus-->>CreateAccHandler: Result of command processing + CreateAccHandler-->>CreateAccHandler: Prepare HTTP Response (e.g., 204) + CreateAccHandler->>AliceApp: HTTP 204 No Content (or error response) +``` +1. Alice's app sends the `POST` request. +2. The Web Router (Chi) finds the `accounts.Create` handler. +3. The `CreateAccountHandler` parses the JSON into a `CreateAccountCommand`. +4. It sends the command to the `CommandBus`. +5. The `CommandBus` delivers it to the `AccountAggregate`. +6. The `AccountAggregate` processes the command (validates, generates [Events](01_event_.md), updates its state, and ensures the [Events](01_event_.md) are saved via a [Repository](04_repository_.md) - this part was covered in detail in Chapters 1-4). +7. The result (success/error) bubbles back up to the handler. +8. The handler sends the appropriate HTTP response back to Alice's app. + +## Conclusion + +API Handlers are the vital "front doors" of our `corebanking` system. They listen for specific web requests (URLs and HTTP methods), parse the incoming data, and then translate these external requests into actions our internal system understands – usually by creating and dispatching [Commands](02_command_.md) for write operations or calling [Services](07_service_.md) for read operations. They then take the result of these actions and formulate an HTTP response to send back to the original requester. + +By bridging the gap between the web world (HTTP) and our core business logic, API Handlers make our system accessible to external clients like mobile apps, websites, or even other banking systems. + +Now that we've seen how requests enter our system, you might wonder: is there a central place that coordinates these actions, especially when a single request might involve multiple steps or components? That's where our next topic comes in. Get ready to learn about the [Core Facade](06_core_facade_.md)! + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/06_core_facade_.md b/nesto/corebanking/06_core_facade_.md new file mode 100644 index 00000000..30f191b3 --- /dev/null +++ b/nesto/corebanking/06_core_facade_.md @@ -0,0 +1,160 @@ +# Chapter 6: Core Facade + +Welcome to Chapter 6! In our [previous chapter on API Handlers](05_api_handler_.md), we saw how external requests, like Alice wanting to create an account, enter our `corebanking` system through specific API endpoints. These [API Handlers](05_api_handler_.md) then need to interact with various parts of our bank's brain – maybe to send a [Command](02_command_.md) or to fetch some data. + +But imagine an [API Handler](05_api_handler_.md) trying to juggle many different tools: one tool for sending [Commands](02_command_.md), another for looking up customer details, yet another for checking product information. It could get messy quickly! How do we keep this organized and simple for the [API Handler](05_api_handler_.md)? + +This is where the **Core Facade** comes in. + +## What is the Core Facade? The Bank's Main Office + +Think of the `Core` struct (our Core Facade) as the **main reception desk or central switchboard** for our entire `corebanking` application. It's like a central office in a large company. If you (an [API Handler](05_api_handler_.md), for instance) need something from any department, you don't run around the building looking for them. You go to the main office, and they connect you to the right place. + +The Core Facade brings together all the different specialized "departments" – which in our system are various [Services](07_service_.md) (like `AccountsService`, `CustomersService`) and other key components like the `EventSourcingClient` (our [Command](02_command_.md) bus) – into a **single, unified point of access**. + +When an external part of the system, like the [API layer](05_api_handler_.md), needs to interact with the business logic, it simply talks to this `Core` facade. This keeps our overall architecture clean and makes it much easier to manage how different parts of the application interact. + +Essentially, the Core Facade is like a well-organized toolbox for our [API Handlers](05_api_handler_.md). Instead of the handler needing to know about and hold onto dozens of individual tools, it just gets this one `Core` toolbox, which contains everything it might need. + +## What Does the Core Facade Look Like? (The `Core` Struct) + +In our Go code, the Core Facade is represented by a struct named `Core`, defined in `api/core.go`. Let's look at a simplified version: + +```go +// From: api/core.go +// Core represents the entrypoint to call the business logic. +type Core struct { + HealthService HealthService + EventSourcingClient es.EventSourcingClient // To send Commands! + ParametersService ParametersService + AccountsService AccountsService // To read account data! + CustomersService CustomersService + ProductsService ProductsService + // ... and many other services and clients ... +} +``` +* You can see that `Core` is a struct that holds various fields. +* Each field represents a specific "department" or tool: + * `EventSourcingClient`: This is crucial for sending [Commands](02_command_.md) (like `CreateAccountCommand`) to our [Aggregates](03_aggregate_.md). We've seen this in action with [API Handlers](05_api_handler_.md). + * `AccountsService`: If an [API Handler](05_api_handler_.md) needs to fetch a list of accounts or details about a specific account (not through event sourcing, but from a read-friendly data store), it would use this [Service](07_service_.md). We'll learn more about [Services](07_service_.md) in the next chapter! + * `CustomersService`, `ProductsService`, etc.: These provide access to other areas of our banking logic. + +So, the `Core` struct acts as a container, a "bag" holding all these important pieces of our application's core logic. + +## How Do [API Handlers](05_api_handler_.md) Use the Core Facade? + +Remember from [Chapter 5](05_api_handler_.md) how our [API Handlers](05_api_handler_.md) are set up using a router? The router is often given access to this `Core` facade instance. When an [API Handler](05_api_handler_.md) function is defined, it can be passed the specific service or client it needs *from* the `Core` facade. + +Let's revisit a simplified example from `api/cmd/corebanking-api/routes/routes.go` where routes are defined: + +```go +// Simplified from: api/cmd/corebanking-api/routes/routes.go + +// 'core' here is an instance of our api.Core struct (the Facade!) +func getAPIRoutes(core *api.Core) http.Handler { + r := chi.NewRouter() // Our web router + + // For creating accounts, the handler needs the EventSourcingClient (CommandBus) + r.Post("/accounts", accounts.Create(core.EventSourcingClient)) + + // For getting account details, a handler might need the AccountsService + // (Example, actual handler for GET might be different) + // r.Get("/accounts/{accountId}", accounts.GetDetails(core.AccountsService)) + + // ... other routes using different parts of 'core' ... + return r +} +``` +* The `getAPIRoutes` function receives an argument `core *api.Core`. This `core` object is our fully assembled Core Facade! +* When setting up the route for `POST /accounts`, the `accounts.Create` handler function is given `core.EventSourcingClient`. This means the `CreateAccountHandler` doesn't need to find or create an `EventSourcingClient` itself; it gets it directly from the Core Facade. +* Similarly, if we had a handler to fetch account details (e.g., `accounts.GetDetails`), it could be given `core.AccountsService`. + +This way, the `Core` facade acts as a central provider. The [API Handlers](05_api_handler_.md) just ask the `Core` facade for the specific tool they need. + +## How is the Core Facade Built? (The Making of the Toolbox) + +You might be wondering: who creates this `Core` struct and fills it with all these services? + +This happens when our `corebanking` application starts up. There's a special part of our code, primarily in a function called `FromEnv` in `api/pkg/core/core.go`, whose job is to: +1. Initialize all the individual [Services](07_service_.md) (like `AccountsService`, `ProductsService`, etc.). +2. Initialize the `EventSourcingClient`. +3. Initialize [Repositories](04_repository_.md) and other necessary components. +4. Then, it packages all of these initialized components into a single `api.Core` struct instance. + +Let's look at a very simplified snippet from our application's main entry point (`api/cmd/corebanking-api/api.go`) to see where this happens: + +```go +// Simplified from: api/cmd/corebanking-api/api.go main() +func main() { + // ... (lots of initial setup: logging, configuration, etc.) ... + + // tenantsConfigManager is used for multi-tenancy configurations + tenantsConfigManager := wiggumauto.NewConfigManager() + tenantsConfigManager.MustInitialize(ctx) + + // Here! 'c' becomes our fully assembled Core Facade instance. + // core.FromEnv does all the hard work of creating and wiring up services. + c := core.FromEnv(tenantsConfigManager) + + // The Core Facade 'c' is then passed to the router setup. + r := routes.GetRouter(c, tenantsConfigManager, monitoring) + + // ... (starts the web server with these routes) ... + // ... (handles graceful shutdown, including c.EventSourcingClient.Close()) +} +``` +* `c := core.FromEnv(tenantsConfigManager)`: This is the key line. The `core.FromEnv()` function (from `api/pkg/core/core.go`) is called. It sets up everything needed for the core logic (databases, event buses, all services) and returns a pointer to a fully populated `api.Core` struct. This `c` is our Core Facade. +* `r := routes.GetRouter(c, ...)`: This `c` (our Core Facade) is then passed to the function that sets up all the API routes, as we saw in the previous section. + +The `FromEnv` function itself is quite complex because it initializes many parts of the system, connecting them together. For a beginner, the important takeaway is that *there is a dedicated place* where the `Core` facade is carefully assembled when the application starts. + +## The Flow: API Handler, Core Facade, and Beyond + +Let's visualize how an [API Handler](05_api_handler_.md) uses the Core Facade: + +```mermaid +sequenceDiagram + participant APIHandler as API Handler (e.g., Create Account) + participant CoreFacade as Core Facade (api.Core instance) + participant AccService as Accounts Service + participant ESClient as EventSourcingClient (CommandBus) + participant AccAgg as Account Aggregate + + APIHandler->>CoreFacade: I need to create an account. Here's the data (CreateAccountCommand). + Note over APIHandler: Handler gets ESClient from CoreFacade + CoreFacade->>ESClient: HandleCommand(CreateAccountCommand) + ESClient->>AccAgg: Process CreateAccountCommand + AccAgg-->>ESClient: Success (or Error) + ESClient-->>CoreFacade: Command processed! + CoreFacade-->>APIHandler: Account creation initiated (Result) + + APIHandler->>CoreFacade: I need details for account "acc-xyz". + Note over APIHandler: Handler gets AccountsService from CoreFacade + CoreFacade->>AccService: GetAccountByID("acc-xyz") + AccService-->>CoreFacade: Account Details + CoreFacade-->>APIHandler: Here are the Account Details. +``` +This diagram shows two scenarios: +1. **Creating an Account (a [Command](02_command_.md)):** The [API Handler](05_api_handler_.md) uses the `EventSourcingClient` *obtained from* the `CoreFacade` to send the `CreateAccountCommand`. The `CoreFacade` itself doesn't process the command but provides access to the component that does. +2. **Getting Account Details (a Query):** The [API Handler](05_api_handler_.md) uses the `AccountsService` *obtained from* the `CoreFacade` to request account information. The `CoreFacade` directs this to the actual `AccountsService`. + +## Benefits of the Core Facade + +Using this Core Facade pattern provides several advantages: + +1. **Organization:** All core business functionalities are neatly bundled and accessible from one place. +2. **Simplicity for Callers:** Components like [API Handlers](05_api_handler_.md) don't need to know about or manage dependencies on dozens of individual services. They just need the `Core` object. +3. **Centralized Dependency Management:** The creation and wiring of all major services and components happen in one central place (like `core.FromEnv`). This makes it easier to manage the application's startup and overall structure. +4. **Clear Entry Point:** It provides a clear and consistent entry point into the application's business logic for any external interface (like the API). + +## Conclusion + +The **Core Facade**, represented by our `api.Core` struct, acts as the central hub or main reception desk for our `corebanking` application. It doesn't do the business logic itself but holds references to all the specialized [Services](07_service_.md) and clients (like the `EventSourcingClient`) that do. + +For components like [API Handlers](05_api_handler_.md), the Core Facade simplifies interaction with the system's core functionalities by providing a single, organized point of access. This keeps the architecture clean and manageable. + +The Core Facade is filled with various "departments" or tools. One very common type of tool it holds is a "Service." In the next chapter, we'll dive deeper into what these [Services](07_service_.md) are and what role they play in our `corebanking` system. + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/07_service_.md b/nesto/corebanking/07_service_.md new file mode 100644 index 00000000..a768a62d --- /dev/null +++ b/nesto/corebanking/07_service_.md @@ -0,0 +1,264 @@ +# Chapter 7: Service + +Welcome to Chapter 7! In the [previous chapter](06_core_facade_.md), we learned about the [Core Facade](06_core_facade_.md), which acts as a central hub providing access to various parts of our `corebanking` system. One of the key components the [Core Facade](06_core_facade_.md) gives us access to are "Services." + +But what exactly is a Service? And why do we need them? + +## What's a Service? The Bank's Specialized Departments + +Imagine you're at a bank. There isn't just one giant room where everything happens. Instead, there are specialized departments: +* The "Account Opening" department helps new customers. +* The "Loans" department handles loan applications and processing. +* The "Customer Support" department answers your questions. + +In our `corebanking` system, a **Service** is like one of these specialized departments. It's a component that groups together related business operations and logic for a specific area of the bank. + +For example, we might have an `AccountsService`. This service would be responsible for all things related to bank accounts, such as: +* Fetching the details of a specific account. +* Searching for accounts based on certain criteria. +* Potentially, initiating the creation of a new account (though, as we'll see, this is often done by sending a [Command](02_command_.md)). + +Services help us organize the application's capabilities, making the system easier to understand, manage, and extend. They often act as a "facade" or an intermediary, receiving requests, interacting with [Repositories](04_repository_.md) to fetch or save data, or dispatching [Commands](02_command_.md) to [Aggregates](03_aggregate_.md) to perform actions. + +## Key Ideas About Services + +1. **Specialized Focus:** Each Service handles a specific domain or business area (e.g., accounts, customers, product definitions). +2. **Groups Business Operations:** It bundles related functions. For instance, an `AccountsService` might have functions like `GetAccountByID()`, `SearchAccounts()`, etc. +3. **Orchestrates Logic:** A Service can coordinate multiple steps to fulfill a request. It might talk to a [Repository](04_repository_.md) to get data, perform some calculations or checks, and then return a result. +4. **Interacts with Other Components:** Services often use [Repositories](04_repository_.md) to access data and can dispatch [Commands](02_command_.md) to [Aggregates](03_aggregate_.md) if changes to the system's state are needed. +5. **Simplifies Interaction:** For other parts of the system (like [API Handlers](05_api_handler_.md)), Services provide a clean and focused interface to the business logic, hiding the underlying complexity. + +## Using a Service: Fetching Account Details + +Let's say an [API Handler](05_api_handler_.md) needs to display Alice's account details. It wouldn't directly query the database. Instead, it would use the `AccountsService`. + +### The Contract: The `AccountsService` Interface + +First, there's an interface that defines what operations the `AccountsService` offers. This is like the list of services a bank department provides. + +```go +// From: api/accounts.go + +// AccountsService manages accounts +type AccountsService interface { + // ... (methods for creating/updating accounts - more on this later) + Search(ctx context.Context, params *AccountSearchParameters) (Accounts, error) + GetByID(ctx context.Context, accountID uuid.UUID) (*Account, error) + GetByIDs(ctx context.Context, accountIDs []uuid.UUID) ([]*Account, error) + // ... other methods ... +} +``` +* This interface declares methods like `Search`, `GetByID`, and `GetByIDs`. +* Notice the `GetByID` method: it takes a `context` and an `accountID` and returns an `*Account` (a struct with account details) and an `error`. + +### Calling the Service + +An [API Handler](05_api_handler_.md) (or another part of the system) that has access to an `AccountsService` instance (perhaps via the [Core Facade](06_core_facade_.md)) can then call its methods: + +```go +// Somewhere in an API Handler (conceptual) +var accountsSvc api.AccountsService // This would be provided by the CoreFacade + +// ... +accountID := uuid.FromString("acc-alice-123") // Alice's account ID +accountDetails, err := accountsSvc.GetByID(context.Background(), accountID) + +if err != nil { + // Handle error (e.g., account not found) + fmt.Println("Error fetching account:", err) + return +} + +// Now we can use accountDetails +fmt.Println("Account Status:", accountDetails.Status) +fmt.Println("Account Currency:", accountDetails.AvailableCurrencies[0]) +``` +* **Input:** The `accountID` for Alice's account. +* **Output:** The `accountDetails` struct (if found) or an `err`. + +The [API Handler](05_api_handler_.md) doesn't need to know *how* the `AccountsService` gets this information. That's the Service's job! + +## Under the Hood: How `AccountsServiceDefault.GetByID` Works + +Our project has a concrete implementation of the `AccountsService` interface called `AccountsServiceDefault` (in `api/accounts.go`). Let's see how it's structured and how its `GetByID` method works. + +First, the `AccountsServiceDefault` struct holds a reference to an `AccountsRepository`: + +```go +// From: api/accounts.go +// AccountsServiceDefault represents the instance of a AccountsService. +type AccountsServiceDefault struct { + connections repository.ConnectionsProvider // Manages DB connections + repository AccountsRepository // The tool to talk to the database +} +``` +When an `AccountsServiceDefault` is created, it's given an `AccountsRepository`: + +```go +// From: api/accounts.go +// NewAccountsService return a new instance of the accounts service. +func NewAccountsService( + connections repository.ConnectionsProvider, + repository AccountsRepository, +) AccountsService { + // ... (nil checks omitted for brevity) ... + return &AccountsServiceDefault{ + connections: connections, + repository: repository, + } +} +``` +This means the service has the necessary tool ([Repository](04_repository_.md)) to fetch account data. + +Now, let's look at a simplified version of its `GetByID` method: + +```go +// Simplified from: api/accounts.go AccountsServiceDefault.GetByID +func (s *AccountsServiceDefault) GetByID( + ctx context.Context, + accountID uuid.UUID, +) (*Account, error) { + // 1. Potentially start a database transaction (simplified) + ctx, err := s.connections.NewTx(ctx) + if err != nil { + return nil, err + } + defer s.connections.Rollback(ctx) // Ensure transaction is rolled back if not committed + + // 2. Use the repository to fetch the account data + account, err := s.repository.GetByID(ctx, accountID) + if err != nil { + return nil, err // Could be "not found" or other DB error + } + + // 3. (A real service might do more here, like checking permissions + // or enriching the data, before returning it) + + return account, nil +} +``` +Here's what happens: +1. **Manage Transaction (Simplified):** It might start a database transaction. This is often good practice for read operations too, to ensure consistency, though for a single `GetByID` it might be optional depending on the database setup. +2. **Use Repository:** The crucial step! It calls `s.repository.GetByID(ctx, accountID)`. The service delegates the actual data fetching to its [Repository](04_repository_.md). +3. **Return Data:** It returns the `account` (or an error) obtained from the [Repository](04_repository_.md). + +Here's a sequence diagram illustrating this flow: + +```mermaid +sequenceDiagram + participant APIHandler as API Handler + participant AccSvc as AccountsService + participant AccRepo as AccountsRepository + participant DB as Database + + APIHandler->>AccSvc: GetByID("acc-alice-123") + AccSvc->>AccSvc: Start DB Transaction (conceptual) + AccSvc->>AccRepo: GetByID(ctx, "acc-alice-123") + AccRepo->>DB: SELECT * FROM accounts WHERE id = "acc-alice-123" + DB-->>AccRepo: Account Row Data + AccRepo-->>AccSvc: *Account (struct) + AccSvc->>AccSvc: Commit/Rollback Transaction (conceptual) + AccSvc-->>APIHandler: *Account (details for Alice) +``` + +## Services and Write Operations (Changes to Data) + +So far, we've focused on how Services help read data. What about when we want to *change* data, like creating an account or updating its status? + +The `AccountsService` interface in `api/accounts.go` actually defines methods like `Create`, `Activate`, etc.: +```go +// From: api/accounts.go +type AccountsService interface { + Create(ctx context.Context, command CreateAccountCommand) error + Activate(ctx context.Context, command ActivateAccountCommand) error + // ... other methods like Close, LinkPaymentDevice ... + // ... and the read methods we saw earlier ... +} +``` +However, if you look at the `AccountsServiceDefault` implementation for these methods, you'll find something like this: +```go +// From: api/accounts.go AccountsServiceDefault.Create +func (s *AccountsServiceDefault) Create( + _ context.Context, + _ CreateAccountCommand, +) error { + // writes should be done through the commandBus + return e.NewNotImplemented() +} +``` +It returns `e.NewNotImplemented()`! This tells us that, in this specific design, the `AccountsServiceDefault` *itself* doesn't directly handle account creation logic. Instead, such "write" operations are intended to be performed by sending a [Command](02_command_.md) (like `CreateAccountCommand`) directly to the `EventSourcingClient` (our Command Bus), which then routes it to the appropriate [Aggregate](03_aggregate_.md) (e.g., `AccountAggregate`). This is often done by the [API Handler](05_api_handler_.md) using the `EventSourcingClient` from the [Core Facade](06_core_facade_.md). + +**Why this separation?** +* **Centralized Command Processing:** Keeps all state-changing logic within [Aggregates](03_aggregate_.md), ensuring business rules are consistently enforced. +* **Event Sourcing Purity:** [Aggregates](03_aggregate_.md) are the source of [Events](01_event_.md), and [Commands](02_command_.md) are the standard way to trigger changes in them. + +### When Services *Do* Orchestrate Writes + +However, this doesn't mean Services *never* handle logic related to writes. A Service might: +1. **Perform preliminary checks or gather data** before a [Command](02_command_.md) is dispatched. +2. **Dispatch a [Command](02_command_.md) itself** if the operation is more complex or involves coordination. +3. **Orchestrate multiple [Commands](02_command_.md)** or interactions with different [Aggregates](03_aggregate_.md) or [Repositories](04_repository_.md). + +For example, consider the `PostingsTransactionsServiceDefault` in `api/postings_transactions.go`. Its `RevertChildTransactions` method shows a service performing more complex orchestration: + +```go +// Simplified from: api/postings_transactions.go +// PostingsTransactionsServiceDefault holds a commandBus +type PostingsTransactionsServiceDefault struct { + // ... other fields ... + commandBus es.CommandBus +} + +// RevertChildTransactions reverts child transactions for a specific transaction. +func (s *PostingsTransactionsServiceDefault) RevertChildTransactions( + ctx context.Context, + transactionID, rootTransactionID, parentTransactionID uuid.UUID, + /* ... other params ... */ +) error { + // 1. Use repository to get child transactions + postingsTransactions, err := s.repository.GetPostingsTransactionByParentID(ctx, transactionID) + // ... error handling ... + + // 2. For each child, dispatch a RevertSettledPostingsTransactionCommand + for _, postingsTransaction := range postingsTransactions { + err := s.commandBus.HandleCommand(ctx, &RevertSettledPostingsTransactionCommand{ + BaseCommand: es.BaseCommand{AggregateID: postingsTransaction.ID}, + // ... populate command details ... + }) + // ... error handling ... + } + return nil +} +``` +In this case, the `PostingsTransactionsService` reads data using its [Repository](04_repository_.md) and then dispatches multiple [Commands](02_command_.md) using its `commandBus`. This is a perfect example of a Service acting as an orchestrator for a business process. + +So, a Service can: +* Directly use [Repositories](04_repository_.md) for reading data (common). +* Dispatch [Commands](02_command_.md) to [Aggregates](03_aggregate_.md) for writing data (also common, especially for complex orchestrations). +* Sometimes, for simpler writes, an [API Handler](05_api_handler_.md) might dispatch a [Command](02_command_.md) directly using the `EventSourcingClient` from the [Core Facade](06_core_facade_.md), bypassing a specific write method on a service if the service doesn't add extra orchestration. + +## Why are Services Important? + +1. **Clear Business API:** They offer a higher-level, business-oriented set of operations compared to raw [Repositories](04_repository_.md) or direct [Command](02_command_.md) dispatching. +2. **Encapsulation:** They group related business logic, making it easier to find and manage. If you need to know how account searching works, you look in the `AccountsService`. +3. **Decoupling:** [API Handlers](05_api_handler_.md) or other clients depend on the Service interface, not on the nitty-gritty details of data storage or [Command](02_command_.md) handling for every single operation. +4. **Orchestration Point:** They are the natural place to put logic that coordinates multiple steps or involves several other components to achieve a business goal (like the `RevertChildTransactions` example). +5. **Testability:** Services can be tested in isolation. For example, when testing `AccountsServiceDefault`, you can provide a "mock" `AccountsRepository` to simulate database interactions. + +## Conclusion + +Services are the specialized "departments" within our `corebanking` application. They: +* Group related business operations for a specific domain (like accounts or transactions). +* Often use [Repositories](04_repository_.md) to fetch data needed for read operations. +* Can orchestrate more complex business processes, sometimes by dispatching [Commands](02_command_.md) to [Aggregates](03_aggregate_.md). +* Provide a clean and organized way for other parts of the system, like [API Handlers](05_api_handler_.md), to interact with the core business logic. + +They help keep our application well-structured and maintainable by ensuring that responsibilities are clearly defined. + +So far, we've seen how requests come in ([API Handler](05_api_handler_.md)), how they might use a [Core Facade](06_core_facade_.md) to access Services, and how Services can interact with [Repositories](04_repository_.md) or dispatch [Commands](02_command_.md) that affect [Aggregates](03_aggregate_.md) and generate [Events](01_event_.md). + +But what happens *after* an [Event](01_event_.md) is generated? Sometimes, other parts of the system need to react to these [Events](01_event_.md) automatically. For example, when an `AccountCreatedEvent` occurs, maybe we need to update a search index or send a welcome email. How does that happen? In the next chapter, we'll explore the [Consumer](08_consumer_.md). + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/08_consumer_.md b/nesto/corebanking/08_consumer_.md new file mode 100644 index 00000000..530c6987 --- /dev/null +++ b/nesto/corebanking/08_consumer_.md @@ -0,0 +1,204 @@ +# Chapter 8: Consumer + +Welcome to Chapter 8! In our [previous chapter on Services](07_service_.md), we learned how specialized "departments" in our bank help organize business logic, often by reading data using [Repositories](04_repository_.md) or orchestrating actions, sometimes by sending [Commands](02_command_.md). These [Commands](02_command_.md), when processed by [Aggregates](03_aggregate_.md), result in [Events](01_event_.md) – records of things that have happened. + +But what happens *after* an [Event](01_event_.md) is generated? For instance, if a `PostingCommittedEvent` (signaling a financial transaction leg) occurs, we probably need to update the customer's account balance so they can see it. Or, if an `AccountCreatedEvent` happens, maybe we need to send a welcome email. How do these follow-up actions happen automatically without slowing down the original process? + +This is where the **Consumer** steps in! + +## What is a Consumer? The Bank's Automated Event Watcher + +Imagine in our bank, every time an important announcement is made (an [Event](01_event_.md) happens), there are dedicated staff members who are always listening for specific types of announcements. +* One staff member listens for "New Account Opened" announcements. When they hear one, they immediately prepare and send a welcome pack to the new customer. +* Another staff member listens for "Transaction Completed" announcements. When they hear one, they quickly update the account ledger display. + +A **Consumer** in our `corebanking` system is like one ofthese automated staff members. It's a component that **listens for specific types of [Events](01_event_.md)** that occur within the system. When it "hears" an [Event](01_event_.md) it's interested in, it performs a predefined action. + +For example, when a `PostingCommittedEvent` occurs (meaning a part of a financial transaction has been finalized), a `BalancesConsumer` might listen for this. Upon receiving it, the consumer will update the account's balance that is stored for quick display (this stored balance is often called a "read model" or "projection"). + +Consumers are crucial because they allow different parts of the system to react to business happenings **asynchronously**. This means the original action (like processing the transaction) can finish quickly, and the follow-up tasks (like updating the balance display or sending a notification) happen separately, a moment later. + +## Key Ideas About Consumers + +1. **Event Listener:** A Consumer "subscribes" to a stream of [Events](01_event_.md) and pays attention only to the types of [Events](01_event_.md) it cares about. +2. **Asynchronous Operation:** Consumers typically run in the background, separate from the process that originally created the [Event](01_event_.md). This keeps the main operations fast and responsive. The part of the system that created the event doesn't have to wait for the Consumer to finish its job. +3. **Performs Actions:** When a relevant [Event](01_event_.md) is received, the Consumer executes specific logic. Common actions include: + * **Updating Read Models:** Calculating and storing data in a format optimized for fast reading. For example, updating an account balance summary after a transaction. + * **Triggering Follow-up Processes:** Starting new workflows, sending notifications (like emails or SMS), or calling other [Services](07_service_.md). +4. **Decoupling:** The component that produces an [Event](01_event_.md) (e.g., an `AccountAggregate` producing `AccountCreatedEvent`) doesn't need to know anything about the Consumers that might be listening to that [Event](01_event_.md). This makes the system more flexible and easier to change. +5. **Data Consistency (Eventually):** Consumers help maintain consistency across different views of data. For example, after a transaction, the balance read model will *eventually* reflect the change. + +## A Consumer in Action: Updating Account Balances + +Let's say a customer makes a deposit. This might result in one or more `PostingCommittedEvent`s. We need to update the account balance that is shown to the customer on their app. + +This is a perfect job for a `BalancesConsumer` (like the `BalancesTimeSeriesConsumerDefault` found in `api/balances_consumer.go`): + +1. **[Event](01_event_.md) Happens:** An [Aggregate](03_aggregate_.md) processes a transaction, and a `PostingCommittedEvent` is generated and saved. This event contains details about the amount, currency, and account involved. +2. **Consumer is Notified:** The `BalancesConsumer` is subscribed to listen for `PostingCommittedEvent`s. An "event bus" (a system for distributing events) delivers this event to the consumer. +3. **Consumer Processes the [Event](01_event_.md):** + * The `BalancesConsumer` receives the `PostingCommittedEvent`. + * It extracts the necessary information (account ID, amount, type of posting - debit/credit). + * It might fetch the current stored balance for that account using a `BalancesRepository` (a type of [Repository](04_repository_.md)). + * It calculates the new balance. +4. **Consumer Updates Read Model:** The consumer then saves this new, updated balance back using the `BalancesRepository`. This updated balance is now ready to be quickly displayed to the user. +5. **Optional: Further [Events](01_event_.md):** The consumer might even publish a new [Event](01_event_.md) itself, like a `BalanceUpdatedEvent`, to signal that the read model balance has changed. + +This entire process happens *after* the original deposit transaction was confirmed. The customer got a quick confirmation of their deposit, and in the background, the consumer ensures their displayed balance is updated. + +## What Does a Consumer Look Like? (A Peek at the Code) + +Consumers are often structs that hold dependencies like [Repositories](04_repository_.md) or other [Services](07_service_.md). They typically have a `Start()` method to begin listening and a processing method that gets called when relevant [Events](01_event_.md) arrive. + +Let's look at a simplified structure for `BalancesTimeSeriesConsumerDefault` from `api/balances_consumer.go`: + +```go +// Simplified from: api/balances_consumer.go +type BalancesTimeSeriesConsumerDefault struct { + repository BalancesRepository // To read/write balance data + events es.EventPublisher // To publish new events (optional) + eventConsumer es.EventConsumer // The mechanism to receive events + // ... other dependencies ... +} + +// NewBalancesTimeSeriesConsumer creates the consumer +func NewBalancesTimeSeriesConsumer( + repository BalancesRepository, + events es.EventPublisher, + eventConsumer es.EventConsumer, + // ... other args ... +) BalancesTimeSeriesConsumer { + // ... (nil checks) ... + return &BalancesTimeSeriesConsumerDefault{ + repository: repository, + events: events, + eventConsumer: eventConsumer, + // ... + } +} +``` +* `repository`: This is a `BalancesRepository`, used to get the current balance and save the new balance. +* `events`: An `EventPublisher` which this consumer might use to send out new [Events](01_event_.md) (like `BalanceUpdatedEvent`). +* `eventConsumer`: This is the component from our event sourcing library (`es`) that actually delivers [Events](01_event_.md) from the event bus to this consumer. + +### Starting the Consumer and Processing Events + +The consumer needs to be "turned on" to start listening. This is usually done by its `Start()` method: + +```go +// Simplified from: api/balances_consumer.go +func (c *BalancesTimeSeriesConsumerDefault) Start() error { + // Tell the eventConsumer to start sending batches of events + // to our 'processBalancesTimeSeriesEvents' method. + err := c.eventConsumer.ConsumeTenantMultiAggregateBatch( + context.Background(), + c.processBalancesTimeSeriesEvents, // Our callback function + ) + return err +} +``` +* `c.eventConsumer.ConsumeTenantMultiAggregateBatch(...)`: This line registers the `processBalancesTimeSeriesEvents` method with the underlying event consumption mechanism. Now, whenever new [Events](01_event_.md) (that this consumer is configured to listen to) arrive, this method will be called with a batch of those [Events](01_event_.md). + +The actual work happens in the callback method, `processBalancesTimeSeriesEvents`. Here's a highly simplified conceptual view of what it does when it receives event data related to postings: + +```go +// Highly simplified concept of processBalancesTimeSeriesEvents +func (c *BalancesTimeSeriesConsumerDefault) processBalancesTimeSeriesEvents( + ctx context.Context, tenantID int, events []*es.Event, +) error { + // For each relevant event (e.g., indirectly from a PostingCommittedEvent): + for _, event := range events { + // 1. Extract posting details from the event + // (The real code uses a helper 'orderedPostingsFromEvents' + // to get 'Posting' data from events like PostingCommittedEvent) + postingData := extractPostingFrom(event) // Conceptual + accountID := postingData.AccountID + + // 2. Get current balances for the account (simplified) + accountBalances, _ := c.repository.GetBalancesByAccountIDs(ctx, []uuid.UUID{accountID}) + // (Real code handles 'not found' and groups by account) + currentBalanceInfo := accountBalances[0] // Simplified + + // 3. Apply the posting to update the balance + // (The real code uses 'ApplyPostingToAccountBalances') + newBalance := calculateNewBalance(currentBalanceInfo, postingData) // Conceptual + + // 4. Save the updated current balance + // (The real code accumulates 'updatedCurrentBalances' and saves in a batch) + err := c.repository.SaveCurrent(ctx, []*Balance{newBalance}) + if err != nil { + // Handle error + return err + } + + // 5. Optionally, publish a new event + balanceUpdatedEvt := createBalanceUpdatedEvent(newBalance) // Conceptual + c.events.PublishBatch(ctx, []es.Event{balanceUpdatedEvt}) + } + return nil +} +``` +Let's break down this conceptual flow: +1. **Extract Posting Details:** The consumer gets the necessary details from the incoming [Event](01_event_.md) (like account ID, amount, currency, debit/credit). In the actual `corebanking` code, `PostingCommittedEvent`s are processed to extract `Posting` objects. +2. **Get Current Balance:** It uses its `BalancesRepository` to fetch the most recent balance record for the affected account. +3. **Apply Posting:** It performs the calculation to update the balance. The actual function `ApplyPostingToAccountBalances` in `api/balances_consumer.go` handles the logic of adding or subtracting based on posting type and account category. +4. **Save Updated Balance:** The new balance is saved back to the read model storage using the `BalancesRepository`'s `SaveCurrent` method. +5. **Publish New Event (Optional):** The `BalancesConsumer` in our project also creates and publishes a `BalanceUpdatedEvent`. This allows other parts of the system to know that a balance read model has been updated. + +The actual `processBalancesTimeSeriesEvents` is more complex because it handles batches of events, deals with time-series data for historical balances, and manages database transactions. But the core idea is: **receive event -> process it -> update read model.** + +## How a Consumer Gets Triggered: The Event Flow + +Here’s a simplified sequence diagram showing how a `BalancesConsumer` might react to a `PostingCommittedEvent`: + +```mermaid +sequenceDiagram + participant TxnProcessor as Transaction Processor (e.g., Aggregate) + participant EventBus as Event Bus / Message Queue + participant BalConsumer as Balances Consumer + participant BalRepo as Balances Repository + participant ReadModelDB as Read Model Database + + TxnProcessor->>EventBus: Publish PostingCommittedEvent (for Account X, Amount $10) + EventBus->>BalConsumer: Deliver PostingCommittedEvent + BalConsumer->>BalRepo: Get Current Balance (Account X) + BalRepo->>ReadModelDB: Fetch balance for Account X + ReadModelDB-->>BalRepo: Current Balance: $50 + BalRepo-->>BalConsumer: Current Balance: $50 + BalConsumer-->>BalConsumer: Calculate New Balance ($50 + $10 = $60) + BalConsumer->>BalRepo: Save Balance (Account X, New Balance $60) + BalRepo->>ReadModelDB: Update balance for Account X to $60 + ReadModelDB-->>BalRepo: Success + BalRepo-->>BalConsumer: Success + BalConsumer->>EventBus: (Optional) Publish BalanceUpdatedEvent +``` +This shows the asynchronous nature: the `TxnProcessor` fires the event and moves on. The `BalConsumer` picks it up later and does its work. + +## Other Types of Consumers + +Our `corebanking` system has other consumers too: + +* **`PostingsTransactionsConsumer`** (from `api/postings_transactions_consumer.go`): + This consumer listens to events related to financial transactions (like `PostingsTransactionCreatedEvent`, `PostingsTransactionSettledEvent`). It updates a read model that stores details about these transactions, making it easy to query their status, amount, etc. + +* **`ProductEnginesConsumer`** (from `api/product_engines_consumer.go`): + This is an interesting one! It listens for events like `ScheduledEvent` (e.g., "it's end of day for account X") or even `PostingCommittedEvent`. When it receives such an event, it triggers the [Product Engine](09_product_engine_.md) (via `productEngines.HandleEvents()`). The [Product Engine](09_product_engine_.md) might then apply interest, charge fees, or perform other account-specific logic defined by the banking product. This shows a consumer triggering a more complex follow-up process. + +## Why Are Consumers So Useful? + +1. **Improved Performance & Responsiveness:** The system part that creates an [Event](01_event_.md) (e.g., an [Aggregate](03_aggregate_.md) handling a [Command](02_command_.md)) can finish its job quickly without waiting for all side effects (like updating multiple read models or sending emails) to complete. +2. **Decoupling & Modularity:** Event producers don't need to know about event consumers. You can add new consumers or change existing ones without affecting the code that generates the events. This makes the system very flexible. +3. **Resilience:** If a consumer temporarily fails while processing an event (e.g., a network issue while saving to a database), the event can often be re-processed later without losing the original data, as the [Event](01_event_.md) itself is already safely stored. +4. **Scalability:** Different consumers can often be scaled independently. If updating balances becomes a bottleneck, you might be able to run more instances of the `BalancesConsumer`. + +## Conclusion + +Consumers are the unsung heroes working diligently in the background of our `corebanking` system. They are **automated listeners** that subscribe to specific types of [Events](01_event_.md). When they receive an [Event](01_event_.md) they're interested in, they swing into action – perhaps updating a read-model for fast data display (like account balances) or triggering follow-up processes (like complex product calculations). + +By reacting to business [Events](01_event_.md) asynchronously, Consumers help keep our system responsive, flexible, and robust. They play a vital role in maintaining data consistency across different views and enabling complex workflows. + +One example we saw was the `ProductEnginesConsumer`, which listens for events and then invokes specific business logic defined by a banking product. What is this "Product Engine" that it calls? Let's explore that in our next chapter: the [Product Engine](09_product_engine_.md). + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/09_product_engine_.md b/nesto/corebanking/09_product_engine_.md new file mode 100644 index 00000000..11c45b49 --- /dev/null +++ b/nesto/corebanking/09_product_engine_.md @@ -0,0 +1,265 @@ +# Chapter 9: Product Engine + +Welcome to Chapter 9! In the [previous chapter](08_consumer_.md), we learned about [Consumers](08_consumer_.md) and how they listen for [Events](01_event_.md) to trigger follow-up actions. For example, a [Consumer](08_consumer_.md) might notice that an account needs its monthly interest calculated. But how does the system know *how* to calculate that interest, especially if different types of accounts (like a "Youth Savings Account" vs. a "Standard Mortgage") have vastly different rules? + +This is where the **Product Engine** comes in. + +## What's a Product Engine? The Bank's Specialized Rulebook + +Imagine a bank offers many different financial products: +* A "Basic Savings Account" with simple interest. +* A "Youth Savings Account" that gets bonus interest if no money is withdrawn during the month. +* A "Standard Mortgage" with complex payment schedules and fee structures. +* A "HELOC Remittance" account with its own unique remittance rules. + +Each of these products behaves differently. They have their own specific features, ways to calculate interest, fee rules, and reactions to events like an account being activated or a transaction occurring. + +A **Product Engine** is like a **detailed rulebook and automated process manager** specifically designed for *one particular financial product*. It defines exactly how that product, like "Standard Mortgage" or "Youth Savings Account," should work. + +When a customer opens an account (say, a "Youth Savings Account"), that account is linked to its specific product. From then on, whenever something relevant happens to that account, the system consults the "Youth Savings Account" Product Engine to: +* Check if an operation is allowed (validation). +* Determine any product-specific actions or outcomes (like calculating that special bonus interest). + +## Key Ideas About Product Engines + +1. **Product-Specific Logic:** Each Product Engine contains the unique rules and behaviors for one type of financial product. +2. **Defines Product Behavior:** It specifies features, fee structures, interest calculations, and how the product responds to lifecycle events (e.g., account activation, transaction posting, scheduled events like "end of month"). +3. **Consulted for Operations:** When an action needs to be performed on an account, or an event occurs related to it, the corresponding Product Engine is invoked. +4. **Automates Processes:** It automates tasks like applying interest, charging fees, or managing scheduled product-specific activities. +5. **Produces Actions:** Based on its rules and the current situation (e.g., an incoming [Event](01_event_.md) or [Command](02_command_.md)), a Product Engine can decide that certain actions need to happen. These actions are often new [Commands](02_command_.md) (e.g., "Create a transaction to credit bonus interest"). + +## The Product Engine in Action: Youth Savings Bonus Interest + +Let's use our "Youth Savings Account" example. This account type offers bonus interest if no withdrawals are made in a month. + +1. **Account Setup:** Alice has a "Youth Savings Account." This account is linked to the "Youth Savings Product Engine." +2. **End of Month [Event](01_event_.md):** A `ScheduledEvent` (like "EndOfMonthProcessing") occurs for Alice's account. +3. **[Consumer](08_consumer_.md) Routes to Service:** A [Consumer](08_consumer_.md) (like the `ProductEnginesConsumer` we touched upon in the [previous chapter](08_consumer_.md)) picks up this `ScheduledEvent`. It knows this event might require product-specific logic, so it passes the event to the `ProductEnginesService`. +4. **Service Invokes Product Engine:** The `ProductEnginesService`: + * Looks at Alice's account and sees it's a "Youth Savings Account." + * Retrieves the specific "Youth Savings Product Engine." + * Prepares a `ProductEngineContext` for Alice's account. This context contains information the engine might need, like Alice's current account details, balances, and the product's configuration. + * Calls a method on the "Youth Savings Product Engine," like `HandleScheduledEvent(context, scheduledEvent)`. +5. **Product Engine Applies Rules:** The "Youth Savings Product Engine" executes its logic for the "EndOfMonthProcessing" event: + * It checks Alice's transaction history for the past month (perhaps by looking at data in the `ProductEngineContext` or by querying for recent postings). + * It determines if any withdrawals were made. + * If no withdrawals, it calculates the bonus interest. +6. **Product Engine Returns Actions:** The engine decides that bonus interest should be applied. It returns a list of `ProductEngineAction`s. One of these actions might be a `HandleCommandAction` containing a `CreatePostingsTransactionCommand` to credit the bonus interest to Alice's account. +7. **Service Executes Actions:** The `ProductEnginesService` takes these actions and processes them. For a `HandleCommandAction`, it would send the `CreatePostingsTransactionCommand` to the [Command Bus](02_command_.md), which then ensures the transaction is created and Alice's balance is updated. + +This way, the specific bonus interest logic for the "Youth Savings Account" is neatly contained within its own Product Engine. + +## A Peek at the Code: Interfaces and Services + +Let's look at some of the key pieces involved. + +### 1. The `ProductEngine` Interface + +At its core, every product engine implements the `ProductEngine` interface (from `api/product_engines.go`). This interface defines the "hooks" or methods that can be called by the system for different situations. + +```go +// Simplified from: api/product_engines.go +type ProductEngine interface { + ProductEngineID() string // E.g., "YOUTH_SAVINGS_ACCOUNT" + ProductEngineVersion() string // E.g., "1.0.0" + + // Called when an account is activated + ActivateAccount(ctx context.Context, engineCtx ProductEngineContext) ([]ProductEngineAction, error) + + // Called for scheduled events (like our "EndOfMonthProcessing") + HandleScheduledEvent(ctx context.Context, engineCtx ProductEngineContext, scheduledEvent *ScheduledEvent) ([]ProductEngineAction, error) + + // Called when a posting (transaction leg) is accepted for an account + AcceptPosting(ctx context.Context, engineCtx ProductEngineContext, posting *Posting) ([]ProductEngineAction, error) + + // ... other methods for validating postings, updating parameters, closing accounts etc. +} +``` +* `ProductEngineID()` and `ProductEngineVersion()`: Identify the specific engine. +* `ActivateAccount`, `HandleScheduledEvent`, `AcceptPosting`: These are examples of methods that get called at different points in an account's lifecycle or when certain [Events](01_event_.md) occur. +* `ProductEngineContext`: This argument provides all the necessary information about the current account, its product settings, balances, etc., that the engine needs to make decisions. +* `[]ProductEngineAction`: The method returns a list of actions the system should take as a result of the engine's logic. + +### 2. The `ProductEngineContext` + +This struct (also from `api/product_engines.go`) is passed to the Product Engine methods. It's like a data package giving the engine all the relevant details for the current operation. + +```go +// Simplified from: api/product_engines.go +type ProductEngineContext struct { + ReferenceDate *date.Date // The current date for processing + Account *Account // Details of the specific account + Product *ResolvedProduct // Details & parameters of the banking product + CurrentBalances *BalancesReader // Access to the account's current balances + // ... and other fields like PostingsByPartition, BalancesByPartition for historical data +} +``` +* `Account`: The specific account being processed. +* `Product`: The resolved product definition, containing parameters like interest rates, fee schedules, etc. +* `CurrentBalances`: A way to read the account's current balances. +The engine uses this information to apply its rules. + +### 3. `ProductEngineAction` + +When a Product Engine finishes its logic, it needs to tell the system what to do next. It does this by returning a slice of `ProductEngineAction` (defined in `api/product_engines.go`). + +A common action is to issue a [Command](02_command_.md): +```go +// Simplified from: api/product_engines.go + +// ProductEngineAction represents a product engine action. +type ProductEngineAction interface { + Actuate() // A marker method +} + +// HandleCommandAction implements product engine action to handle command. +type HandleCommandAction struct { + command es.Command // The command to be executed +} + +// NewHandleCommandAction returns a new HandleCommandAction. +func NewHandleCommandAction(command es.Command) HandleCommandAction { + return HandleCommandAction{command} +} + +// GetCommand returns the underlying command +func (a HandleCommandAction) GetCommand() es.Command { + return a.command +} +``` +So, if our "Youth Savings Product Engine" decides to apply bonus interest, it would create a `CreatePostingsTransactionCommand` and wrap it in a `NewHandleCommandAction`. + +### 4. The `ProductEnginesService` + +The `ProductEnginesServiceDefault` (from `api/product_engines.go`) is the orchestrator. It's responsible for receiving [Events](01_event_.md) (often from a [Consumer](08_consumer_.md)), finding the right Product Engine, and invoking it. + +```go +// Conceptual flow within ProductEnginesServiceDefault.HandleEvents +// (Actual code is more complex, handling batches and various event types) + +func (s *ProductEnginesServiceDefault) HandleEvents(ctx context.Context, events []*es.Event) error { + for _, event := range events { + // 1. Determine which account this event is for (from event.AggregateID) + accountID := event.AggregateID + account, _ := s.accounts.GetByID(ctx, accountID) // Fetch account details + + // 2. Get the Product Engine for this account's product + productEngine, _ := s.GetProductEngine(ctx, account.Product) + if productEngine == nil { + continue // No engine for this product, or product not set + } + + // 3. Create an Event Handler wrapper (e.g., for a ScheduledEvent) + // The hook file (api/product_engines_hooks.go) has different handlers for different events. + var eventHandler ProductEngineEventHandler // Defined in api/product_engines_hooks.go + if se, ok := event.Data.(*ScheduledEvent); ok && eventHandlerNameFromAction(se.Action) == "HandleScheduledEvent" { + eventHandler = NewProductEngineEventHandlerHandleScheduledEvent(event.ID, se, /*...location...*/) + } + // ... (else if for AccountActivatedEvent, PostingCommittedEvent, etc.) ... + + if eventHandler == nil { + continue + } + + // 4. Build the ProductEngineContext (simplified) + engineCtx, _ := s.buildProductEngineContext(ctx, /*...details...*/, account, /*balances*/, /*config*/) + + // 5. Call the specific hook on the event handler, which calls the engine + actions, err := eventHandler.Hook(ctx, *engineCtx, productEngine) + // ... error handling ... + + // 6. Run the actions returned by the Product Engine + s.runActions(ctx, actions) // This would dispatch commands, schedule things, etc. + } + return nil +} +``` +* **Get Account & Engine:** It fetches the account and determines the correct Product Engine based on the account's product information (`account.Product.EngineID`, `account.Product.EngineVersion`). +* **Event Handler Wrapper:** An `ProductEngineEventHandler` (from `api/product_engines_hooks.go`) is chosen based on the type of incoming [Event](01_event_.md). This wrapper standardizes how different events are passed to the Product Engine. For example, `ProductEngineEventHandlerHandleScheduledEvent` has a `Hook` method that will ultimately call `engine.HandleScheduledEvent(...)`. +* **Build Context:** It prepares the `ProductEngineContext` with all necessary data. +* **Invoke Engine (via Hook):** It calls the `Hook` method on the event handler, which in turn calls the relevant method on the `ProductEngine` instance (e.g., `HandleScheduledEvent`). +* **Run Actions:** It takes the `ProductEngineAction`s returned by the engine and executes them (e.g., sends [Commands](02_command_.md) to the [Command Bus](02_command_.md)). + +### Example: A Simple Product Engine's `HandleScheduledEvent` + +Let's imagine a very basic "MonthlyFeeProductEngine": +```go +// Hypothetical MonthlyFeeProductEngine +type MonthlyFeeProductEngine struct { /* ... */ } + +func (pe *MonthlyFeeProductEngine) HandleScheduledEvent( + ctx context.Context, + engineCtx api.ProductEngineContext, + scheduledEvent *api.ScheduledEvent, +) ([]api.ProductEngineAction, error) { + + if scheduledEvent.Action == "APPLY_MONTHLY_FEE" { + feeAmountStr, _ := engineCtx.Product.Parameters["monthlyFeeAmount"] // Get fee from product config + feeAmount, _ := decimal.NewFromString(feeAmountStr) + + if feeAmount.IsPositive() { + // Create a command to charge the fee + feeTxCmd := &api.CreatePostingsTransactionCommand{ + BaseCommand: es.BaseCommand{AggregateID: engineCtx.Account.ID}, + Action: api.PostingActionTransfer, + Code: "MONTHLY_FEE", + Amount: feeAmount, + Debtor: &api.PostingDestination{AccountID: engineCtx.Account.ID}, // Debit customer + Creditor: &api.PostingDestination{AccountID: uuid.FromString("bank-fee-income-account-id")}, // Credit bank + } + return []api.ProductEngineAction{api.NewHandleCommandAction(feeTxCmd)}, nil + } + } + return nil, nil // No action +} + +// ... other required methods of ProductEngine interface ... +``` +When a `ScheduledEvent` with action "APPLY_MONTHLY_FEE" comes in for an account using this product: +1. The engine checks if the action matches. +2. It reads the `monthlyFeeAmount` from the product's parameters (available in `engineCtx.Product.Parameters`). +3. If a fee is due, it creates a `CreatePostingsTransactionCommand` to transfer the fee from the customer's account to the bank's fee income account. +4. It returns this command wrapped in a `HandleCommandAction`. + +## Visualizing the Flow + +Here's how an event for Alice's "Youth Savings Account" (YSA) might trigger its Product Engine to apply bonus interest: + +```mermaid +sequenceDiagram + participant EventBus + participant YSAConsumer as YSA Product Consumer + participant PEService as ProductEnginesService + participant YSAPEngine as YouthSavingsProductEngine + participant CmdBus as CommandBus + + EventBus->>YSAConsumer: ScheduledEvent (EndOfMonth, Account: AliceYSA) + YSAConsumer->>PEService: HandleEvents([ScheduledEvent]) + PEService->>YSAPEngine: GetEngine("YouthSavingsProductEngine", "v1") + Note over PEService: Builds ProductEngineContext for AliceYSA + PEService->>YSAPEngine: HandleScheduledEvent(ctx, engineCtx, ScheduledEvent) + YSAPEngine-->>YSAPEngine: Logic: Check no withdrawals, calculate bonus. + YSAPEngine-->>PEService: Return [HandleCommandAction(CreateBonusTxCmd)] + PEService->>CmdBus: HandleCommand(CreateBonusTxCmd) + CmdBus-->>PEService: Command processed (Alice gets bonus) +``` + +## Why Product Engines are Powerful + +* **Flexibility:** Easily define unique behaviors for many different banking products without cluttering the core system. +* **Modularity:** Product-specific logic is isolated. Changes to one product's rules don't affect others. +* **Maintainability:** Easier to understand and update the logic for a specific product. +* **Testability:** Individual Product Engines can be tested in isolation with a mocked `ProductEngineContext`. +* **Rapid Development:** New financial products can often be introduced by creating a new Product Engine and its configuration, rather than changing deep parts of the core system. + +## Conclusion + +Product Engines are the specialized "brains" behind each financial product offered by the bank. They act as detailed rulebooks and automated process managers, defining how a product like a "Standard Mortgage" or a "Youth Savings Account" behaves – its features, fees, interest calculations, and responses to lifecycle events. + +When an [Event](01_event_.md) occurs for an account, the `ProductEnginesService` (often triggered by a [Consumer](08_consumer_.md)) identifies the account's product and invokes the corresponding Product Engine. The engine then applies its specific rules and can return a list of actions, such as [Commands](02_command_.md) to create transactions, for the system to execute. This powerful mechanism allows `corebanking` to support a diverse range of financial products with complex, individualized behaviors. + +With all these intricate product behaviors and event-driven interactions, how can we be sure everything works as expected? How can we test different scenarios, perhaps even simulating future events? In our next and final chapter, we'll look into [Simulation Services and Repositories](10_simulation_services_and_repositories_.md). + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/10_simulation_services_and_repositories_.md b/nesto/corebanking/10_simulation_services_and_repositories_.md new file mode 100644 index 00000000..dd0db4cd --- /dev/null +++ b/nesto/corebanking/10_simulation_services_and_repositories_.md @@ -0,0 +1,205 @@ +# Chapter 10: Simulation Services and Repositories + +Welcome to the final chapter of our conceptual overview! In [Chapter 9: Product Engine](09_product_engine_.md), we saw how our `corebanking` system can support diverse financial products, each with its own complex set of rules for things like interest calculation, fee application, and behavior over time. + +Now, imagine we've just designed a sophisticated 30-year mortgage product using our [Product Engine](09_product_engine_.md). How can we be absolutely sure it works correctly under all sorts of conditions? How do we test its behavior over the full 30 years, perhaps with fluctuating interest rates, without actually waiting for three decades or messing with real customer data? This is where **Simulation Services and Repositories** come to the rescue! + +## What are Simulation Services and Repositories? Your Bank's Flight Simulator! + +Think of Simulation Services and Repositories as your bank's very own **flight simulator** or **practice mode**. They are special "test-double" or "fake" versions of the regular [Services](07_service_.md) and [Repositories](04_repository_.md) we've learned about. + +Instead of connecting to real databases and affecting real money or customer data, these simulators: +* **Work "in-memory":** They store their data temporarily, right in the computer's memory, for the duration of a test or simulation. When the simulation ends, this temporary data is gone. This is great because it's fast and completely isolated from the live system. +* **Mimic real behavior:** They are designed to act just like their real counterparts, but give us much more control. +* **Allow controlled testing:** We can use them to test complex business logic, like our 30-year mortgage product, under various scenarios (e.g., "what if interest rates double next year?") in a safe and repeatable way. + +Essentially, these simulators let us "play out" complex financial scenarios, fast-forward through time, and see what happens, all without any real-world consequences. + +## Key Ideas Behind Simulation + +How do these simulators achieve this magic? There are a few key ingredients: + +1. **In-Memory Storage:** + * Simulation [Repositories](04_repository_.md) (like `AccountRepositorySimulator` or `PostingsTransactionsRepositorySimulator`) don't save data to a permanent database. Instead, they use simple Go data structures like maps and slices to hold information temporarily. + * This makes them very fast and ensures that each simulation run starts fresh. + +2. **Controlled Time (The "Time Machine"):** + * A special `ClockServiceSimulator` lets us control the "current date" within the simulation. We can tell it to jump forward day by day, month by month, or even set it to a specific date in the future. This is crucial for testing products that behave differently over long periods. + +3. **Deterministic Behavior:** + * For a given starting point and set of actions, a simulation should ideally produce the exact same results every time it's run. This makes testing reliable and helps us pinpoint issues if something changes unexpectedly. + +4. **Isolation:** + * Simulations run completely separately from the live banking system. There's no risk of accidentally creating a real account or moving real money. + +## Using the Simulators: Testing Our Mortgage Product + +Let's go back to our 30-year mortgage product. How would we use the simulation tools to test it? The main entry point for complex product simulations is often the `ProductEnginesServiceSimulatorDefault` (found in `api/pkg/productengines/simulator/simulator.go`). + +Here's a conceptual walkthrough: + +1. **Setup the Simulation:** + * First, we'd create an instance of `ProductEnginesServiceSimulatorDefault`. This master simulator internally sets up other necessary simulator components like `AccountsServiceSimulator`, `BalancesServiceSimulator`, `PostingsServiceSimulator`, a `ClockServiceSimulator`, and a `CommandBusSimulator`. + * We'd tell the simulator about our "Mortgage Product" definition (its rules, interest rates, fee structures, etc.) by registering it. + * We'd then create a "Simulated Account" – let's say, "Alice's Mortgage Account" – and link it to our mortgage product. We might also set its initial loan amount and other starting conditions. + * We'd define a time period for our simulation, for example, from "2024-01-01" to "2025-01-01" to see how it behaves over one year. + +2. **Run the Simulation:** + * We'd call the `Simulate` method on our `ProductEnginesServiceSimulatorDefault`, giving it Alice's simulated account and our desired start and end dates. + +3. **What Happens Inside the `Simulate` Method (Simplified):** + * The simulator enters a loop, advancing day by day from the start date to the end date using its internal `ClockServiceSimulator`. + * On each simulated day: + * It checks if any scheduled events are due for Alice's account (e.g., "end of month interest calculation," "payment due date"). + * If so, it triggers these events. These events are then passed to the (real) [Product Engine](09_product_engine_.md) logic for our mortgage product. + * The mortgage [Product Engine](09_product_engine_.md) runs its rules based on the current simulated date and account state. It might decide to: + * Calculate interest due. + * Generate a [Command](02_command_.md) like `CreatePostingsTransactionCommand` to add this interest to Alice's loan balance. + * This `CreatePostingsTransactionCommand` is then handled by the `CommandBusSimulator`. The `CommandBusSimulator` doesn't send it to the real system; instead, it directs it to other *simulated* services. + * For example, the `PostingsServiceSimulator` would "commit" these interest postings to its in-memory store. + * The `BalancesServiceSimulator` would update Alice's simulated loan balance in its in-memory store. + * This loop continues until the simulation end date is reached. + +4. **Get the Results:** + * Once the simulation is complete, the `Simulate` method returns a `SimulationResult` struct. + * This `SimulationResult` contains a snapshot of Alice's mortgage account at the end of the simulated year: her final loan balance, a list of all transactions (postings) that occurred (like interest charges, payments), any scheduled items, and potentially any errors that happened during the simulation. + +5. **Check the Results:** + * We can then examine this `SimulationResult` to verify if our mortgage product behaved as expected. Did the interest calculate correctly? Were payments applied properly? Is the final balance what we predicted? + +## A Peek at Some Simulator Code + +Let's look at tiny, simplified snippets to get a feel for how these simulators are built. + +### 1. The `AccountRepositorySimulator` (Storing Data In-Memory) + +This simulator (from `api/pkg/productengines/simulator/accounts_repository.go`) fakes a [Repository](04_repository_.md) for accounts. + +```go +// Simplified from: api/pkg/productengines/simulator/accounts_repository.go +type AccountRepositorySimulator struct { + // Accounts are stored in a map, with account ID as the key. + accounts map[uuid.UUID]*api.Account + // ... other fields for simulation ... +} + +// GetByID tries to find an account in its in-memory map. +func (r *AccountRepositorySimulator) GetByID( + ctx context.Context, accountID uuid.UUID, +) (*api.Account, error) { + account, found := r.accounts[accountID] + if !found { + // Return an error if not found (simplified error) + return nil, errors.New("simulated account not found") + } + return account, nil // Return the account from memory +} +``` +* Instead of database queries, it uses a simple Go `map` (`r.accounts`) to store account data. +* The `GetByID` method just looks up the account in this map. Very fast! + +### 2. The `ClockServiceSimulator` (Controlling Time) + +The `ProductEnginesServiceSimulatorDefault` uses a clock service (like `api.ClockServiceDefault`) and controls it. A key method for the simulation loop is `SetDate`: + +```go +// Conceptual use of ClockService's SetDate within the simulation loop +// (s.clock is an instance of a clock service) +// s.clock.SetDate(currentSimulatedDate, timeZone) +``` +* Inside the `Simulate` method's loop, this `SetDate` is called repeatedly to advance the simulation's "current time" one day at a time. This makes the whole system believe it's that specific day. + +### 3. The `ProductEnginesServiceSimulatorDefault`'s `Simulate` Method + +This is the heart of the product simulation (from `api/pkg/productengines/simulator/simulator.go`). + +```go +// Highly simplified concept of Simulate method's loop +// (s is *ProductEnginesServiceSimulatorDefault) +func (s *ProductEnginesServiceSimulatorDefault) Simulate(/*...params...*/) (*SimulationResult, error) { + // ... (lots of initial setup: create account in simulator, set initial balances) ... + + // Loop from start date to end date + for refDate := startDate; refDate.Before(endDate); refDate = refDate.AddDays(1) { + s.clock.SetDate(refDate, s.c.ReferenceLocation) // Advance simulated time! + + // 1. Trigger scheduled events for this 'refDate' + // This involves calling s.HandleEvents(), which eventually + // invokes the Product Engine for the account. + // (Simplified representation) + s.processScheduledItemsForDate(ctx, account.ID, refDate) + + // 2. Settle any simulated transactions due on this 'refDate' + // (e.g., if a payment was scheduled to clear today) + s.settleTransactions(ctx, simulationConfig, account.ID, refDate) + } + + // ... (gather all simulated data: final account state, postings, balances) ... + return &SimulationResult{ /* ... collected data ... */ }, nil +} +``` +* The loop iterates day by day. +* `s.clock.SetDate(...)` tells the entire simulation what "today" is. +* `processScheduledItemsForDate` (a conceptual helper representing logic within `Simulate` and `GetSchedules/HandleEvents` from `simulator.go`) finds any scheduled tasks for the account on this simulated day (like "calculate monthly interest"). It then uses `s.HandleEvents()` to pass these to the [Product Engine](09_product_engine_.md). The Product Engine might then generate [Commands](02_command_.md). +* These [Commands](02_command_.md) are handled by the `CommandBusSimulator` (also part of `ProductEnginesServiceSimulatorDefault`), which ensures they are processed by *other simulated services and repositories*, updating the in-memory state. +* `s.settleTransactions` handles any pre-registered transaction settlements for that day. + +## How it All Connects: A Simulation Day + +Here's a simplified sequence of what might happen on one simulated day for our mortgage account: + +```mermaid +sequenceDiagram + participant SimLoop as Simulation Loop (in Simulate method) + participant ClockSim as ClockServiceSimulator + participant PESimService as ProductEnginesServiceSimulator + participant MortgagePE as Mortgage Product Engine + participant CmdBusSim as CommandBusSimulator + participant PostingRepoSim as PostingRepositorySimulator (In-Memory) + participant BalanceRepoSim as BalanceRepositorySimulator (In-Memory) + + SimLoop->>ClockSim: SetDate(currentSimulatedDay) + SimLoop->>PESimService: ProcessScheduledEvents(account, currentSimulatedDay) + PESimService->>MortgagePE: HandleScheduledEvent(context, scheduledEvent) + MortgagePE-->>MortgagePE: Calculate interest... + MortgagePE-->>PESimService: Return [Action: CreateInterestTxCommand] + PESimService->>CmdBusSim: HandleCommand(CreateInterestTxCommand) + CmdBusSim->>PostingRepoSim: Save new interest postings (in-memory) + CmdBusSim->>BalanceRepoSim: Update loan balance (in-memory) +``` +1. The simulation loop sets the "current day" using the `ClockSim`. +2. It asks the `PESimService` (our main simulator) to process any events for that day. +3. The `PESimService` invokes the actual `MortgagePE` (Product Engine) logic. +4. The `MortgagePE` decides interest needs to be charged and returns an action to create a transaction command. +5. The `PESimService` uses the `CmdBusSim` to handle this command. +6. The `CmdBusSim` ensures that the resulting postings and balance updates are stored in the *in-memory* repositories (`PostingRepoSim`, `BalanceRepoSim`). + +This cycle repeats for every day in the simulation period. + +## Why are these Simulators So Valuable? + +* **Safety First:** We can test the riskiest financial logic without any danger to real operations or data. +* **Blazing Speed:** Simulating years of financial activity can take just seconds or minutes, not actual years. +* **Perfect Repetition:** If a test fails, we can run the exact same simulation again to understand and fix the problem. This is called deterministic testing. +* **"What If?" Scenarios:** We can easily explore complex situations: What if interest rates soar? What if a customer misses payments? How does our product react? +* **Debugging Superpowers:** When something goes wrong in a simulation, it's often easier to trace the problem step-by-step through the in-memory state changes. +* **Building Confidence:** Thorough simulation gives us much higher confidence that our banking products will work correctly in the real world. + +## Conclusion + +Simulation Services and Repositories are powerful, specialized tools within the `corebanking` project. They create a safe, controlled, and fast "practice environment" where we can: +* Test complex financial products (especially those managed by [Product Engines](09_product_engine_.md)) over long periods. +* Simulate various market conditions and customer behaviors. +* Verify that our business logic is correct and robust. + +By using in-memory data storage, a controllable clock, and simulated versions of core components, we can gain deep insights into how our system will behave, all before it ever touches real customer information or funds. This is essential for building a reliable and trustworthy core banking system. + +--- + +Congratulations on completing this tour of the core concepts in the `corebanking` project! From the fundamental [Events](01_event_.md) and [Commands](02_command_.md) that drive the system, through [Aggregates](03_aggregate_.md) that protect business rules, [Repositories](04_repository_.md) that manage data, to the [API Handlers](05_api_handler_.md) that welcome external requests, the coordinating [Core Facade](06_core_facade_.md), specialized [Services](07_service_.md), reactive [Consumers](08_consumer_.md), intelligent [Product Engines](09_product_engine_.md), and finally, these invaluable Simulation tools, you now have a solid foundation. + +While this marks the end of our conceptual overview, your journey of exploration and contribution is just beginning. You can now revisit earlier chapters with a deeper understanding or start exploring the codebase to see these concepts in full action! + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/index.md b/nesto/corebanking/index.md new file mode 100644 index 00000000..162cbff4 --- /dev/null +++ b/nesto/corebanking/index.md @@ -0,0 +1,78 @@ +# Tutorial: corebanking + +The project is a **core banking system** designed to manage financial operations such as *account creation, transaction processing (postings), and financial product lifecycle management*. +It heavily relies on an *Event Sourcing* architecture, where system state changes are captured as a sequence of immutable **Events**. +*Aggregates* (like `AccountAggregate` or `PostingsTransactionAggregate`) are central entities that process **Commands** (requests for change) and emit these Events. +*Services* (e.g., `AccountsService`, `ProductEnginesService`) encapsulate business logic and often orchestrate interactions between Aggregates, Repositories, and Product Engines. +*Repositories* provide an abstraction layer for data persistence. +*Product Engines* define the specific rules, behaviors, and automated processes for different financial products (e.g., mortgages, HELOCs). + + +**Source Repository:** [None](None) + +```mermaid +flowchart TD + A0["Aggregate +"] + A1["Command +"] + A2["Event +"] + A3["Service +"] + A4["Repository +"] + A5["Product Engine +"] + A6["Consumer +"] + A7["API Handler +"] + A8["Core Facade +"] + A9["Simulation Services and Repositories +"] + A0 -- "Processes" --> A1 + A0 -- "Produces" --> A2 + A0 -- "Uses for persistence" --> A4 + A0 -- "Uses dependent" --> A3 + A7 -- "Uses" --> A8 + A8 -- "Dispatches (via EventSourci..." --> A1 + A8 -- "Provides access to" --> A3 + A3 -- "Uses for data access" --> A4 + A3 -- "Invokes/Manages" --> A5 + A6 -- "Consumes" --> A2 + A6 -- "Updates read-models via" --> A4 + A6 -- "Uses (e.g., ProductEnginesS..." --> A3 + A5 -- "Reacts to (via ProductEngin..." --> A2 + A9 -- "Simulates" --> A3 + A9 -- "Simulates" --> A4 +``` + +## Chapters + +1. [Event +](01_event_.md) +2. [Command +](02_command_.md) +3. [Aggregate +](03_aggregate_.md) +4. [Repository +](04_repository_.md) +5. [API Handler +](05_api_handler_.md) +6. [Core Facade +](06_core_facade_.md) +7. [Service +](07_service_.md) +8. [Consumer +](08_consumer_.md) +9. [Product Engine +](09_product_engine_.md) +10. [Simulation Services and Repositories +](10_simulation_services_and_repositories_.md) + + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file From b2d6239e880619c1096d9df433e443b9b2118539 Mon Sep 17 00:00:00 2001 From: Louis-David Coulombe Date: Sat, 21 Jun 2025 14:13:02 -0400 Subject: [PATCH 03/15] Added corebanking review --- nesto/corebanking/review.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 nesto/corebanking/review.md diff --git a/nesto/corebanking/review.md b/nesto/corebanking/review.md new file mode 100644 index 00000000..d70a0d95 --- /dev/null +++ b/nesto/corebanking/review.md @@ -0,0 +1,36 @@ +# Core Banking Documentation Review + +## Section-by-Section Analysis + +- **Event**: Not bad overall, except the diagram which is really off +- **Command**: Not too bad +- **Aggregate**: Generally good, but hallucinates a command that doesn't exist (DepositFundsCommand) and from there it goes off the rails, claiming that the account manages transactions and balances... +- **Repository**: Not too bad, but the 2nd diagram is really off +- **API Handler**: Not too bad +- **Core Facade**: Not too bad +- **Service**: There are some impressive parts in this section, like the explanation of why services don't implement writes and an exception where the service does implement the write +- **Consumer**: Same thing - impressive analysis of the balances consumer. Some weird naming though: TxnProcessor/BalConsumer. +- **Product Engine**: Really good too (I'm curious if it will be as good with the latest changes I made), diagram is off again... +- **Simulation Services and Repos**: Weird product example, didn't catch that each product engine must implement its own simulator and allows offering features specific to the product, for example lump sum on a mortgage, drawdown request on a HELOC, etc. But overall conceptually it explains what can be done by implementing the simulator for a specific product engine. + +## Overall Assessment + +It's a really weird feeling because overall it's really not bad - there are even places where it surpasses the analysis I would have thought possible by an LLM. But it seems like there are enough small errors everywhere that it just feels like work done by someone who "doesn't care"... I'm not sure how to explain it. + +## Specific Issues + +For example, it generates code that's a bit more than pseudo-code, but outputs something like `uuid.FromString("acc-alice-123")`. The naming doesn't follow the conventions of the analyzed code like `accountsSvc` or `accountsRepo`. The diagrams invent layers that don't exist, and there are comments in the repo struct that are wrong. + +Otherwise, there are subtleties that I gloss over without thinking too much when reading because they make more or less sense, but not enough to really derail the reading. But I don't know if it would harm someone new to the codebase or not. + +## Major Concerns + +But there are some more major issues, like the invention of the DepositFundsCommand command which would really harm someone new to the codebase in understanding why transaction commands are sent directly to the account aggregate. + +## Missing Sections + +Otherwise, there would probably need to be a section for products, parameters, and customers. + +## Final Thoughts + +Overall, if there's a "parental advisory" sign on the documentation that it's auto-generated and to take certain details with a grain of salt, I think it covers the different concepts in corebanking and can probably help someone who doesn't know the project. It hammers the concepts of CQRS/event sourcing quite a bit. \ No newline at end of file From d1b464009b3277523489ca3a3399ac2f4f1852e4 Mon Sep 17 00:00:00 2001 From: Louis-David Coulombe Date: Sat, 21 Jun 2025 16:12:14 -0400 Subject: [PATCH 04/15] Added abstractions hints --- main.py | 5 ++++- nodes.py | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index dbba523f..4a460565 100644 --- a/main.py +++ b/main.py @@ -56,6 +56,8 @@ def main(): parser.add_argument("--no-cache", action="store_true", help="Disable LLM response caching (default: caching enabled)") # Add max_abstraction_num parameter to control the number of abstractions parser.add_argument("--max-abstractions", type=int, default=10, help="Maximum number of abstractions to identify (default: 10)") + # Add abstractions_hints parameter to control the abstractions to include + parser.add_argument("--abstractions-hints", nargs="+", help="Abstractions to include (e.g. 'Event' 'Command' 'Aggregate'). Defaults to all abstractions if not specified.") args = parser.parse_args() @@ -86,7 +88,8 @@ def main(): "use_cache": not args.no_cache, # Add max_abstraction_num parameter - "max_abstraction_num": args.max_abstractions, + "abstractions_hints": args.abstractions_hints, + "max_abstraction_num": args.max_abstractions if args.abstractions_hints is None else len(args.abstractions_hints), # Outputs will be populated by the nodes "files": [], diff --git a/nodes.py b/nodes.py index 0e3fa587..029fce76 100644 --- a/nodes.py +++ b/nodes.py @@ -88,6 +88,7 @@ def prep(self, shared): language = shared.get("language", "english") # Get language use_cache = shared.get("use_cache", True) # Get use_cache flag, default to True max_abstraction_num = shared.get("max_abstraction_num", 10) # Get max_abstraction_num, default to 10 + abstractions_hints = shared.get("abstractions_hints", []) # List of abstractions to include, if empty, all abstractions will be included # Helper to create context from files, respecting limits (basic example) def create_llm_context(files_data): @@ -113,6 +114,7 @@ def create_llm_context(files_data): language, use_cache, max_abstraction_num, + abstractions_hints, ) # Return all parameters def exec(self, prep_res): @@ -124,6 +126,7 @@ def exec(self, prep_res): language, use_cache, max_abstraction_num, + abstractions_hints, ) = prep_res # Unpack all parameters print(f"Identifying abstractions using LLM...") @@ -131,6 +134,10 @@ def exec(self, prep_res): language_instruction = "" name_lang_hint = "" desc_lang_hint = "" + if len(abstractions_hints) > 0: + abstractions_hints_str = f"IMPORTANT: Specific abstractions to include: {abstractions_hints}" + max_abstraction_num = len(abstractions_hints) + if language.lower() != "english": language_instruction = f"IMPORTANT: Generate the `name` and `description` for each abstraction in **{language.capitalize()}** language. Do NOT use English for these fields.\n\n" # Keep specific hints here as name/description are primary targets @@ -143,6 +150,8 @@ def exec(self, prep_res): Codebase Context: {context} +{abstractions_hints_str} + {language_instruction}Analyze the codebase context. Identify the top 5-{max_abstraction_num} core most important abstractions to help those new to the codebase. From f0e6a4cdd56eb3f92bdc7feacad5510922ce9a0a Mon Sep 17 00:00:00 2001 From: Louis-David Coulombe Date: Sat, 21 Jun 2025 16:12:25 -0400 Subject: [PATCH 05/15] Added shell to create corebanking --- run_corebanking_analysis.sh | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100755 run_corebanking_analysis.sh diff --git a/run_corebanking_analysis.sh b/run_corebanking_analysis.sh new file mode 100755 index 00000000..49195c82 --- /dev/null +++ b/run_corebanking_analysis.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Shell script to analyze the local corebanking repository +# with specific abstraction hints + +echo "Starting corebanking tutorial generation..." + +python main.py \ + --dir "/Users/louis-davidcoulombe/github/corebanking" \ + --name "corebanking" \ + --output "nesto/corebanking/analysis_output" \ + --abstractions-hints \ + "Event" \ + "Command" \ + "Aggregate" \ + "Repository" \ + "API Handler" \ + "Core Facade" \ + "Service" \ + "Consumer" \ + "Product Engine" \ + "Simulation Services and Repositories" \ + "products" \ + "parameters" \ + "customers" \ + --language "english" \ + --max-size 150000 + +echo "Corebanking analysis completed!" \ No newline at end of file From de4015f949af0f3bbc08addc0681ddab8847067d Mon Sep 17 00:00:00 2001 From: Louis-David Coulombe Date: Sat, 21 Jun 2025 19:30:09 -0400 Subject: [PATCH 06/15] Added command to include feedback --- main.py | 19 ++++++++ nodes.py | 88 ++++++++++++++++++++++++++++++++----- run_corebanking_analysis.sh | 3 +- 3 files changed, 99 insertions(+), 11 deletions(-) diff --git a/main.py b/main.py index 4a460565..e10d6e86 100644 --- a/main.py +++ b/main.py @@ -58,6 +58,8 @@ def main(): parser.add_argument("--max-abstractions", type=int, default=10, help="Maximum number of abstractions to identify (default: 10)") # Add abstractions_hints parameter to control the abstractions to include parser.add_argument("--abstractions-hints", nargs="+", help="Abstractions to include (e.g. 'Event' 'Command' 'Aggregate'). Defaults to all abstractions if not specified.") + # Add feedback parameter to provide feedback from previous runs + parser.add_argument("--feedback", help="Path to markdown file with feedback from previous analysis runs to help improve results.") args = parser.parse_args() @@ -68,6 +70,18 @@ def main(): if not github_token: print("Warning: No GitHub token provided. You might hit rate limits for public repositories.") + # Load feedback from file if provided + feedback_content = None + if args.feedback: + try: + with open(args.feedback, 'r', encoding='utf-8') as f: + feedback_content = f.read() + print(f"Loaded feedback from: {args.feedback}") + except FileNotFoundError: + print(f"Warning: Feedback file not found: {args.feedback}") + except Exception as e: + print(f"Warning: Could not read feedback file {args.feedback}: {e}") + # Initialize the shared dictionary with inputs shared = { "repo_url": args.repo, @@ -91,6 +105,9 @@ def main(): "abstractions_hints": args.abstractions_hints, "max_abstraction_num": args.max_abstractions if args.abstractions_hints is None else len(args.abstractions_hints), + # Add feedback content + "feedback_content": feedback_content, + # Outputs will be populated by the nodes "files": [], "abstractions": [], @@ -103,6 +120,8 @@ def main(): # Display starting message with repository/directory and language print(f"Starting tutorial generation for: {args.repo or args.dir} in {args.language.capitalize()} language") print(f"LLM caching: {'Disabled' if args.no_cache else 'Enabled'}") + if feedback_content: + print(f"Using feedback from previous run to improve results") # Create the flow instance tutorial_flow = create_tutorial_flow() diff --git a/nodes.py b/nodes.py index 029fce76..181938ca 100644 --- a/nodes.py +++ b/nodes.py @@ -71,6 +71,9 @@ def exec(self, prep_res): ) # Convert dict to list of tuples: [(path, content), ...] + if result is None: + raise ValueError("Failed to fetch files - crawling function returned None") + files_list = list(result.get("files", {}).items()) if len(files_list) == 0: raise (ValueError("Failed to fetch files")) @@ -89,6 +92,7 @@ def prep(self, shared): use_cache = shared.get("use_cache", True) # Get use_cache flag, default to True max_abstraction_num = shared.get("max_abstraction_num", 10) # Get max_abstraction_num, default to 10 abstractions_hints = shared.get("abstractions_hints", []) # List of abstractions to include, if empty, all abstractions will be included + feedback_content = shared.get("feedback_content") # Get feedback from previous runs # Helper to create context from files, respecting limits (basic example) def create_llm_context(files_data): @@ -115,7 +119,8 @@ def create_llm_context(files_data): use_cache, max_abstraction_num, abstractions_hints, - ) # Return all parameters + feedback_content, + ) # Return all parameters including feedback def exec(self, prep_res): ( @@ -127,6 +132,7 @@ def exec(self, prep_res): use_cache, max_abstraction_num, abstractions_hints, + feedback_content, ) = prep_res # Unpack all parameters print(f"Identifying abstractions using LLM...") @@ -134,10 +140,29 @@ def exec(self, prep_res): language_instruction = "" name_lang_hint = "" desc_lang_hint = "" + + abstractions_hints_str = "" if len(abstractions_hints) > 0: - abstractions_hints_str = f"IMPORTANT: Specific abstractions to include: {abstractions_hints}" + abstractions_hints_str = f"IMPORTANT: Specific abstractions to include: {abstractions_hints}\n\n" max_abstraction_num = len(abstractions_hints) + # Add feedback section if available + feedback_section = "" + if feedback_content: + feedback_section = f""" +CRITICAL: Learn from Previous Analysis Feedback +The following feedback was provided from a previous analysis of this codebase. Please carefully consider these points to avoid repeating the same mistakes: + +{feedback_content} + +Based on this feedback, pay special attention to: +- Avoiding hallucinated commands or concepts that don't exist in the codebase +- Following proper naming conventions used in the code +- Ensuring accuracy in descriptions and relationships +- Including all important abstractions mentioned in the feedback + +""" + if language.lower() != "english": language_instruction = f"IMPORTANT: Generate the `name` and `description` for each abstraction in **{language.capitalize()}** language. Do NOT use English for these fields.\n\n" # Keep specific hints here as name/description are primary targets @@ -147,12 +172,10 @@ def exec(self, prep_res): prompt = f""" For the project `{project_name}`: -Codebase Context: +{feedback_section}Codebase Context: {context} -{abstractions_hints_str} - -{language_instruction}Analyze the codebase context. +{abstractions_hints_str}{language_instruction}Analyze the codebase context. Identify the top 5-{max_abstraction_num} core most important abstractions to help those new to the codebase. For each abstraction, provide: @@ -255,6 +278,7 @@ def prep(self, shared): project_name = shared["project_name"] # Get project name language = shared.get("language", "english") # Get language use_cache = shared.get("use_cache", True) # Get use_cache flag, default to True + feedback_content = shared.get("feedback_content") # Get feedback from previous runs # Get the actual number of abstractions directly num_abstractions = len(abstractions) @@ -293,7 +317,8 @@ def prep(self, shared): project_name, language, use_cache, - ) # Return use_cache + feedback_content, + ) # Return feedback_content def exec(self, prep_res): ( @@ -303,7 +328,8 @@ def exec(self, prep_res): project_name, language, use_cache, - ) = prep_res # Unpack use_cache + feedback_content, + ) = prep_res # Unpack use_cache and feedback_content print(f"Analyzing relationships using LLM...") # Add language instruction and hints only if not English @@ -315,6 +341,24 @@ def exec(self, prep_res): lang_hint = f" (in {language.capitalize()})" list_lang_note = f" (Names might be in {language.capitalize()})" # Note for the input list + # Add feedback section if available + feedback_section = "" + if feedback_content: + feedback_section = f""" +CRITICAL: Learn from Previous Analysis Feedback +The following feedback was provided from a previous analysis of this codebase. Please carefully consider these points to avoid repeating the same mistakes: + +{feedback_content} + +Based on this feedback, pay special attention to: +- Avoiding hallucinated commands or concepts that don't exist in the codebase +- Ensuring relationships are accurate and based on actual code interactions +- Following proper naming conventions used in the code +- Being precise about how different components actually interact in the codebase +- Including all important relationships mentioned in the feedback + +""" + prompt = f""" Based on the following abstractions and relevant code snippets from the project `{project_name}`: @@ -324,7 +368,7 @@ def exec(self, prep_res): Context (Abstractions, Descriptions, Code): {context} -{language_instruction}Please provide: +{feedback_section}{language_instruction}Please provide: 1. A high-level `summary` of the project's main purpose and functionality in a few beginner-friendly sentences{lang_hint}. Use markdown formatting with **bold** and *italic* text to highlight important concepts. 2. A list (`relationships`) describing the key interactions between these abstractions. For each relationship, specify: - `from_abstraction`: Index of the source abstraction (e.g., `0 # AbstractionName1`) @@ -553,6 +597,7 @@ def prep(self, shared): project_name = shared["project_name"] language = shared.get("language", "english") use_cache = shared.get("use_cache", True) # Get use_cache flag, default to True + feedback_content = shared.get("feedback_content") # Get feedback from previous runs # Get already written chapters to provide context # We store them temporarily during the batch run, not in shared memory yet @@ -625,6 +670,7 @@ def prep(self, shared): "next_chapter": next_chapter, # Add next chapter info (uses potentially translated name) "language": language, # Add language for multi-language support "use_cache": use_cache, # Pass use_cache flag + "feedback_content": feedback_content, # Add feedback content # previous_chapters_summary will be added dynamically in exec } ) @@ -648,6 +694,7 @@ def exec(self, item): project_name = item.get("project_name") language = item.get("language", "english") use_cache = item.get("use_cache", True) # Read use_cache from item + feedback_content = item.get("feedback_content") # Get feedback content print(f"Writing chapter {chapter_num} for: {abstraction_name} using LLM...") # Prepare file context string from the map @@ -684,6 +731,27 @@ def exec(self, item): ) tone_note = f" (appropriate for {lang_cap} readers)" + # Add feedback section if available + feedback_section = "" + if feedback_content: + feedback_section = f""" + +CRITICAL: Learn from Previous Analysis Feedback +The following feedback was provided from a previous analysis of this codebase. Please carefully consider these points to avoid repeating the same mistakes in this chapter: + +{feedback_content} + +Based on this feedback, pay special attention to: +- Avoiding hallucinated commands, classes, or concepts that don't exist in the codebase +- Following proper naming conventions used in the actual code (like `accountsSvc`, `accountsRepo`) +- Ensuring code examples are realistic and follow the patterns seen in the actual codebase +- Creating accurate diagrams that reflect the actual architecture, not invented layers +- Being precise about how components actually work based on the code, not assumptions +- Including accurate comments and struct descriptions +- Avoiding pseudo-code that might mislead readers about the actual implementation + +""" + prompt = f""" {language_instruction}Write a very beginner-friendly tutorial chapter (in Markdown format) for the project `{project_name}` about the concept: "{abstraction_name}". This is Chapter {chapter_num}. @@ -700,7 +768,7 @@ def exec(self, item): Relevant Code Snippets (Code itself remains unchanged): {file_context_str if file_context_str else "No specific code snippets provided for this abstraction."} - +{feedback_section} Instructions for the chapter (Generate content in {language.capitalize()} unless specified otherwise): - Start with a clear heading (e.g., `# Chapter {chapter_num}: {abstraction_name}`). Use the provided concept name. diff --git a/run_corebanking_analysis.sh b/run_corebanking_analysis.sh index 49195c82..4d3538eb 100755 --- a/run_corebanking_analysis.sh +++ b/run_corebanking_analysis.sh @@ -1,7 +1,7 @@ #!/bin/bash # Shell script to analyze the local corebanking repository -# with specific abstraction hints +# with specific abstraction hints and feedback from previous analysis echo "Starting corebanking tutorial generation..." @@ -23,6 +23,7 @@ python main.py \ "products" \ "parameters" \ "customers" \ + --feedback "nesto/corebanking/review.md" \ --language "english" \ --max-size 150000 From 77856206f580d003bdf7d0209ea6658a723d717e Mon Sep 17 00:00:00 2001 From: Louis-David Coulombe Date: Mon, 27 Oct 2025 15:54:04 -0400 Subject: [PATCH 07/15] Update to file configuration --- configs/corebanking/config.yaml | 91 ++++++++++++++++ main.py | 184 ++++++++++++++++++++------------ run_corebanking_analysis.sh | 35 +++--- 3 files changed, 221 insertions(+), 89 deletions(-) create mode 100644 configs/corebanking/config.yaml diff --git a/configs/corebanking/config.yaml b/configs/corebanking/config.yaml new file mode 100644 index 00000000..9e12e3f5 --- /dev/null +++ b/configs/corebanking/config.yaml @@ -0,0 +1,91 @@ +# Corebanking Project Analysis Configuration + +# Source configuration - specify either repo or local_dir +source: + local_dir: "/Users/louis-davidcoulombe/github/corebanking" + # repo: "https://github.com/example/corebanking" # Use this instead of local_dir for GitHub repos + +# Project settings +project: + name: "corebanking" + language: "english" + output_dir: "nesto/corebanking/analysis_output" + +# File processing settings +file_processing: + max_file_size: 150000 # Maximum file size in bytes + include_patterns: + - "*.py" + - "*.js" + - "*.jsx" + - "*.ts" + - "*.tsx" + - "*.go" + - "*.java" + - "*.pyi" + - "*.pyx" + - "*.c" + - "*.cc" + - "*.cpp" + - "*.h" + - "*.md" + - "*.rst" + - "*Dockerfile" + - "*Makefile" + - "*.yaml" + - "*.yml" + exclude_patterns: + - "assets/*" + - "data/*" + - "images/*" + - "public/*" + - "static/*" + - "temp/*" + - "*docs/*" + - "*venv/*" + - "*.venv/*" + - "*test*" + - "*tests/*" + - "*examples/*" + - "v1/*" + - "*dist/*" + - "*build/*" + - "*experimental/*" + - "*deprecated/*" + - "*misc/*" + - "*legacy/*" + - ".git/*" + - ".github/*" + - ".next/*" + - ".vscode/*" + - "*obj/*" + - "*bin/*" + - "*node_modules/*" + - "*.log" + +# Analysis settings +analysis: + max_abstractions: 12 # Based on number of abstraction hints provided + abstractions_hints: + - "Event" + - "Command" + - "Aggregate" + - "Repository" + - "API Handler" + - "Core Facade" + - "Service" + - "Consumer" + - "Product Engine" + - "Simulation Services and Repositories" + - "products" + - "parameters" + - "customers" + feedback_file: "nesto/corebanking/review.md" + +# LLM settings +llm: + use_cache: true # Set to false to disable caching + +# GitHub settings (only needed if using repo instead of local_dir) +github: + # token: "your_github_token_here" # Or set GITHUB_TOKEN environment variable diff --git a/main.py b/main.py index e10d6e86..59f80e38 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,7 @@ import dotenv import os import argparse +import yaml # Import the function that creates the flow from flow import create_tutorial_flow @@ -35,77 +36,98 @@ "*.log" } -# --- Main Function --- -def main(): - parser = argparse.ArgumentParser(description="Generate a tutorial for a GitHub codebase or local directory.") - - # Create mutually exclusive group for source - source_group = parser.add_mutually_exclusive_group(required=True) - source_group.add_argument("--repo", help="URL of the public GitHub repository.") - source_group.add_argument("--dir", help="Path to local directory.") - - parser.add_argument("-n", "--name", help="Project name (optional, derived from repo/directory if omitted).") - parser.add_argument("-t", "--token", help="GitHub personal access token (optional, reads from GITHUB_TOKEN env var if not provided).") - parser.add_argument("-o", "--output", default="output", help="Base directory for output (default: ./output).") - parser.add_argument("-i", "--include", nargs="+", help="Include file patterns (e.g. '*.py' '*.js'). Defaults to common code files if not specified.") - parser.add_argument("-e", "--exclude", nargs="+", help="Exclude file patterns (e.g. 'tests/*' 'docs/*'). Defaults to test/build directories if not specified.") - parser.add_argument("-s", "--max-size", type=int, default=100000, help="Maximum file size in bytes (default: 100000, about 100KB).") - # Add language parameter for multi-language support - parser.add_argument("--language", default="english", help="Language for the generated tutorial (default: english)") - # Add use_cache parameter to control LLM caching - parser.add_argument("--no-cache", action="store_true", help="Disable LLM response caching (default: caching enabled)") - # Add max_abstraction_num parameter to control the number of abstractions - parser.add_argument("--max-abstractions", type=int, default=10, help="Maximum number of abstractions to identify (default: 10)") - # Add abstractions_hints parameter to control the abstractions to include - parser.add_argument("--abstractions-hints", nargs="+", help="Abstractions to include (e.g. 'Event' 'Command' 'Aggregate'). Defaults to all abstractions if not specified.") - # Add feedback parameter to provide feedback from previous runs - parser.add_argument("--feedback", help="Path to markdown file with feedback from previous analysis runs to help improve results.") - - args = parser.parse_args() - - # Get GitHub token from argument or environment variable if using repo +def load_config(config_path): + """Load and validate YAML configuration file.""" + try: + with open(config_path, 'r', encoding='utf-8') as f: + config = yaml.safe_load(f) + + # Validate required sections + required_sections = ['source', 'project'] + for section in required_sections: + if section not in config: + raise ValueError(f"Missing required section '{section}' in config file") + + # Validate source (must have either repo or local_dir) + source = config['source'] + if not ('repo' in source or 'local_dir' in source): + raise ValueError("Source section must contain either 'repo' or 'local_dir'") + + # Set defaults for optional sections + if 'file_processing' not in config: + config['file_processing'] = {} + if 'analysis' not in config: + config['analysis'] = {} + if 'llm' not in config: + config['llm'] = {} + if 'github' not in config: + config['github'] = {} + + return config + + except FileNotFoundError: + raise FileNotFoundError(f"Configuration file not found: {config_path}") + except yaml.YAMLError as e: + raise ValueError(f"Invalid YAML in configuration file: {e}") + except Exception as e: + raise Exception(f"Error loading configuration: {e}") + +def config_to_shared(config): + """Convert YAML config to shared dictionary format.""" + source = config['source'] + project = config['project'] + file_proc = config.get('file_processing', {}) + analysis = config.get('analysis', {}) + llm_config = config.get('llm', {}) + github_config = config.get('github', {}) + + # Get GitHub token from config or environment variable github_token = None - if args.repo: - github_token = args.token or os.environ.get('GITHUB_TOKEN') + if 'repo' in source: + github_token = github_config.get('token') or os.environ.get('GITHUB_TOKEN') if not github_token: print("Warning: No GitHub token provided. You might hit rate limits for public repositories.") - + # Load feedback from file if provided feedback_content = None - if args.feedback: + feedback_file = analysis.get('feedback_file') + if feedback_file: try: - with open(args.feedback, 'r', encoding='utf-8') as f: + with open(feedback_file, 'r', encoding='utf-8') as f: feedback_content = f.read() - print(f"Loaded feedback from: {args.feedback}") + print(f"Loaded feedback from: {feedback_file}") except FileNotFoundError: - print(f"Warning: Feedback file not found: {args.feedback}") + print(f"Warning: Feedback file not found: {feedback_file}") except Exception as e: - print(f"Warning: Could not read feedback file {args.feedback}: {e}") - - # Initialize the shared dictionary with inputs + print(f"Warning: Could not read feedback file {feedback_file}: {e}") + + # Get abstractions hints and calculate max abstractions + abstractions_hints = analysis.get('abstractions_hints') + max_abstractions = analysis.get('max_abstractions', 10) + if abstractions_hints: + max_abstractions = len(abstractions_hints) + shared = { - "repo_url": args.repo, - "local_dir": args.dir, - "project_name": args.name, # Can be None, FetchRepo will derive it + "repo_url": source.get('repo'), + "local_dir": source.get('local_dir'), + "project_name": project.get('name'), "github_token": github_token, - "output_dir": args.output, # Base directory for CombineTutorial output + "output_dir": project.get('output_dir', 'output'), - # Add include/exclude patterns and max file size - "include_patterns": set(args.include) if args.include else DEFAULT_INCLUDE_PATTERNS, - "exclude_patterns": set(args.exclude) if args.exclude else DEFAULT_EXCLUDE_PATTERNS, - "max_file_size": args.max_size, + # File processing settings + "include_patterns": set(file_proc.get('include_patterns', DEFAULT_INCLUDE_PATTERNS)), + "exclude_patterns": set(file_proc.get('exclude_patterns', DEFAULT_EXCLUDE_PATTERNS)), + "max_file_size": file_proc.get('max_file_size', 100000), - # Add language for multi-language support - "language": args.language, + # Project settings + "language": project.get('language', 'english'), - # Add use_cache flag (inverse of no-cache flag) - "use_cache": not args.no_cache, + # LLM settings + "use_cache": llm_config.get('use_cache', True), - # Add max_abstraction_num parameter - "abstractions_hints": args.abstractions_hints, - "max_abstraction_num": args.max_abstractions if args.abstractions_hints is None else len(args.abstractions_hints), - - # Add feedback content + # Analysis settings + "abstractions_hints": abstractions_hints, + "max_abstraction_num": max_abstractions, "feedback_content": feedback_content, # Outputs will be populated by the nodes @@ -116,18 +138,46 @@ def main(): "chapters": [], "final_output_dir": None } + + return shared - # Display starting message with repository/directory and language - print(f"Starting tutorial generation for: {args.repo or args.dir} in {args.language.capitalize()} language") - print(f"LLM caching: {'Disabled' if args.no_cache else 'Enabled'}") - if feedback_content: - print(f"Using feedback from previous run to improve results") - - # Create the flow instance - tutorial_flow = create_tutorial_flow() - - # Run the flow - tutorial_flow.run(shared) +# --- Main Function --- +def main(): + parser = argparse.ArgumentParser(description="Generate a tutorial for a GitHub codebase or local directory using YAML configuration.") + parser.add_argument("config", help="Path to YAML configuration file") + parser.add_argument("--validate-only", action="store_true", help="Only validate the configuration file without running the analysis") + + args = parser.parse_args() + + try: + # Load and validate configuration + config = load_config(args.config) + print(f"✓ Configuration loaded successfully from: {args.config}") + + if args.validate_only: + print("✓ Configuration is valid!") + return + + # Convert config to shared dictionary + shared = config_to_shared(config) + + # Display starting message + source_info = shared.get('repo_url') or shared.get('local_dir') + language = shared.get('language', 'english') + print(f"Starting tutorial generation for: {source_info} in {language.capitalize()} language") + print(f"LLM caching: {'Enabled' if shared.get('use_cache') else 'Disabled'}") + if shared.get('feedback_content'): + print("Using feedback from previous run to improve results") + + # Create the flow instance + tutorial_flow = create_tutorial_flow() + + # Run the flow + tutorial_flow.run(shared) + + except Exception as e: + print(f"Error: {e}") + exit(1) if __name__ == "__main__": main() diff --git a/run_corebanking_analysis.sh b/run_corebanking_analysis.sh index 4d3538eb..6a0a7b01 100755 --- a/run_corebanking_analysis.sh +++ b/run_corebanking_analysis.sh @@ -1,30 +1,21 @@ #!/bin/bash # Shell script to analyze the local corebanking repository -# with specific abstraction hints and feedback from previous analysis +# using YAML configuration file echo "Starting corebanking tutorial generation..." +source .venv/bin/activate -python main.py \ - --dir "/Users/louis-davidcoulombe/github/corebanking" \ - --name "corebanking" \ - --output "nesto/corebanking/analysis_output" \ - --abstractions-hints \ - "Event" \ - "Command" \ - "Aggregate" \ - "Repository" \ - "API Handler" \ - "Core Facade" \ - "Service" \ - "Consumer" \ - "Product Engine" \ - "Simulation Services and Repositories" \ - "products" \ - "parameters" \ - "customers" \ - --feedback "nesto/corebanking/review.md" \ - --language "english" \ - --max-size 150000 +# Validate configuration before running +echo "Validating configuration..." +python main.py configs/corebanking/config.yaml --validate-only + +if [ $? -eq 0 ]; then + echo "Configuration is valid. Running analysis..." + python main.py configs/corebanking/config.yaml +else + echo "Configuration validation failed. Please check the config file." + exit 1 +fi echo "Corebanking analysis completed!" \ No newline at end of file From d27a921bf8babedccd15c750fe5a16c9e9af1260 Mon Sep 17 00:00:00 2001 From: Louis-David Coulombe Date: Mon, 27 Oct 2025 15:54:39 -0400 Subject: [PATCH 08/15] Corebanking config --- configs/{corebanking/config.yaml => corebanking.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename configs/{corebanking/config.yaml => corebanking.yaml} (100%) diff --git a/configs/corebanking/config.yaml b/configs/corebanking.yaml similarity index 100% rename from configs/corebanking/config.yaml rename to configs/corebanking.yaml From aa09fc0b777b09a98541ee9b288445677b180dfd Mon Sep 17 00:00:00 2001 From: Louis-David Coulombe Date: Mon, 27 Oct 2025 15:55:11 -0400 Subject: [PATCH 09/15] Example config --- configs/example_config.yaml | 72 +++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 configs/example_config.yaml diff --git a/configs/example_config.yaml b/configs/example_config.yaml new file mode 100644 index 00000000..e9c8d04c --- /dev/null +++ b/configs/example_config.yaml @@ -0,0 +1,72 @@ +# Example Configuration File for Tutorial Generation +# Copy this file and modify for your specific project + +# Source configuration - specify either repo or local_dir (not both) +source: + # For local directories: + local_dir: "/path/to/your/local/project" + + # For GitHub repositories (comment out local_dir if using this): + # repo: "https://github.com/username/repository" + +# Project settings +project: + name: "your_project_name" + language: "english" # or "french", "spanish", etc. + output_dir: "output/your_project" # where to save the generated tutorial + +# File processing settings +file_processing: + max_file_size: 100000 # Maximum file size in bytes (default: 100KB) + + # File patterns to include in analysis + include_patterns: + - "*.py" + - "*.js" + - "*.jsx" + - "*.ts" + - "*.tsx" + - "*.go" + - "*.java" + - "*.md" + - "*.rst" + - "*.yaml" + - "*.yml" + + # File patterns to exclude from analysis + exclude_patterns: + - "*test*" + - "*tests/*" + - "*node_modules/*" + - "*venv/*" + - "*.venv/*" + - "*dist/*" + - "*build/*" + - ".git/*" + - "*.log" + +# Analysis settings +analysis: + max_abstractions: 10 # Maximum number of abstractions to identify + + # Specific abstractions to look for (optional) + abstractions_hints: + - "Class" + - "Interface" + - "Service" + - "Controller" + - "Repository" + - "Model" + - "Component" + - "Module" + + # Path to feedback file from previous runs (optional) + # feedback_file: "path/to/feedback.md" + +# LLM settings +llm: + use_cache: true # Set to false to disable response caching + +# GitHub settings (only needed if using repo instead of local_dir) +github: + # token: "your_github_token_here" # Or set GITHUB_TOKEN environment variable From bb9be2cb6dbbd4f22b8dbfd64a0ad652e323ad0c Mon Sep 17 00:00:00 2001 From: Louis-David Coulombe Date: Mon, 27 Oct 2025 16:21:55 -0400 Subject: [PATCH 10/15] Error hanlding and more verbose progress --- main.py | 25 ++++++++++--- nodes.py | 36 +++++++++++++++++-- ...corebanking_analysis.sh => run_analysis.sh | 6 ++-- 3 files changed, 57 insertions(+), 10 deletions(-) rename run_corebanking_analysis.sh => run_analysis.sh (73%) diff --git a/main.py b/main.py index 59f80e38..40c0b2cc 100644 --- a/main.py +++ b/main.py @@ -104,8 +104,17 @@ def config_to_shared(config): # Get abstractions hints and calculate max abstractions abstractions_hints = analysis.get('abstractions_hints') max_abstractions = analysis.get('max_abstractions', 10) + + # Ensure abstractions_hints is never None, default to empty list + if abstractions_hints is None: + abstractions_hints = [] + + # If specific abstractions are provided, use their count as max if abstractions_hints: max_abstractions = len(abstractions_hints) + print(f"✓ Using {len(abstractions_hints)} specific abstraction hints") + else: + print(f"✓ No specific abstraction hints provided, will identify up to {max_abstractions} abstractions") shared = { "repo_url": source.get('repo'), @@ -164,10 +173,18 @@ def main(): # Display starting message source_info = shared.get('repo_url') or shared.get('local_dir') language = shared.get('language', 'english') - print(f"Starting tutorial generation for: {source_info} in {language.capitalize()} language") - print(f"LLM caching: {'Enabled' if shared.get('use_cache') else 'Disabled'}") - if shared.get('feedback_content'): - print("Using feedback from previous run to improve results") + print(f"🚀 Starting tutorial generation for: {source_info} in {language.capitalize()} language") + print(f"📊 Configuration:") + print(f" ├─ Project name: {shared.get('project_name', 'Auto-detected')}") + print(f" ├─ Output directory: {shared.get('output_dir')}") + print(f" ├─ Max file size: {shared.get('max_file_size'):,} bytes") + print(f" ├─ LLM caching: {'Enabled' if shared.get('use_cache') else 'Disabled'}") + print(f" ├─ Language: {language.capitalize()}") + print(f" └─ Feedback from previous run: {'Yes' if shared.get('feedback_content') else 'No'}") + + include_count = len(shared.get('include_patterns', [])) + exclude_count = len(shared.get('exclude_patterns', [])) + print(f"📁 File patterns: {include_count} include, {exclude_count} exclude") # Create the flow instance tutorial_flow = create_tutorial_flow() diff --git a/nodes.py b/nodes.py index 181938ca..3eef25b0 100644 --- a/nodes.py +++ b/nodes.py @@ -77,7 +77,14 @@ def exec(self, prep_res): files_list = list(result.get("files", {}).items()) if len(files_list) == 0: raise (ValueError("Failed to fetch files")) - print(f"Fetched {len(files_list)} files.") + print(f"✅ Fetched {len(files_list)} files successfully") + + # Show some stats about the files + total_size = sum(len(content) for _, content in files_list) + avg_size = total_size // len(files_list) if files_list else 0 + print(f" ├─ Total content size: {total_size:,} characters") + print(f" ├─ Average file size: {avg_size:,} characters") + print(f" └─ Largest file: {max((len(content) for _, content in files_list), default=0):,} characters") return files_list def post(self, shared, prep_res, exec_res): @@ -134,21 +141,38 @@ def exec(self, prep_res): abstractions_hints, feedback_content, ) = prep_res # Unpack all parameters - print(f"Identifying abstractions using LLM...") + + print(f"🔍 Identifying abstractions using LLM...") + print(f" ├─ Project: {project_name}") + print(f" ├─ Files analyzed: {file_count}") + print(f" ├─ Language: {language}") + print(f" ├─ Max abstractions: {max_abstraction_num}") + print(f" ├─ Cache enabled: {use_cache}") + print(f" ├─ Feedback provided: {'Yes' if feedback_content else 'No'}") + print(f" └─ Abstraction hints: {len(abstractions_hints) if abstractions_hints else 0} provided") # Add language instruction and hints only if not English language_instruction = "" name_lang_hint = "" desc_lang_hint = "" + # Safety check for abstractions_hints + if abstractions_hints is None: + abstractions_hints = [] + print("Warning: abstractions_hints was None, defaulting to empty list") + abstractions_hints_str = "" if len(abstractions_hints) > 0: + print(f"✓ Using specific abstraction hints: {abstractions_hints}") abstractions_hints_str = f"IMPORTANT: Specific abstractions to include: {abstractions_hints}\n\n" max_abstraction_num = len(abstractions_hints) + else: + print(f"✓ No specific abstraction hints, identifying up to {max_abstraction_num} abstractions") # Add feedback section if available feedback_section = "" if feedback_content: + print(f"📝 Including feedback from previous analysis run") feedback_section = f""" CRITICAL: Learn from Previous Analysis Feedback The following feedback was provided from a previous analysis of this codebase. Please carefully consider these points to avoid repeating the same mistakes: @@ -205,9 +229,12 @@ def exec(self, prep_res): - 5 # path/to/another.js # ... up to {max_abstraction_num} abstractions ```""" + print(f"🤖 Calling LLM for abstraction identification (attempt {self.cur_retry + 1})") response = call_llm(prompt, use_cache=(use_cache and self.cur_retry == 0)) # Use cache only if enabled and not retrying + print(f"✅ Received LLM response ({len(response)} characters)") # --- Validation --- + print(f"🔍 Parsing and validating LLM response...") yaml_str = response.strip().split("```yaml")[1].split("```")[0].strip() abstractions = yaml.safe_load(yaml_str) @@ -260,7 +287,10 @@ def exec(self, prep_res): } ) - print(f"Identified {len(validated_abstractions)} abstractions.") + print(f"✅ Successfully identified {len(validated_abstractions)} abstractions:") + for i, abstr in enumerate(validated_abstractions): + file_count = len(abstr.get('files', [])) + print(f" {i+1}. {abstr['name']} ({file_count} files)") return validated_abstractions def post(self, shared, prep_res, exec_res): diff --git a/run_corebanking_analysis.sh b/run_analysis.sh similarity index 73% rename from run_corebanking_analysis.sh rename to run_analysis.sh index 6a0a7b01..3e5b3b22 100755 --- a/run_corebanking_analysis.sh +++ b/run_analysis.sh @@ -8,14 +8,14 @@ source .venv/bin/activate # Validate configuration before running echo "Validating configuration..." -python main.py configs/corebanking/config.yaml --validate-only +python main.py "$1" --validate-only if [ $? -eq 0 ]; then echo "Configuration is valid. Running analysis..." - python main.py configs/corebanking/config.yaml + python main.py "$1" else echo "Configuration validation failed. Please check the config file." exit 1 fi -echo "Corebanking analysis completed!" \ No newline at end of file +echo "Done" \ No newline at end of file From b39085834a0ccda595bc44506df253cd2c5efce0 Mon Sep 17 00:00:00 2001 From: Louis-David Coulombe Date: Mon, 27 Oct 2025 16:22:54 -0400 Subject: [PATCH 11/15] Updated documentation --- WorkflowDocumentation.md | 109 +++++++++++++++++++ configs/corebanking.yaml | 2 +- configs/documents-extractor.yaml | 91 ++++++++++++++++ prompt.md | 177 +++++++++++++++++++++++++++++++ 4 files changed, 378 insertions(+), 1 deletion(-) create mode 100644 WorkflowDocumentation.md create mode 100644 configs/documents-extractor.yaml create mode 100644 prompt.md diff --git a/WorkflowDocumentation.md b/WorkflowDocumentation.md new file mode 100644 index 00000000..2d5bb9a7 --- /dev/null +++ b/WorkflowDocumentation.md @@ -0,0 +1,109 @@ + # Documentation Generation Workflow + +This document explains the workflow of the documentation generation process, focusing on how the Large Language Model (LLM) prompts chain together to create a comprehensive tutorial for a given codebase. + +## Overview + +The process is a pipeline of six sequential steps. Four of these steps involve interactions with an LLM to analyze the code, structure the content, and write the documentation. + +```mermaid +flowchart TD + A[1. Fetch Repository] --> B[2. Identify Abstractions] + B --> C[3. Analyze Relationships] + C --> D[4. Order Chapters] + D --> E[5. Write Chapters] + E --> F[6. Combine Tutorial] +``` + +## Detailed Steps & Prompt Chaining + +Here’s a detailed breakdown of each step, focusing on the inputs and outputs of the LLM prompts. + +### 1. Fetch Repository (`FetchRepo`) +This initial step does not use an LLM. It's responsible for gathering the source code. + +- **Input**: A repository URL or a local directory path. +- **Output**: A list of all relevant code files and their content. This list becomes the foundational input for the entire workflow. +- **Chaining**: The raw codebase content is passed to the next step. + +--- + +### 2. Identify Abstractions (`IdentifyAbstractions`) +This is the first interaction with the LLM. Its goal is to identify the most important, high-level concepts in the codebase. + +- **Input to Prompt**: The entire codebase content fetched in the previous step. +- **LLM Prompt Goal**: The LLM is asked to analyze the code and identify the top 5-10 core abstractions. For each abstraction, it must provide: + 1. A concise `name`. + 2. A beginner-friendly `description` with an analogy. + 3. A list of relevant `file_indices` that implement or define the abstraction. +- **Output of Prompt**: The LLM returns a YAML-formatted string. + ```yaml + - name: | + Core Concept A + description: | + An explanation of what this concept does, like a central controller. + file_indices: + - 0 # path/to/file1.py + - 3 # path/to/file2.py + ``` +- **Chaining**: The validated list of abstractions (name, description, file indices) is passed to the next step. + +--- + +### 3. Analyze Relationships (`AnalyzeRelationships`) +The second LLM interaction focuses on understanding how the identified abstractions interact. + +- **Input to Prompt**: The list of abstractions (names and descriptions) and the code snippets from their relevant files. +- **LLM Prompt Goal**: The LLM is prompted to: + 1. Create a high-level `summary` of the project's purpose. + 2. Define the `relationships` between the abstractions, describing how they interact (e.g., "Manages", "Inherits from", "Uses"). +- **Output of Prompt**: A YAML object containing the summary and a list of relationships. + ```yaml + summary: | + A brief, simple explanation of the project's purpose. + relationships: + - from_abstraction: 0 # Core Concept A + to_abstraction: 1 # Core Concept B + label: "Manages" + ``` +- **Chaining**: The project summary and the list of relationships are passed to the next step. + +--- + +### 4. Order Chapters (`OrderChapters`) +The third LLM interaction determines the best pedagogical order to present the concepts. + +- **Input to Prompt**: The project summary, the list of abstractions, and their relationships. +- **LLM Prompt Goal**: The LLM is asked to determine the optimal sequence for a tutorial. It's instructed to start with foundational or user-facing concepts and then move to lower-level implementation details, respecting dependencies revealed in the relationships. +- **Output of Prompt**: A YAML list of the abstraction indices, sorted in the recommended chapter order. + ```yaml + - 2 # FoundationalConcept + - 0 # CoreClassA + - 1 # CoreClassB (uses CoreClassA) + ``` +- **Chaining**: This ordered list of indices dictates the structure of the final tutorial and is passed to the chapter writing step. + +--- + +### 5. Write Chapters (`WriteChapters`) +This is the most intensive LLM step, where the actual tutorial content is generated for each abstraction, one by one. + +- **Input to Prompt (for each chapter)**: + - The specific abstraction's details (name, description). + - The full tutorial structure (for linking to other chapters). + - The content of previously written chapters (to ensure smooth transitions). + - Relevant code snippets for the current abstraction. +- **LLM Prompt Goal**: The LLM is given a detailed set of instructions to write a beginner-friendly Markdown chapter. This includes creating a heading, explaining the concept with analogies, providing simplified code examples (under 10 lines), using Mermaid diagrams for illustration, and writing transitions to the previous and next chapters. +- **Output of Prompt**: A Markdown-formatted string for each chapter. +- **Chaining**: The list of all generated Markdown chapter strings is passed to the final step. + +--- + +### 6. Combine Tutorial (`CombineTutorial`) +This final step does not use an LLM. It assembles all the generated pieces into the final documentation. + +- **Input**: The project summary, relationship graph, chapter order, and the content of all chapters. +- **Output**: A directory containing: + - `index.md`: An overview page with the project summary, a Mermaid diagram of the relationships, and a linked table of contents. + - `01_concept.md`, `02_another.md`, etc.: Individual chapter files in Markdown format. +- **Chaining**: This is the final step, and the output is the completed tutorial saved to the filesystem. \ No newline at end of file diff --git a/configs/corebanking.yaml b/configs/corebanking.yaml index 9e12e3f5..a93a1063 100644 --- a/configs/corebanking.yaml +++ b/configs/corebanking.yaml @@ -65,7 +65,7 @@ file_processing: # Analysis settings analysis: - max_abstractions: 12 # Based on number of abstraction hints provided + max_abstractions: 13 # Based on number of abstraction hints provided abstractions_hints: - "Event" - "Command" diff --git a/configs/documents-extractor.yaml b/configs/documents-extractor.yaml new file mode 100644 index 00000000..3fafdcf5 --- /dev/null +++ b/configs/documents-extractor.yaml @@ -0,0 +1,91 @@ +# Corebanking Project Analysis Configuration + +# Source configuration - specify either repo or local_dir +source: + local_dir: "/Users/louis-davidcoulombe/src/github.com/nestoca/documents-extractor" + # repo: "https://github.com/example/corebanking" # Use this instead of local_dir for GitHub repos + +# Project settings +project: + name: "documents-extractor" + language: "english" + output_dir: "nesto/corebanking/documents-extractor" + +# File processing settings +file_processing: + max_file_size: 150000 # Maximum file size in bytes + include_patterns: + - "*.py" + - "*.js" + - "*.jsx" + - "*.ts" + - "*.tsx" + - "*.go" + - "*.java" + - "*.pyi" + - "*.pyx" + - "*.c" + - "*.cc" + - "*.cpp" + - "*.h" + - "*.md" + - "*.rst" + - "*Dockerfile" + - "*Makefile" + - "*.yaml" + - "*.yml" + exclude_patterns: + - "assets/*" + - "data/*" + - "images/*" + - "public/*" + - "static/*" + - "temp/*" + - "*docs/*" + - "*venv/*" + - "*.venv/*" + - "*test*" + - "*tests/*" + - "*examples/*" + - "v1/*" + - "*dist/*" + - "*build/*" + - "*experimental/*" + - "*deprecated/*" + - "*misc/*" + - "*legacy/*" + - ".git/*" + - ".github/*" + - ".next/*" + - ".vscode/*" + - "*obj/*" + - "*bin/*" + - "*node_modules/*" + - "*.log" + +# Analysis settings +# analysis: +# max_abstractions: 13 # Based on number of abstraction hints provided +# abstractions_hints: +# - "Event" +# - "Command" +# - "Aggregate" +# - "Repository" +# - "API Handler" +# - "Core Facade" +# - "Service" +# - "Consumer" +# - "Product Engine" +# - "Simulation Services and Repositories" +# - "products" +# - "parameters" +# - "customers" +# feedback_file: "nesto/corebanking/review.md" + +# LLM settings +llm: + use_cache: false # Set to false to disable caching + +# GitHub settings (only needed if using repo instead of local_dir) +github: + # token: "your_github_token_here" # Or set GITHUB_TOKEN environment variable diff --git a/prompt.md b/prompt.md new file mode 100644 index 00000000..af69e582 --- /dev/null +++ b/prompt.md @@ -0,0 +1,177 @@ +Of course. I will extract all the LLM prompts from the `nodes.py` file and present them in a markdown format. First, I need to read the `nodes.py` file. +Read file: nodes.py +Okay, I have the content of `nodes.py`. Now I will extract the prompts and format them into a markdown file named `prompts.md`. +My apologies, it seems I tried to use a tool that isn't available to me. I will now output the extracted prompts directly into our conversation. + +# LLM Prompts from `nodes.py` + +This file contains all the LLM prompts used in the documentation generation workflow, extracted from the `nodes.py` file. + +## 1. `IdentifyAbstractions` Node + +This prompt analyzes the entire codebase to identify the most important core abstractions. + +```python +f""" +For the project `{project_name}`: + +Codebase Context: +{context} + +{language_instruction}Analyze the codebase context. +Identify the top 5-{max_abstraction_num} core most important abstractions to help those new to the codebase. + +For each abstraction, provide: +1. A concise `name`{name_lang_hint}. +2. A beginner-friendly `description` explaining what it is with a simple analogy, in around 100 words{desc_lang_hint}. +3. A list of relevant `file_indices` (integers) using the format `idx # path/comment`. + +List of file indices and paths present in the context: +{file_listing_for_prompt} + +Format the output as a YAML list of dictionaries: + +```yaml +- name: | + Query Processing{name_lang_hint} + description: | + Explains what the abstraction does. + It's like a central dispatcher routing requests.{desc_lang_hint} + file_indices: + - 0 # path/to/file1.py + - 3 # path/to/related.py +- name: | + Query Optimization{name_lang_hint} + description: | + Another core concept, similar to a blueprint for objects.{desc_lang_hint} + file_indices: + - 5 # path/to/another.js +# ... up to {max_abstraction_num} abstractions +```""" +``` + +## 2. `AnalyzeRelationships` Node + +This prompt takes the identified abstractions and analyzes their relationships to build a high-level overview of the project. + +```python +f""" +Based on the following abstractions and relevant code snippets from the project `{project_name}`: + +List of Abstraction Indices and Names{list_lang_note}: +{abstraction_listing} + +Context (Abstractions, Descriptions, Code): +{context} + +{language_instruction}Please provide: +1. A high-level `summary` of the project's main purpose and functionality in a few beginner-friendly sentences{lang_hint}. Use markdown formatting with **bold** and *italic* text to highlight important concepts. +2. A list (`relationships`) describing the key interactions between these abstractions. For each relationship, specify: + - `from_abstraction`: Index of the source abstraction (e.g., `0 # AbstractionName1`) + - `to_abstraction`: Index of the target abstraction (e.g., `1 # AbstractionName2`) + - `label`: A brief label for the interaction **in just a few words**{lang_hint} (e.g., "Manages", "Inherits", "Uses"). + Ideally the relationship should be backed by one abstraction calling or passing parameters to another. + Simplify the relationship and exclude those non-important ones. + +IMPORTANT: Make sure EVERY abstraction is involved in at least ONE relationship (either as source or target). Each abstraction index must appear at least once across all relationships. + +Format the output as YAML: + +```yaml +summary: | + A brief, simple explanation of the project{lang_hint}. + Can span multiple lines with **bold** and *italic* for emphasis. +relationships: + - from_abstraction: 0 # AbstractionName1 + to_abstraction: 1 # AbstractionName2 + label: "Manages"{lang_hint} + - from_abstraction: 2 # AbstractionName3 + to_abstraction: 0 # AbstractionName1 + label: "Provides config"{lang_hint} + # ... other relationships +``` + + +## 3. `OrderChapters` Node + +This prompt determines the best pedagogical order to explain the identified abstractions to a newcomer. + +```python +f""" +Given the following project abstractions and their relationships for the project `{project_name}`: + +Abstractions (Index # Name){list_lang_note}: +{abstraction_listing} + +Context about relationships and project summary: +{context} + +If you are going to make a tutorial for ```` {project_name} ````, what is the best order to explain these abstractions, from first to last? +Ideally, first explain those that are the most important or foundational, perhaps user-facing concepts or entry points. Then move to more detailed, lower-level implementation details or supporting concepts. + +Output the ordered list of abstraction indices, including the name in a comment for clarity. Use the format `idx # AbstractionName`. + +```yaml +- 2 # FoundationalConcept +- 0 # CoreClassA +- 1 # CoreClassB (uses CoreClassA) +- ... +``` + + + + +## 4. `WriteChapters` Node + +This prompt is executed for each abstraction to generate a detailed, beginner-friendly tutorial chapter in Markdown. + +```python +f""" +{language_instruction}Write a very beginner-friendly tutorial chapter (in Markdown format) for the project `{project_name}` about the concept: "{abstraction_name}". This is Chapter {chapter_num}. + +Concept Details{concept_details_note}: +- Name: {abstraction_name} +- Description: +{abstraction_description} + +Complete Tutorial Structure{structure_note}: +{item["full_chapter_listing"]} + +Context from previous chapters{prev_summary_note}: +{previous_chapters_summary if previous_chapters_summary else "This is the first chapter."} + +Relevant Code Snippets (Code itself remains unchanged): +{file_context_str if file_context_str else "No specific code snippets provided for this abstraction."} + +Instructions for the chapter (Generate content in {language.capitalize()} unless specified otherwise): +- Start with a clear heading (e.g., `# Chapter {chapter_num}: {abstraction_name}`). Use the provided concept name. + +- If this is not the first chapter, begin with a brief transition from the previous chapter{instruction_lang_note}, referencing it with a proper Markdown link using its name{link_lang_note}. + +- Begin with a high-level motivation explaining what problem this abstraction solves{instruction_lang_note}. Start with a central use case as a concrete example. The whole chapter should guide the reader to understand how to solve this use case. Make it very minimal and friendly to beginners. + +- If the abstraction is complex, break it down into key concepts. Explain each concept one-by-one in a very beginner-friendly way{instruction_lang_note}. + +- Explain how to use this abstraction to solve the use case{instruction_lang_note}. Give example inputs and outputs for code snippets (if the output isn't values, describe at a high level what will happen{instruction_lang_note}). + +- Each code block should be BELOW 10 lines! If longer code blocks are needed, break them down into smaller pieces and walk through them one-by-one. Aggresively simplify the code to make it minimal. Use comments{code_comment_note} to skip non-important implementation details. Each code block should have a beginner friendly explanation right after it{instruction_lang_note}. + +- Describe the internal implementation to help understand what's under the hood{instruction_lang_note}. First provide a non-code or code-light walkthrough on what happens step-by-step when the abstraction is called{instruction_lang_note}. It's recommended to use a simple sequenceDiagram with a dummy example - keep it minimal with at most 5 participants to ensure clarity. If participant name has space, use: `participant QP as Query Processing`. {mermaid_lang_note}. + +- Then dive deeper into code for the internal implementation with references to files. Provide example code blocks, but make them similarly simple and beginner-friendly. Explain{instruction_lang_note}. + +- IMPORTANT: When you need to refer to other core abstractions covered in other chapters, ALWAYS use proper Markdown links like this: [Chapter Title](filename.md). Use the Complete Tutorial Structure above to find the correct filename and the chapter title{link_lang_note}. Translate the surrounding text. + +- Use mermaid diagrams to illustrate complex concepts (```mermaid``` format). {mermaid_lang_note}. + +- Heavily use analogies and examples throughout{instruction_lang_note} to help beginners understand. + +- End the chapter with a brief conclusion that summarizes what was learned{instruction_lang_note} and provides a transition to the next chapter{instruction_lang_note}. If there is a next chapter, use a proper Markdown link: [Next Chapter Title](next_chapter_filename){link_lang_note}. + +- Ensure the tone is welcoming and easy for a newcomer to understand{tone_note}. + +- Output *only* the Markdown content for this chapter. + +Now, directly provide a super beginner-friendly Markdown output (DON'T need ```markdown``` tags): +""" +``` \ No newline at end of file From 9048afb27af6f79cbb5b0a5d8c01996f995a73be Mon Sep 17 00:00:00 2001 From: Louis-David Coulombe Date: Mon, 27 Oct 2025 21:10:25 -0400 Subject: [PATCH 12/15] Storing changes --- tmp/WorkflowDocumentation.md | 109 +++++++ tmp/configs/corebanking.yaml | 91 ++++++ tmp/configs/documents-extractor.yaml | 91 ++++++ tmp/configs/example_config.yaml | 72 +++++ tmp/nesto/corebanking/01_event_.md | 179 ++++++++++++ tmp/nesto/corebanking/02_command_.md | 179 ++++++++++++ tmp/nesto/corebanking/03_aggregate_.md | 241 ++++++++++++++++ tmp/nesto/corebanking/04_repository_.md | 229 +++++++++++++++ tmp/nesto/corebanking/05_api_handler_.md | 225 +++++++++++++++ tmp/nesto/corebanking/06_core_facade_.md | 160 +++++++++++ tmp/nesto/corebanking/07_service_.md | 264 +++++++++++++++++ tmp/nesto/corebanking/08_consumer_.md | 204 ++++++++++++++ tmp/nesto/corebanking/09_product_engine_.md | 265 ++++++++++++++++++ ...0_simulation_services_and_repositories_.md | 205 ++++++++++++++ tmp/nesto/corebanking/index.md | 78 ++++++ tmp/nesto/corebanking/review.md | 36 +++ tmp/prompt.md | 177 ++++++++++++ tmp/run_analysis.sh | 21 ++ 18 files changed, 2826 insertions(+) create mode 100644 tmp/WorkflowDocumentation.md create mode 100644 tmp/configs/corebanking.yaml create mode 100644 tmp/configs/documents-extractor.yaml create mode 100644 tmp/configs/example_config.yaml create mode 100644 tmp/nesto/corebanking/01_event_.md create mode 100644 tmp/nesto/corebanking/02_command_.md create mode 100644 tmp/nesto/corebanking/03_aggregate_.md create mode 100644 tmp/nesto/corebanking/04_repository_.md create mode 100644 tmp/nesto/corebanking/05_api_handler_.md create mode 100644 tmp/nesto/corebanking/06_core_facade_.md create mode 100644 tmp/nesto/corebanking/07_service_.md create mode 100644 tmp/nesto/corebanking/08_consumer_.md create mode 100644 tmp/nesto/corebanking/09_product_engine_.md create mode 100644 tmp/nesto/corebanking/10_simulation_services_and_repositories_.md create mode 100644 tmp/nesto/corebanking/index.md create mode 100644 tmp/nesto/corebanking/review.md create mode 100644 tmp/prompt.md create mode 100755 tmp/run_analysis.sh diff --git a/tmp/WorkflowDocumentation.md b/tmp/WorkflowDocumentation.md new file mode 100644 index 00000000..2d5bb9a7 --- /dev/null +++ b/tmp/WorkflowDocumentation.md @@ -0,0 +1,109 @@ + # Documentation Generation Workflow + +This document explains the workflow of the documentation generation process, focusing on how the Large Language Model (LLM) prompts chain together to create a comprehensive tutorial for a given codebase. + +## Overview + +The process is a pipeline of six sequential steps. Four of these steps involve interactions with an LLM to analyze the code, structure the content, and write the documentation. + +```mermaid +flowchart TD + A[1. Fetch Repository] --> B[2. Identify Abstractions] + B --> C[3. Analyze Relationships] + C --> D[4. Order Chapters] + D --> E[5. Write Chapters] + E --> F[6. Combine Tutorial] +``` + +## Detailed Steps & Prompt Chaining + +Here’s a detailed breakdown of each step, focusing on the inputs and outputs of the LLM prompts. + +### 1. Fetch Repository (`FetchRepo`) +This initial step does not use an LLM. It's responsible for gathering the source code. + +- **Input**: A repository URL or a local directory path. +- **Output**: A list of all relevant code files and their content. This list becomes the foundational input for the entire workflow. +- **Chaining**: The raw codebase content is passed to the next step. + +--- + +### 2. Identify Abstractions (`IdentifyAbstractions`) +This is the first interaction with the LLM. Its goal is to identify the most important, high-level concepts in the codebase. + +- **Input to Prompt**: The entire codebase content fetched in the previous step. +- **LLM Prompt Goal**: The LLM is asked to analyze the code and identify the top 5-10 core abstractions. For each abstraction, it must provide: + 1. A concise `name`. + 2. A beginner-friendly `description` with an analogy. + 3. A list of relevant `file_indices` that implement or define the abstraction. +- **Output of Prompt**: The LLM returns a YAML-formatted string. + ```yaml + - name: | + Core Concept A + description: | + An explanation of what this concept does, like a central controller. + file_indices: + - 0 # path/to/file1.py + - 3 # path/to/file2.py + ``` +- **Chaining**: The validated list of abstractions (name, description, file indices) is passed to the next step. + +--- + +### 3. Analyze Relationships (`AnalyzeRelationships`) +The second LLM interaction focuses on understanding how the identified abstractions interact. + +- **Input to Prompt**: The list of abstractions (names and descriptions) and the code snippets from their relevant files. +- **LLM Prompt Goal**: The LLM is prompted to: + 1. Create a high-level `summary` of the project's purpose. + 2. Define the `relationships` between the abstractions, describing how they interact (e.g., "Manages", "Inherits from", "Uses"). +- **Output of Prompt**: A YAML object containing the summary and a list of relationships. + ```yaml + summary: | + A brief, simple explanation of the project's purpose. + relationships: + - from_abstraction: 0 # Core Concept A + to_abstraction: 1 # Core Concept B + label: "Manages" + ``` +- **Chaining**: The project summary and the list of relationships are passed to the next step. + +--- + +### 4. Order Chapters (`OrderChapters`) +The third LLM interaction determines the best pedagogical order to present the concepts. + +- **Input to Prompt**: The project summary, the list of abstractions, and their relationships. +- **LLM Prompt Goal**: The LLM is asked to determine the optimal sequence for a tutorial. It's instructed to start with foundational or user-facing concepts and then move to lower-level implementation details, respecting dependencies revealed in the relationships. +- **Output of Prompt**: A YAML list of the abstraction indices, sorted in the recommended chapter order. + ```yaml + - 2 # FoundationalConcept + - 0 # CoreClassA + - 1 # CoreClassB (uses CoreClassA) + ``` +- **Chaining**: This ordered list of indices dictates the structure of the final tutorial and is passed to the chapter writing step. + +--- + +### 5. Write Chapters (`WriteChapters`) +This is the most intensive LLM step, where the actual tutorial content is generated for each abstraction, one by one. + +- **Input to Prompt (for each chapter)**: + - The specific abstraction's details (name, description). + - The full tutorial structure (for linking to other chapters). + - The content of previously written chapters (to ensure smooth transitions). + - Relevant code snippets for the current abstraction. +- **LLM Prompt Goal**: The LLM is given a detailed set of instructions to write a beginner-friendly Markdown chapter. This includes creating a heading, explaining the concept with analogies, providing simplified code examples (under 10 lines), using Mermaid diagrams for illustration, and writing transitions to the previous and next chapters. +- **Output of Prompt**: A Markdown-formatted string for each chapter. +- **Chaining**: The list of all generated Markdown chapter strings is passed to the final step. + +--- + +### 6. Combine Tutorial (`CombineTutorial`) +This final step does not use an LLM. It assembles all the generated pieces into the final documentation. + +- **Input**: The project summary, relationship graph, chapter order, and the content of all chapters. +- **Output**: A directory containing: + - `index.md`: An overview page with the project summary, a Mermaid diagram of the relationships, and a linked table of contents. + - `01_concept.md`, `02_another.md`, etc.: Individual chapter files in Markdown format. +- **Chaining**: This is the final step, and the output is the completed tutorial saved to the filesystem. \ No newline at end of file diff --git a/tmp/configs/corebanking.yaml b/tmp/configs/corebanking.yaml new file mode 100644 index 00000000..a93a1063 --- /dev/null +++ b/tmp/configs/corebanking.yaml @@ -0,0 +1,91 @@ +# Corebanking Project Analysis Configuration + +# Source configuration - specify either repo or local_dir +source: + local_dir: "/Users/louis-davidcoulombe/github/corebanking" + # repo: "https://github.com/example/corebanking" # Use this instead of local_dir for GitHub repos + +# Project settings +project: + name: "corebanking" + language: "english" + output_dir: "nesto/corebanking/analysis_output" + +# File processing settings +file_processing: + max_file_size: 150000 # Maximum file size in bytes + include_patterns: + - "*.py" + - "*.js" + - "*.jsx" + - "*.ts" + - "*.tsx" + - "*.go" + - "*.java" + - "*.pyi" + - "*.pyx" + - "*.c" + - "*.cc" + - "*.cpp" + - "*.h" + - "*.md" + - "*.rst" + - "*Dockerfile" + - "*Makefile" + - "*.yaml" + - "*.yml" + exclude_patterns: + - "assets/*" + - "data/*" + - "images/*" + - "public/*" + - "static/*" + - "temp/*" + - "*docs/*" + - "*venv/*" + - "*.venv/*" + - "*test*" + - "*tests/*" + - "*examples/*" + - "v1/*" + - "*dist/*" + - "*build/*" + - "*experimental/*" + - "*deprecated/*" + - "*misc/*" + - "*legacy/*" + - ".git/*" + - ".github/*" + - ".next/*" + - ".vscode/*" + - "*obj/*" + - "*bin/*" + - "*node_modules/*" + - "*.log" + +# Analysis settings +analysis: + max_abstractions: 13 # Based on number of abstraction hints provided + abstractions_hints: + - "Event" + - "Command" + - "Aggregate" + - "Repository" + - "API Handler" + - "Core Facade" + - "Service" + - "Consumer" + - "Product Engine" + - "Simulation Services and Repositories" + - "products" + - "parameters" + - "customers" + feedback_file: "nesto/corebanking/review.md" + +# LLM settings +llm: + use_cache: true # Set to false to disable caching + +# GitHub settings (only needed if using repo instead of local_dir) +github: + # token: "your_github_token_here" # Or set GITHUB_TOKEN environment variable diff --git a/tmp/configs/documents-extractor.yaml b/tmp/configs/documents-extractor.yaml new file mode 100644 index 00000000..3fafdcf5 --- /dev/null +++ b/tmp/configs/documents-extractor.yaml @@ -0,0 +1,91 @@ +# Corebanking Project Analysis Configuration + +# Source configuration - specify either repo or local_dir +source: + local_dir: "/Users/louis-davidcoulombe/src/github.com/nestoca/documents-extractor" + # repo: "https://github.com/example/corebanking" # Use this instead of local_dir for GitHub repos + +# Project settings +project: + name: "documents-extractor" + language: "english" + output_dir: "nesto/corebanking/documents-extractor" + +# File processing settings +file_processing: + max_file_size: 150000 # Maximum file size in bytes + include_patterns: + - "*.py" + - "*.js" + - "*.jsx" + - "*.ts" + - "*.tsx" + - "*.go" + - "*.java" + - "*.pyi" + - "*.pyx" + - "*.c" + - "*.cc" + - "*.cpp" + - "*.h" + - "*.md" + - "*.rst" + - "*Dockerfile" + - "*Makefile" + - "*.yaml" + - "*.yml" + exclude_patterns: + - "assets/*" + - "data/*" + - "images/*" + - "public/*" + - "static/*" + - "temp/*" + - "*docs/*" + - "*venv/*" + - "*.venv/*" + - "*test*" + - "*tests/*" + - "*examples/*" + - "v1/*" + - "*dist/*" + - "*build/*" + - "*experimental/*" + - "*deprecated/*" + - "*misc/*" + - "*legacy/*" + - ".git/*" + - ".github/*" + - ".next/*" + - ".vscode/*" + - "*obj/*" + - "*bin/*" + - "*node_modules/*" + - "*.log" + +# Analysis settings +# analysis: +# max_abstractions: 13 # Based on number of abstraction hints provided +# abstractions_hints: +# - "Event" +# - "Command" +# - "Aggregate" +# - "Repository" +# - "API Handler" +# - "Core Facade" +# - "Service" +# - "Consumer" +# - "Product Engine" +# - "Simulation Services and Repositories" +# - "products" +# - "parameters" +# - "customers" +# feedback_file: "nesto/corebanking/review.md" + +# LLM settings +llm: + use_cache: false # Set to false to disable caching + +# GitHub settings (only needed if using repo instead of local_dir) +github: + # token: "your_github_token_here" # Or set GITHUB_TOKEN environment variable diff --git a/tmp/configs/example_config.yaml b/tmp/configs/example_config.yaml new file mode 100644 index 00000000..e9c8d04c --- /dev/null +++ b/tmp/configs/example_config.yaml @@ -0,0 +1,72 @@ +# Example Configuration File for Tutorial Generation +# Copy this file and modify for your specific project + +# Source configuration - specify either repo or local_dir (not both) +source: + # For local directories: + local_dir: "/path/to/your/local/project" + + # For GitHub repositories (comment out local_dir if using this): + # repo: "https://github.com/username/repository" + +# Project settings +project: + name: "your_project_name" + language: "english" # or "french", "spanish", etc. + output_dir: "output/your_project" # where to save the generated tutorial + +# File processing settings +file_processing: + max_file_size: 100000 # Maximum file size in bytes (default: 100KB) + + # File patterns to include in analysis + include_patterns: + - "*.py" + - "*.js" + - "*.jsx" + - "*.ts" + - "*.tsx" + - "*.go" + - "*.java" + - "*.md" + - "*.rst" + - "*.yaml" + - "*.yml" + + # File patterns to exclude from analysis + exclude_patterns: + - "*test*" + - "*tests/*" + - "*node_modules/*" + - "*venv/*" + - "*.venv/*" + - "*dist/*" + - "*build/*" + - ".git/*" + - "*.log" + +# Analysis settings +analysis: + max_abstractions: 10 # Maximum number of abstractions to identify + + # Specific abstractions to look for (optional) + abstractions_hints: + - "Class" + - "Interface" + - "Service" + - "Controller" + - "Repository" + - "Model" + - "Component" + - "Module" + + # Path to feedback file from previous runs (optional) + # feedback_file: "path/to/feedback.md" + +# LLM settings +llm: + use_cache: true # Set to false to disable response caching + +# GitHub settings (only needed if using repo instead of local_dir) +github: + # token: "your_github_token_here" # Or set GITHUB_TOKEN environment variable diff --git a/tmp/nesto/corebanking/01_event_.md b/tmp/nesto/corebanking/01_event_.md new file mode 100644 index 00000000..8d1f7c5c --- /dev/null +++ b/tmp/nesto/corebanking/01_event_.md @@ -0,0 +1,179 @@ +# Chapter 1: Event + +Welcome to the `corebanking` project tutorial! We're thrilled to have you on board. In this first chapter, we'll dive into one of the most fundamental concepts in our system: the **Event**. + +## What's the Big Deal with Events? + +Imagine you're building a brand new digital bank. A customer, let's call her Alice, wants to open a new savings account. She fills out a form, clicks "Submit," and voilà, her account is created. + +But how does the bank *remember* that this account was created? How does it keep a permanent, trustworthy record of this action? What if we need to look back and see exactly when Alice's account was opened or what its initial details were? + +This is where **Events** come into play. + +In our `corebanking` system, an **Event** is an **immutable record of something significant that has already happened**. Think of it like an official entry in a historical logbook or a diary. Once an entry is written, it's not changed. + +When something important occurs in the system, like Alice's account being created, we record this fact as an Event. For example, after successfully processing Alice's request to create an account (which we'll later learn is a [Command](02_command_.md)), an `AccountCreatedEvent` is generated. + +## Key Characteristics of an Event + +Events have a few defining features: + +1. **Immutable Record:** Once an Event is recorded, it **cannot be changed**. It's a statement of fact about something that happened in the past. Like carving something in stone. +2. **Significant Happenings:** We don't create Events for every tiny operation. They are reserved for actions that have a meaningful impact on the system's state. Examples: `AccountCreatedEvent`, `FundsDepositedEvent`, `CustomerAddressUpdatedEvent`. +3. **Past Tense:** Event names usually reflect that the action has already completed. Notice the "ed" or "d" at the end: `AccountCreatedEvent`, `PaymentDeviceLinkedEvent`. +4. **Source of Truth:** Events are the ultimate, undeniable truth for what has occurred. If we ever need to understand the current state of something (like an account's balance or status), we can, in theory, replay all the Events related to it from the very beginning. It's like re-reading a ship's log to know its entire journey. +5. **Result of a [Command](02_command_.md):** Typically, an Event is produced after the system processes a [Command](02_command_.md). A [Command](02_command_.md) is a request to do something (e.g., "Create Account"). If the command is successful and changes the system's state, one or more Events are generated as a result. + +## Events in Action: Creating Alice's Account + +Let's revisit Alice creating her savings account: + +1. Alice submits her request (this is a [Command](02_command_.md)). +2. The `corebanking` system processes this request. It checks if Alice can open an account, if she provided all necessary details, etc. +3. If everything is okay, the system changes its state: a new account for Alice now exists! +4. To record this fact, the system generates an `AccountCreatedEvent`. This Event will contain all the crucial information: Alice's customer ID, the new account ID, the type of account, the currency, the date and time of creation, etc. +5. This `AccountCreatedEvent` is then durably stored, perhaps in a special database. It becomes part of the bank's permanent history. + +##What Does an Event Look Like? (A Peek at the Code) + +In Go, an Event is often represented as a struct. There's a general structure for all events, and then specific data for each type of event. + +Here's a simplified general `Event` structure from our system: + +```go +// From: api/events.go + +// Event represents an event +type Event struct { + ID uuid.UUID // Unique ID for this specific event instance + AggregateID uuid.UUID // ID of the entity this event pertains to (e.g., Account ID) + AggregateType string // Type of the entity (e.g., "Account") + Type string // Specific type of event (e.g., "AccountCreatedEvent") + Data interface{} // The actual data specific to this event type + Created time.Time // When the event was created + // ... other general fields like Actor, TenantID ... +} +``` + +* `ID`: Every single event occurrence gets its own unique identifier. +* `AggregateID`: This tells us which specific entity the event is about. For an `AccountCreatedEvent`, this would be the ID of the newly created account. We'll learn more about [Aggregates](03_aggregate_.md) later. +* `AggregateType`: The kind of entity, like "Account" or "Customer". +* `Type`: A string that clearly states what kind of event this is, e.g., `com.nesto.corebanking.accounts.events.AccountCreatedEvent`. +* `Data`: This is the payload, containing the specific details of *what happened*. For an `AccountCreatedEvent`, this would hold the account category, currency, etc. +* `Created`: A timestamp indicating when the event occurred. + +The `Data` field itself would be another struct, specific to the event type. For example, an `AccountCreatedEvent` might have data like this (simplified): + +```go +// From: api/account_aggregate.go + +// AccountCreatedEvent represents a created account. +type AccountCreatedEvent struct { + Category string `avro:"category"` + Product *AccountProduct `avro:"product"` + AvailableCurrencies []string `avro:"availableCurrencies"` + CustomerIDs []uuid.UUID `avro:"customerIds"` + // ... other fields like Parameters, Metadata ... +} +``` +This `AccountCreatedEvent` struct holds all the specific details that are relevant when a new account is made. + +## How an Event is "Born" - The Internals + +Let's look under the hood to see how an `AccountCreatedEvent` comes to life. + +1. **A Request Arrives:** Alice's request to create an account comes in as a [Command](02_command_.md) (e.g., `CreateAccountCommand`). +2. **Processing by an [Aggregate](03_aggregate_.md):** An [Aggregate](03_aggregate_.md) is responsible for handling commands related to a specific entity (like an `AccountAggregate` for accounts). It takes the `CreateAccountCommand`. +3. **Validation and State Change:** The `AccountAggregate` checks if the command is valid (e.g., are all required fields present?). If valid, it determines what state changes are needed. For a new account, this means setting its initial properties. +4. **Event Generation:** Because the state changed, the `AccountAggregate` now creates an `AccountCreatedEvent`, filling it with details from the command and any system-generated data (like the creation timestamp). +5. **Event Storage:** This newly minted event is then passed to an "Event Store" (a specialized database or system component) to be saved permanently. + +Here's a simplified sequence diagram: + +```mermaid +sequenceDiagram + participant Alice + participant CoreBankingAPI as API + participant AccountAgg as Account Aggregate + participant EventStore + + Alice->>API: Submit Create Account Form + API->>AccountAgg: Process CreateAccountCommand + AccountAgg-->>AccountAgg: Validate, decide to create account + AccountAgg->>EventStore: New AccountCreatedEvent(details...) + EventStore-->>AccountAgg: Event Saved! + AccountAgg-->>API: Account Creation Successful + API-->>Alice: Your account is created! +``` + +Let's look at a snippet from `api/account_aggregate.go` where an event is created within the `HandleCommand` method. This method is part of the [Aggregate](03_aggregate_.md) (which we'll cover in detail later). + +```go +// Simplified from api/account_aggregate.go +// Inside AccountAggregate's HandleCommand method: + +case *CreateAccountCommand: // This is the request to create an account + // ... (some validation and setup logic) ... + + // Prepare the specific data for our AccountCreatedEvent + data := &AccountCreatedEvent{ + Category: c.Category, + Product: c.Product, + AvailableCurrencies: c.AvailableCurrencies, + CustomerIDs: c.CustomerIDs, + // ... other relevant details from the command ... + } + + // This is the magic moment! An Event is created. + // a.NewEvent() wraps 'data' with general event info (ID, timestamp, etc.) + event = a.NewEvent(command, data) + + // ... (logic to apply this event to the aggregate's state and save it) ... + a.ApplyChangeHelper(a, event, true) +``` +In this code: +* We receive a `CreateAccountCommand` (aliased as `c`). +* We gather the necessary details into an `AccountCreatedEvent` struct (`data`). +* `a.NewEvent(command, data)` is a helper method that takes the original command and the event-specific `data` to construct the full `Event` object, including its unique ID, timestamp, and type. +* `a.ApplyChangeHelper` is another crucial step. After an event is "born," the [Aggregate](03_aggregate_.md) itself uses this event to update its own internal state. This ensures the [Aggregate's](03_aggregate_.md) in-memory representation reflects the new reality. + +The `ApplyChange` method (or a helper it calls) looks at the type of event and updates the [Aggregate's](03_aggregate_.md) fields accordingly: + +```go +// Simplified from api/account_aggregate.go +// Inside AccountAggregate's ApplyChange method: + +func (a *AccountAggregate) ApplyChange(event es.Event) { + switch e := event.Data.(type) { // e is the specific event data + case *AccountCreatedEvent: + a.ID = event.AggregateID // The aggregate now knows its ID + a.Category = e.Category + a.Status = AccountStatusCreated // Set initial status + a.Created = event.Created // Record creation time + // ... update other fields based on AccountCreatedEvent ... + case *AccountActivatedEvent: + a.Status = AccountStatusActivated + // ... + // ... other event types ... + } +} +``` +This ensures that the `AccountAggregate`'s state is consistent with the events it has produced and processed. + +## Why Immutability and "Facts" Matter So Much + +The fact that Events are immutable historical records is incredibly powerful: + +* **Reliability & Auditability:** You have a perfect, trustworthy audit trail of everything significant that has happened. Need to know why an account is in a particular state? Replay its events! This is invaluable for debugging, compliance, and understanding system behavior. +* **State Reconstruction:** If, for some reason, the current "snapshot" of an account's data gets corrupted, you can rebuild it by replaying all its historical Events in order. +* **Decoupling:** Other parts of the system can subscribe to events and react to them independently, without needing to know the intricate details of how the event was produced. For example, a notification service could listen for `AccountCreatedEvent` and send a welcome email to Alice. + +## Conclusion + +Events are the bedrock of our `corebanking` system's memory. They are **immutable facts** representing **significant past occurrences**. Each Event tells a small part of a larger story, like the story of Alice's bank account. By recording and storing these Events, we build a reliable, auditable, and resilient system. + +We've seen that Events are often born from processing requests. In the next chapter, we'll take a closer look at these requests themselves. Get ready to learn about the [Command](02_command_.md)! + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/02_command_.md b/tmp/nesto/corebanking/02_command_.md new file mode 100644 index 00000000..d8113c5b --- /dev/null +++ b/tmp/nesto/corebanking/02_command_.md @@ -0,0 +1,179 @@ +# Chapter 2: Command + +In our [previous chapter](01_event_.md), we learned about [Events](01_event_.md) – the immutable records of things that have already happened in our `corebanking` system. For example, when Alice successfully created her bank account, an `AccountCreatedEvent` was generated and stored. + +But how did Alice *tell* the bank she wanted to create an account in the first place? How does she ask the bank to do something, like deposit money or change her address? This is where **Commands** come in. + +## What's a Command? The Bank's Instruction Slip + +Imagine you walk into a physical bank. To make a deposit, you fill out a deposit slip. To open an account, you fill out an application form. These forms are your formal instructions to the bank. + +In the `corebanking` system, a **Command** is very similar. It's like an **instruction or a formal request** you submit to the system. It represents an **intent** to change something. + +Think of it this way: +* An [Event](01_event_.md) says: "This happened!" (e.g., `AccountCreatedEvent`) +* A **Command** says: "I want this to happen!" (e.g., `CreateAccountCommand`) + +Commands are specific, named operations that carry all the necessary data to perform an action. They are the primary way external users or other systems interact with the core logic of our bank to request changes. + +## Key Characteristics of a Command + +1. **Intent to Act:** A Command expresses a desire for the system to perform an action. It's a request, not a statement of fact. +2. **Specific Operation:** Each Command has a clear name that describes the intended action, like `CreateAccountCommand`, `DepositFundsCommand`, or `UpdateCustomerAddressCommand`. +3. **Carries Data:** A Command bundles all the information needed to execute the desired action. For instance, a `CreateAccountCommand` would include details like who the customer is, what type of account they want, and in what currency. +4. **Sent for Processing:** Commands are typically sent to a specific part of our system called an [Aggregate](03_aggregate_.md) (we'll learn about Aggregates in the next chapter!). The [Aggregate](03_aggregate_.md) is responsible for deciding if the Command is valid and then carrying out the action. +5. **May Result in [Events](01_event_.md):** If a Command is successfully processed and leads to a change in the system's state, one or more [Events](01_event_.md) are usually generated to record that change. If the Command is invalid (e.g., trying to deposit into a non-existent account), it might be rejected, and no [Events](01_event_.md) related to the intended change would be created. + +## Commands in Action: Alice's Account Creation (The "How") + +Let's go back to Alice wanting to open her savings account: + +1. **Alice Expresses Her Intent:** Alice, through a web form or a mobile app, indicates she wants a new account. She provides her details (customer ID, desired currency, etc.). +2. **The `CreateAccountCommand` is Born:** The application she's using takes this information and packages it into a `CreateAccountCommand`. This command now holds all the data needed to open the account. +3. **Command is Sent:** This `CreateAccountCommand` is sent to the `corebanking` system. +4. **Processing by an [Aggregate](03_aggregate_.md):** An `AccountAggregate` (which is responsible for managing account-related operations) receives the `CreateAccountCommand`. +5. **Validation & Logic:** The `AccountAggregate` checks: + * Is the customer ID valid? + * Is the currency supported? + * Are all required fields present? + * Any other business rules? +6. **Success and [Event](01_event_.md) Generation:** If everything is valid, the `AccountAggregate` proceeds to create the account. As a result of this successful operation, it generates an `AccountCreatedEvent` (which we saw in Chapter 1!). +7. **Failure:** If the command is invalid (e.g., Alice provides an unsupported currency), the `AccountAggregate` rejects the command, and no `AccountCreatedEvent` is generated. An error message would typically be returned to Alice. + +## What Does a Command Look Like? (A Peek at the Code) + +In Go, just like [Events](01_event_.md), Commands are often represented as structs. They carry the data necessary for the operation. + +Here's a simplified look at the `CreateAccountCommand` from our `corebanking` project (`api/account_aggregate.go`): + +```go +// From: api/account_aggregate.go + +// CreateAccountCommand represents the creation of an account. +type CreateAccountCommand struct { + es.BaseCommand // Provides common fields for all commands + Category string `json:"category"` + Product *AccountProduct `json:"product"` // Details about the specific account product + AvailableCurrencies []string `json:"availableCurrencies"` + CustomerIDs []uuid.UUID `json:"customerIds"` + // ... other fields like Parameters, Metadata ... +} +``` + +Let's break this down: +* `es.BaseCommand`: This is a standard part of many commands in our system. It usually includes an `AggregateID`. For `CreateAccountCommand`, this `AggregateID` will be the ID of the *new* account we want to create. For a command like `DepositFundsCommand`, the `AggregateID` would be the ID of the *existing* account to deposit into. +* `Category`: What kind of account is it (e.g., "ASSET", "LIABILITY"). +* `Product`: Specifies the type of banking product (e.g., "Savings Account Basic", "Current Account Premium"). +* `AvailableCurrencies`: Which currencies can this account hold (e.g., `["USD", "EUR"]`). +* `CustomerIDs`: A list of customer IDs associated with this account. + +Each field in the command provides a piece of information that the system needs to fulfill the request. + +Another example, a `CloseAccountCommand`, would be simpler: + +```go +// From: api/account_aggregate.go + +// CloseAccountCommand represents the closing of an account. +type CloseAccountCommand struct { + es.BaseCommand // Contains the ID of the account to close + Reason string `json:"reason"` // Why is the account being closed? +} +``` +This command needs to know *which* account to close (via `BaseCommand`'s `AggregateID`) and optionally, a reason for closing it. + +## How a Command is Processed: Under the Hood + +When a command arrives, how does the system handle it? + +1. **Dispatch:** The command is typically sent from an API layer (like a web server handling Alice's request) to the core business logic. +2. **Targeting an [Aggregate](03_aggregate_.md):** The command is routed to the correct [Aggregate](03_aggregate_.md). An [Aggregate](03_aggregate_.md) is like a guardian for a specific piece of data (e.g., an `AccountAggregate` guards a specific bank account). If it's a `CreateAccountCommand`, a new `AccountAggregate` instance might be effectively created to handle it. If it's `DepositFundsCommand`, it's sent to the existing `AccountAggregate` for that account. +3. **Validation:** The [Aggregate](03_aggregate_.md) first validates the command. Can this action be performed? Is all the data correct and complete? +4. **Business Logic:** If valid, the [Aggregate](03_aggregate_.md) executes the business rules associated with the command. +5. **[Event](01_event_.md) Generation:** If the business logic results in a state change, the [Aggregate](03_aggregate_.md) creates one or more [Events](01_event_.md) to describe what happened. + +Here's a simplified sequence diagram illustrating the flow for creating an account: + +```mermaid +sequenceDiagram + participant AliceApp as Alice's App + participant CoreAPI as Core Banking API + participant AccountAgg as Account Aggregate + participant EventStore + + AliceApp->>CoreAPI: I want to create an account (details...) + CoreAPI->>AccountAgg: CreateAccountCommand{details...} + AccountAgg-->>AccountAgg: Validate command data (e.g., currency OK?) + AccountAgg-->>AccountAgg: Business logic: Okay, let's create it! + AccountAgg->>EventStore: Store this new AccountCreatedEvent(...) + EventStore-->>AccountAgg: Event saved! + AccountAgg-->>CoreAPI: Success! Account created. + CoreAPI-->>AliceApp: Your account is ready! +``` + +Let's look at a snippet from `api/account_aggregate.go` where an `AccountAggregate` handles a `CreateAccountCommand`. This happens inside a method called `HandleCommand`: + +```go +// Simplified from AccountAggregate.HandleCommand in api/account_aggregate.go + +// ... (inside HandleCommand method) ... +switch c := command.(type) { // 'c' is the incoming command +case *CreateAccountCommand: + // 1. Check if a product is specified and valid (simplified) + if c.Product != nil { + // ... (logic to validate product exists) ... + } + + // 2. The command carries the intended ID for the new account + a.ID = command.GetAggregateID() // 'a' is the AccountAggregate + + // 3. Prepare the data for the Event that will be created + eventData := &AccountCreatedEvent{ + Category: c.Category, + Product: c.Product, + CustomerIDs: c.CustomerIDs, + AvailableCurrencies: c.AvailableCurrencies, + // ... other details from the command 'c' ... + } + + // 4. Create the actual Event (as we saw in Chapter 1) + event := a.NewEvent(command, eventData) + + // 5. Apply the event to change the aggregate's state and record it + a.ApplyChangeHelper(a, event, true) // This stores the event for saving +``` + +In this snippet: +1. The `HandleCommand` method receives the `CreateAccountCommand` (aliased as `c`). +2. It performs some initial validation (like checking the product). +3. It uses the data *from the command* (`c.Category`, `c.Product`, etc.) to populate an `AccountCreatedEvent`. +4. The `a.NewEvent(...)` function (which we touched on in Chapter 1) creates the full [Event](01_event_.md) object, adding things like a unique event ID and timestamp. +5. `a.ApplyChangeHelper(...)` is a crucial step where the [Aggregate](03_aggregate_.md) updates its own state based on this new event and adds the event to a list of changes to be saved. + +So, the **Command** provides the *intent* and the *data*, and the [Aggregate's](03_aggregate_.md) `HandleCommand` method uses that to *do the work* and produce [Events](01_event_.md). + +## Commands vs. [Events](01_event_.md): A Quick Reminder + +It's vital to distinguish between Commands and [Events](01_event_.md): + +| Feature | Command | [Event](01_event_.md) | +| :-------------- | :-------------------------------------------- | :------------------------------------------------------ | +| **Purpose** | Request an action (intent) | Record a fact (something that happened) | +| **Tense** | Present/Future (e.g., "Create Account") | Past (e.g., "Account Created") | +| **Outcome** | May succeed or fail | Is a record of a successful state change | +| **Result of** | User input, system process | Successful command processing | +| **Analogy** | Filling out a request form | An entry in a historical logbook | + +You send a `CreateAccountCommand` (your request). If it's successful, an `AccountCreatedEvent` (the historical record) is generated. + +## Conclusion + +Commands are the way we tell our `corebanking` system what we want to do. They are formal, data-rich instructions representing an **intent to change the system's state**. They are named operations like `CreateAccountCommand` or `DepositFundsCommand`. + +When a Command is received, it's typically handled by an [Aggregate](03_aggregate_.md), which validates it, executes business logic, and, if successful, produces [Events](01_event_.md) to record the changes. + +Now that we understand how we *ask* the system to do things (Commands) and how the system *remembers* what happened ([Events](01_event_.md)), it's time to meet the component responsible for processing these Commands and generating those Events. Let's dive into the world of the [Aggregate](03_aggregate_.md)! + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/03_aggregate_.md b/tmp/nesto/corebanking/03_aggregate_.md new file mode 100644 index 00000000..4d605b58 --- /dev/null +++ b/tmp/nesto/corebanking/03_aggregate_.md @@ -0,0 +1,241 @@ +# Chapter 3: Aggregate + +Welcome to Chapter 3! In our [previous chapter](02_command_.md), we learned about [Commands](02_command_.md) – how we tell the `corebanking` system what we want to do, like "Create an Account" or "Deposit Funds." We also know from [Chapter 1](01_event_.md) that when these [Commands](02_command_.md) are successful, [Events](01_event_.md) are generated to record what happened. + +But who is in charge of receiving a [Command](02_command_.md), deciding if it's okay, and then creating the [Event](01_event_.md)? If Alice wants to deposit money into *her specific account*, how does the system make sure the change happens only to *her* account and follows all the rules? This is where the **Aggregate** steps in. + +## Meet the Main Character: The Aggregate + +Think of an **Aggregate** as the main character in a story, like a specific bank account (Account #12345) or a particular customer (Alice Wonderland). It's a self-contained unit that looks after its own information (its "state") and its own rules. + +When a [Command](02_command_.md) arrives—like a request to deposit money into Account #12345—it's the `AccountAggregate` for Account #12345 that handles it. This Aggregate will: +1. Check if the command is valid (e.g., "Is the account open?", "Is the deposit amount positive?"). +2. If everything is okay, it will change its state (e.g., increase its balance). +3. Then, it will record this change as a fact by producing an [Event](01_event_.md) (e.g., `FundsDepositedEvent`). + +This way, the Aggregate ensures that all changes to *its* data are consistent and follow the rules. It also provides a historical record of everything that has happened to it through the [Events](01_event_.md) it produces. Aggregates are a cornerstone of the Event Sourcing pattern used in our `corebanking` system. + +## Key Ideas About Aggregates + +1. **A Guardian of Data:** An Aggregate is responsible for a specific piece of data or a small group of related data. For example, an `AccountAggregate` is responsible for *one specific bank account*. It manages the account's balance, status, customer links, etc. +2. **Unique Identity:** Each Aggregate instance has a unique ID. So, `AccountAggregate` with ID `acc-123` is different from `AccountAggregate` with ID `acc-456`. +3. **Manages Its Own State:** The "state" is the current information about the entity. For an account, this includes its balance, status (CREATED, ACTIVATED, CLOSED), currency, etc. +4. **Enforces Business Rules:** Aggregates contain the logic to check if a requested action (a [Command](02_command_.md)) is allowed. Can you withdraw more money than you have? Can you close an account that has a pending transaction? The Aggregate knows! +5. **Processes [Commands](02_command_.md):** It takes a [Command](02_command_.md) as input. +6. **Produces [Events](01_event_.md):** If a [Command](02_command_.md) is valid and changes the state, the Aggregate creates one or more [Events](01_event_.md) to describe what happened. +7. **Consistency Boundary:** All changes within a single Aggregate happen together (atomically). Either all changes are applied, or none are. This keeps the data for that specific entity consistent. + +## Example: Alice Deposits Money + +Let's imagine Alice (customer `cust-001`) has an account (`acc-123`) and wants to deposit $50. + +1. **A [Command](02_command_.md) is Sent:** Alice's banking app creates a `DepositFundsCommand` with details: `AccountID: "acc-123"`, `Amount: 50.00`. +2. **The Right Aggregate is Found:** The system directs this command to the `AccountAggregate` responsible for account `acc-123`. +3. **The Aggregate Works:** + * The `AccountAggregate` for `acc-123` loads its current state (perhaps its current balance is $100 and its status is "ACTIVATED"). + * It checks business rules: Is the account "ACTIVATED"? Yes. Is $50 a valid amount? Yes. + * Since the command is valid, the Aggregate decides an `AccountCreditedEvent` (or similar) needs to happen. +4. **An [Event](01_event_.md) is Born:** It creates an `AccountCreditedEvent` with details: `AmountCredited: 50.00`, `NewBalance: 150.00` (or the event might just state the amount credited, and the balance is recalculated). +5. **State is Updated:** The Aggregate updates its own internal state. Its balance is now $150. +6. **[Event](01_event_.md) is Stored:** The `AccountCreditedEvent` is saved permanently. + +Now, account `acc-123` correctly reflects the deposit, and there's a permanent record of it. + +## What Does an Aggregate Look Like? (A Peek at the Code) + +In Go, an Aggregate is typically a struct. It often embeds a base struct that provides common functionalities and includes fields to hold its state. + +Here's a very simplified view of an `AccountAggregate` from `api/account_aggregate.go`: + +```go +// From: api/account_aggregate.go +// Simplified AccountAggregate structure +type AccountAggregate struct { + es.BaseAggregate // Provides common aggregate features (like ID, event helpers) + + // State of the account (the actual data for *this* account instance) + *AccountAggregateState + + // (Dependencies like repositories are omitted for simplicity here) +} +``` +* `es.BaseAggregate`: This is a helper struct from our event sourcing library (`es`). It provides common things an aggregate needs, like a place to store its ID, version, and methods for creating new [Events](01_event_.md). +* `*AccountAggregateState`: This pointer field is where the actual data specific to *this* account instance is stored. + +The `AccountAggregateState` itself is another struct: + +```go +// From: api/account_aggregate.go +// AccountAggregateState holds the data for an account +type AccountAggregateState struct { + Category string // e.g., "ASSET", "LIABILITY" + Status string // e.g., "CREATED", "ACTIVATED", "CLOSED" + CustomerIDs []uuid.UUID // Which customers own this account + // ... other fields like product info, currency, parameters ... + Created time.Time // When the account was created + // (The actual balance is often managed by postings and calculated, + // but for conceptual understanding, imagine it's part of the state) +} +``` +This `AccountAggregateState` holds all the important details about one specific account. When we talk about an Aggregate's "state," we're talking about the values in these fields. + +## How an Aggregate Handles a [Command](02_command_.md) + +There are two main methods in an Aggregate that work together: +1. `HandleCommand`: This is the entry point. It receives a [Command](02_command_.md), applies business rules, and if successful, creates [Event(s)](01_event_.md). +2. `ApplyChange`: This method takes an [Event](01_event_.md) and uses it to update the Aggregate's internal state. + +Let's look at simplified versions: + +### 1. `HandleCommand` + +When a [Command](02_command_.md) like `CreateAccountCommand` comes in: + +```go +// Simplified from AccountAggregate.HandleCommand in api/account_aggregate.go +// This method decides what to do when a command arrives. +func (a *AccountAggregate) HandleCommand(ctx context.Context, command es.Command) error { + var event es.Event // This will hold the event we generate + + switch c := command.(type) { // 'c' is the specific command (e.g., CreateAccountCommand) + case *CreateAccountCommand: + // 1. Validations (e.g., is product specified? are customers valid?) + // (Detailed validation logic omitted for brevity) + if c.Product == nil { + return errors.New("product is required") // Simplified error + } + + // 2. Set the ID for this new account aggregate from the command + // The command carries the intended ID for the new entity. + a.ID = command.GetAggregateID() + + // 3. Prepare data for the AccountCreatedEvent using info from the command + eventData := &AccountCreatedEvent{ + Category: c.Category, + Product: c.Product, // Details of the account type + CustomerIDs: c.CustomerIDs, + // ... other details from the CreateAccountCommand 'c' ... + } + + // 4. Create the actual Event object using a helper from BaseAggregate + event = a.NewEvent(command, eventData) // 'command' provides context + + // 5. Tell the aggregate to apply this change to itself + // and mark the event to be saved. + a.ApplyChangeHelper(a, event, true) + + // case *DepositFundsCommand: + // // ... logic to handle deposit, create FundsDepositedEvent ... + // case *ActivateAccountCommand: + // // ... logic to handle activation, create AccountActivatedEvent ... + } + return nil // If successful +} +``` +* The `switch` statement checks the type of [Command](02_command_.md). +* For `CreateAccountCommand`, it performs validations. +* It then prepares the data for an `AccountCreatedEvent` using information from the command. +* `a.NewEvent(...)` is a helper (from `es.BaseAggregate`) that wraps `eventData` with standard event fields like a unique Event ID, timestamp, Aggregate ID, etc. +* `a.ApplyChangeHelper(...)` is another crucial helper. It does two things: + 1. Calls `a.ApplyChange(event)` (see below) to update the aggregate's in-memory state. + 2. Adds the `event` to a list of "changes" that will be saved to the Event Store later. + +### 2. `ApplyChange` + +After an [Event](01_event_.md) is created by `HandleCommand`, the `ApplyChange` method is called to update the Aggregate's state. **The state of an Aggregate is *always* a result of applying its [Events](01_event_.md).** + +```go +// Simplified from AccountAggregate.ApplyChange in api/account_aggregate.go +// This method updates the aggregate's state based on an event. +func (a *AccountAggregate) ApplyChange(event es.Event) { + // 'e' is the specific event data (like AccountCreatedEvent details) + switch e := event.Data.(type) { + case *AccountCreatedEvent: + // The aggregate's ID is usually set by BaseAggregate from the event + a.ID = event.AggregateID + // Now, update the AccountAggregateState fields + a.Category = e.Category + a.Product = e.Product // Store product info + a.Status = AccountStatusCreated // Set initial status + a.CustomerIDs = e.CustomerIDs + a.Created = event.Created // Record creation time + // ... update other AccountAggregateState fields based on the event ... + + case *AccountActivatedEvent: // If an AccountActivatedEvent occurred + a.Status = AccountStatusActivated // Update the status + a.Updated = event.Created // Update modification time + + // case *FundsDepositedEvent: + // a.Balance = a.Balance + e.AmountDeposited // Hypothetical balance update + // a.Updated = event.Created + } +} +``` +* This method also uses a `switch` on the *type* of [Event](01_event_.md). +* For an `AccountCreatedEvent`, it sets the initial properties of the `AccountAggregateState` (like `Category`, `Status`, `Created` timestamp). +* For an `AccountActivatedEvent`, it would just update the `Status` field. +* This ensures that the Aggregate's in-memory state accurately reflects all the [Events](01_event_.md) that have happened to it. + +## The Life Cycle: Loading an Existing Aggregate + +What if Alice wants to deposit into an *existing* account? The system doesn't just create a new `AccountAggregate`. Instead: +1. It knows the `AccountID` (e.g., `acc-123`) from the `DepositFundsCommand`. +2. It goes to the Event Store (where all [Events](01_event_.md) are saved). +3. It fetches *all* the past [Events](01_event_.md) for `acc-123` in the order they happened. +4. It creates a fresh `AccountAggregate` instance. +5. It then "replays" each historical [Event](01_event_.md) by calling `ApplyChange` for each one. + * `ApplyChange(AccountCreatedEvent{...})` -> sets initial state. + * `ApplyChange(AccountActivatedEvent{...})` -> sets status to "ACTIVATED". + * `ApplyChange(SomeOtherEvent{...})` -> updates state further. +6. After replaying all its history, the `AccountAggregate` is now in its correct, current state, ready to handle the new `DepositFundsCommand`. + +This process of rebuilding state from [Events](01_event_.md) is fundamental to Event Sourcing. + +## Why Aggregates are So Important + +* **Encapsulation:** They bundle data (state) and behavior (rules) together, making the system easier to understand and manage. +* **Consistency:** They ensure that the data for a specific entity (like an account) is always valid and consistent according to business rules. +* **Clear Responsibilities:** It's clear which Aggregate is responsible for which [Commands](02_command_.md) and which part of the system's data. +* **Event Sourcing Enabler:** They are the primary producers of [Events](01_event_.md) in an Event Sourced system. The history of [Events](01_event_.md) an Aggregate produces *is* the history of that entity. + +## The Big Picture: Processing a Command + +Let's visualize how a command flows through an Aggregate to produce an Event. + +```mermaid +sequenceDiagram + participant ClientApp as Alice's App + participant CoreAPI as Core Banking API + participant AccountAgg as Account Aggregate (for acc-123) + participant EventStore + + ClientApp->>CoreAPI: Request: Deposit $50 into Account 123 + CoreAPI->>AccountAgg: Process DepositFundsCommand{AccountID: 123, Amount: 50} + Note over AccountAgg: 1. Load/Rehydrate Account 123's state
by replaying its past events from EventStore. + AccountAgg-->>AccountAgg: 2. HandleCommand: Validate Command (e.g., account active?) + AccountAgg-->>AccountAgg: 3. HandleCommand: Business Logic: Okay to deposit. + AccountAgg->>AccountAgg: 4. HandleCommand: Create FundsDepositedEvent{Amount: 50} + AccountAgg->>AccountAgg: 5. HandleCommand calls ApplyChange(FundsDepositedEvent)
--> Update internal state (e.g., balance) + Note over AccountAgg: 6. Event is marked for saving. + AccountAgg->>EventStore: 7. System saves FundsDepositedEvent + EventStore-->>AccountAgg: Event Saved! + AccountAgg-->>CoreAPI: Success! Deposit processed. + CoreAPI-->>ClientApp: Your deposit is successful! +``` +This diagram shows that the Aggregate is the central processor. It loads its state, validates the [Command](02_command_.md), applies logic, creates an [Event](01_event_.md), updates itself based on that [Event](01_event_.md), and then the [Event](01_event_.md) gets stored. + +## Conclusion + +Aggregates are like mini-managers for specific entities in our system, such as individual bank accounts or customers. They are responsible for: +* Guarding their own data (state). +* Enforcing business rules. +* Processing incoming [Commands](02_command_.md). +* Producing [Events](01_event_.md) as a record of successful changes. + +By doing this, Aggregates ensure data consistency and are the heart of how our `corebanking` system uses Event Sourcing. They take the "intent" from a [Command](02_command_.md) and, if valid, turn it into a historical "fact" as an [Event](01_event_.md). + +Now that we understand how Aggregates work and how their state can be built from [Events](01_event_.md), you might be wondering: how are these Aggregates (and their [Events](01_event_.md)) actually loaded from and saved to storage? That's where our next topic comes in. Get ready to learn about the [Repository](04_repository_.md)! + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/04_repository_.md b/tmp/nesto/corebanking/04_repository_.md new file mode 100644 index 00000000..1c6b2e27 --- /dev/null +++ b/tmp/nesto/corebanking/04_repository_.md @@ -0,0 +1,229 @@ +# Chapter 4: Repository + +Welcome to Chapter 4! In the [previous chapter](03_aggregate_.md), we learned about [Aggregates](03_aggregate_.md), like our `AccountAggregate`. We saw how they manage their own state (like an account's balance or status) by processing [Commands](02_command_.md) and then producing [Events](01_event_.md) to record what happened. + +Now, a couple of important questions arise: +1. When an `AccountAggregate` needs to process a [Command](02_command_.md) for an *existing* account, how does it get its current state? Remember, its state is built by replaying its past [Events](01_event_.md). Where do these [Events](01_event_.md) come from? +2. After an `AccountAggregate` processes a [Command](02_command_.md) and creates new [Events](01_event_.md), where do these new [Events](01_event_.md) get stored so they are not forgotten? +3. Beyond [Aggregates](03_aggregate_.md), what if other parts of our bank system simply need to *read* account information – say, to display a list of Alice's accounts on a web page? How do they get this data without needing to understand all the [Events](01_event_.md)? + +This is where the **Repository** pattern comes to our rescue! + +## What is a Repository? The Bank's Diligent Archivist + +Imagine a bank's large, old-fashioned archive room, filled with rows and rows of filing cabinets. This room has a team of dedicated archivists. + +* If a bank teller (let's say, an [Aggregate](03_aggregate_.md)) needs the history of a specific account to understand its current situation, they don't go rummaging through the cabinets themselves. They fill out a request slip and give it to an archivist. The archivist knows exactly where to find the files (the [Events](01_event_.md) or account data) and brings them back. +* If the teller completes a transaction and needs to file away the record (a new [Event](01_event_.md) or updated account information), they hand it to the archivist, who ensures it's stored in the right place, safely and correctly. + +A **Repository** in our `corebanking` system works just like this team of archivists. It's a component responsible for **persisting (saving) and retrieving data**. This data could be account details, customer information, transaction histories, or the [Events](01_event_.md) that make up an [Aggregate's](03_aggregate_.md) history. + +The crucial part is that the Repository provides an **abstraction** over the actual data storage mechanism. This means other parts of the system (like [Aggregates](03_aggregate_.md) or [Services](07_service_.md) which we'll see later) don't need to know if the data is stored in a MySQL database, a different type of database, or even text files. They just talk to the Repository using simple, clear methods. + +## Key Ideas About Repositories + +1. **Hides Storage Details:** The rest of the application doesn't care *how* or *where* data is stored. Is it SQL? Is it a NoSQL database? The Repository handles those details. This is like not needing to know the archivist's specific filing system, just that they can get you the file you need. +2. **Defined Contract (Interface):** A Repository offers a clear set of operations, like `GetByID()`, `Save()`, or `Search()`. In Go, this "contract" is usually defined by an `interface`. +3. **One Repository per Data Type (Usually):** You'll often have a specific repository for each main type of data you're managing. For example: + * An `AccountsRepository` to manage `Account` data. + * A `CustomersRepository` to manage `Customer` data. +4. **Different Repositories for Different Needs:** + * **For [Aggregates](03_aggregate_.md) (Event Sourcing):** In an Event Sourced system like ours, [Aggregates](03_aggregate_.md) (like `AccountAggregate`) are special. They are not stored directly as a single row in a database. Instead, their state is derived from their history of [Events](01_event_.md). A specialized type of repository (often called an `EventSourcedRepository` or similar) is used: + * To **load** an [Aggregate](03_aggregate_.md): It fetches all its past [Events](01_event_.md) from an "Event Store" (a database optimized for storing events). + * To **save** an [Aggregate](03_aggregate_.md): It takes any new [Events](01_event_.md) the [Aggregate](03_aggregate_.md) has produced and stores them in the Event Store. + * **For Read Data (Projections/Views):** Sometimes, we need to query data that's already nicely formatted for display or reporting – this is often called a "read model" or "projection." For example, when displaying account details on a screen, we want the current status, balance, etc., directly, not a list of [Events](01_event_.md). Repositories are also used to fetch this kind of data. Our `AccountsRepository` in `api/accounts.go` is an example of this type. + +## Using a Repository: Let's Look at Account Data + +Let's focus on how we might get information about an account that's easy to read and display, using the `AccountsRepository` from our project. Imagine a [Service](07_service_.md) (a component we'll discuss later) needs to fetch Alice's account details. + +### The Contract: The `AccountsRepository` Interface + +First, there's an interface that defines what operations can be performed for accounts. This is like the list of services the archivists offer. + +```go +// From: api/accounts.go + +// AccountsRepository defines methods for account data storage +type AccountsRepository interface { + Save(ctx context.Context, account *Account) error + // UpdateStatus updates the status of an account + UpdateStatus(ctx context.Context, accountID uuid.UUID, status string, updated time.Time) error + GetByID(ctx context.Context, accountID uuid.UUID) (*Account, error) + // ... other methods like Search, GetByIDs, SavePaymentDeviceLink etc. +} +``` +* This interface declares methods like `Save`, `UpdateStatus`, and `GetByID`. +* It uses `*Account`, which is a struct (`api.Account`) representing the "read model" of an account – a snapshot of its current, easily readable state. This is different from the `AccountAggregate` which is focused on processing [Commands](02_command_.md) and [Events](01_event_.md). + +### Getting Account Data + +If a part of our system needs to get details for account `acc-123`, it would use an implementation of `AccountsRepository`: + +```go +// Somewhere in our application (e.g., inside an Account Service) +var accountsRepo api.AccountsRepository // This would be a concrete implementation + +// ... +accountID := uuid.FromString("acc-123") // The ID of the account we want +account, err := accountsRepo.GetByID(context.Background(), accountID) +if err != nil { + // Handle error, maybe the account wasn't found + fmt.Println("Error fetching account:", err) + return +} + +fmt.Println("Fetched Account Category:", account.Category) +fmt.Println("Fetched Account Status:", account.Status) +``` +* **Input:** The `GetByID` method takes a `context` and the `accountID`. +* **Output:** It returns an `*api.Account` struct (containing the account's details like category, status, etc.) and an error (which will be `nil` if successful). + +The code calling `accountsRepo.GetByID` doesn't know or care if the data came from MySQL, a different database, or even a text file. That's the beauty of the repository abstraction! + +## Under the Hood: The `MySQLRepository` for Accounts + +Our project has a concrete implementation of the `AccountsRepository` interface that uses a MySQL database. It's located in `api/pkg/accounts/mysql_repository.go`. + +Let's see a simplified version of how its `GetByID` method might work: + +```go +// Simplified from: api/pkg/accounts/mysql_repository.go + +// MySQLRepository implements AccountsRepository using MySQL. +type MySQLRepository struct { + // (Internal details, like how it gets a database connection, are hidden here) +} + +// GetByID retrieves an account's projection (read model) by its ID from MySQL. +func (r *MySQLRepository) GetByID(ctx context.Context, accountID uuid.UUID) (*api.Account, error) { + // 1. Get a database connection (simplified) + conn, _ := repository.SQLConnection(ctx) + + // 2. This struct holds the raw data fetched from the 'account' table + var dbAccountData struct { // In real code, this is 'accountDbEntry' + ID uuid.UUID `db:"id"` + Category string `db:"category"` + Status string `db:"status"` + // ... other fields matching the database table columns ... + } + + // 3. Build the SQL query to select account data + sqlQuery := "SELECT id, category, status /*, ...other columns... */ FROM account WHERE id = ?" + + // 4. Execute the query against the MySQL database + err := conn.GetContext(ctx, &dbAccountData, sqlQuery, accountID) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, e.NewEntityIDNotFound("account", accountID.String()) // Specific error for "not found" + } + // Some other database error occurred + return nil, e.NewInternal(err) + } + + // 5. Convert the raw database data (dbAccountData) into the application's api.Account struct + apiAccount := &api.Account{ + ID: dbAccountData.ID, + Category: dbAccountData.Category, + Status: dbAccountData.Status, + // ... populate other fields ... + } + + // 6. In the real code, it also fetches related data like customer links, parameters, + // payment device links and populates them into apiAccount. + // Example (highly simplified): + // apiAccount.CustomerIDs, _ = r.getCustomersForAccount(ctx, accountID) + // apiAccount.Parameters, _ = r.getParametersForAccount(ctx, accountID) + + return apiAccount, nil +} +``` +Let's break this down step-by-step: +1. **Get Connection:** It first obtains a connection to the MySQL database. +2. **Data Holder:** `dbAccountData` is a temporary struct to hold the data exactly as it comes from the database table. +3. **SQL Query:** It defines the SQL `SELECT` statement to fetch data from the `account` table based on the `id`. +4. **Execute Query:** The query is executed. If no account is found, it returns a "not found" error. Other database errors are also handled. +5. **Map Data:** If data is found, the raw `dbAccountData` is converted into an `*api.Account` struct. This `api.Account` struct is what the rest of our application understands and expects. +6. **Fetch Related Data:** An account might have associated customers, specific parameters, or linked payment devices. The full repository method would also query other tables to fetch this related information and populate the `api.Account` struct completely. + +Here's a sequence diagram showing this: + +```mermaid +sequenceDiagram + participant AppCode as Application Code + participant AccRepo as AccountsRepository (MySQL) + participant MySQLDB as MySQL Database + + AppCode->>AccRepo: GetByID("acc-123") + AccRepo->>MySQLDB: SELECT id, category, status FROM account WHERE id = "acc-123" + MySQLDB-->>AccRepo: Row data (ID, Category, Status) + AccRepo-->>AccRepo: Map row to api.Account struct + AccRepo->>MySQLDB: SELECT customer_id FROM account_customer WHERE account_id = "acc-123" (etc. for other related data) + MySQLDB-->>AccRepo: Related data + AccRepo-->>AccRepo: Add related data to api.Account struct + AccRepo-->>AppCode: *api.Account (fully populated) +``` + +Similarly, a `Save` method in `MySQLRepository` would take an `*api.Account` struct, convert its fields into a format suitable for the database, and then execute an `INSERT` or `UPDATE` SQL statement. + +## What About [Aggregates](03_aggregate_.md) and Their [Events](01_event_.md)? + +As mentioned earlier, [Aggregates](03_aggregate_.md) like `AccountAggregate` are handled a bit differently in an Event Sourced system. They also use a Repository, but it's a specialized one. + +* **Loading an [Aggregate](03_aggregate_.md):** + When the system needs to load, say, `AccountAggregate` for `acc-123`: + 1. It asks an `EventSourcedRepository` to `Load("acc-123")`. + 2. This repository connects to an **Event Store** (a special database designed to store sequences of [Events](01_event_.md)). + 3. It fetches all [Events](01_event_.md) ever recorded for `acc-123`. + 4. It creates a new, empty `AccountAggregate` instance. + 5. It then "replays" each historical [Event](01_event_.md) on this instance by calling its `ApplyChange(event)` method. This rebuilds the [Aggregate's](03_aggregate_.md) current state. + 6. The fully rehydrated `AccountAggregate` is returned. + +* **Saving an [Aggregate](03_aggregate_.md):** + After `AccountAggregate` processes a [Command](02_command_.md) and generates new [Events](01_event_.md): + 1. The system calls `Save(accountAggregate)` on the `EventSourcedRepository`. + 2. The repository gets the list of newly generated (uncommitted) [Events](01_event_.md) from the `accountAggregate`. + 3. It appends these new [Events](01_event_.md) to the Event Store, associated with `acc-123`. + 4. It then clears the list of uncommitted [Events](01_event_.md) from the `accountAggregate`. + +Here's how loading an [Aggregate](03_aggregate_.md) via an EventSourced Repository looks: +```mermaid +sequenceDiagram + participant AppLogic as Application Logic + participant ESRepo as EventSourced Repository + participant EventStoreDB as Event Store Database + participant AccAgg as Account Aggregate + + AppLogic->>ESRepo: Load AccountAggregate (ID: "acc-123") + ESRepo->>EventStoreDB: GetEvents("acc-123") + EventStoreDB-->>ESRepo: [Event1, Event2, Event3...] + ESRepo->>AccAgg: New AccountAggregate() + loop For each Event in List + ESRepo->>AccAgg: ApplyChange(CurrentEvent) + end + ESRepo-->>AppLogic: AccountAggregate (state rehydrated) +``` +This shows that the Repository pattern is flexible. We can have different repository implementations for different kinds of data (read models like `api.Account`) and different storage strategies (like an Event Store for [Aggregates](03_aggregate_.md)). + +## Why is This Abstraction So Useful? + +Using the Repository pattern brings several benefits: + +1. **Testability:** When testing parts of your application that need data, you don't always want to connect to a real database. It can be slow and complicated to set up. With repositories, you can create a "mock" or "fake" repository for your tests. This fake repository can pretend to be a database, returning predefined data or checking if `Save` was called correctly, all without any actual database interaction. +2. **Flexibility (Change Your Database Later):** Imagine your bank starts with MySQL but later decides to switch to a different database, say PostgreSQL. If your application code directly uses MySQL-specific queries everywhere, changing the database would be a nightmare! With repositories, you only need to write a *new implementation* of the `AccountsRepository` interface (e.g., `PostgreSQLRepository`). The rest of your application code that uses the `AccountsRepository` interface doesn't need to change at all because it was only depending on the interface, not the specific MySQL details. +3. **Clear Separation of Concerns:** Business logic (what the bank *does*) is kept separate from data access logic (how data is saved and retrieved). This makes the code cleaner, easier to understand, and maintain. [Aggregates](03_aggregate_.md) and [Services](07_service_.md) focus on their tasks without being cluttered by SQL queries. + +## Conclusion + +Repositories are like the organized and efficient archivists of our `corebanking` system. They handle all the details of storing and retrieving data, whether it's: +* Fetching historical [Events](01_event_.md) to rebuild an [Aggregate's](03_aggregate_.md) state. +* Saving new [Events](01_event_.md) produced by an [Aggregate](03_aggregate_.md). +* Querying user-friendly "read models" (like an `api.Account`) from a database for display or reporting. + +By providing a clean abstraction layer over data storage, Repositories make our system more maintainable, testable, and flexible. They ensure that other parts of the system can request or save data using simple methods, without needing to know the nitty-gritty details of how or where that data is actually stored. + +Now that we know how data can be persisted and retrieved, how do external requests (like Alice trying to create an account through a mobile app) actually reach our core system and make use of these components? In the next chapter, we'll look at the entry points for such requests: the [API Handler](05_api_handler_.md). + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/05_api_handler_.md b/tmp/nesto/corebanking/05_api_handler_.md new file mode 100644 index 00000000..756e8914 --- /dev/null +++ b/tmp/nesto/corebanking/05_api_handler_.md @@ -0,0 +1,225 @@ +# Chapter 5: API Handler + +Welcome to Chapter 5! In the [previous chapter](04_repository_.md), we explored [Repositories](04_repository_.md) and how they help us save and retrieve data, like account information or the history of [Events](01_event_.md) for an [Aggregate](03_aggregate_.md). We now know how our system manages its data. + +But how does a request from the outside world, like Alice trying to open a new bank account using her mobile app, actually reach our `corebanking` system and start this whole process? How does Alice's app "talk" to our bank's brain? This is where the **API Handler** comes into play. + +## What's an API Handler? The Bank's Welcome Desk + +Imagine walking into a bank. You don't just wander into the vault or start talking to random employees. You usually go to a specific counter or desk: +* "Account Opening" desk. +* "Deposits & Withdrawals" window. +* A specific online form for "Loan Application." + +An **API Handler** in our `corebanking` system is just like that specific desk or form. It's the **official entry point** for any external request that wants to interact with our system. "API" stands for Application Programming Interface, which is a way for different software programs to communicate with each other. + +Each API Handler is responsible for: +1. A specific **URL path** (like a web address, e.g., `/api/accounts`). +2. A specific **HTTP method** (like the type of action, e.g., `POST` for creating something, `GET` for fetching something). + +When a request arrives matching its designated URL and method, the API Handler takes charge. Its job is to: +* Understand the incoming request (e.g., read the data Alice submitted). +* Translate this request into an instruction our system understands, like a [Command](02_command_.md) (e.g., "Please create an account with these details"). +* Or, it might ask a [Service](07_service_.md) to fetch some information. +* Finally, it sends back an HTTP response to tell the requester what happened (e.g., "Account created successfully!" or "Sorry, there was an error."). + +Essentially, API Handlers are the bridge connecting the "web world" (HTTP requests and responses) to the internal business logic of our bank. + +## Key Responsibilities of an API Handler + +Let's break down what an API Handler typically does: + +1. **Listens at a Specific Address:** Each handler is tied to a unique combination of a URL path and an HTTP method. For example: + * `POST /api/accounts`: Handler for creating new accounts. + * `GET /api/accounts/{accountId}`: Handler for fetching details of a specific account. + * `PUT /api/accounts/{accountId}/close`: Handler for closing an account. +2. **Parses the Incoming Request:** When Alice submits her account application form, her details (name, desired currency, etc.) are often sent as a JSON payload in the body of an HTTP request. The API Handler needs to read this data and convert it into a format our Go code can understand (like a Go struct). +3. **Basic Validation (Sometimes):** The handler might do some very basic checks, like "Is the JSON data correctly formatted?" or "Are essential fields like 'currency' present?". However, deep business rule validation (e.g., "Is this currency supported by our bank?") is usually done further inside the system by [Aggregates](03_aggregate_.md) or [Services](07_service_.md). +4. **Translates to Internal Actions:** This is a crucial step. + * **For actions that change data (Writes):** The handler often creates a [Command](02_command_.md) object (like `CreateAccountCommand`) and populates it with the data from the request. It then sends this [Command](02_command_.md) to a "Command Bus," which is like a central dispatcher that routes the [Command](02_command_.md) to the correct [Aggregate](03_aggregate_.md) (e.g., `AccountAggregate`) for processing. + * **For actions that only read data (Reads):** The handler might directly call a [Service](07_service_.md) (e.g., an `AccountsService`) to fetch the requested information (like account details from a [Repository](04_repository_.md)). +5. **Formats and Sends the Response:** After the [Command](02_command_.md) is processed or the [Service](07_service_.md) returns data, the API Handler takes the result and crafts an HTTP response. This includes: + * An **HTTP Status Code** (e.g., `200 OK` for success, `201 Created` if something new was made, `400 Bad Request` if the input was wrong, `500 Internal Server Error` if something went wrong on our side). + * A **Response Body** (often JSON data, like the details of the created account, or an error message). + +## Alice Creates an Account: The API Handler's Role + +Let's see how an API Handler helps Alice open her account: + +1. **Alice's App Sends a Request:** Alice fills out the "New Account" form on her banking app and taps "Submit." Her app sends an HTTP `POST` request to the URL `/api/accounts`. The body of this request contains her details in JSON format, like: + ```json + { + "category": "ASSET", + "product": { "id": "prod-savings-basic", "name": "Basic Savings" }, + "availableCurrencies": ["USD"], + "customerIds": ["cust-alice-123"] + } + ``` +2. **The Correct Handler is Activated:** Our `corebanking` system's web server receives this request. It looks at the method (`POST`) and the path (`/api/accounts`) and finds the specific API Handler responsible for this combination. +3. **Handler Parses the Request:** This API Handler (let's call it the `CreateAccountHandler`) takes the JSON data from the request's body. It uses a JSON parser to convert this data into a Go struct that matches the structure of a `CreateAccountCommand`. +4. **Handler Creates and Dispatches a [Command](02_command_.md):** The handler now creates an instance of `api.CreateAccountCommand` and fills it with the details parsed from Alice's request. It then passes this command to something called a `CommandBus` (part of `core.EventSourcingClient` in our project). + ```go + // Inside the handler (conceptual) + var cmd api.CreateAccountCommand + // ... (parse JSON from request into cmd) ... + err := commandBus.HandleCommand(context, &cmd) + ``` +5. **Handler Waits and Responds:** The Command Bus ensures the `CreateAccountCommand` is processed (by an `AccountAggregate`, which generates an `AccountCreatedEvent`, as we saw in earlier chapters). + * If the command is processed successfully, the handler sends back an HTTP response like `204 No Content` (meaning "I did what you asked, and there's no specific content to return") or `201 Created` (if it were to return details of the created account). + * If there's an error (e.g., Alice provided an invalid customer ID), the [Aggregate](03_aggregate_.md) would signal an error, and the handler would send back an HTTP response like `400 Bad Request` with a JSON body describing the error. + +## A Peek at the Code: An Account Creation Handler + +Our project has API handlers defined in packages under `api/cmd/corebanking-api/`. For creating accounts, the relevant file is `api/cmd/corebanking-api/accounts/create.go`. + +Let's look at a simplified version of the `Create` function, which acts as our API Handler: + +```go +// Simplified from: api/cmd/corebanking-api/accounts/create.go + +// 'cb' is a CommandBus, which knows how to send commands +// to the right part of our system. +func Create(cb es.CommandBus) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // 1. Parse the request to get command data + command, err := parseCreateAccountCommand(r) + if err != nil { + // If parsing fails, send an error response + e.MarshallHTTP(err, w) // Helper to send formatted error + return + } + + // 2. Send the command for processing + err = cb.HandleCommand(r.Context(), &command) + if err != nil { + // If command processing fails, send an error response + e.MarshallHTTP(err, w) + return + } + + // 3. If everything is OK, send a success response + w.WriteHeader(http.StatusNoContent) // "204 No Content" + } +} +``` +Let's break this down: +* `func Create(cb es.CommandBus) http.HandlerFunc`: This function `Create` takes a `CommandBus` (our `es.EventSourcingClient`) as an argument and *returns* another function. This returned function is the actual handler that Go's HTTP server will call for each incoming request. + * `w http.ResponseWriter`: Used to write the HTTP response back to Alice's app. + * `r *http.Request`: Represents the incoming HTTP request from Alice's app. + +Inside the returned function: +1. **Parse Request:** `command, err := parseCreateAccountCommand(r)` + This calls a helper function (we'll see it next) to read Alice's data from the request (`r`) and turn it into our `api.CreateAccountCommand` struct. If there's an error during parsing (e.g., malformed JSON), it sends an error response using `e.MarshallHTTP(err, w)`. +2. **Send Command:** `err = cb.HandleCommand(r.Context(), &command)` + This is where the magic happens! The parsed `command` is handed off to the `CommandBus` (`cb`). The `CommandBus` will make sure it gets to the right [Aggregate](03_aggregate_.md) for processing. If the [Aggregate](03_aggregate_.md) rejects the command (e.g., business rule violation), `HandleCommand` returns an error, which is then sent back as an HTTP error. +3. **Success Response:** `w.WriteHeader(http.StatusNoContent)` + If both parsing and command processing are successful, this line sends an HTTP status code `204 No Content` back to Alice's app, indicating success. + +### Parsing the Request Data + +The `parseCreateAccountCommand` helper function looks something like this (simplified): + +```go +// Simplified from: api/cmd/corebanking-api/accounts/create.go +func parseCreateAccountCommand(r *http.Request) (api.CreateAccountCommand, error) { + var command api.CreateAccountCommand // Prepare an empty command struct + + // Decode the JSON data from the request body into our 'command' struct + if err := json.NewDecoder(r.Body).Decode(&command); err != nil { + // If decoding fails, log it and return a "bad format" error + log.G(r.Context()).WithError(err).Error("could not decode json") + return command, e.NewBadFormat(err.Error()) + } + + return command, nil // Return the populated command and no error +} +``` +* `var command api.CreateAccountCommand`: An empty `CreateAccountCommand` struct is created. +* `json.NewDecoder(r.Body).Decode(&command)`: This is standard Go for reading JSON. It takes the request's body (`r.Body`), creates a JSON decoder, and tries to fill the `command` struct with the data from the JSON. +* If `Decode` fails (e.g., the JSON is broken or doesn't match the command structure), an error is logged and a specific "bad format" error is returned. + +So, the handler first uses this parsing logic to understand the request, then dispatches the [Command](02_command_.md). + +## How Does the System Know Which Handler to Call? Routing! + +You might be wondering: if Alice's app sends a `POST` request to `/api/accounts`, how does our `corebanking` application know to call the `accounts.Create` handler function we just looked at? + +This is done by a **router**. A router is like a traffic controller for web requests. It looks at the URL and HTTP method of an incoming request and directs it to the correct handler function. Our project uses a popular Go router called `chi`. + +The setup for these routes is typically done in a file like `api/cmd/corebanking-api/routes/routes.go`. Here's a tiny snippet showing how the account creation route is defined: + +```go +// Simplified from: api/cmd/corebanking-api/routes/routes.go + +// 'core' contains things like our CommandBus (EventSourcingClient) +// and Services. +func getAPIRoutes(core *api.Core) http.Handler { + r := chi.NewRouter() // Create a new router + + // ... other routes ... + + // This line tells the router: + // If a POST request comes to "/accounts", + // call the handler returned by accounts.Create(), + // passing it the EventSourcingClient from 'core'. + r.Post("/accounts", accounts.Create(core.EventSourcingClient)) + + // Example of a GET request handler: + // r.Get("/accounts/{id}", accounts.Get(core.AccountsService)) + + // ... many other routes ... + + return r +} +``` +* `r := chi.NewRouter()`: Initializes a new router. +* `r.Post("/accounts", accounts.Create(core.EventSourcingClient))`: This is the key line. + * `r.Post` means this rule applies to HTTP `POST` requests. + * `"/accounts"` is the URL path. + * `accounts.Create(core.EventSourcingClient)` is the handler function we want to execute. Notice that we *call* `accounts.Create(...)` here. This function, as we saw, *returns* the actual `http.HandlerFunc` that `chi` will use. We pass `core.EventSourcingClient` (which is our `CommandBus`) to it so the handler can dispatch commands. + +For requests that read data, like getting account details, you'd see something like `r.Get("/accounts/{id}", accounts.Get(core.AccountsService))`. Here, `accounts.Get` would be another handler function, likely taking an `AccountsService` to fetch data. + +## The Journey of a Request: From App to Response + +Let's visualize the entire flow for Alice creating an account: + +```mermaid +sequenceDiagram + participant AliceApp as Alice's App + participant WebRouter as API Router (Chi) + participant CreateAccHandler as accounts.Create Handler + participant CmdBus as CommandBus (EventSourcingClient) + participant AccAgg as AccountAggregate + + AliceApp->>WebRouter: POST /api/accounts (with JSON data) + WebRouter->>CreateAccHandler: Execute handler for this route + CreateAccHandler-->>CreateAccHandler: parseCreateAccountCommand(request) + CreateAccHandler->>CmdBus: HandleCommand(CreateAccountCommand) + CmdBus->>AccAgg: Process CreateAccountCommand + Note over AccAgg: Validates, creates AccountCreatedEvent,
updates state, saves event (via Repository). + AccAgg-->>CmdBus: Success (or Error) + CmdBus-->>CreateAccHandler: Result of command processing + CreateAccHandler-->>CreateAccHandler: Prepare HTTP Response (e.g., 204) + CreateAccHandler->>AliceApp: HTTP 204 No Content (or error response) +``` +1. Alice's app sends the `POST` request. +2. The Web Router (Chi) finds the `accounts.Create` handler. +3. The `CreateAccountHandler` parses the JSON into a `CreateAccountCommand`. +4. It sends the command to the `CommandBus`. +5. The `CommandBus` delivers it to the `AccountAggregate`. +6. The `AccountAggregate` processes the command (validates, generates [Events](01_event_.md), updates its state, and ensures the [Events](01_event_.md) are saved via a [Repository](04_repository_.md) - this part was covered in detail in Chapters 1-4). +7. The result (success/error) bubbles back up to the handler. +8. The handler sends the appropriate HTTP response back to Alice's app. + +## Conclusion + +API Handlers are the vital "front doors" of our `corebanking` system. They listen for specific web requests (URLs and HTTP methods), parse the incoming data, and then translate these external requests into actions our internal system understands – usually by creating and dispatching [Commands](02_command_.md) for write operations or calling [Services](07_service_.md) for read operations. They then take the result of these actions and formulate an HTTP response to send back to the original requester. + +By bridging the gap between the web world (HTTP) and our core business logic, API Handlers make our system accessible to external clients like mobile apps, websites, or even other banking systems. + +Now that we've seen how requests enter our system, you might wonder: is there a central place that coordinates these actions, especially when a single request might involve multiple steps or components? That's where our next topic comes in. Get ready to learn about the [Core Facade](06_core_facade_.md)! + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/06_core_facade_.md b/tmp/nesto/corebanking/06_core_facade_.md new file mode 100644 index 00000000..30f191b3 --- /dev/null +++ b/tmp/nesto/corebanking/06_core_facade_.md @@ -0,0 +1,160 @@ +# Chapter 6: Core Facade + +Welcome to Chapter 6! In our [previous chapter on API Handlers](05_api_handler_.md), we saw how external requests, like Alice wanting to create an account, enter our `corebanking` system through specific API endpoints. These [API Handlers](05_api_handler_.md) then need to interact with various parts of our bank's brain – maybe to send a [Command](02_command_.md) or to fetch some data. + +But imagine an [API Handler](05_api_handler_.md) trying to juggle many different tools: one tool for sending [Commands](02_command_.md), another for looking up customer details, yet another for checking product information. It could get messy quickly! How do we keep this organized and simple for the [API Handler](05_api_handler_.md)? + +This is where the **Core Facade** comes in. + +## What is the Core Facade? The Bank's Main Office + +Think of the `Core` struct (our Core Facade) as the **main reception desk or central switchboard** for our entire `corebanking` application. It's like a central office in a large company. If you (an [API Handler](05_api_handler_.md), for instance) need something from any department, you don't run around the building looking for them. You go to the main office, and they connect you to the right place. + +The Core Facade brings together all the different specialized "departments" – which in our system are various [Services](07_service_.md) (like `AccountsService`, `CustomersService`) and other key components like the `EventSourcingClient` (our [Command](02_command_.md) bus) – into a **single, unified point of access**. + +When an external part of the system, like the [API layer](05_api_handler_.md), needs to interact with the business logic, it simply talks to this `Core` facade. This keeps our overall architecture clean and makes it much easier to manage how different parts of the application interact. + +Essentially, the Core Facade is like a well-organized toolbox for our [API Handlers](05_api_handler_.md). Instead of the handler needing to know about and hold onto dozens of individual tools, it just gets this one `Core` toolbox, which contains everything it might need. + +## What Does the Core Facade Look Like? (The `Core` Struct) + +In our Go code, the Core Facade is represented by a struct named `Core`, defined in `api/core.go`. Let's look at a simplified version: + +```go +// From: api/core.go +// Core represents the entrypoint to call the business logic. +type Core struct { + HealthService HealthService + EventSourcingClient es.EventSourcingClient // To send Commands! + ParametersService ParametersService + AccountsService AccountsService // To read account data! + CustomersService CustomersService + ProductsService ProductsService + // ... and many other services and clients ... +} +``` +* You can see that `Core` is a struct that holds various fields. +* Each field represents a specific "department" or tool: + * `EventSourcingClient`: This is crucial for sending [Commands](02_command_.md) (like `CreateAccountCommand`) to our [Aggregates](03_aggregate_.md). We've seen this in action with [API Handlers](05_api_handler_.md). + * `AccountsService`: If an [API Handler](05_api_handler_.md) needs to fetch a list of accounts or details about a specific account (not through event sourcing, but from a read-friendly data store), it would use this [Service](07_service_.md). We'll learn more about [Services](07_service_.md) in the next chapter! + * `CustomersService`, `ProductsService`, etc.: These provide access to other areas of our banking logic. + +So, the `Core` struct acts as a container, a "bag" holding all these important pieces of our application's core logic. + +## How Do [API Handlers](05_api_handler_.md) Use the Core Facade? + +Remember from [Chapter 5](05_api_handler_.md) how our [API Handlers](05_api_handler_.md) are set up using a router? The router is often given access to this `Core` facade instance. When an [API Handler](05_api_handler_.md) function is defined, it can be passed the specific service or client it needs *from* the `Core` facade. + +Let's revisit a simplified example from `api/cmd/corebanking-api/routes/routes.go` where routes are defined: + +```go +// Simplified from: api/cmd/corebanking-api/routes/routes.go + +// 'core' here is an instance of our api.Core struct (the Facade!) +func getAPIRoutes(core *api.Core) http.Handler { + r := chi.NewRouter() // Our web router + + // For creating accounts, the handler needs the EventSourcingClient (CommandBus) + r.Post("/accounts", accounts.Create(core.EventSourcingClient)) + + // For getting account details, a handler might need the AccountsService + // (Example, actual handler for GET might be different) + // r.Get("/accounts/{accountId}", accounts.GetDetails(core.AccountsService)) + + // ... other routes using different parts of 'core' ... + return r +} +``` +* The `getAPIRoutes` function receives an argument `core *api.Core`. This `core` object is our fully assembled Core Facade! +* When setting up the route for `POST /accounts`, the `accounts.Create` handler function is given `core.EventSourcingClient`. This means the `CreateAccountHandler` doesn't need to find or create an `EventSourcingClient` itself; it gets it directly from the Core Facade. +* Similarly, if we had a handler to fetch account details (e.g., `accounts.GetDetails`), it could be given `core.AccountsService`. + +This way, the `Core` facade acts as a central provider. The [API Handlers](05_api_handler_.md) just ask the `Core` facade for the specific tool they need. + +## How is the Core Facade Built? (The Making of the Toolbox) + +You might be wondering: who creates this `Core` struct and fills it with all these services? + +This happens when our `corebanking` application starts up. There's a special part of our code, primarily in a function called `FromEnv` in `api/pkg/core/core.go`, whose job is to: +1. Initialize all the individual [Services](07_service_.md) (like `AccountsService`, `ProductsService`, etc.). +2. Initialize the `EventSourcingClient`. +3. Initialize [Repositories](04_repository_.md) and other necessary components. +4. Then, it packages all of these initialized components into a single `api.Core` struct instance. + +Let's look at a very simplified snippet from our application's main entry point (`api/cmd/corebanking-api/api.go`) to see where this happens: + +```go +// Simplified from: api/cmd/corebanking-api/api.go main() +func main() { + // ... (lots of initial setup: logging, configuration, etc.) ... + + // tenantsConfigManager is used for multi-tenancy configurations + tenantsConfigManager := wiggumauto.NewConfigManager() + tenantsConfigManager.MustInitialize(ctx) + + // Here! 'c' becomes our fully assembled Core Facade instance. + // core.FromEnv does all the hard work of creating and wiring up services. + c := core.FromEnv(tenantsConfigManager) + + // The Core Facade 'c' is then passed to the router setup. + r := routes.GetRouter(c, tenantsConfigManager, monitoring) + + // ... (starts the web server with these routes) ... + // ... (handles graceful shutdown, including c.EventSourcingClient.Close()) +} +``` +* `c := core.FromEnv(tenantsConfigManager)`: This is the key line. The `core.FromEnv()` function (from `api/pkg/core/core.go`) is called. It sets up everything needed for the core logic (databases, event buses, all services) and returns a pointer to a fully populated `api.Core` struct. This `c` is our Core Facade. +* `r := routes.GetRouter(c, ...)`: This `c` (our Core Facade) is then passed to the function that sets up all the API routes, as we saw in the previous section. + +The `FromEnv` function itself is quite complex because it initializes many parts of the system, connecting them together. For a beginner, the important takeaway is that *there is a dedicated place* where the `Core` facade is carefully assembled when the application starts. + +## The Flow: API Handler, Core Facade, and Beyond + +Let's visualize how an [API Handler](05_api_handler_.md) uses the Core Facade: + +```mermaid +sequenceDiagram + participant APIHandler as API Handler (e.g., Create Account) + participant CoreFacade as Core Facade (api.Core instance) + participant AccService as Accounts Service + participant ESClient as EventSourcingClient (CommandBus) + participant AccAgg as Account Aggregate + + APIHandler->>CoreFacade: I need to create an account. Here's the data (CreateAccountCommand). + Note over APIHandler: Handler gets ESClient from CoreFacade + CoreFacade->>ESClient: HandleCommand(CreateAccountCommand) + ESClient->>AccAgg: Process CreateAccountCommand + AccAgg-->>ESClient: Success (or Error) + ESClient-->>CoreFacade: Command processed! + CoreFacade-->>APIHandler: Account creation initiated (Result) + + APIHandler->>CoreFacade: I need details for account "acc-xyz". + Note over APIHandler: Handler gets AccountsService from CoreFacade + CoreFacade->>AccService: GetAccountByID("acc-xyz") + AccService-->>CoreFacade: Account Details + CoreFacade-->>APIHandler: Here are the Account Details. +``` +This diagram shows two scenarios: +1. **Creating an Account (a [Command](02_command_.md)):** The [API Handler](05_api_handler_.md) uses the `EventSourcingClient` *obtained from* the `CoreFacade` to send the `CreateAccountCommand`. The `CoreFacade` itself doesn't process the command but provides access to the component that does. +2. **Getting Account Details (a Query):** The [API Handler](05_api_handler_.md) uses the `AccountsService` *obtained from* the `CoreFacade` to request account information. The `CoreFacade` directs this to the actual `AccountsService`. + +## Benefits of the Core Facade + +Using this Core Facade pattern provides several advantages: + +1. **Organization:** All core business functionalities are neatly bundled and accessible from one place. +2. **Simplicity for Callers:** Components like [API Handlers](05_api_handler_.md) don't need to know about or manage dependencies on dozens of individual services. They just need the `Core` object. +3. **Centralized Dependency Management:** The creation and wiring of all major services and components happen in one central place (like `core.FromEnv`). This makes it easier to manage the application's startup and overall structure. +4. **Clear Entry Point:** It provides a clear and consistent entry point into the application's business logic for any external interface (like the API). + +## Conclusion + +The **Core Facade**, represented by our `api.Core` struct, acts as the central hub or main reception desk for our `corebanking` application. It doesn't do the business logic itself but holds references to all the specialized [Services](07_service_.md) and clients (like the `EventSourcingClient`) that do. + +For components like [API Handlers](05_api_handler_.md), the Core Facade simplifies interaction with the system's core functionalities by providing a single, organized point of access. This keeps the architecture clean and manageable. + +The Core Facade is filled with various "departments" or tools. One very common type of tool it holds is a "Service." In the next chapter, we'll dive deeper into what these [Services](07_service_.md) are and what role they play in our `corebanking` system. + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/07_service_.md b/tmp/nesto/corebanking/07_service_.md new file mode 100644 index 00000000..a768a62d --- /dev/null +++ b/tmp/nesto/corebanking/07_service_.md @@ -0,0 +1,264 @@ +# Chapter 7: Service + +Welcome to Chapter 7! In the [previous chapter](06_core_facade_.md), we learned about the [Core Facade](06_core_facade_.md), which acts as a central hub providing access to various parts of our `corebanking` system. One of the key components the [Core Facade](06_core_facade_.md) gives us access to are "Services." + +But what exactly is a Service? And why do we need them? + +## What's a Service? The Bank's Specialized Departments + +Imagine you're at a bank. There isn't just one giant room where everything happens. Instead, there are specialized departments: +* The "Account Opening" department helps new customers. +* The "Loans" department handles loan applications and processing. +* The "Customer Support" department answers your questions. + +In our `corebanking` system, a **Service** is like one of these specialized departments. It's a component that groups together related business operations and logic for a specific area of the bank. + +For example, we might have an `AccountsService`. This service would be responsible for all things related to bank accounts, such as: +* Fetching the details of a specific account. +* Searching for accounts based on certain criteria. +* Potentially, initiating the creation of a new account (though, as we'll see, this is often done by sending a [Command](02_command_.md)). + +Services help us organize the application's capabilities, making the system easier to understand, manage, and extend. They often act as a "facade" or an intermediary, receiving requests, interacting with [Repositories](04_repository_.md) to fetch or save data, or dispatching [Commands](02_command_.md) to [Aggregates](03_aggregate_.md) to perform actions. + +## Key Ideas About Services + +1. **Specialized Focus:** Each Service handles a specific domain or business area (e.g., accounts, customers, product definitions). +2. **Groups Business Operations:** It bundles related functions. For instance, an `AccountsService` might have functions like `GetAccountByID()`, `SearchAccounts()`, etc. +3. **Orchestrates Logic:** A Service can coordinate multiple steps to fulfill a request. It might talk to a [Repository](04_repository_.md) to get data, perform some calculations or checks, and then return a result. +4. **Interacts with Other Components:** Services often use [Repositories](04_repository_.md) to access data and can dispatch [Commands](02_command_.md) to [Aggregates](03_aggregate_.md) if changes to the system's state are needed. +5. **Simplifies Interaction:** For other parts of the system (like [API Handlers](05_api_handler_.md)), Services provide a clean and focused interface to the business logic, hiding the underlying complexity. + +## Using a Service: Fetching Account Details + +Let's say an [API Handler](05_api_handler_.md) needs to display Alice's account details. It wouldn't directly query the database. Instead, it would use the `AccountsService`. + +### The Contract: The `AccountsService` Interface + +First, there's an interface that defines what operations the `AccountsService` offers. This is like the list of services a bank department provides. + +```go +// From: api/accounts.go + +// AccountsService manages accounts +type AccountsService interface { + // ... (methods for creating/updating accounts - more on this later) + Search(ctx context.Context, params *AccountSearchParameters) (Accounts, error) + GetByID(ctx context.Context, accountID uuid.UUID) (*Account, error) + GetByIDs(ctx context.Context, accountIDs []uuid.UUID) ([]*Account, error) + // ... other methods ... +} +``` +* This interface declares methods like `Search`, `GetByID`, and `GetByIDs`. +* Notice the `GetByID` method: it takes a `context` and an `accountID` and returns an `*Account` (a struct with account details) and an `error`. + +### Calling the Service + +An [API Handler](05_api_handler_.md) (or another part of the system) that has access to an `AccountsService` instance (perhaps via the [Core Facade](06_core_facade_.md)) can then call its methods: + +```go +// Somewhere in an API Handler (conceptual) +var accountsSvc api.AccountsService // This would be provided by the CoreFacade + +// ... +accountID := uuid.FromString("acc-alice-123") // Alice's account ID +accountDetails, err := accountsSvc.GetByID(context.Background(), accountID) + +if err != nil { + // Handle error (e.g., account not found) + fmt.Println("Error fetching account:", err) + return +} + +// Now we can use accountDetails +fmt.Println("Account Status:", accountDetails.Status) +fmt.Println("Account Currency:", accountDetails.AvailableCurrencies[0]) +``` +* **Input:** The `accountID` for Alice's account. +* **Output:** The `accountDetails` struct (if found) or an `err`. + +The [API Handler](05_api_handler_.md) doesn't need to know *how* the `AccountsService` gets this information. That's the Service's job! + +## Under the Hood: How `AccountsServiceDefault.GetByID` Works + +Our project has a concrete implementation of the `AccountsService` interface called `AccountsServiceDefault` (in `api/accounts.go`). Let's see how it's structured and how its `GetByID` method works. + +First, the `AccountsServiceDefault` struct holds a reference to an `AccountsRepository`: + +```go +// From: api/accounts.go +// AccountsServiceDefault represents the instance of a AccountsService. +type AccountsServiceDefault struct { + connections repository.ConnectionsProvider // Manages DB connections + repository AccountsRepository // The tool to talk to the database +} +``` +When an `AccountsServiceDefault` is created, it's given an `AccountsRepository`: + +```go +// From: api/accounts.go +// NewAccountsService return a new instance of the accounts service. +func NewAccountsService( + connections repository.ConnectionsProvider, + repository AccountsRepository, +) AccountsService { + // ... (nil checks omitted for brevity) ... + return &AccountsServiceDefault{ + connections: connections, + repository: repository, + } +} +``` +This means the service has the necessary tool ([Repository](04_repository_.md)) to fetch account data. + +Now, let's look at a simplified version of its `GetByID` method: + +```go +// Simplified from: api/accounts.go AccountsServiceDefault.GetByID +func (s *AccountsServiceDefault) GetByID( + ctx context.Context, + accountID uuid.UUID, +) (*Account, error) { + // 1. Potentially start a database transaction (simplified) + ctx, err := s.connections.NewTx(ctx) + if err != nil { + return nil, err + } + defer s.connections.Rollback(ctx) // Ensure transaction is rolled back if not committed + + // 2. Use the repository to fetch the account data + account, err := s.repository.GetByID(ctx, accountID) + if err != nil { + return nil, err // Could be "not found" or other DB error + } + + // 3. (A real service might do more here, like checking permissions + // or enriching the data, before returning it) + + return account, nil +} +``` +Here's what happens: +1. **Manage Transaction (Simplified):** It might start a database transaction. This is often good practice for read operations too, to ensure consistency, though for a single `GetByID` it might be optional depending on the database setup. +2. **Use Repository:** The crucial step! It calls `s.repository.GetByID(ctx, accountID)`. The service delegates the actual data fetching to its [Repository](04_repository_.md). +3. **Return Data:** It returns the `account` (or an error) obtained from the [Repository](04_repository_.md). + +Here's a sequence diagram illustrating this flow: + +```mermaid +sequenceDiagram + participant APIHandler as API Handler + participant AccSvc as AccountsService + participant AccRepo as AccountsRepository + participant DB as Database + + APIHandler->>AccSvc: GetByID("acc-alice-123") + AccSvc->>AccSvc: Start DB Transaction (conceptual) + AccSvc->>AccRepo: GetByID(ctx, "acc-alice-123") + AccRepo->>DB: SELECT * FROM accounts WHERE id = "acc-alice-123" + DB-->>AccRepo: Account Row Data + AccRepo-->>AccSvc: *Account (struct) + AccSvc->>AccSvc: Commit/Rollback Transaction (conceptual) + AccSvc-->>APIHandler: *Account (details for Alice) +``` + +## Services and Write Operations (Changes to Data) + +So far, we've focused on how Services help read data. What about when we want to *change* data, like creating an account or updating its status? + +The `AccountsService` interface in `api/accounts.go` actually defines methods like `Create`, `Activate`, etc.: +```go +// From: api/accounts.go +type AccountsService interface { + Create(ctx context.Context, command CreateAccountCommand) error + Activate(ctx context.Context, command ActivateAccountCommand) error + // ... other methods like Close, LinkPaymentDevice ... + // ... and the read methods we saw earlier ... +} +``` +However, if you look at the `AccountsServiceDefault` implementation for these methods, you'll find something like this: +```go +// From: api/accounts.go AccountsServiceDefault.Create +func (s *AccountsServiceDefault) Create( + _ context.Context, + _ CreateAccountCommand, +) error { + // writes should be done through the commandBus + return e.NewNotImplemented() +} +``` +It returns `e.NewNotImplemented()`! This tells us that, in this specific design, the `AccountsServiceDefault` *itself* doesn't directly handle account creation logic. Instead, such "write" operations are intended to be performed by sending a [Command](02_command_.md) (like `CreateAccountCommand`) directly to the `EventSourcingClient` (our Command Bus), which then routes it to the appropriate [Aggregate](03_aggregate_.md) (e.g., `AccountAggregate`). This is often done by the [API Handler](05_api_handler_.md) using the `EventSourcingClient` from the [Core Facade](06_core_facade_.md). + +**Why this separation?** +* **Centralized Command Processing:** Keeps all state-changing logic within [Aggregates](03_aggregate_.md), ensuring business rules are consistently enforced. +* **Event Sourcing Purity:** [Aggregates](03_aggregate_.md) are the source of [Events](01_event_.md), and [Commands](02_command_.md) are the standard way to trigger changes in them. + +### When Services *Do* Orchestrate Writes + +However, this doesn't mean Services *never* handle logic related to writes. A Service might: +1. **Perform preliminary checks or gather data** before a [Command](02_command_.md) is dispatched. +2. **Dispatch a [Command](02_command_.md) itself** if the operation is more complex or involves coordination. +3. **Orchestrate multiple [Commands](02_command_.md)** or interactions with different [Aggregates](03_aggregate_.md) or [Repositories](04_repository_.md). + +For example, consider the `PostingsTransactionsServiceDefault` in `api/postings_transactions.go`. Its `RevertChildTransactions` method shows a service performing more complex orchestration: + +```go +// Simplified from: api/postings_transactions.go +// PostingsTransactionsServiceDefault holds a commandBus +type PostingsTransactionsServiceDefault struct { + // ... other fields ... + commandBus es.CommandBus +} + +// RevertChildTransactions reverts child transactions for a specific transaction. +func (s *PostingsTransactionsServiceDefault) RevertChildTransactions( + ctx context.Context, + transactionID, rootTransactionID, parentTransactionID uuid.UUID, + /* ... other params ... */ +) error { + // 1. Use repository to get child transactions + postingsTransactions, err := s.repository.GetPostingsTransactionByParentID(ctx, transactionID) + // ... error handling ... + + // 2. For each child, dispatch a RevertSettledPostingsTransactionCommand + for _, postingsTransaction := range postingsTransactions { + err := s.commandBus.HandleCommand(ctx, &RevertSettledPostingsTransactionCommand{ + BaseCommand: es.BaseCommand{AggregateID: postingsTransaction.ID}, + // ... populate command details ... + }) + // ... error handling ... + } + return nil +} +``` +In this case, the `PostingsTransactionsService` reads data using its [Repository](04_repository_.md) and then dispatches multiple [Commands](02_command_.md) using its `commandBus`. This is a perfect example of a Service acting as an orchestrator for a business process. + +So, a Service can: +* Directly use [Repositories](04_repository_.md) for reading data (common). +* Dispatch [Commands](02_command_.md) to [Aggregates](03_aggregate_.md) for writing data (also common, especially for complex orchestrations). +* Sometimes, for simpler writes, an [API Handler](05_api_handler_.md) might dispatch a [Command](02_command_.md) directly using the `EventSourcingClient` from the [Core Facade](06_core_facade_.md), bypassing a specific write method on a service if the service doesn't add extra orchestration. + +## Why are Services Important? + +1. **Clear Business API:** They offer a higher-level, business-oriented set of operations compared to raw [Repositories](04_repository_.md) or direct [Command](02_command_.md) dispatching. +2. **Encapsulation:** They group related business logic, making it easier to find and manage. If you need to know how account searching works, you look in the `AccountsService`. +3. **Decoupling:** [API Handlers](05_api_handler_.md) or other clients depend on the Service interface, not on the nitty-gritty details of data storage or [Command](02_command_.md) handling for every single operation. +4. **Orchestration Point:** They are the natural place to put logic that coordinates multiple steps or involves several other components to achieve a business goal (like the `RevertChildTransactions` example). +5. **Testability:** Services can be tested in isolation. For example, when testing `AccountsServiceDefault`, you can provide a "mock" `AccountsRepository` to simulate database interactions. + +## Conclusion + +Services are the specialized "departments" within our `corebanking` application. They: +* Group related business operations for a specific domain (like accounts or transactions). +* Often use [Repositories](04_repository_.md) to fetch data needed for read operations. +* Can orchestrate more complex business processes, sometimes by dispatching [Commands](02_command_.md) to [Aggregates](03_aggregate_.md). +* Provide a clean and organized way for other parts of the system, like [API Handlers](05_api_handler_.md), to interact with the core business logic. + +They help keep our application well-structured and maintainable by ensuring that responsibilities are clearly defined. + +So far, we've seen how requests come in ([API Handler](05_api_handler_.md)), how they might use a [Core Facade](06_core_facade_.md) to access Services, and how Services can interact with [Repositories](04_repository_.md) or dispatch [Commands](02_command_.md) that affect [Aggregates](03_aggregate_.md) and generate [Events](01_event_.md). + +But what happens *after* an [Event](01_event_.md) is generated? Sometimes, other parts of the system need to react to these [Events](01_event_.md) automatically. For example, when an `AccountCreatedEvent` occurs, maybe we need to update a search index or send a welcome email. How does that happen? In the next chapter, we'll explore the [Consumer](08_consumer_.md). + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/08_consumer_.md b/tmp/nesto/corebanking/08_consumer_.md new file mode 100644 index 00000000..530c6987 --- /dev/null +++ b/tmp/nesto/corebanking/08_consumer_.md @@ -0,0 +1,204 @@ +# Chapter 8: Consumer + +Welcome to Chapter 8! In our [previous chapter on Services](07_service_.md), we learned how specialized "departments" in our bank help organize business logic, often by reading data using [Repositories](04_repository_.md) or orchestrating actions, sometimes by sending [Commands](02_command_.md). These [Commands](02_command_.md), when processed by [Aggregates](03_aggregate_.md), result in [Events](01_event_.md) – records of things that have happened. + +But what happens *after* an [Event](01_event_.md) is generated? For instance, if a `PostingCommittedEvent` (signaling a financial transaction leg) occurs, we probably need to update the customer's account balance so they can see it. Or, if an `AccountCreatedEvent` happens, maybe we need to send a welcome email. How do these follow-up actions happen automatically without slowing down the original process? + +This is where the **Consumer** steps in! + +## What is a Consumer? The Bank's Automated Event Watcher + +Imagine in our bank, every time an important announcement is made (an [Event](01_event_.md) happens), there are dedicated staff members who are always listening for specific types of announcements. +* One staff member listens for "New Account Opened" announcements. When they hear one, they immediately prepare and send a welcome pack to the new customer. +* Another staff member listens for "Transaction Completed" announcements. When they hear one, they quickly update the account ledger display. + +A **Consumer** in our `corebanking` system is like one ofthese automated staff members. It's a component that **listens for specific types of [Events](01_event_.md)** that occur within the system. When it "hears" an [Event](01_event_.md) it's interested in, it performs a predefined action. + +For example, when a `PostingCommittedEvent` occurs (meaning a part of a financial transaction has been finalized), a `BalancesConsumer` might listen for this. Upon receiving it, the consumer will update the account's balance that is stored for quick display (this stored balance is often called a "read model" or "projection"). + +Consumers are crucial because they allow different parts of the system to react to business happenings **asynchronously**. This means the original action (like processing the transaction) can finish quickly, and the follow-up tasks (like updating the balance display or sending a notification) happen separately, a moment later. + +## Key Ideas About Consumers + +1. **Event Listener:** A Consumer "subscribes" to a stream of [Events](01_event_.md) and pays attention only to the types of [Events](01_event_.md) it cares about. +2. **Asynchronous Operation:** Consumers typically run in the background, separate from the process that originally created the [Event](01_event_.md). This keeps the main operations fast and responsive. The part of the system that created the event doesn't have to wait for the Consumer to finish its job. +3. **Performs Actions:** When a relevant [Event](01_event_.md) is received, the Consumer executes specific logic. Common actions include: + * **Updating Read Models:** Calculating and storing data in a format optimized for fast reading. For example, updating an account balance summary after a transaction. + * **Triggering Follow-up Processes:** Starting new workflows, sending notifications (like emails or SMS), or calling other [Services](07_service_.md). +4. **Decoupling:** The component that produces an [Event](01_event_.md) (e.g., an `AccountAggregate` producing `AccountCreatedEvent`) doesn't need to know anything about the Consumers that might be listening to that [Event](01_event_.md). This makes the system more flexible and easier to change. +5. **Data Consistency (Eventually):** Consumers help maintain consistency across different views of data. For example, after a transaction, the balance read model will *eventually* reflect the change. + +## A Consumer in Action: Updating Account Balances + +Let's say a customer makes a deposit. This might result in one or more `PostingCommittedEvent`s. We need to update the account balance that is shown to the customer on their app. + +This is a perfect job for a `BalancesConsumer` (like the `BalancesTimeSeriesConsumerDefault` found in `api/balances_consumer.go`): + +1. **[Event](01_event_.md) Happens:** An [Aggregate](03_aggregate_.md) processes a transaction, and a `PostingCommittedEvent` is generated and saved. This event contains details about the amount, currency, and account involved. +2. **Consumer is Notified:** The `BalancesConsumer` is subscribed to listen for `PostingCommittedEvent`s. An "event bus" (a system for distributing events) delivers this event to the consumer. +3. **Consumer Processes the [Event](01_event_.md):** + * The `BalancesConsumer` receives the `PostingCommittedEvent`. + * It extracts the necessary information (account ID, amount, type of posting - debit/credit). + * It might fetch the current stored balance for that account using a `BalancesRepository` (a type of [Repository](04_repository_.md)). + * It calculates the new balance. +4. **Consumer Updates Read Model:** The consumer then saves this new, updated balance back using the `BalancesRepository`. This updated balance is now ready to be quickly displayed to the user. +5. **Optional: Further [Events](01_event_.md):** The consumer might even publish a new [Event](01_event_.md) itself, like a `BalanceUpdatedEvent`, to signal that the read model balance has changed. + +This entire process happens *after* the original deposit transaction was confirmed. The customer got a quick confirmation of their deposit, and in the background, the consumer ensures their displayed balance is updated. + +## What Does a Consumer Look Like? (A Peek at the Code) + +Consumers are often structs that hold dependencies like [Repositories](04_repository_.md) or other [Services](07_service_.md). They typically have a `Start()` method to begin listening and a processing method that gets called when relevant [Events](01_event_.md) arrive. + +Let's look at a simplified structure for `BalancesTimeSeriesConsumerDefault` from `api/balances_consumer.go`: + +```go +// Simplified from: api/balances_consumer.go +type BalancesTimeSeriesConsumerDefault struct { + repository BalancesRepository // To read/write balance data + events es.EventPublisher // To publish new events (optional) + eventConsumer es.EventConsumer // The mechanism to receive events + // ... other dependencies ... +} + +// NewBalancesTimeSeriesConsumer creates the consumer +func NewBalancesTimeSeriesConsumer( + repository BalancesRepository, + events es.EventPublisher, + eventConsumer es.EventConsumer, + // ... other args ... +) BalancesTimeSeriesConsumer { + // ... (nil checks) ... + return &BalancesTimeSeriesConsumerDefault{ + repository: repository, + events: events, + eventConsumer: eventConsumer, + // ... + } +} +``` +* `repository`: This is a `BalancesRepository`, used to get the current balance and save the new balance. +* `events`: An `EventPublisher` which this consumer might use to send out new [Events](01_event_.md) (like `BalanceUpdatedEvent`). +* `eventConsumer`: This is the component from our event sourcing library (`es`) that actually delivers [Events](01_event_.md) from the event bus to this consumer. + +### Starting the Consumer and Processing Events + +The consumer needs to be "turned on" to start listening. This is usually done by its `Start()` method: + +```go +// Simplified from: api/balances_consumer.go +func (c *BalancesTimeSeriesConsumerDefault) Start() error { + // Tell the eventConsumer to start sending batches of events + // to our 'processBalancesTimeSeriesEvents' method. + err := c.eventConsumer.ConsumeTenantMultiAggregateBatch( + context.Background(), + c.processBalancesTimeSeriesEvents, // Our callback function + ) + return err +} +``` +* `c.eventConsumer.ConsumeTenantMultiAggregateBatch(...)`: This line registers the `processBalancesTimeSeriesEvents` method with the underlying event consumption mechanism. Now, whenever new [Events](01_event_.md) (that this consumer is configured to listen to) arrive, this method will be called with a batch of those [Events](01_event_.md). + +The actual work happens in the callback method, `processBalancesTimeSeriesEvents`. Here's a highly simplified conceptual view of what it does when it receives event data related to postings: + +```go +// Highly simplified concept of processBalancesTimeSeriesEvents +func (c *BalancesTimeSeriesConsumerDefault) processBalancesTimeSeriesEvents( + ctx context.Context, tenantID int, events []*es.Event, +) error { + // For each relevant event (e.g., indirectly from a PostingCommittedEvent): + for _, event := range events { + // 1. Extract posting details from the event + // (The real code uses a helper 'orderedPostingsFromEvents' + // to get 'Posting' data from events like PostingCommittedEvent) + postingData := extractPostingFrom(event) // Conceptual + accountID := postingData.AccountID + + // 2. Get current balances for the account (simplified) + accountBalances, _ := c.repository.GetBalancesByAccountIDs(ctx, []uuid.UUID{accountID}) + // (Real code handles 'not found' and groups by account) + currentBalanceInfo := accountBalances[0] // Simplified + + // 3. Apply the posting to update the balance + // (The real code uses 'ApplyPostingToAccountBalances') + newBalance := calculateNewBalance(currentBalanceInfo, postingData) // Conceptual + + // 4. Save the updated current balance + // (The real code accumulates 'updatedCurrentBalances' and saves in a batch) + err := c.repository.SaveCurrent(ctx, []*Balance{newBalance}) + if err != nil { + // Handle error + return err + } + + // 5. Optionally, publish a new event + balanceUpdatedEvt := createBalanceUpdatedEvent(newBalance) // Conceptual + c.events.PublishBatch(ctx, []es.Event{balanceUpdatedEvt}) + } + return nil +} +``` +Let's break down this conceptual flow: +1. **Extract Posting Details:** The consumer gets the necessary details from the incoming [Event](01_event_.md) (like account ID, amount, currency, debit/credit). In the actual `corebanking` code, `PostingCommittedEvent`s are processed to extract `Posting` objects. +2. **Get Current Balance:** It uses its `BalancesRepository` to fetch the most recent balance record for the affected account. +3. **Apply Posting:** It performs the calculation to update the balance. The actual function `ApplyPostingToAccountBalances` in `api/balances_consumer.go` handles the logic of adding or subtracting based on posting type and account category. +4. **Save Updated Balance:** The new balance is saved back to the read model storage using the `BalancesRepository`'s `SaveCurrent` method. +5. **Publish New Event (Optional):** The `BalancesConsumer` in our project also creates and publishes a `BalanceUpdatedEvent`. This allows other parts of the system to know that a balance read model has been updated. + +The actual `processBalancesTimeSeriesEvents` is more complex because it handles batches of events, deals with time-series data for historical balances, and manages database transactions. But the core idea is: **receive event -> process it -> update read model.** + +## How a Consumer Gets Triggered: The Event Flow + +Here’s a simplified sequence diagram showing how a `BalancesConsumer` might react to a `PostingCommittedEvent`: + +```mermaid +sequenceDiagram + participant TxnProcessor as Transaction Processor (e.g., Aggregate) + participant EventBus as Event Bus / Message Queue + participant BalConsumer as Balances Consumer + participant BalRepo as Balances Repository + participant ReadModelDB as Read Model Database + + TxnProcessor->>EventBus: Publish PostingCommittedEvent (for Account X, Amount $10) + EventBus->>BalConsumer: Deliver PostingCommittedEvent + BalConsumer->>BalRepo: Get Current Balance (Account X) + BalRepo->>ReadModelDB: Fetch balance for Account X + ReadModelDB-->>BalRepo: Current Balance: $50 + BalRepo-->>BalConsumer: Current Balance: $50 + BalConsumer-->>BalConsumer: Calculate New Balance ($50 + $10 = $60) + BalConsumer->>BalRepo: Save Balance (Account X, New Balance $60) + BalRepo->>ReadModelDB: Update balance for Account X to $60 + ReadModelDB-->>BalRepo: Success + BalRepo-->>BalConsumer: Success + BalConsumer->>EventBus: (Optional) Publish BalanceUpdatedEvent +``` +This shows the asynchronous nature: the `TxnProcessor` fires the event and moves on. The `BalConsumer` picks it up later and does its work. + +## Other Types of Consumers + +Our `corebanking` system has other consumers too: + +* **`PostingsTransactionsConsumer`** (from `api/postings_transactions_consumer.go`): + This consumer listens to events related to financial transactions (like `PostingsTransactionCreatedEvent`, `PostingsTransactionSettledEvent`). It updates a read model that stores details about these transactions, making it easy to query their status, amount, etc. + +* **`ProductEnginesConsumer`** (from `api/product_engines_consumer.go`): + This is an interesting one! It listens for events like `ScheduledEvent` (e.g., "it's end of day for account X") or even `PostingCommittedEvent`. When it receives such an event, it triggers the [Product Engine](09_product_engine_.md) (via `productEngines.HandleEvents()`). The [Product Engine](09_product_engine_.md) might then apply interest, charge fees, or perform other account-specific logic defined by the banking product. This shows a consumer triggering a more complex follow-up process. + +## Why Are Consumers So Useful? + +1. **Improved Performance & Responsiveness:** The system part that creates an [Event](01_event_.md) (e.g., an [Aggregate](03_aggregate_.md) handling a [Command](02_command_.md)) can finish its job quickly without waiting for all side effects (like updating multiple read models or sending emails) to complete. +2. **Decoupling & Modularity:** Event producers don't need to know about event consumers. You can add new consumers or change existing ones without affecting the code that generates the events. This makes the system very flexible. +3. **Resilience:** If a consumer temporarily fails while processing an event (e.g., a network issue while saving to a database), the event can often be re-processed later without losing the original data, as the [Event](01_event_.md) itself is already safely stored. +4. **Scalability:** Different consumers can often be scaled independently. If updating balances becomes a bottleneck, you might be able to run more instances of the `BalancesConsumer`. + +## Conclusion + +Consumers are the unsung heroes working diligently in the background of our `corebanking` system. They are **automated listeners** that subscribe to specific types of [Events](01_event_.md). When they receive an [Event](01_event_.md) they're interested in, they swing into action – perhaps updating a read-model for fast data display (like account balances) or triggering follow-up processes (like complex product calculations). + +By reacting to business [Events](01_event_.md) asynchronously, Consumers help keep our system responsive, flexible, and robust. They play a vital role in maintaining data consistency across different views and enabling complex workflows. + +One example we saw was the `ProductEnginesConsumer`, which listens for events and then invokes specific business logic defined by a banking product. What is this "Product Engine" that it calls? Let's explore that in our next chapter: the [Product Engine](09_product_engine_.md). + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/09_product_engine_.md b/tmp/nesto/corebanking/09_product_engine_.md new file mode 100644 index 00000000..11c45b49 --- /dev/null +++ b/tmp/nesto/corebanking/09_product_engine_.md @@ -0,0 +1,265 @@ +# Chapter 9: Product Engine + +Welcome to Chapter 9! In the [previous chapter](08_consumer_.md), we learned about [Consumers](08_consumer_.md) and how they listen for [Events](01_event_.md) to trigger follow-up actions. For example, a [Consumer](08_consumer_.md) might notice that an account needs its monthly interest calculated. But how does the system know *how* to calculate that interest, especially if different types of accounts (like a "Youth Savings Account" vs. a "Standard Mortgage") have vastly different rules? + +This is where the **Product Engine** comes in. + +## What's a Product Engine? The Bank's Specialized Rulebook + +Imagine a bank offers many different financial products: +* A "Basic Savings Account" with simple interest. +* A "Youth Savings Account" that gets bonus interest if no money is withdrawn during the month. +* A "Standard Mortgage" with complex payment schedules and fee structures. +* A "HELOC Remittance" account with its own unique remittance rules. + +Each of these products behaves differently. They have their own specific features, ways to calculate interest, fee rules, and reactions to events like an account being activated or a transaction occurring. + +A **Product Engine** is like a **detailed rulebook and automated process manager** specifically designed for *one particular financial product*. It defines exactly how that product, like "Standard Mortgage" or "Youth Savings Account," should work. + +When a customer opens an account (say, a "Youth Savings Account"), that account is linked to its specific product. From then on, whenever something relevant happens to that account, the system consults the "Youth Savings Account" Product Engine to: +* Check if an operation is allowed (validation). +* Determine any product-specific actions or outcomes (like calculating that special bonus interest). + +## Key Ideas About Product Engines + +1. **Product-Specific Logic:** Each Product Engine contains the unique rules and behaviors for one type of financial product. +2. **Defines Product Behavior:** It specifies features, fee structures, interest calculations, and how the product responds to lifecycle events (e.g., account activation, transaction posting, scheduled events like "end of month"). +3. **Consulted for Operations:** When an action needs to be performed on an account, or an event occurs related to it, the corresponding Product Engine is invoked. +4. **Automates Processes:** It automates tasks like applying interest, charging fees, or managing scheduled product-specific activities. +5. **Produces Actions:** Based on its rules and the current situation (e.g., an incoming [Event](01_event_.md) or [Command](02_command_.md)), a Product Engine can decide that certain actions need to happen. These actions are often new [Commands](02_command_.md) (e.g., "Create a transaction to credit bonus interest"). + +## The Product Engine in Action: Youth Savings Bonus Interest + +Let's use our "Youth Savings Account" example. This account type offers bonus interest if no withdrawals are made in a month. + +1. **Account Setup:** Alice has a "Youth Savings Account." This account is linked to the "Youth Savings Product Engine." +2. **End of Month [Event](01_event_.md):** A `ScheduledEvent` (like "EndOfMonthProcessing") occurs for Alice's account. +3. **[Consumer](08_consumer_.md) Routes to Service:** A [Consumer](08_consumer_.md) (like the `ProductEnginesConsumer` we touched upon in the [previous chapter](08_consumer_.md)) picks up this `ScheduledEvent`. It knows this event might require product-specific logic, so it passes the event to the `ProductEnginesService`. +4. **Service Invokes Product Engine:** The `ProductEnginesService`: + * Looks at Alice's account and sees it's a "Youth Savings Account." + * Retrieves the specific "Youth Savings Product Engine." + * Prepares a `ProductEngineContext` for Alice's account. This context contains information the engine might need, like Alice's current account details, balances, and the product's configuration. + * Calls a method on the "Youth Savings Product Engine," like `HandleScheduledEvent(context, scheduledEvent)`. +5. **Product Engine Applies Rules:** The "Youth Savings Product Engine" executes its logic for the "EndOfMonthProcessing" event: + * It checks Alice's transaction history for the past month (perhaps by looking at data in the `ProductEngineContext` or by querying for recent postings). + * It determines if any withdrawals were made. + * If no withdrawals, it calculates the bonus interest. +6. **Product Engine Returns Actions:** The engine decides that bonus interest should be applied. It returns a list of `ProductEngineAction`s. One of these actions might be a `HandleCommandAction` containing a `CreatePostingsTransactionCommand` to credit the bonus interest to Alice's account. +7. **Service Executes Actions:** The `ProductEnginesService` takes these actions and processes them. For a `HandleCommandAction`, it would send the `CreatePostingsTransactionCommand` to the [Command Bus](02_command_.md), which then ensures the transaction is created and Alice's balance is updated. + +This way, the specific bonus interest logic for the "Youth Savings Account" is neatly contained within its own Product Engine. + +## A Peek at the Code: Interfaces and Services + +Let's look at some of the key pieces involved. + +### 1. The `ProductEngine` Interface + +At its core, every product engine implements the `ProductEngine` interface (from `api/product_engines.go`). This interface defines the "hooks" or methods that can be called by the system for different situations. + +```go +// Simplified from: api/product_engines.go +type ProductEngine interface { + ProductEngineID() string // E.g., "YOUTH_SAVINGS_ACCOUNT" + ProductEngineVersion() string // E.g., "1.0.0" + + // Called when an account is activated + ActivateAccount(ctx context.Context, engineCtx ProductEngineContext) ([]ProductEngineAction, error) + + // Called for scheduled events (like our "EndOfMonthProcessing") + HandleScheduledEvent(ctx context.Context, engineCtx ProductEngineContext, scheduledEvent *ScheduledEvent) ([]ProductEngineAction, error) + + // Called when a posting (transaction leg) is accepted for an account + AcceptPosting(ctx context.Context, engineCtx ProductEngineContext, posting *Posting) ([]ProductEngineAction, error) + + // ... other methods for validating postings, updating parameters, closing accounts etc. +} +``` +* `ProductEngineID()` and `ProductEngineVersion()`: Identify the specific engine. +* `ActivateAccount`, `HandleScheduledEvent`, `AcceptPosting`: These are examples of methods that get called at different points in an account's lifecycle or when certain [Events](01_event_.md) occur. +* `ProductEngineContext`: This argument provides all the necessary information about the current account, its product settings, balances, etc., that the engine needs to make decisions. +* `[]ProductEngineAction`: The method returns a list of actions the system should take as a result of the engine's logic. + +### 2. The `ProductEngineContext` + +This struct (also from `api/product_engines.go`) is passed to the Product Engine methods. It's like a data package giving the engine all the relevant details for the current operation. + +```go +// Simplified from: api/product_engines.go +type ProductEngineContext struct { + ReferenceDate *date.Date // The current date for processing + Account *Account // Details of the specific account + Product *ResolvedProduct // Details & parameters of the banking product + CurrentBalances *BalancesReader // Access to the account's current balances + // ... and other fields like PostingsByPartition, BalancesByPartition for historical data +} +``` +* `Account`: The specific account being processed. +* `Product`: The resolved product definition, containing parameters like interest rates, fee schedules, etc. +* `CurrentBalances`: A way to read the account's current balances. +The engine uses this information to apply its rules. + +### 3. `ProductEngineAction` + +When a Product Engine finishes its logic, it needs to tell the system what to do next. It does this by returning a slice of `ProductEngineAction` (defined in `api/product_engines.go`). + +A common action is to issue a [Command](02_command_.md): +```go +// Simplified from: api/product_engines.go + +// ProductEngineAction represents a product engine action. +type ProductEngineAction interface { + Actuate() // A marker method +} + +// HandleCommandAction implements product engine action to handle command. +type HandleCommandAction struct { + command es.Command // The command to be executed +} + +// NewHandleCommandAction returns a new HandleCommandAction. +func NewHandleCommandAction(command es.Command) HandleCommandAction { + return HandleCommandAction{command} +} + +// GetCommand returns the underlying command +func (a HandleCommandAction) GetCommand() es.Command { + return a.command +} +``` +So, if our "Youth Savings Product Engine" decides to apply bonus interest, it would create a `CreatePostingsTransactionCommand` and wrap it in a `NewHandleCommandAction`. + +### 4. The `ProductEnginesService` + +The `ProductEnginesServiceDefault` (from `api/product_engines.go`) is the orchestrator. It's responsible for receiving [Events](01_event_.md) (often from a [Consumer](08_consumer_.md)), finding the right Product Engine, and invoking it. + +```go +// Conceptual flow within ProductEnginesServiceDefault.HandleEvents +// (Actual code is more complex, handling batches and various event types) + +func (s *ProductEnginesServiceDefault) HandleEvents(ctx context.Context, events []*es.Event) error { + for _, event := range events { + // 1. Determine which account this event is for (from event.AggregateID) + accountID := event.AggregateID + account, _ := s.accounts.GetByID(ctx, accountID) // Fetch account details + + // 2. Get the Product Engine for this account's product + productEngine, _ := s.GetProductEngine(ctx, account.Product) + if productEngine == nil { + continue // No engine for this product, or product not set + } + + // 3. Create an Event Handler wrapper (e.g., for a ScheduledEvent) + // The hook file (api/product_engines_hooks.go) has different handlers for different events. + var eventHandler ProductEngineEventHandler // Defined in api/product_engines_hooks.go + if se, ok := event.Data.(*ScheduledEvent); ok && eventHandlerNameFromAction(se.Action) == "HandleScheduledEvent" { + eventHandler = NewProductEngineEventHandlerHandleScheduledEvent(event.ID, se, /*...location...*/) + } + // ... (else if for AccountActivatedEvent, PostingCommittedEvent, etc.) ... + + if eventHandler == nil { + continue + } + + // 4. Build the ProductEngineContext (simplified) + engineCtx, _ := s.buildProductEngineContext(ctx, /*...details...*/, account, /*balances*/, /*config*/) + + // 5. Call the specific hook on the event handler, which calls the engine + actions, err := eventHandler.Hook(ctx, *engineCtx, productEngine) + // ... error handling ... + + // 6. Run the actions returned by the Product Engine + s.runActions(ctx, actions) // This would dispatch commands, schedule things, etc. + } + return nil +} +``` +* **Get Account & Engine:** It fetches the account and determines the correct Product Engine based on the account's product information (`account.Product.EngineID`, `account.Product.EngineVersion`). +* **Event Handler Wrapper:** An `ProductEngineEventHandler` (from `api/product_engines_hooks.go`) is chosen based on the type of incoming [Event](01_event_.md). This wrapper standardizes how different events are passed to the Product Engine. For example, `ProductEngineEventHandlerHandleScheduledEvent` has a `Hook` method that will ultimately call `engine.HandleScheduledEvent(...)`. +* **Build Context:** It prepares the `ProductEngineContext` with all necessary data. +* **Invoke Engine (via Hook):** It calls the `Hook` method on the event handler, which in turn calls the relevant method on the `ProductEngine` instance (e.g., `HandleScheduledEvent`). +* **Run Actions:** It takes the `ProductEngineAction`s returned by the engine and executes them (e.g., sends [Commands](02_command_.md) to the [Command Bus](02_command_.md)). + +### Example: A Simple Product Engine's `HandleScheduledEvent` + +Let's imagine a very basic "MonthlyFeeProductEngine": +```go +// Hypothetical MonthlyFeeProductEngine +type MonthlyFeeProductEngine struct { /* ... */ } + +func (pe *MonthlyFeeProductEngine) HandleScheduledEvent( + ctx context.Context, + engineCtx api.ProductEngineContext, + scheduledEvent *api.ScheduledEvent, +) ([]api.ProductEngineAction, error) { + + if scheduledEvent.Action == "APPLY_MONTHLY_FEE" { + feeAmountStr, _ := engineCtx.Product.Parameters["monthlyFeeAmount"] // Get fee from product config + feeAmount, _ := decimal.NewFromString(feeAmountStr) + + if feeAmount.IsPositive() { + // Create a command to charge the fee + feeTxCmd := &api.CreatePostingsTransactionCommand{ + BaseCommand: es.BaseCommand{AggregateID: engineCtx.Account.ID}, + Action: api.PostingActionTransfer, + Code: "MONTHLY_FEE", + Amount: feeAmount, + Debtor: &api.PostingDestination{AccountID: engineCtx.Account.ID}, // Debit customer + Creditor: &api.PostingDestination{AccountID: uuid.FromString("bank-fee-income-account-id")}, // Credit bank + } + return []api.ProductEngineAction{api.NewHandleCommandAction(feeTxCmd)}, nil + } + } + return nil, nil // No action +} + +// ... other required methods of ProductEngine interface ... +``` +When a `ScheduledEvent` with action "APPLY_MONTHLY_FEE" comes in for an account using this product: +1. The engine checks if the action matches. +2. It reads the `monthlyFeeAmount` from the product's parameters (available in `engineCtx.Product.Parameters`). +3. If a fee is due, it creates a `CreatePostingsTransactionCommand` to transfer the fee from the customer's account to the bank's fee income account. +4. It returns this command wrapped in a `HandleCommandAction`. + +## Visualizing the Flow + +Here's how an event for Alice's "Youth Savings Account" (YSA) might trigger its Product Engine to apply bonus interest: + +```mermaid +sequenceDiagram + participant EventBus + participant YSAConsumer as YSA Product Consumer + participant PEService as ProductEnginesService + participant YSAPEngine as YouthSavingsProductEngine + participant CmdBus as CommandBus + + EventBus->>YSAConsumer: ScheduledEvent (EndOfMonth, Account: AliceYSA) + YSAConsumer->>PEService: HandleEvents([ScheduledEvent]) + PEService->>YSAPEngine: GetEngine("YouthSavingsProductEngine", "v1") + Note over PEService: Builds ProductEngineContext for AliceYSA + PEService->>YSAPEngine: HandleScheduledEvent(ctx, engineCtx, ScheduledEvent) + YSAPEngine-->>YSAPEngine: Logic: Check no withdrawals, calculate bonus. + YSAPEngine-->>PEService: Return [HandleCommandAction(CreateBonusTxCmd)] + PEService->>CmdBus: HandleCommand(CreateBonusTxCmd) + CmdBus-->>PEService: Command processed (Alice gets bonus) +``` + +## Why Product Engines are Powerful + +* **Flexibility:** Easily define unique behaviors for many different banking products without cluttering the core system. +* **Modularity:** Product-specific logic is isolated. Changes to one product's rules don't affect others. +* **Maintainability:** Easier to understand and update the logic for a specific product. +* **Testability:** Individual Product Engines can be tested in isolation with a mocked `ProductEngineContext`. +* **Rapid Development:** New financial products can often be introduced by creating a new Product Engine and its configuration, rather than changing deep parts of the core system. + +## Conclusion + +Product Engines are the specialized "brains" behind each financial product offered by the bank. They act as detailed rulebooks and automated process managers, defining how a product like a "Standard Mortgage" or a "Youth Savings Account" behaves – its features, fees, interest calculations, and responses to lifecycle events. + +When an [Event](01_event_.md) occurs for an account, the `ProductEnginesService` (often triggered by a [Consumer](08_consumer_.md)) identifies the account's product and invokes the corresponding Product Engine. The engine then applies its specific rules and can return a list of actions, such as [Commands](02_command_.md) to create transactions, for the system to execute. This powerful mechanism allows `corebanking` to support a diverse range of financial products with complex, individualized behaviors. + +With all these intricate product behaviors and event-driven interactions, how can we be sure everything works as expected? How can we test different scenarios, perhaps even simulating future events? In our next and final chapter, we'll look into [Simulation Services and Repositories](10_simulation_services_and_repositories_.md). + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/10_simulation_services_and_repositories_.md b/tmp/nesto/corebanking/10_simulation_services_and_repositories_.md new file mode 100644 index 00000000..dd0db4cd --- /dev/null +++ b/tmp/nesto/corebanking/10_simulation_services_and_repositories_.md @@ -0,0 +1,205 @@ +# Chapter 10: Simulation Services and Repositories + +Welcome to the final chapter of our conceptual overview! In [Chapter 9: Product Engine](09_product_engine_.md), we saw how our `corebanking` system can support diverse financial products, each with its own complex set of rules for things like interest calculation, fee application, and behavior over time. + +Now, imagine we've just designed a sophisticated 30-year mortgage product using our [Product Engine](09_product_engine_.md). How can we be absolutely sure it works correctly under all sorts of conditions? How do we test its behavior over the full 30 years, perhaps with fluctuating interest rates, without actually waiting for three decades or messing with real customer data? This is where **Simulation Services and Repositories** come to the rescue! + +## What are Simulation Services and Repositories? Your Bank's Flight Simulator! + +Think of Simulation Services and Repositories as your bank's very own **flight simulator** or **practice mode**. They are special "test-double" or "fake" versions of the regular [Services](07_service_.md) and [Repositories](04_repository_.md) we've learned about. + +Instead of connecting to real databases and affecting real money or customer data, these simulators: +* **Work "in-memory":** They store their data temporarily, right in the computer's memory, for the duration of a test or simulation. When the simulation ends, this temporary data is gone. This is great because it's fast and completely isolated from the live system. +* **Mimic real behavior:** They are designed to act just like their real counterparts, but give us much more control. +* **Allow controlled testing:** We can use them to test complex business logic, like our 30-year mortgage product, under various scenarios (e.g., "what if interest rates double next year?") in a safe and repeatable way. + +Essentially, these simulators let us "play out" complex financial scenarios, fast-forward through time, and see what happens, all without any real-world consequences. + +## Key Ideas Behind Simulation + +How do these simulators achieve this magic? There are a few key ingredients: + +1. **In-Memory Storage:** + * Simulation [Repositories](04_repository_.md) (like `AccountRepositorySimulator` or `PostingsTransactionsRepositorySimulator`) don't save data to a permanent database. Instead, they use simple Go data structures like maps and slices to hold information temporarily. + * This makes them very fast and ensures that each simulation run starts fresh. + +2. **Controlled Time (The "Time Machine"):** + * A special `ClockServiceSimulator` lets us control the "current date" within the simulation. We can tell it to jump forward day by day, month by month, or even set it to a specific date in the future. This is crucial for testing products that behave differently over long periods. + +3. **Deterministic Behavior:** + * For a given starting point and set of actions, a simulation should ideally produce the exact same results every time it's run. This makes testing reliable and helps us pinpoint issues if something changes unexpectedly. + +4. **Isolation:** + * Simulations run completely separately from the live banking system. There's no risk of accidentally creating a real account or moving real money. + +## Using the Simulators: Testing Our Mortgage Product + +Let's go back to our 30-year mortgage product. How would we use the simulation tools to test it? The main entry point for complex product simulations is often the `ProductEnginesServiceSimulatorDefault` (found in `api/pkg/productengines/simulator/simulator.go`). + +Here's a conceptual walkthrough: + +1. **Setup the Simulation:** + * First, we'd create an instance of `ProductEnginesServiceSimulatorDefault`. This master simulator internally sets up other necessary simulator components like `AccountsServiceSimulator`, `BalancesServiceSimulator`, `PostingsServiceSimulator`, a `ClockServiceSimulator`, and a `CommandBusSimulator`. + * We'd tell the simulator about our "Mortgage Product" definition (its rules, interest rates, fee structures, etc.) by registering it. + * We'd then create a "Simulated Account" – let's say, "Alice's Mortgage Account" – and link it to our mortgage product. We might also set its initial loan amount and other starting conditions. + * We'd define a time period for our simulation, for example, from "2024-01-01" to "2025-01-01" to see how it behaves over one year. + +2. **Run the Simulation:** + * We'd call the `Simulate` method on our `ProductEnginesServiceSimulatorDefault`, giving it Alice's simulated account and our desired start and end dates. + +3. **What Happens Inside the `Simulate` Method (Simplified):** + * The simulator enters a loop, advancing day by day from the start date to the end date using its internal `ClockServiceSimulator`. + * On each simulated day: + * It checks if any scheduled events are due for Alice's account (e.g., "end of month interest calculation," "payment due date"). + * If so, it triggers these events. These events are then passed to the (real) [Product Engine](09_product_engine_.md) logic for our mortgage product. + * The mortgage [Product Engine](09_product_engine_.md) runs its rules based on the current simulated date and account state. It might decide to: + * Calculate interest due. + * Generate a [Command](02_command_.md) like `CreatePostingsTransactionCommand` to add this interest to Alice's loan balance. + * This `CreatePostingsTransactionCommand` is then handled by the `CommandBusSimulator`. The `CommandBusSimulator` doesn't send it to the real system; instead, it directs it to other *simulated* services. + * For example, the `PostingsServiceSimulator` would "commit" these interest postings to its in-memory store. + * The `BalancesServiceSimulator` would update Alice's simulated loan balance in its in-memory store. + * This loop continues until the simulation end date is reached. + +4. **Get the Results:** + * Once the simulation is complete, the `Simulate` method returns a `SimulationResult` struct. + * This `SimulationResult` contains a snapshot of Alice's mortgage account at the end of the simulated year: her final loan balance, a list of all transactions (postings) that occurred (like interest charges, payments), any scheduled items, and potentially any errors that happened during the simulation. + +5. **Check the Results:** + * We can then examine this `SimulationResult` to verify if our mortgage product behaved as expected. Did the interest calculate correctly? Were payments applied properly? Is the final balance what we predicted? + +## A Peek at Some Simulator Code + +Let's look at tiny, simplified snippets to get a feel for how these simulators are built. + +### 1. The `AccountRepositorySimulator` (Storing Data In-Memory) + +This simulator (from `api/pkg/productengines/simulator/accounts_repository.go`) fakes a [Repository](04_repository_.md) for accounts. + +```go +// Simplified from: api/pkg/productengines/simulator/accounts_repository.go +type AccountRepositorySimulator struct { + // Accounts are stored in a map, with account ID as the key. + accounts map[uuid.UUID]*api.Account + // ... other fields for simulation ... +} + +// GetByID tries to find an account in its in-memory map. +func (r *AccountRepositorySimulator) GetByID( + ctx context.Context, accountID uuid.UUID, +) (*api.Account, error) { + account, found := r.accounts[accountID] + if !found { + // Return an error if not found (simplified error) + return nil, errors.New("simulated account not found") + } + return account, nil // Return the account from memory +} +``` +* Instead of database queries, it uses a simple Go `map` (`r.accounts`) to store account data. +* The `GetByID` method just looks up the account in this map. Very fast! + +### 2. The `ClockServiceSimulator` (Controlling Time) + +The `ProductEnginesServiceSimulatorDefault` uses a clock service (like `api.ClockServiceDefault`) and controls it. A key method for the simulation loop is `SetDate`: + +```go +// Conceptual use of ClockService's SetDate within the simulation loop +// (s.clock is an instance of a clock service) +// s.clock.SetDate(currentSimulatedDate, timeZone) +``` +* Inside the `Simulate` method's loop, this `SetDate` is called repeatedly to advance the simulation's "current time" one day at a time. This makes the whole system believe it's that specific day. + +### 3. The `ProductEnginesServiceSimulatorDefault`'s `Simulate` Method + +This is the heart of the product simulation (from `api/pkg/productengines/simulator/simulator.go`). + +```go +// Highly simplified concept of Simulate method's loop +// (s is *ProductEnginesServiceSimulatorDefault) +func (s *ProductEnginesServiceSimulatorDefault) Simulate(/*...params...*/) (*SimulationResult, error) { + // ... (lots of initial setup: create account in simulator, set initial balances) ... + + // Loop from start date to end date + for refDate := startDate; refDate.Before(endDate); refDate = refDate.AddDays(1) { + s.clock.SetDate(refDate, s.c.ReferenceLocation) // Advance simulated time! + + // 1. Trigger scheduled events for this 'refDate' + // This involves calling s.HandleEvents(), which eventually + // invokes the Product Engine for the account. + // (Simplified representation) + s.processScheduledItemsForDate(ctx, account.ID, refDate) + + // 2. Settle any simulated transactions due on this 'refDate' + // (e.g., if a payment was scheduled to clear today) + s.settleTransactions(ctx, simulationConfig, account.ID, refDate) + } + + // ... (gather all simulated data: final account state, postings, balances) ... + return &SimulationResult{ /* ... collected data ... */ }, nil +} +``` +* The loop iterates day by day. +* `s.clock.SetDate(...)` tells the entire simulation what "today" is. +* `processScheduledItemsForDate` (a conceptual helper representing logic within `Simulate` and `GetSchedules/HandleEvents` from `simulator.go`) finds any scheduled tasks for the account on this simulated day (like "calculate monthly interest"). It then uses `s.HandleEvents()` to pass these to the [Product Engine](09_product_engine_.md). The Product Engine might then generate [Commands](02_command_.md). +* These [Commands](02_command_.md) are handled by the `CommandBusSimulator` (also part of `ProductEnginesServiceSimulatorDefault`), which ensures they are processed by *other simulated services and repositories*, updating the in-memory state. +* `s.settleTransactions` handles any pre-registered transaction settlements for that day. + +## How it All Connects: A Simulation Day + +Here's a simplified sequence of what might happen on one simulated day for our mortgage account: + +```mermaid +sequenceDiagram + participant SimLoop as Simulation Loop (in Simulate method) + participant ClockSim as ClockServiceSimulator + participant PESimService as ProductEnginesServiceSimulator + participant MortgagePE as Mortgage Product Engine + participant CmdBusSim as CommandBusSimulator + participant PostingRepoSim as PostingRepositorySimulator (In-Memory) + participant BalanceRepoSim as BalanceRepositorySimulator (In-Memory) + + SimLoop->>ClockSim: SetDate(currentSimulatedDay) + SimLoop->>PESimService: ProcessScheduledEvents(account, currentSimulatedDay) + PESimService->>MortgagePE: HandleScheduledEvent(context, scheduledEvent) + MortgagePE-->>MortgagePE: Calculate interest... + MortgagePE-->>PESimService: Return [Action: CreateInterestTxCommand] + PESimService->>CmdBusSim: HandleCommand(CreateInterestTxCommand) + CmdBusSim->>PostingRepoSim: Save new interest postings (in-memory) + CmdBusSim->>BalanceRepoSim: Update loan balance (in-memory) +``` +1. The simulation loop sets the "current day" using the `ClockSim`. +2. It asks the `PESimService` (our main simulator) to process any events for that day. +3. The `PESimService` invokes the actual `MortgagePE` (Product Engine) logic. +4. The `MortgagePE` decides interest needs to be charged and returns an action to create a transaction command. +5. The `PESimService` uses the `CmdBusSim` to handle this command. +6. The `CmdBusSim` ensures that the resulting postings and balance updates are stored in the *in-memory* repositories (`PostingRepoSim`, `BalanceRepoSim`). + +This cycle repeats for every day in the simulation period. + +## Why are these Simulators So Valuable? + +* **Safety First:** We can test the riskiest financial logic without any danger to real operations or data. +* **Blazing Speed:** Simulating years of financial activity can take just seconds or minutes, not actual years. +* **Perfect Repetition:** If a test fails, we can run the exact same simulation again to understand and fix the problem. This is called deterministic testing. +* **"What If?" Scenarios:** We can easily explore complex situations: What if interest rates soar? What if a customer misses payments? How does our product react? +* **Debugging Superpowers:** When something goes wrong in a simulation, it's often easier to trace the problem step-by-step through the in-memory state changes. +* **Building Confidence:** Thorough simulation gives us much higher confidence that our banking products will work correctly in the real world. + +## Conclusion + +Simulation Services and Repositories are powerful, specialized tools within the `corebanking` project. They create a safe, controlled, and fast "practice environment" where we can: +* Test complex financial products (especially those managed by [Product Engines](09_product_engine_.md)) over long periods. +* Simulate various market conditions and customer behaviors. +* Verify that our business logic is correct and robust. + +By using in-memory data storage, a controllable clock, and simulated versions of core components, we can gain deep insights into how our system will behave, all before it ever touches real customer information or funds. This is essential for building a reliable and trustworthy core banking system. + +--- + +Congratulations on completing this tour of the core concepts in the `corebanking` project! From the fundamental [Events](01_event_.md) and [Commands](02_command_.md) that drive the system, through [Aggregates](03_aggregate_.md) that protect business rules, [Repositories](04_repository_.md) that manage data, to the [API Handlers](05_api_handler_.md) that welcome external requests, the coordinating [Core Facade](06_core_facade_.md), specialized [Services](07_service_.md), reactive [Consumers](08_consumer_.md), intelligent [Product Engines](09_product_engine_.md), and finally, these invaluable Simulation tools, you now have a solid foundation. + +While this marks the end of our conceptual overview, your journey of exploration and contribution is just beginning. You can now revisit earlier chapters with a deeper understanding or start exploring the codebase to see these concepts in full action! + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/index.md b/tmp/nesto/corebanking/index.md new file mode 100644 index 00000000..162cbff4 --- /dev/null +++ b/tmp/nesto/corebanking/index.md @@ -0,0 +1,78 @@ +# Tutorial: corebanking + +The project is a **core banking system** designed to manage financial operations such as *account creation, transaction processing (postings), and financial product lifecycle management*. +It heavily relies on an *Event Sourcing* architecture, where system state changes are captured as a sequence of immutable **Events**. +*Aggregates* (like `AccountAggregate` or `PostingsTransactionAggregate`) are central entities that process **Commands** (requests for change) and emit these Events. +*Services* (e.g., `AccountsService`, `ProductEnginesService`) encapsulate business logic and often orchestrate interactions between Aggregates, Repositories, and Product Engines. +*Repositories* provide an abstraction layer for data persistence. +*Product Engines* define the specific rules, behaviors, and automated processes for different financial products (e.g., mortgages, HELOCs). + + +**Source Repository:** [None](None) + +```mermaid +flowchart TD + A0["Aggregate +"] + A1["Command +"] + A2["Event +"] + A3["Service +"] + A4["Repository +"] + A5["Product Engine +"] + A6["Consumer +"] + A7["API Handler +"] + A8["Core Facade +"] + A9["Simulation Services and Repositories +"] + A0 -- "Processes" --> A1 + A0 -- "Produces" --> A2 + A0 -- "Uses for persistence" --> A4 + A0 -- "Uses dependent" --> A3 + A7 -- "Uses" --> A8 + A8 -- "Dispatches (via EventSourci..." --> A1 + A8 -- "Provides access to" --> A3 + A3 -- "Uses for data access" --> A4 + A3 -- "Invokes/Manages" --> A5 + A6 -- "Consumes" --> A2 + A6 -- "Updates read-models via" --> A4 + A6 -- "Uses (e.g., ProductEnginesS..." --> A3 + A5 -- "Reacts to (via ProductEngin..." --> A2 + A9 -- "Simulates" --> A3 + A9 -- "Simulates" --> A4 +``` + +## Chapters + +1. [Event +](01_event_.md) +2. [Command +](02_command_.md) +3. [Aggregate +](03_aggregate_.md) +4. [Repository +](04_repository_.md) +5. [API Handler +](05_api_handler_.md) +6. [Core Facade +](06_core_facade_.md) +7. [Service +](07_service_.md) +8. [Consumer +](08_consumer_.md) +9. [Product Engine +](09_product_engine_.md) +10. [Simulation Services and Repositories +](10_simulation_services_and_repositories_.md) + + +--- + +Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/review.md b/tmp/nesto/corebanking/review.md new file mode 100644 index 00000000..d70a0d95 --- /dev/null +++ b/tmp/nesto/corebanking/review.md @@ -0,0 +1,36 @@ +# Core Banking Documentation Review + +## Section-by-Section Analysis + +- **Event**: Not bad overall, except the diagram which is really off +- **Command**: Not too bad +- **Aggregate**: Generally good, but hallucinates a command that doesn't exist (DepositFundsCommand) and from there it goes off the rails, claiming that the account manages transactions and balances... +- **Repository**: Not too bad, but the 2nd diagram is really off +- **API Handler**: Not too bad +- **Core Facade**: Not too bad +- **Service**: There are some impressive parts in this section, like the explanation of why services don't implement writes and an exception where the service does implement the write +- **Consumer**: Same thing - impressive analysis of the balances consumer. Some weird naming though: TxnProcessor/BalConsumer. +- **Product Engine**: Really good too (I'm curious if it will be as good with the latest changes I made), diagram is off again... +- **Simulation Services and Repos**: Weird product example, didn't catch that each product engine must implement its own simulator and allows offering features specific to the product, for example lump sum on a mortgage, drawdown request on a HELOC, etc. But overall conceptually it explains what can be done by implementing the simulator for a specific product engine. + +## Overall Assessment + +It's a really weird feeling because overall it's really not bad - there are even places where it surpasses the analysis I would have thought possible by an LLM. But it seems like there are enough small errors everywhere that it just feels like work done by someone who "doesn't care"... I'm not sure how to explain it. + +## Specific Issues + +For example, it generates code that's a bit more than pseudo-code, but outputs something like `uuid.FromString("acc-alice-123")`. The naming doesn't follow the conventions of the analyzed code like `accountsSvc` or `accountsRepo`. The diagrams invent layers that don't exist, and there are comments in the repo struct that are wrong. + +Otherwise, there are subtleties that I gloss over without thinking too much when reading because they make more or less sense, but not enough to really derail the reading. But I don't know if it would harm someone new to the codebase or not. + +## Major Concerns + +But there are some more major issues, like the invention of the DepositFundsCommand command which would really harm someone new to the codebase in understanding why transaction commands are sent directly to the account aggregate. + +## Missing Sections + +Otherwise, there would probably need to be a section for products, parameters, and customers. + +## Final Thoughts + +Overall, if there's a "parental advisory" sign on the documentation that it's auto-generated and to take certain details with a grain of salt, I think it covers the different concepts in corebanking and can probably help someone who doesn't know the project. It hammers the concepts of CQRS/event sourcing quite a bit. \ No newline at end of file diff --git a/tmp/prompt.md b/tmp/prompt.md new file mode 100644 index 00000000..af69e582 --- /dev/null +++ b/tmp/prompt.md @@ -0,0 +1,177 @@ +Of course. I will extract all the LLM prompts from the `nodes.py` file and present them in a markdown format. First, I need to read the `nodes.py` file. +Read file: nodes.py +Okay, I have the content of `nodes.py`. Now I will extract the prompts and format them into a markdown file named `prompts.md`. +My apologies, it seems I tried to use a tool that isn't available to me. I will now output the extracted prompts directly into our conversation. + +# LLM Prompts from `nodes.py` + +This file contains all the LLM prompts used in the documentation generation workflow, extracted from the `nodes.py` file. + +## 1. `IdentifyAbstractions` Node + +This prompt analyzes the entire codebase to identify the most important core abstractions. + +```python +f""" +For the project `{project_name}`: + +Codebase Context: +{context} + +{language_instruction}Analyze the codebase context. +Identify the top 5-{max_abstraction_num} core most important abstractions to help those new to the codebase. + +For each abstraction, provide: +1. A concise `name`{name_lang_hint}. +2. A beginner-friendly `description` explaining what it is with a simple analogy, in around 100 words{desc_lang_hint}. +3. A list of relevant `file_indices` (integers) using the format `idx # path/comment`. + +List of file indices and paths present in the context: +{file_listing_for_prompt} + +Format the output as a YAML list of dictionaries: + +```yaml +- name: | + Query Processing{name_lang_hint} + description: | + Explains what the abstraction does. + It's like a central dispatcher routing requests.{desc_lang_hint} + file_indices: + - 0 # path/to/file1.py + - 3 # path/to/related.py +- name: | + Query Optimization{name_lang_hint} + description: | + Another core concept, similar to a blueprint for objects.{desc_lang_hint} + file_indices: + - 5 # path/to/another.js +# ... up to {max_abstraction_num} abstractions +```""" +``` + +## 2. `AnalyzeRelationships` Node + +This prompt takes the identified abstractions and analyzes their relationships to build a high-level overview of the project. + +```python +f""" +Based on the following abstractions and relevant code snippets from the project `{project_name}`: + +List of Abstraction Indices and Names{list_lang_note}: +{abstraction_listing} + +Context (Abstractions, Descriptions, Code): +{context} + +{language_instruction}Please provide: +1. A high-level `summary` of the project's main purpose and functionality in a few beginner-friendly sentences{lang_hint}. Use markdown formatting with **bold** and *italic* text to highlight important concepts. +2. A list (`relationships`) describing the key interactions between these abstractions. For each relationship, specify: + - `from_abstraction`: Index of the source abstraction (e.g., `0 # AbstractionName1`) + - `to_abstraction`: Index of the target abstraction (e.g., `1 # AbstractionName2`) + - `label`: A brief label for the interaction **in just a few words**{lang_hint} (e.g., "Manages", "Inherits", "Uses"). + Ideally the relationship should be backed by one abstraction calling or passing parameters to another. + Simplify the relationship and exclude those non-important ones. + +IMPORTANT: Make sure EVERY abstraction is involved in at least ONE relationship (either as source or target). Each abstraction index must appear at least once across all relationships. + +Format the output as YAML: + +```yaml +summary: | + A brief, simple explanation of the project{lang_hint}. + Can span multiple lines with **bold** and *italic* for emphasis. +relationships: + - from_abstraction: 0 # AbstractionName1 + to_abstraction: 1 # AbstractionName2 + label: "Manages"{lang_hint} + - from_abstraction: 2 # AbstractionName3 + to_abstraction: 0 # AbstractionName1 + label: "Provides config"{lang_hint} + # ... other relationships +``` + + +## 3. `OrderChapters` Node + +This prompt determines the best pedagogical order to explain the identified abstractions to a newcomer. + +```python +f""" +Given the following project abstractions and their relationships for the project `{project_name}`: + +Abstractions (Index # Name){list_lang_note}: +{abstraction_listing} + +Context about relationships and project summary: +{context} + +If you are going to make a tutorial for ```` {project_name} ````, what is the best order to explain these abstractions, from first to last? +Ideally, first explain those that are the most important or foundational, perhaps user-facing concepts or entry points. Then move to more detailed, lower-level implementation details or supporting concepts. + +Output the ordered list of abstraction indices, including the name in a comment for clarity. Use the format `idx # AbstractionName`. + +```yaml +- 2 # FoundationalConcept +- 0 # CoreClassA +- 1 # CoreClassB (uses CoreClassA) +- ... +``` + + + + +## 4. `WriteChapters` Node + +This prompt is executed for each abstraction to generate a detailed, beginner-friendly tutorial chapter in Markdown. + +```python +f""" +{language_instruction}Write a very beginner-friendly tutorial chapter (in Markdown format) for the project `{project_name}` about the concept: "{abstraction_name}". This is Chapter {chapter_num}. + +Concept Details{concept_details_note}: +- Name: {abstraction_name} +- Description: +{abstraction_description} + +Complete Tutorial Structure{structure_note}: +{item["full_chapter_listing"]} + +Context from previous chapters{prev_summary_note}: +{previous_chapters_summary if previous_chapters_summary else "This is the first chapter."} + +Relevant Code Snippets (Code itself remains unchanged): +{file_context_str if file_context_str else "No specific code snippets provided for this abstraction."} + +Instructions for the chapter (Generate content in {language.capitalize()} unless specified otherwise): +- Start with a clear heading (e.g., `# Chapter {chapter_num}: {abstraction_name}`). Use the provided concept name. + +- If this is not the first chapter, begin with a brief transition from the previous chapter{instruction_lang_note}, referencing it with a proper Markdown link using its name{link_lang_note}. + +- Begin with a high-level motivation explaining what problem this abstraction solves{instruction_lang_note}. Start with a central use case as a concrete example. The whole chapter should guide the reader to understand how to solve this use case. Make it very minimal and friendly to beginners. + +- If the abstraction is complex, break it down into key concepts. Explain each concept one-by-one in a very beginner-friendly way{instruction_lang_note}. + +- Explain how to use this abstraction to solve the use case{instruction_lang_note}. Give example inputs and outputs for code snippets (if the output isn't values, describe at a high level what will happen{instruction_lang_note}). + +- Each code block should be BELOW 10 lines! If longer code blocks are needed, break them down into smaller pieces and walk through them one-by-one. Aggresively simplify the code to make it minimal. Use comments{code_comment_note} to skip non-important implementation details. Each code block should have a beginner friendly explanation right after it{instruction_lang_note}. + +- Describe the internal implementation to help understand what's under the hood{instruction_lang_note}. First provide a non-code or code-light walkthrough on what happens step-by-step when the abstraction is called{instruction_lang_note}. It's recommended to use a simple sequenceDiagram with a dummy example - keep it minimal with at most 5 participants to ensure clarity. If participant name has space, use: `participant QP as Query Processing`. {mermaid_lang_note}. + +- Then dive deeper into code for the internal implementation with references to files. Provide example code blocks, but make them similarly simple and beginner-friendly. Explain{instruction_lang_note}. + +- IMPORTANT: When you need to refer to other core abstractions covered in other chapters, ALWAYS use proper Markdown links like this: [Chapter Title](filename.md). Use the Complete Tutorial Structure above to find the correct filename and the chapter title{link_lang_note}. Translate the surrounding text. + +- Use mermaid diagrams to illustrate complex concepts (```mermaid``` format). {mermaid_lang_note}. + +- Heavily use analogies and examples throughout{instruction_lang_note} to help beginners understand. + +- End the chapter with a brief conclusion that summarizes what was learned{instruction_lang_note} and provides a transition to the next chapter{instruction_lang_note}. If there is a next chapter, use a proper Markdown link: [Next Chapter Title](next_chapter_filename){link_lang_note}. + +- Ensure the tone is welcoming and easy for a newcomer to understand{tone_note}. + +- Output *only* the Markdown content for this chapter. + +Now, directly provide a super beginner-friendly Markdown output (DON'T need ```markdown``` tags): +""" +``` \ No newline at end of file diff --git a/tmp/run_analysis.sh b/tmp/run_analysis.sh new file mode 100755 index 00000000..3e5b3b22 --- /dev/null +++ b/tmp/run_analysis.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Shell script to analyze the local corebanking repository +# using YAML configuration file + +echo "Starting corebanking tutorial generation..." +source .venv/bin/activate + +# Validate configuration before running +echo "Validating configuration..." +python main.py "$1" --validate-only + +if [ $? -eq 0 ]; then + echo "Configuration is valid. Running analysis..." + python main.py "$1" +else + echo "Configuration validation failed. Please check the config file." + exit 1 +fi + +echo "Done" \ No newline at end of file From 63bec7b17e89fcb5d671d809480793739e3ea825 Mon Sep 17 00:00:00 2001 From: Louis-David Coulombe Date: Mon, 27 Oct 2025 21:11:42 -0400 Subject: [PATCH 13/15] Remove generated doc --- nesto/corebanking/01_event_.md | 179 ------------ nesto/corebanking/02_command_.md | 179 ------------ nesto/corebanking/03_aggregate_.md | 241 ---------------- nesto/corebanking/04_repository_.md | 229 --------------- nesto/corebanking/05_api_handler_.md | 225 --------------- nesto/corebanking/06_core_facade_.md | 160 ----------- nesto/corebanking/07_service_.md | 264 ----------------- nesto/corebanking/08_consumer_.md | 204 -------------- nesto/corebanking/09_product_engine_.md | 265 ------------------ ...0_simulation_services_and_repositories_.md | 205 -------------- nesto/corebanking/index.md | 78 ------ nesto/corebanking/review.md | 36 --- 12 files changed, 2265 deletions(-) delete mode 100644 nesto/corebanking/01_event_.md delete mode 100644 nesto/corebanking/02_command_.md delete mode 100644 nesto/corebanking/03_aggregate_.md delete mode 100644 nesto/corebanking/04_repository_.md delete mode 100644 nesto/corebanking/05_api_handler_.md delete mode 100644 nesto/corebanking/06_core_facade_.md delete mode 100644 nesto/corebanking/07_service_.md delete mode 100644 nesto/corebanking/08_consumer_.md delete mode 100644 nesto/corebanking/09_product_engine_.md delete mode 100644 nesto/corebanking/10_simulation_services_and_repositories_.md delete mode 100644 nesto/corebanking/index.md delete mode 100644 nesto/corebanking/review.md diff --git a/nesto/corebanking/01_event_.md b/nesto/corebanking/01_event_.md deleted file mode 100644 index 8d1f7c5c..00000000 --- a/nesto/corebanking/01_event_.md +++ /dev/null @@ -1,179 +0,0 @@ -# Chapter 1: Event - -Welcome to the `corebanking` project tutorial! We're thrilled to have you on board. In this first chapter, we'll dive into one of the most fundamental concepts in our system: the **Event**. - -## What's the Big Deal with Events? - -Imagine you're building a brand new digital bank. A customer, let's call her Alice, wants to open a new savings account. She fills out a form, clicks "Submit," and voilà, her account is created. - -But how does the bank *remember* that this account was created? How does it keep a permanent, trustworthy record of this action? What if we need to look back and see exactly when Alice's account was opened or what its initial details were? - -This is where **Events** come into play. - -In our `corebanking` system, an **Event** is an **immutable record of something significant that has already happened**. Think of it like an official entry in a historical logbook or a diary. Once an entry is written, it's not changed. - -When something important occurs in the system, like Alice's account being created, we record this fact as an Event. For example, after successfully processing Alice's request to create an account (which we'll later learn is a [Command](02_command_.md)), an `AccountCreatedEvent` is generated. - -## Key Characteristics of an Event - -Events have a few defining features: - -1. **Immutable Record:** Once an Event is recorded, it **cannot be changed**. It's a statement of fact about something that happened in the past. Like carving something in stone. -2. **Significant Happenings:** We don't create Events for every tiny operation. They are reserved for actions that have a meaningful impact on the system's state. Examples: `AccountCreatedEvent`, `FundsDepositedEvent`, `CustomerAddressUpdatedEvent`. -3. **Past Tense:** Event names usually reflect that the action has already completed. Notice the "ed" or "d" at the end: `AccountCreatedEvent`, `PaymentDeviceLinkedEvent`. -4. **Source of Truth:** Events are the ultimate, undeniable truth for what has occurred. If we ever need to understand the current state of something (like an account's balance or status), we can, in theory, replay all the Events related to it from the very beginning. It's like re-reading a ship's log to know its entire journey. -5. **Result of a [Command](02_command_.md):** Typically, an Event is produced after the system processes a [Command](02_command_.md). A [Command](02_command_.md) is a request to do something (e.g., "Create Account"). If the command is successful and changes the system's state, one or more Events are generated as a result. - -## Events in Action: Creating Alice's Account - -Let's revisit Alice creating her savings account: - -1. Alice submits her request (this is a [Command](02_command_.md)). -2. The `corebanking` system processes this request. It checks if Alice can open an account, if she provided all necessary details, etc. -3. If everything is okay, the system changes its state: a new account for Alice now exists! -4. To record this fact, the system generates an `AccountCreatedEvent`. This Event will contain all the crucial information: Alice's customer ID, the new account ID, the type of account, the currency, the date and time of creation, etc. -5. This `AccountCreatedEvent` is then durably stored, perhaps in a special database. It becomes part of the bank's permanent history. - -##What Does an Event Look Like? (A Peek at the Code) - -In Go, an Event is often represented as a struct. There's a general structure for all events, and then specific data for each type of event. - -Here's a simplified general `Event` structure from our system: - -```go -// From: api/events.go - -// Event represents an event -type Event struct { - ID uuid.UUID // Unique ID for this specific event instance - AggregateID uuid.UUID // ID of the entity this event pertains to (e.g., Account ID) - AggregateType string // Type of the entity (e.g., "Account") - Type string // Specific type of event (e.g., "AccountCreatedEvent") - Data interface{} // The actual data specific to this event type - Created time.Time // When the event was created - // ... other general fields like Actor, TenantID ... -} -``` - -* `ID`: Every single event occurrence gets its own unique identifier. -* `AggregateID`: This tells us which specific entity the event is about. For an `AccountCreatedEvent`, this would be the ID of the newly created account. We'll learn more about [Aggregates](03_aggregate_.md) later. -* `AggregateType`: The kind of entity, like "Account" or "Customer". -* `Type`: A string that clearly states what kind of event this is, e.g., `com.nesto.corebanking.accounts.events.AccountCreatedEvent`. -* `Data`: This is the payload, containing the specific details of *what happened*. For an `AccountCreatedEvent`, this would hold the account category, currency, etc. -* `Created`: A timestamp indicating when the event occurred. - -The `Data` field itself would be another struct, specific to the event type. For example, an `AccountCreatedEvent` might have data like this (simplified): - -```go -// From: api/account_aggregate.go - -// AccountCreatedEvent represents a created account. -type AccountCreatedEvent struct { - Category string `avro:"category"` - Product *AccountProduct `avro:"product"` - AvailableCurrencies []string `avro:"availableCurrencies"` - CustomerIDs []uuid.UUID `avro:"customerIds"` - // ... other fields like Parameters, Metadata ... -} -``` -This `AccountCreatedEvent` struct holds all the specific details that are relevant when a new account is made. - -## How an Event is "Born" - The Internals - -Let's look under the hood to see how an `AccountCreatedEvent` comes to life. - -1. **A Request Arrives:** Alice's request to create an account comes in as a [Command](02_command_.md) (e.g., `CreateAccountCommand`). -2. **Processing by an [Aggregate](03_aggregate_.md):** An [Aggregate](03_aggregate_.md) is responsible for handling commands related to a specific entity (like an `AccountAggregate` for accounts). It takes the `CreateAccountCommand`. -3. **Validation and State Change:** The `AccountAggregate` checks if the command is valid (e.g., are all required fields present?). If valid, it determines what state changes are needed. For a new account, this means setting its initial properties. -4. **Event Generation:** Because the state changed, the `AccountAggregate` now creates an `AccountCreatedEvent`, filling it with details from the command and any system-generated data (like the creation timestamp). -5. **Event Storage:** This newly minted event is then passed to an "Event Store" (a specialized database or system component) to be saved permanently. - -Here's a simplified sequence diagram: - -```mermaid -sequenceDiagram - participant Alice - participant CoreBankingAPI as API - participant AccountAgg as Account Aggregate - participant EventStore - - Alice->>API: Submit Create Account Form - API->>AccountAgg: Process CreateAccountCommand - AccountAgg-->>AccountAgg: Validate, decide to create account - AccountAgg->>EventStore: New AccountCreatedEvent(details...) - EventStore-->>AccountAgg: Event Saved! - AccountAgg-->>API: Account Creation Successful - API-->>Alice: Your account is created! -``` - -Let's look at a snippet from `api/account_aggregate.go` where an event is created within the `HandleCommand` method. This method is part of the [Aggregate](03_aggregate_.md) (which we'll cover in detail later). - -```go -// Simplified from api/account_aggregate.go -// Inside AccountAggregate's HandleCommand method: - -case *CreateAccountCommand: // This is the request to create an account - // ... (some validation and setup logic) ... - - // Prepare the specific data for our AccountCreatedEvent - data := &AccountCreatedEvent{ - Category: c.Category, - Product: c.Product, - AvailableCurrencies: c.AvailableCurrencies, - CustomerIDs: c.CustomerIDs, - // ... other relevant details from the command ... - } - - // This is the magic moment! An Event is created. - // a.NewEvent() wraps 'data' with general event info (ID, timestamp, etc.) - event = a.NewEvent(command, data) - - // ... (logic to apply this event to the aggregate's state and save it) ... - a.ApplyChangeHelper(a, event, true) -``` -In this code: -* We receive a `CreateAccountCommand` (aliased as `c`). -* We gather the necessary details into an `AccountCreatedEvent` struct (`data`). -* `a.NewEvent(command, data)` is a helper method that takes the original command and the event-specific `data` to construct the full `Event` object, including its unique ID, timestamp, and type. -* `a.ApplyChangeHelper` is another crucial step. After an event is "born," the [Aggregate](03_aggregate_.md) itself uses this event to update its own internal state. This ensures the [Aggregate's](03_aggregate_.md) in-memory representation reflects the new reality. - -The `ApplyChange` method (or a helper it calls) looks at the type of event and updates the [Aggregate's](03_aggregate_.md) fields accordingly: - -```go -// Simplified from api/account_aggregate.go -// Inside AccountAggregate's ApplyChange method: - -func (a *AccountAggregate) ApplyChange(event es.Event) { - switch e := event.Data.(type) { // e is the specific event data - case *AccountCreatedEvent: - a.ID = event.AggregateID // The aggregate now knows its ID - a.Category = e.Category - a.Status = AccountStatusCreated // Set initial status - a.Created = event.Created // Record creation time - // ... update other fields based on AccountCreatedEvent ... - case *AccountActivatedEvent: - a.Status = AccountStatusActivated - // ... - // ... other event types ... - } -} -``` -This ensures that the `AccountAggregate`'s state is consistent with the events it has produced and processed. - -## Why Immutability and "Facts" Matter So Much - -The fact that Events are immutable historical records is incredibly powerful: - -* **Reliability & Auditability:** You have a perfect, trustworthy audit trail of everything significant that has happened. Need to know why an account is in a particular state? Replay its events! This is invaluable for debugging, compliance, and understanding system behavior. -* **State Reconstruction:** If, for some reason, the current "snapshot" of an account's data gets corrupted, you can rebuild it by replaying all its historical Events in order. -* **Decoupling:** Other parts of the system can subscribe to events and react to them independently, without needing to know the intricate details of how the event was produced. For example, a notification service could listen for `AccountCreatedEvent` and send a welcome email to Alice. - -## Conclusion - -Events are the bedrock of our `corebanking` system's memory. They are **immutable facts** representing **significant past occurrences**. Each Event tells a small part of a larger story, like the story of Alice's bank account. By recording and storing these Events, we build a reliable, auditable, and resilient system. - -We've seen that Events are often born from processing requests. In the next chapter, we'll take a closer look at these requests themselves. Get ready to learn about the [Command](02_command_.md)! - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/02_command_.md b/nesto/corebanking/02_command_.md deleted file mode 100644 index d8113c5b..00000000 --- a/nesto/corebanking/02_command_.md +++ /dev/null @@ -1,179 +0,0 @@ -# Chapter 2: Command - -In our [previous chapter](01_event_.md), we learned about [Events](01_event_.md) – the immutable records of things that have already happened in our `corebanking` system. For example, when Alice successfully created her bank account, an `AccountCreatedEvent` was generated and stored. - -But how did Alice *tell* the bank she wanted to create an account in the first place? How does she ask the bank to do something, like deposit money or change her address? This is where **Commands** come in. - -## What's a Command? The Bank's Instruction Slip - -Imagine you walk into a physical bank. To make a deposit, you fill out a deposit slip. To open an account, you fill out an application form. These forms are your formal instructions to the bank. - -In the `corebanking` system, a **Command** is very similar. It's like an **instruction or a formal request** you submit to the system. It represents an **intent** to change something. - -Think of it this way: -* An [Event](01_event_.md) says: "This happened!" (e.g., `AccountCreatedEvent`) -* A **Command** says: "I want this to happen!" (e.g., `CreateAccountCommand`) - -Commands are specific, named operations that carry all the necessary data to perform an action. They are the primary way external users or other systems interact with the core logic of our bank to request changes. - -## Key Characteristics of a Command - -1. **Intent to Act:** A Command expresses a desire for the system to perform an action. It's a request, not a statement of fact. -2. **Specific Operation:** Each Command has a clear name that describes the intended action, like `CreateAccountCommand`, `DepositFundsCommand`, or `UpdateCustomerAddressCommand`. -3. **Carries Data:** A Command bundles all the information needed to execute the desired action. For instance, a `CreateAccountCommand` would include details like who the customer is, what type of account they want, and in what currency. -4. **Sent for Processing:** Commands are typically sent to a specific part of our system called an [Aggregate](03_aggregate_.md) (we'll learn about Aggregates in the next chapter!). The [Aggregate](03_aggregate_.md) is responsible for deciding if the Command is valid and then carrying out the action. -5. **May Result in [Events](01_event_.md):** If a Command is successfully processed and leads to a change in the system's state, one or more [Events](01_event_.md) are usually generated to record that change. If the Command is invalid (e.g., trying to deposit into a non-existent account), it might be rejected, and no [Events](01_event_.md) related to the intended change would be created. - -## Commands in Action: Alice's Account Creation (The "How") - -Let's go back to Alice wanting to open her savings account: - -1. **Alice Expresses Her Intent:** Alice, through a web form or a mobile app, indicates she wants a new account. She provides her details (customer ID, desired currency, etc.). -2. **The `CreateAccountCommand` is Born:** The application she's using takes this information and packages it into a `CreateAccountCommand`. This command now holds all the data needed to open the account. -3. **Command is Sent:** This `CreateAccountCommand` is sent to the `corebanking` system. -4. **Processing by an [Aggregate](03_aggregate_.md):** An `AccountAggregate` (which is responsible for managing account-related operations) receives the `CreateAccountCommand`. -5. **Validation & Logic:** The `AccountAggregate` checks: - * Is the customer ID valid? - * Is the currency supported? - * Are all required fields present? - * Any other business rules? -6. **Success and [Event](01_event_.md) Generation:** If everything is valid, the `AccountAggregate` proceeds to create the account. As a result of this successful operation, it generates an `AccountCreatedEvent` (which we saw in Chapter 1!). -7. **Failure:** If the command is invalid (e.g., Alice provides an unsupported currency), the `AccountAggregate` rejects the command, and no `AccountCreatedEvent` is generated. An error message would typically be returned to Alice. - -## What Does a Command Look Like? (A Peek at the Code) - -In Go, just like [Events](01_event_.md), Commands are often represented as structs. They carry the data necessary for the operation. - -Here's a simplified look at the `CreateAccountCommand` from our `corebanking` project (`api/account_aggregate.go`): - -```go -// From: api/account_aggregate.go - -// CreateAccountCommand represents the creation of an account. -type CreateAccountCommand struct { - es.BaseCommand // Provides common fields for all commands - Category string `json:"category"` - Product *AccountProduct `json:"product"` // Details about the specific account product - AvailableCurrencies []string `json:"availableCurrencies"` - CustomerIDs []uuid.UUID `json:"customerIds"` - // ... other fields like Parameters, Metadata ... -} -``` - -Let's break this down: -* `es.BaseCommand`: This is a standard part of many commands in our system. It usually includes an `AggregateID`. For `CreateAccountCommand`, this `AggregateID` will be the ID of the *new* account we want to create. For a command like `DepositFundsCommand`, the `AggregateID` would be the ID of the *existing* account to deposit into. -* `Category`: What kind of account is it (e.g., "ASSET", "LIABILITY"). -* `Product`: Specifies the type of banking product (e.g., "Savings Account Basic", "Current Account Premium"). -* `AvailableCurrencies`: Which currencies can this account hold (e.g., `["USD", "EUR"]`). -* `CustomerIDs`: A list of customer IDs associated with this account. - -Each field in the command provides a piece of information that the system needs to fulfill the request. - -Another example, a `CloseAccountCommand`, would be simpler: - -```go -// From: api/account_aggregate.go - -// CloseAccountCommand represents the closing of an account. -type CloseAccountCommand struct { - es.BaseCommand // Contains the ID of the account to close - Reason string `json:"reason"` // Why is the account being closed? -} -``` -This command needs to know *which* account to close (via `BaseCommand`'s `AggregateID`) and optionally, a reason for closing it. - -## How a Command is Processed: Under the Hood - -When a command arrives, how does the system handle it? - -1. **Dispatch:** The command is typically sent from an API layer (like a web server handling Alice's request) to the core business logic. -2. **Targeting an [Aggregate](03_aggregate_.md):** The command is routed to the correct [Aggregate](03_aggregate_.md). An [Aggregate](03_aggregate_.md) is like a guardian for a specific piece of data (e.g., an `AccountAggregate` guards a specific bank account). If it's a `CreateAccountCommand`, a new `AccountAggregate` instance might be effectively created to handle it. If it's `DepositFundsCommand`, it's sent to the existing `AccountAggregate` for that account. -3. **Validation:** The [Aggregate](03_aggregate_.md) first validates the command. Can this action be performed? Is all the data correct and complete? -4. **Business Logic:** If valid, the [Aggregate](03_aggregate_.md) executes the business rules associated with the command. -5. **[Event](01_event_.md) Generation:** If the business logic results in a state change, the [Aggregate](03_aggregate_.md) creates one or more [Events](01_event_.md) to describe what happened. - -Here's a simplified sequence diagram illustrating the flow for creating an account: - -```mermaid -sequenceDiagram - participant AliceApp as Alice's App - participant CoreAPI as Core Banking API - participant AccountAgg as Account Aggregate - participant EventStore - - AliceApp->>CoreAPI: I want to create an account (details...) - CoreAPI->>AccountAgg: CreateAccountCommand{details...} - AccountAgg-->>AccountAgg: Validate command data (e.g., currency OK?) - AccountAgg-->>AccountAgg: Business logic: Okay, let's create it! - AccountAgg->>EventStore: Store this new AccountCreatedEvent(...) - EventStore-->>AccountAgg: Event saved! - AccountAgg-->>CoreAPI: Success! Account created. - CoreAPI-->>AliceApp: Your account is ready! -``` - -Let's look at a snippet from `api/account_aggregate.go` where an `AccountAggregate` handles a `CreateAccountCommand`. This happens inside a method called `HandleCommand`: - -```go -// Simplified from AccountAggregate.HandleCommand in api/account_aggregate.go - -// ... (inside HandleCommand method) ... -switch c := command.(type) { // 'c' is the incoming command -case *CreateAccountCommand: - // 1. Check if a product is specified and valid (simplified) - if c.Product != nil { - // ... (logic to validate product exists) ... - } - - // 2. The command carries the intended ID for the new account - a.ID = command.GetAggregateID() // 'a' is the AccountAggregate - - // 3. Prepare the data for the Event that will be created - eventData := &AccountCreatedEvent{ - Category: c.Category, - Product: c.Product, - CustomerIDs: c.CustomerIDs, - AvailableCurrencies: c.AvailableCurrencies, - // ... other details from the command 'c' ... - } - - // 4. Create the actual Event (as we saw in Chapter 1) - event := a.NewEvent(command, eventData) - - // 5. Apply the event to change the aggregate's state and record it - a.ApplyChangeHelper(a, event, true) // This stores the event for saving -``` - -In this snippet: -1. The `HandleCommand` method receives the `CreateAccountCommand` (aliased as `c`). -2. It performs some initial validation (like checking the product). -3. It uses the data *from the command* (`c.Category`, `c.Product`, etc.) to populate an `AccountCreatedEvent`. -4. The `a.NewEvent(...)` function (which we touched on in Chapter 1) creates the full [Event](01_event_.md) object, adding things like a unique event ID and timestamp. -5. `a.ApplyChangeHelper(...)` is a crucial step where the [Aggregate](03_aggregate_.md) updates its own state based on this new event and adds the event to a list of changes to be saved. - -So, the **Command** provides the *intent* and the *data*, and the [Aggregate's](03_aggregate_.md) `HandleCommand` method uses that to *do the work* and produce [Events](01_event_.md). - -## Commands vs. [Events](01_event_.md): A Quick Reminder - -It's vital to distinguish between Commands and [Events](01_event_.md): - -| Feature | Command | [Event](01_event_.md) | -| :-------------- | :-------------------------------------------- | :------------------------------------------------------ | -| **Purpose** | Request an action (intent) | Record a fact (something that happened) | -| **Tense** | Present/Future (e.g., "Create Account") | Past (e.g., "Account Created") | -| **Outcome** | May succeed or fail | Is a record of a successful state change | -| **Result of** | User input, system process | Successful command processing | -| **Analogy** | Filling out a request form | An entry in a historical logbook | - -You send a `CreateAccountCommand` (your request). If it's successful, an `AccountCreatedEvent` (the historical record) is generated. - -## Conclusion - -Commands are the way we tell our `corebanking` system what we want to do. They are formal, data-rich instructions representing an **intent to change the system's state**. They are named operations like `CreateAccountCommand` or `DepositFundsCommand`. - -When a Command is received, it's typically handled by an [Aggregate](03_aggregate_.md), which validates it, executes business logic, and, if successful, produces [Events](01_event_.md) to record the changes. - -Now that we understand how we *ask* the system to do things (Commands) and how the system *remembers* what happened ([Events](01_event_.md)), it's time to meet the component responsible for processing these Commands and generating those Events. Let's dive into the world of the [Aggregate](03_aggregate_.md)! - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/03_aggregate_.md b/nesto/corebanking/03_aggregate_.md deleted file mode 100644 index 4d605b58..00000000 --- a/nesto/corebanking/03_aggregate_.md +++ /dev/null @@ -1,241 +0,0 @@ -# Chapter 3: Aggregate - -Welcome to Chapter 3! In our [previous chapter](02_command_.md), we learned about [Commands](02_command_.md) – how we tell the `corebanking` system what we want to do, like "Create an Account" or "Deposit Funds." We also know from [Chapter 1](01_event_.md) that when these [Commands](02_command_.md) are successful, [Events](01_event_.md) are generated to record what happened. - -But who is in charge of receiving a [Command](02_command_.md), deciding if it's okay, and then creating the [Event](01_event_.md)? If Alice wants to deposit money into *her specific account*, how does the system make sure the change happens only to *her* account and follows all the rules? This is where the **Aggregate** steps in. - -## Meet the Main Character: The Aggregate - -Think of an **Aggregate** as the main character in a story, like a specific bank account (Account #12345) or a particular customer (Alice Wonderland). It's a self-contained unit that looks after its own information (its "state") and its own rules. - -When a [Command](02_command_.md) arrives—like a request to deposit money into Account #12345—it's the `AccountAggregate` for Account #12345 that handles it. This Aggregate will: -1. Check if the command is valid (e.g., "Is the account open?", "Is the deposit amount positive?"). -2. If everything is okay, it will change its state (e.g., increase its balance). -3. Then, it will record this change as a fact by producing an [Event](01_event_.md) (e.g., `FundsDepositedEvent`). - -This way, the Aggregate ensures that all changes to *its* data are consistent and follow the rules. It also provides a historical record of everything that has happened to it through the [Events](01_event_.md) it produces. Aggregates are a cornerstone of the Event Sourcing pattern used in our `corebanking` system. - -## Key Ideas About Aggregates - -1. **A Guardian of Data:** An Aggregate is responsible for a specific piece of data or a small group of related data. For example, an `AccountAggregate` is responsible for *one specific bank account*. It manages the account's balance, status, customer links, etc. -2. **Unique Identity:** Each Aggregate instance has a unique ID. So, `AccountAggregate` with ID `acc-123` is different from `AccountAggregate` with ID `acc-456`. -3. **Manages Its Own State:** The "state" is the current information about the entity. For an account, this includes its balance, status (CREATED, ACTIVATED, CLOSED), currency, etc. -4. **Enforces Business Rules:** Aggregates contain the logic to check if a requested action (a [Command](02_command_.md)) is allowed. Can you withdraw more money than you have? Can you close an account that has a pending transaction? The Aggregate knows! -5. **Processes [Commands](02_command_.md):** It takes a [Command](02_command_.md) as input. -6. **Produces [Events](01_event_.md):** If a [Command](02_command_.md) is valid and changes the state, the Aggregate creates one or more [Events](01_event_.md) to describe what happened. -7. **Consistency Boundary:** All changes within a single Aggregate happen together (atomically). Either all changes are applied, or none are. This keeps the data for that specific entity consistent. - -## Example: Alice Deposits Money - -Let's imagine Alice (customer `cust-001`) has an account (`acc-123`) and wants to deposit $50. - -1. **A [Command](02_command_.md) is Sent:** Alice's banking app creates a `DepositFundsCommand` with details: `AccountID: "acc-123"`, `Amount: 50.00`. -2. **The Right Aggregate is Found:** The system directs this command to the `AccountAggregate` responsible for account `acc-123`. -3. **The Aggregate Works:** - * The `AccountAggregate` for `acc-123` loads its current state (perhaps its current balance is $100 and its status is "ACTIVATED"). - * It checks business rules: Is the account "ACTIVATED"? Yes. Is $50 a valid amount? Yes. - * Since the command is valid, the Aggregate decides an `AccountCreditedEvent` (or similar) needs to happen. -4. **An [Event](01_event_.md) is Born:** It creates an `AccountCreditedEvent` with details: `AmountCredited: 50.00`, `NewBalance: 150.00` (or the event might just state the amount credited, and the balance is recalculated). -5. **State is Updated:** The Aggregate updates its own internal state. Its balance is now $150. -6. **[Event](01_event_.md) is Stored:** The `AccountCreditedEvent` is saved permanently. - -Now, account `acc-123` correctly reflects the deposit, and there's a permanent record of it. - -## What Does an Aggregate Look Like? (A Peek at the Code) - -In Go, an Aggregate is typically a struct. It often embeds a base struct that provides common functionalities and includes fields to hold its state. - -Here's a very simplified view of an `AccountAggregate` from `api/account_aggregate.go`: - -```go -// From: api/account_aggregate.go -// Simplified AccountAggregate structure -type AccountAggregate struct { - es.BaseAggregate // Provides common aggregate features (like ID, event helpers) - - // State of the account (the actual data for *this* account instance) - *AccountAggregateState - - // (Dependencies like repositories are omitted for simplicity here) -} -``` -* `es.BaseAggregate`: This is a helper struct from our event sourcing library (`es`). It provides common things an aggregate needs, like a place to store its ID, version, and methods for creating new [Events](01_event_.md). -* `*AccountAggregateState`: This pointer field is where the actual data specific to *this* account instance is stored. - -The `AccountAggregateState` itself is another struct: - -```go -// From: api/account_aggregate.go -// AccountAggregateState holds the data for an account -type AccountAggregateState struct { - Category string // e.g., "ASSET", "LIABILITY" - Status string // e.g., "CREATED", "ACTIVATED", "CLOSED" - CustomerIDs []uuid.UUID // Which customers own this account - // ... other fields like product info, currency, parameters ... - Created time.Time // When the account was created - // (The actual balance is often managed by postings and calculated, - // but for conceptual understanding, imagine it's part of the state) -} -``` -This `AccountAggregateState` holds all the important details about one specific account. When we talk about an Aggregate's "state," we're talking about the values in these fields. - -## How an Aggregate Handles a [Command](02_command_.md) - -There are two main methods in an Aggregate that work together: -1. `HandleCommand`: This is the entry point. It receives a [Command](02_command_.md), applies business rules, and if successful, creates [Event(s)](01_event_.md). -2. `ApplyChange`: This method takes an [Event](01_event_.md) and uses it to update the Aggregate's internal state. - -Let's look at simplified versions: - -### 1. `HandleCommand` - -When a [Command](02_command_.md) like `CreateAccountCommand` comes in: - -```go -// Simplified from AccountAggregate.HandleCommand in api/account_aggregate.go -// This method decides what to do when a command arrives. -func (a *AccountAggregate) HandleCommand(ctx context.Context, command es.Command) error { - var event es.Event // This will hold the event we generate - - switch c := command.(type) { // 'c' is the specific command (e.g., CreateAccountCommand) - case *CreateAccountCommand: - // 1. Validations (e.g., is product specified? are customers valid?) - // (Detailed validation logic omitted for brevity) - if c.Product == nil { - return errors.New("product is required") // Simplified error - } - - // 2. Set the ID for this new account aggregate from the command - // The command carries the intended ID for the new entity. - a.ID = command.GetAggregateID() - - // 3. Prepare data for the AccountCreatedEvent using info from the command - eventData := &AccountCreatedEvent{ - Category: c.Category, - Product: c.Product, // Details of the account type - CustomerIDs: c.CustomerIDs, - // ... other details from the CreateAccountCommand 'c' ... - } - - // 4. Create the actual Event object using a helper from BaseAggregate - event = a.NewEvent(command, eventData) // 'command' provides context - - // 5. Tell the aggregate to apply this change to itself - // and mark the event to be saved. - a.ApplyChangeHelper(a, event, true) - - // case *DepositFundsCommand: - // // ... logic to handle deposit, create FundsDepositedEvent ... - // case *ActivateAccountCommand: - // // ... logic to handle activation, create AccountActivatedEvent ... - } - return nil // If successful -} -``` -* The `switch` statement checks the type of [Command](02_command_.md). -* For `CreateAccountCommand`, it performs validations. -* It then prepares the data for an `AccountCreatedEvent` using information from the command. -* `a.NewEvent(...)` is a helper (from `es.BaseAggregate`) that wraps `eventData` with standard event fields like a unique Event ID, timestamp, Aggregate ID, etc. -* `a.ApplyChangeHelper(...)` is another crucial helper. It does two things: - 1. Calls `a.ApplyChange(event)` (see below) to update the aggregate's in-memory state. - 2. Adds the `event` to a list of "changes" that will be saved to the Event Store later. - -### 2. `ApplyChange` - -After an [Event](01_event_.md) is created by `HandleCommand`, the `ApplyChange` method is called to update the Aggregate's state. **The state of an Aggregate is *always* a result of applying its [Events](01_event_.md).** - -```go -// Simplified from AccountAggregate.ApplyChange in api/account_aggregate.go -// This method updates the aggregate's state based on an event. -func (a *AccountAggregate) ApplyChange(event es.Event) { - // 'e' is the specific event data (like AccountCreatedEvent details) - switch e := event.Data.(type) { - case *AccountCreatedEvent: - // The aggregate's ID is usually set by BaseAggregate from the event - a.ID = event.AggregateID - // Now, update the AccountAggregateState fields - a.Category = e.Category - a.Product = e.Product // Store product info - a.Status = AccountStatusCreated // Set initial status - a.CustomerIDs = e.CustomerIDs - a.Created = event.Created // Record creation time - // ... update other AccountAggregateState fields based on the event ... - - case *AccountActivatedEvent: // If an AccountActivatedEvent occurred - a.Status = AccountStatusActivated // Update the status - a.Updated = event.Created // Update modification time - - // case *FundsDepositedEvent: - // a.Balance = a.Balance + e.AmountDeposited // Hypothetical balance update - // a.Updated = event.Created - } -} -``` -* This method also uses a `switch` on the *type* of [Event](01_event_.md). -* For an `AccountCreatedEvent`, it sets the initial properties of the `AccountAggregateState` (like `Category`, `Status`, `Created` timestamp). -* For an `AccountActivatedEvent`, it would just update the `Status` field. -* This ensures that the Aggregate's in-memory state accurately reflects all the [Events](01_event_.md) that have happened to it. - -## The Life Cycle: Loading an Existing Aggregate - -What if Alice wants to deposit into an *existing* account? The system doesn't just create a new `AccountAggregate`. Instead: -1. It knows the `AccountID` (e.g., `acc-123`) from the `DepositFundsCommand`. -2. It goes to the Event Store (where all [Events](01_event_.md) are saved). -3. It fetches *all* the past [Events](01_event_.md) for `acc-123` in the order they happened. -4. It creates a fresh `AccountAggregate` instance. -5. It then "replays" each historical [Event](01_event_.md) by calling `ApplyChange` for each one. - * `ApplyChange(AccountCreatedEvent{...})` -> sets initial state. - * `ApplyChange(AccountActivatedEvent{...})` -> sets status to "ACTIVATED". - * `ApplyChange(SomeOtherEvent{...})` -> updates state further. -6. After replaying all its history, the `AccountAggregate` is now in its correct, current state, ready to handle the new `DepositFundsCommand`. - -This process of rebuilding state from [Events](01_event_.md) is fundamental to Event Sourcing. - -## Why Aggregates are So Important - -* **Encapsulation:** They bundle data (state) and behavior (rules) together, making the system easier to understand and manage. -* **Consistency:** They ensure that the data for a specific entity (like an account) is always valid and consistent according to business rules. -* **Clear Responsibilities:** It's clear which Aggregate is responsible for which [Commands](02_command_.md) and which part of the system's data. -* **Event Sourcing Enabler:** They are the primary producers of [Events](01_event_.md) in an Event Sourced system. The history of [Events](01_event_.md) an Aggregate produces *is* the history of that entity. - -## The Big Picture: Processing a Command - -Let's visualize how a command flows through an Aggregate to produce an Event. - -```mermaid -sequenceDiagram - participant ClientApp as Alice's App - participant CoreAPI as Core Banking API - participant AccountAgg as Account Aggregate (for acc-123) - participant EventStore - - ClientApp->>CoreAPI: Request: Deposit $50 into Account 123 - CoreAPI->>AccountAgg: Process DepositFundsCommand{AccountID: 123, Amount: 50} - Note over AccountAgg: 1. Load/Rehydrate Account 123's state
by replaying its past events from EventStore. - AccountAgg-->>AccountAgg: 2. HandleCommand: Validate Command (e.g., account active?) - AccountAgg-->>AccountAgg: 3. HandleCommand: Business Logic: Okay to deposit. - AccountAgg->>AccountAgg: 4. HandleCommand: Create FundsDepositedEvent{Amount: 50} - AccountAgg->>AccountAgg: 5. HandleCommand calls ApplyChange(FundsDepositedEvent)
--> Update internal state (e.g., balance) - Note over AccountAgg: 6. Event is marked for saving. - AccountAgg->>EventStore: 7. System saves FundsDepositedEvent - EventStore-->>AccountAgg: Event Saved! - AccountAgg-->>CoreAPI: Success! Deposit processed. - CoreAPI-->>ClientApp: Your deposit is successful! -``` -This diagram shows that the Aggregate is the central processor. It loads its state, validates the [Command](02_command_.md), applies logic, creates an [Event](01_event_.md), updates itself based on that [Event](01_event_.md), and then the [Event](01_event_.md) gets stored. - -## Conclusion - -Aggregates are like mini-managers for specific entities in our system, such as individual bank accounts or customers. They are responsible for: -* Guarding their own data (state). -* Enforcing business rules. -* Processing incoming [Commands](02_command_.md). -* Producing [Events](01_event_.md) as a record of successful changes. - -By doing this, Aggregates ensure data consistency and are the heart of how our `corebanking` system uses Event Sourcing. They take the "intent" from a [Command](02_command_.md) and, if valid, turn it into a historical "fact" as an [Event](01_event_.md). - -Now that we understand how Aggregates work and how their state can be built from [Events](01_event_.md), you might be wondering: how are these Aggregates (and their [Events](01_event_.md)) actually loaded from and saved to storage? That's where our next topic comes in. Get ready to learn about the [Repository](04_repository_.md)! - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/04_repository_.md b/nesto/corebanking/04_repository_.md deleted file mode 100644 index 1c6b2e27..00000000 --- a/nesto/corebanking/04_repository_.md +++ /dev/null @@ -1,229 +0,0 @@ -# Chapter 4: Repository - -Welcome to Chapter 4! In the [previous chapter](03_aggregate_.md), we learned about [Aggregates](03_aggregate_.md), like our `AccountAggregate`. We saw how they manage their own state (like an account's balance or status) by processing [Commands](02_command_.md) and then producing [Events](01_event_.md) to record what happened. - -Now, a couple of important questions arise: -1. When an `AccountAggregate` needs to process a [Command](02_command_.md) for an *existing* account, how does it get its current state? Remember, its state is built by replaying its past [Events](01_event_.md). Where do these [Events](01_event_.md) come from? -2. After an `AccountAggregate` processes a [Command](02_command_.md) and creates new [Events](01_event_.md), where do these new [Events](01_event_.md) get stored so they are not forgotten? -3. Beyond [Aggregates](03_aggregate_.md), what if other parts of our bank system simply need to *read* account information – say, to display a list of Alice's accounts on a web page? How do they get this data without needing to understand all the [Events](01_event_.md)? - -This is where the **Repository** pattern comes to our rescue! - -## What is a Repository? The Bank's Diligent Archivist - -Imagine a bank's large, old-fashioned archive room, filled with rows and rows of filing cabinets. This room has a team of dedicated archivists. - -* If a bank teller (let's say, an [Aggregate](03_aggregate_.md)) needs the history of a specific account to understand its current situation, they don't go rummaging through the cabinets themselves. They fill out a request slip and give it to an archivist. The archivist knows exactly where to find the files (the [Events](01_event_.md) or account data) and brings them back. -* If the teller completes a transaction and needs to file away the record (a new [Event](01_event_.md) or updated account information), they hand it to the archivist, who ensures it's stored in the right place, safely and correctly. - -A **Repository** in our `corebanking` system works just like this team of archivists. It's a component responsible for **persisting (saving) and retrieving data**. This data could be account details, customer information, transaction histories, or the [Events](01_event_.md) that make up an [Aggregate's](03_aggregate_.md) history. - -The crucial part is that the Repository provides an **abstraction** over the actual data storage mechanism. This means other parts of the system (like [Aggregates](03_aggregate_.md) or [Services](07_service_.md) which we'll see later) don't need to know if the data is stored in a MySQL database, a different type of database, or even text files. They just talk to the Repository using simple, clear methods. - -## Key Ideas About Repositories - -1. **Hides Storage Details:** The rest of the application doesn't care *how* or *where* data is stored. Is it SQL? Is it a NoSQL database? The Repository handles those details. This is like not needing to know the archivist's specific filing system, just that they can get you the file you need. -2. **Defined Contract (Interface):** A Repository offers a clear set of operations, like `GetByID()`, `Save()`, or `Search()`. In Go, this "contract" is usually defined by an `interface`. -3. **One Repository per Data Type (Usually):** You'll often have a specific repository for each main type of data you're managing. For example: - * An `AccountsRepository` to manage `Account` data. - * A `CustomersRepository` to manage `Customer` data. -4. **Different Repositories for Different Needs:** - * **For [Aggregates](03_aggregate_.md) (Event Sourcing):** In an Event Sourced system like ours, [Aggregates](03_aggregate_.md) (like `AccountAggregate`) are special. They are not stored directly as a single row in a database. Instead, their state is derived from their history of [Events](01_event_.md). A specialized type of repository (often called an `EventSourcedRepository` or similar) is used: - * To **load** an [Aggregate](03_aggregate_.md): It fetches all its past [Events](01_event_.md) from an "Event Store" (a database optimized for storing events). - * To **save** an [Aggregate](03_aggregate_.md): It takes any new [Events](01_event_.md) the [Aggregate](03_aggregate_.md) has produced and stores them in the Event Store. - * **For Read Data (Projections/Views):** Sometimes, we need to query data that's already nicely formatted for display or reporting – this is often called a "read model" or "projection." For example, when displaying account details on a screen, we want the current status, balance, etc., directly, not a list of [Events](01_event_.md). Repositories are also used to fetch this kind of data. Our `AccountsRepository` in `api/accounts.go` is an example of this type. - -## Using a Repository: Let's Look at Account Data - -Let's focus on how we might get information about an account that's easy to read and display, using the `AccountsRepository` from our project. Imagine a [Service](07_service_.md) (a component we'll discuss later) needs to fetch Alice's account details. - -### The Contract: The `AccountsRepository` Interface - -First, there's an interface that defines what operations can be performed for accounts. This is like the list of services the archivists offer. - -```go -// From: api/accounts.go - -// AccountsRepository defines methods for account data storage -type AccountsRepository interface { - Save(ctx context.Context, account *Account) error - // UpdateStatus updates the status of an account - UpdateStatus(ctx context.Context, accountID uuid.UUID, status string, updated time.Time) error - GetByID(ctx context.Context, accountID uuid.UUID) (*Account, error) - // ... other methods like Search, GetByIDs, SavePaymentDeviceLink etc. -} -``` -* This interface declares methods like `Save`, `UpdateStatus`, and `GetByID`. -* It uses `*Account`, which is a struct (`api.Account`) representing the "read model" of an account – a snapshot of its current, easily readable state. This is different from the `AccountAggregate` which is focused on processing [Commands](02_command_.md) and [Events](01_event_.md). - -### Getting Account Data - -If a part of our system needs to get details for account `acc-123`, it would use an implementation of `AccountsRepository`: - -```go -// Somewhere in our application (e.g., inside an Account Service) -var accountsRepo api.AccountsRepository // This would be a concrete implementation - -// ... -accountID := uuid.FromString("acc-123") // The ID of the account we want -account, err := accountsRepo.GetByID(context.Background(), accountID) -if err != nil { - // Handle error, maybe the account wasn't found - fmt.Println("Error fetching account:", err) - return -} - -fmt.Println("Fetched Account Category:", account.Category) -fmt.Println("Fetched Account Status:", account.Status) -``` -* **Input:** The `GetByID` method takes a `context` and the `accountID`. -* **Output:** It returns an `*api.Account` struct (containing the account's details like category, status, etc.) and an error (which will be `nil` if successful). - -The code calling `accountsRepo.GetByID` doesn't know or care if the data came from MySQL, a different database, or even a text file. That's the beauty of the repository abstraction! - -## Under the Hood: The `MySQLRepository` for Accounts - -Our project has a concrete implementation of the `AccountsRepository` interface that uses a MySQL database. It's located in `api/pkg/accounts/mysql_repository.go`. - -Let's see a simplified version of how its `GetByID` method might work: - -```go -// Simplified from: api/pkg/accounts/mysql_repository.go - -// MySQLRepository implements AccountsRepository using MySQL. -type MySQLRepository struct { - // (Internal details, like how it gets a database connection, are hidden here) -} - -// GetByID retrieves an account's projection (read model) by its ID from MySQL. -func (r *MySQLRepository) GetByID(ctx context.Context, accountID uuid.UUID) (*api.Account, error) { - // 1. Get a database connection (simplified) - conn, _ := repository.SQLConnection(ctx) - - // 2. This struct holds the raw data fetched from the 'account' table - var dbAccountData struct { // In real code, this is 'accountDbEntry' - ID uuid.UUID `db:"id"` - Category string `db:"category"` - Status string `db:"status"` - // ... other fields matching the database table columns ... - } - - // 3. Build the SQL query to select account data - sqlQuery := "SELECT id, category, status /*, ...other columns... */ FROM account WHERE id = ?" - - // 4. Execute the query against the MySQL database - err := conn.GetContext(ctx, &dbAccountData, sqlQuery, accountID) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, e.NewEntityIDNotFound("account", accountID.String()) // Specific error for "not found" - } - // Some other database error occurred - return nil, e.NewInternal(err) - } - - // 5. Convert the raw database data (dbAccountData) into the application's api.Account struct - apiAccount := &api.Account{ - ID: dbAccountData.ID, - Category: dbAccountData.Category, - Status: dbAccountData.Status, - // ... populate other fields ... - } - - // 6. In the real code, it also fetches related data like customer links, parameters, - // payment device links and populates them into apiAccount. - // Example (highly simplified): - // apiAccount.CustomerIDs, _ = r.getCustomersForAccount(ctx, accountID) - // apiAccount.Parameters, _ = r.getParametersForAccount(ctx, accountID) - - return apiAccount, nil -} -``` -Let's break this down step-by-step: -1. **Get Connection:** It first obtains a connection to the MySQL database. -2. **Data Holder:** `dbAccountData` is a temporary struct to hold the data exactly as it comes from the database table. -3. **SQL Query:** It defines the SQL `SELECT` statement to fetch data from the `account` table based on the `id`. -4. **Execute Query:** The query is executed. If no account is found, it returns a "not found" error. Other database errors are also handled. -5. **Map Data:** If data is found, the raw `dbAccountData` is converted into an `*api.Account` struct. This `api.Account` struct is what the rest of our application understands and expects. -6. **Fetch Related Data:** An account might have associated customers, specific parameters, or linked payment devices. The full repository method would also query other tables to fetch this related information and populate the `api.Account` struct completely. - -Here's a sequence diagram showing this: - -```mermaid -sequenceDiagram - participant AppCode as Application Code - participant AccRepo as AccountsRepository (MySQL) - participant MySQLDB as MySQL Database - - AppCode->>AccRepo: GetByID("acc-123") - AccRepo->>MySQLDB: SELECT id, category, status FROM account WHERE id = "acc-123" - MySQLDB-->>AccRepo: Row data (ID, Category, Status) - AccRepo-->>AccRepo: Map row to api.Account struct - AccRepo->>MySQLDB: SELECT customer_id FROM account_customer WHERE account_id = "acc-123" (etc. for other related data) - MySQLDB-->>AccRepo: Related data - AccRepo-->>AccRepo: Add related data to api.Account struct - AccRepo-->>AppCode: *api.Account (fully populated) -``` - -Similarly, a `Save` method in `MySQLRepository` would take an `*api.Account` struct, convert its fields into a format suitable for the database, and then execute an `INSERT` or `UPDATE` SQL statement. - -## What About [Aggregates](03_aggregate_.md) and Their [Events](01_event_.md)? - -As mentioned earlier, [Aggregates](03_aggregate_.md) like `AccountAggregate` are handled a bit differently in an Event Sourced system. They also use a Repository, but it's a specialized one. - -* **Loading an [Aggregate](03_aggregate_.md):** - When the system needs to load, say, `AccountAggregate` for `acc-123`: - 1. It asks an `EventSourcedRepository` to `Load("acc-123")`. - 2. This repository connects to an **Event Store** (a special database designed to store sequences of [Events](01_event_.md)). - 3. It fetches all [Events](01_event_.md) ever recorded for `acc-123`. - 4. It creates a new, empty `AccountAggregate` instance. - 5. It then "replays" each historical [Event](01_event_.md) on this instance by calling its `ApplyChange(event)` method. This rebuilds the [Aggregate's](03_aggregate_.md) current state. - 6. The fully rehydrated `AccountAggregate` is returned. - -* **Saving an [Aggregate](03_aggregate_.md):** - After `AccountAggregate` processes a [Command](02_command_.md) and generates new [Events](01_event_.md): - 1. The system calls `Save(accountAggregate)` on the `EventSourcedRepository`. - 2. The repository gets the list of newly generated (uncommitted) [Events](01_event_.md) from the `accountAggregate`. - 3. It appends these new [Events](01_event_.md) to the Event Store, associated with `acc-123`. - 4. It then clears the list of uncommitted [Events](01_event_.md) from the `accountAggregate`. - -Here's how loading an [Aggregate](03_aggregate_.md) via an EventSourced Repository looks: -```mermaid -sequenceDiagram - participant AppLogic as Application Logic - participant ESRepo as EventSourced Repository - participant EventStoreDB as Event Store Database - participant AccAgg as Account Aggregate - - AppLogic->>ESRepo: Load AccountAggregate (ID: "acc-123") - ESRepo->>EventStoreDB: GetEvents("acc-123") - EventStoreDB-->>ESRepo: [Event1, Event2, Event3...] - ESRepo->>AccAgg: New AccountAggregate() - loop For each Event in List - ESRepo->>AccAgg: ApplyChange(CurrentEvent) - end - ESRepo-->>AppLogic: AccountAggregate (state rehydrated) -``` -This shows that the Repository pattern is flexible. We can have different repository implementations for different kinds of data (read models like `api.Account`) and different storage strategies (like an Event Store for [Aggregates](03_aggregate_.md)). - -## Why is This Abstraction So Useful? - -Using the Repository pattern brings several benefits: - -1. **Testability:** When testing parts of your application that need data, you don't always want to connect to a real database. It can be slow and complicated to set up. With repositories, you can create a "mock" or "fake" repository for your tests. This fake repository can pretend to be a database, returning predefined data or checking if `Save` was called correctly, all without any actual database interaction. -2. **Flexibility (Change Your Database Later):** Imagine your bank starts with MySQL but later decides to switch to a different database, say PostgreSQL. If your application code directly uses MySQL-specific queries everywhere, changing the database would be a nightmare! With repositories, you only need to write a *new implementation* of the `AccountsRepository` interface (e.g., `PostgreSQLRepository`). The rest of your application code that uses the `AccountsRepository` interface doesn't need to change at all because it was only depending on the interface, not the specific MySQL details. -3. **Clear Separation of Concerns:** Business logic (what the bank *does*) is kept separate from data access logic (how data is saved and retrieved). This makes the code cleaner, easier to understand, and maintain. [Aggregates](03_aggregate_.md) and [Services](07_service_.md) focus on their tasks without being cluttered by SQL queries. - -## Conclusion - -Repositories are like the organized and efficient archivists of our `corebanking` system. They handle all the details of storing and retrieving data, whether it's: -* Fetching historical [Events](01_event_.md) to rebuild an [Aggregate's](03_aggregate_.md) state. -* Saving new [Events](01_event_.md) produced by an [Aggregate](03_aggregate_.md). -* Querying user-friendly "read models" (like an `api.Account`) from a database for display or reporting. - -By providing a clean abstraction layer over data storage, Repositories make our system more maintainable, testable, and flexible. They ensure that other parts of the system can request or save data using simple methods, without needing to know the nitty-gritty details of how or where that data is actually stored. - -Now that we know how data can be persisted and retrieved, how do external requests (like Alice trying to create an account through a mobile app) actually reach our core system and make use of these components? In the next chapter, we'll look at the entry points for such requests: the [API Handler](05_api_handler_.md). - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/05_api_handler_.md b/nesto/corebanking/05_api_handler_.md deleted file mode 100644 index 756e8914..00000000 --- a/nesto/corebanking/05_api_handler_.md +++ /dev/null @@ -1,225 +0,0 @@ -# Chapter 5: API Handler - -Welcome to Chapter 5! In the [previous chapter](04_repository_.md), we explored [Repositories](04_repository_.md) and how they help us save and retrieve data, like account information or the history of [Events](01_event_.md) for an [Aggregate](03_aggregate_.md). We now know how our system manages its data. - -But how does a request from the outside world, like Alice trying to open a new bank account using her mobile app, actually reach our `corebanking` system and start this whole process? How does Alice's app "talk" to our bank's brain? This is where the **API Handler** comes into play. - -## What's an API Handler? The Bank's Welcome Desk - -Imagine walking into a bank. You don't just wander into the vault or start talking to random employees. You usually go to a specific counter or desk: -* "Account Opening" desk. -* "Deposits & Withdrawals" window. -* A specific online form for "Loan Application." - -An **API Handler** in our `corebanking` system is just like that specific desk or form. It's the **official entry point** for any external request that wants to interact with our system. "API" stands for Application Programming Interface, which is a way for different software programs to communicate with each other. - -Each API Handler is responsible for: -1. A specific **URL path** (like a web address, e.g., `/api/accounts`). -2. A specific **HTTP method** (like the type of action, e.g., `POST` for creating something, `GET` for fetching something). - -When a request arrives matching its designated URL and method, the API Handler takes charge. Its job is to: -* Understand the incoming request (e.g., read the data Alice submitted). -* Translate this request into an instruction our system understands, like a [Command](02_command_.md) (e.g., "Please create an account with these details"). -* Or, it might ask a [Service](07_service_.md) to fetch some information. -* Finally, it sends back an HTTP response to tell the requester what happened (e.g., "Account created successfully!" or "Sorry, there was an error."). - -Essentially, API Handlers are the bridge connecting the "web world" (HTTP requests and responses) to the internal business logic of our bank. - -## Key Responsibilities of an API Handler - -Let's break down what an API Handler typically does: - -1. **Listens at a Specific Address:** Each handler is tied to a unique combination of a URL path and an HTTP method. For example: - * `POST /api/accounts`: Handler for creating new accounts. - * `GET /api/accounts/{accountId}`: Handler for fetching details of a specific account. - * `PUT /api/accounts/{accountId}/close`: Handler for closing an account. -2. **Parses the Incoming Request:** When Alice submits her account application form, her details (name, desired currency, etc.) are often sent as a JSON payload in the body of an HTTP request. The API Handler needs to read this data and convert it into a format our Go code can understand (like a Go struct). -3. **Basic Validation (Sometimes):** The handler might do some very basic checks, like "Is the JSON data correctly formatted?" or "Are essential fields like 'currency' present?". However, deep business rule validation (e.g., "Is this currency supported by our bank?") is usually done further inside the system by [Aggregates](03_aggregate_.md) or [Services](07_service_.md). -4. **Translates to Internal Actions:** This is a crucial step. - * **For actions that change data (Writes):** The handler often creates a [Command](02_command_.md) object (like `CreateAccountCommand`) and populates it with the data from the request. It then sends this [Command](02_command_.md) to a "Command Bus," which is like a central dispatcher that routes the [Command](02_command_.md) to the correct [Aggregate](03_aggregate_.md) (e.g., `AccountAggregate`) for processing. - * **For actions that only read data (Reads):** The handler might directly call a [Service](07_service_.md) (e.g., an `AccountsService`) to fetch the requested information (like account details from a [Repository](04_repository_.md)). -5. **Formats and Sends the Response:** After the [Command](02_command_.md) is processed or the [Service](07_service_.md) returns data, the API Handler takes the result and crafts an HTTP response. This includes: - * An **HTTP Status Code** (e.g., `200 OK` for success, `201 Created` if something new was made, `400 Bad Request` if the input was wrong, `500 Internal Server Error` if something went wrong on our side). - * A **Response Body** (often JSON data, like the details of the created account, or an error message). - -## Alice Creates an Account: The API Handler's Role - -Let's see how an API Handler helps Alice open her account: - -1. **Alice's App Sends a Request:** Alice fills out the "New Account" form on her banking app and taps "Submit." Her app sends an HTTP `POST` request to the URL `/api/accounts`. The body of this request contains her details in JSON format, like: - ```json - { - "category": "ASSET", - "product": { "id": "prod-savings-basic", "name": "Basic Savings" }, - "availableCurrencies": ["USD"], - "customerIds": ["cust-alice-123"] - } - ``` -2. **The Correct Handler is Activated:** Our `corebanking` system's web server receives this request. It looks at the method (`POST`) and the path (`/api/accounts`) and finds the specific API Handler responsible for this combination. -3. **Handler Parses the Request:** This API Handler (let's call it the `CreateAccountHandler`) takes the JSON data from the request's body. It uses a JSON parser to convert this data into a Go struct that matches the structure of a `CreateAccountCommand`. -4. **Handler Creates and Dispatches a [Command](02_command_.md):** The handler now creates an instance of `api.CreateAccountCommand` and fills it with the details parsed from Alice's request. It then passes this command to something called a `CommandBus` (part of `core.EventSourcingClient` in our project). - ```go - // Inside the handler (conceptual) - var cmd api.CreateAccountCommand - // ... (parse JSON from request into cmd) ... - err := commandBus.HandleCommand(context, &cmd) - ``` -5. **Handler Waits and Responds:** The Command Bus ensures the `CreateAccountCommand` is processed (by an `AccountAggregate`, which generates an `AccountCreatedEvent`, as we saw in earlier chapters). - * If the command is processed successfully, the handler sends back an HTTP response like `204 No Content` (meaning "I did what you asked, and there's no specific content to return") or `201 Created` (if it were to return details of the created account). - * If there's an error (e.g., Alice provided an invalid customer ID), the [Aggregate](03_aggregate_.md) would signal an error, and the handler would send back an HTTP response like `400 Bad Request` with a JSON body describing the error. - -## A Peek at the Code: An Account Creation Handler - -Our project has API handlers defined in packages under `api/cmd/corebanking-api/`. For creating accounts, the relevant file is `api/cmd/corebanking-api/accounts/create.go`. - -Let's look at a simplified version of the `Create` function, which acts as our API Handler: - -```go -// Simplified from: api/cmd/corebanking-api/accounts/create.go - -// 'cb' is a CommandBus, which knows how to send commands -// to the right part of our system. -func Create(cb es.CommandBus) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - // 1. Parse the request to get command data - command, err := parseCreateAccountCommand(r) - if err != nil { - // If parsing fails, send an error response - e.MarshallHTTP(err, w) // Helper to send formatted error - return - } - - // 2. Send the command for processing - err = cb.HandleCommand(r.Context(), &command) - if err != nil { - // If command processing fails, send an error response - e.MarshallHTTP(err, w) - return - } - - // 3. If everything is OK, send a success response - w.WriteHeader(http.StatusNoContent) // "204 No Content" - } -} -``` -Let's break this down: -* `func Create(cb es.CommandBus) http.HandlerFunc`: This function `Create` takes a `CommandBus` (our `es.EventSourcingClient`) as an argument and *returns* another function. This returned function is the actual handler that Go's HTTP server will call for each incoming request. - * `w http.ResponseWriter`: Used to write the HTTP response back to Alice's app. - * `r *http.Request`: Represents the incoming HTTP request from Alice's app. - -Inside the returned function: -1. **Parse Request:** `command, err := parseCreateAccountCommand(r)` - This calls a helper function (we'll see it next) to read Alice's data from the request (`r`) and turn it into our `api.CreateAccountCommand` struct. If there's an error during parsing (e.g., malformed JSON), it sends an error response using `e.MarshallHTTP(err, w)`. -2. **Send Command:** `err = cb.HandleCommand(r.Context(), &command)` - This is where the magic happens! The parsed `command` is handed off to the `CommandBus` (`cb`). The `CommandBus` will make sure it gets to the right [Aggregate](03_aggregate_.md) for processing. If the [Aggregate](03_aggregate_.md) rejects the command (e.g., business rule violation), `HandleCommand` returns an error, which is then sent back as an HTTP error. -3. **Success Response:** `w.WriteHeader(http.StatusNoContent)` - If both parsing and command processing are successful, this line sends an HTTP status code `204 No Content` back to Alice's app, indicating success. - -### Parsing the Request Data - -The `parseCreateAccountCommand` helper function looks something like this (simplified): - -```go -// Simplified from: api/cmd/corebanking-api/accounts/create.go -func parseCreateAccountCommand(r *http.Request) (api.CreateAccountCommand, error) { - var command api.CreateAccountCommand // Prepare an empty command struct - - // Decode the JSON data from the request body into our 'command' struct - if err := json.NewDecoder(r.Body).Decode(&command); err != nil { - // If decoding fails, log it and return a "bad format" error - log.G(r.Context()).WithError(err).Error("could not decode json") - return command, e.NewBadFormat(err.Error()) - } - - return command, nil // Return the populated command and no error -} -``` -* `var command api.CreateAccountCommand`: An empty `CreateAccountCommand` struct is created. -* `json.NewDecoder(r.Body).Decode(&command)`: This is standard Go for reading JSON. It takes the request's body (`r.Body`), creates a JSON decoder, and tries to fill the `command` struct with the data from the JSON. -* If `Decode` fails (e.g., the JSON is broken or doesn't match the command structure), an error is logged and a specific "bad format" error is returned. - -So, the handler first uses this parsing logic to understand the request, then dispatches the [Command](02_command_.md). - -## How Does the System Know Which Handler to Call? Routing! - -You might be wondering: if Alice's app sends a `POST` request to `/api/accounts`, how does our `corebanking` application know to call the `accounts.Create` handler function we just looked at? - -This is done by a **router**. A router is like a traffic controller for web requests. It looks at the URL and HTTP method of an incoming request and directs it to the correct handler function. Our project uses a popular Go router called `chi`. - -The setup for these routes is typically done in a file like `api/cmd/corebanking-api/routes/routes.go`. Here's a tiny snippet showing how the account creation route is defined: - -```go -// Simplified from: api/cmd/corebanking-api/routes/routes.go - -// 'core' contains things like our CommandBus (EventSourcingClient) -// and Services. -func getAPIRoutes(core *api.Core) http.Handler { - r := chi.NewRouter() // Create a new router - - // ... other routes ... - - // This line tells the router: - // If a POST request comes to "/accounts", - // call the handler returned by accounts.Create(), - // passing it the EventSourcingClient from 'core'. - r.Post("/accounts", accounts.Create(core.EventSourcingClient)) - - // Example of a GET request handler: - // r.Get("/accounts/{id}", accounts.Get(core.AccountsService)) - - // ... many other routes ... - - return r -} -``` -* `r := chi.NewRouter()`: Initializes a new router. -* `r.Post("/accounts", accounts.Create(core.EventSourcingClient))`: This is the key line. - * `r.Post` means this rule applies to HTTP `POST` requests. - * `"/accounts"` is the URL path. - * `accounts.Create(core.EventSourcingClient)` is the handler function we want to execute. Notice that we *call* `accounts.Create(...)` here. This function, as we saw, *returns* the actual `http.HandlerFunc` that `chi` will use. We pass `core.EventSourcingClient` (which is our `CommandBus`) to it so the handler can dispatch commands. - -For requests that read data, like getting account details, you'd see something like `r.Get("/accounts/{id}", accounts.Get(core.AccountsService))`. Here, `accounts.Get` would be another handler function, likely taking an `AccountsService` to fetch data. - -## The Journey of a Request: From App to Response - -Let's visualize the entire flow for Alice creating an account: - -```mermaid -sequenceDiagram - participant AliceApp as Alice's App - participant WebRouter as API Router (Chi) - participant CreateAccHandler as accounts.Create Handler - participant CmdBus as CommandBus (EventSourcingClient) - participant AccAgg as AccountAggregate - - AliceApp->>WebRouter: POST /api/accounts (with JSON data) - WebRouter->>CreateAccHandler: Execute handler for this route - CreateAccHandler-->>CreateAccHandler: parseCreateAccountCommand(request) - CreateAccHandler->>CmdBus: HandleCommand(CreateAccountCommand) - CmdBus->>AccAgg: Process CreateAccountCommand - Note over AccAgg: Validates, creates AccountCreatedEvent,
updates state, saves event (via Repository). - AccAgg-->>CmdBus: Success (or Error) - CmdBus-->>CreateAccHandler: Result of command processing - CreateAccHandler-->>CreateAccHandler: Prepare HTTP Response (e.g., 204) - CreateAccHandler->>AliceApp: HTTP 204 No Content (or error response) -``` -1. Alice's app sends the `POST` request. -2. The Web Router (Chi) finds the `accounts.Create` handler. -3. The `CreateAccountHandler` parses the JSON into a `CreateAccountCommand`. -4. It sends the command to the `CommandBus`. -5. The `CommandBus` delivers it to the `AccountAggregate`. -6. The `AccountAggregate` processes the command (validates, generates [Events](01_event_.md), updates its state, and ensures the [Events](01_event_.md) are saved via a [Repository](04_repository_.md) - this part was covered in detail in Chapters 1-4). -7. The result (success/error) bubbles back up to the handler. -8. The handler sends the appropriate HTTP response back to Alice's app. - -## Conclusion - -API Handlers are the vital "front doors" of our `corebanking` system. They listen for specific web requests (URLs and HTTP methods), parse the incoming data, and then translate these external requests into actions our internal system understands – usually by creating and dispatching [Commands](02_command_.md) for write operations or calling [Services](07_service_.md) for read operations. They then take the result of these actions and formulate an HTTP response to send back to the original requester. - -By bridging the gap between the web world (HTTP) and our core business logic, API Handlers make our system accessible to external clients like mobile apps, websites, or even other banking systems. - -Now that we've seen how requests enter our system, you might wonder: is there a central place that coordinates these actions, especially when a single request might involve multiple steps or components? That's where our next topic comes in. Get ready to learn about the [Core Facade](06_core_facade_.md)! - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/06_core_facade_.md b/nesto/corebanking/06_core_facade_.md deleted file mode 100644 index 30f191b3..00000000 --- a/nesto/corebanking/06_core_facade_.md +++ /dev/null @@ -1,160 +0,0 @@ -# Chapter 6: Core Facade - -Welcome to Chapter 6! In our [previous chapter on API Handlers](05_api_handler_.md), we saw how external requests, like Alice wanting to create an account, enter our `corebanking` system through specific API endpoints. These [API Handlers](05_api_handler_.md) then need to interact with various parts of our bank's brain – maybe to send a [Command](02_command_.md) or to fetch some data. - -But imagine an [API Handler](05_api_handler_.md) trying to juggle many different tools: one tool for sending [Commands](02_command_.md), another for looking up customer details, yet another for checking product information. It could get messy quickly! How do we keep this organized and simple for the [API Handler](05_api_handler_.md)? - -This is where the **Core Facade** comes in. - -## What is the Core Facade? The Bank's Main Office - -Think of the `Core` struct (our Core Facade) as the **main reception desk or central switchboard** for our entire `corebanking` application. It's like a central office in a large company. If you (an [API Handler](05_api_handler_.md), for instance) need something from any department, you don't run around the building looking for them. You go to the main office, and they connect you to the right place. - -The Core Facade brings together all the different specialized "departments" – which in our system are various [Services](07_service_.md) (like `AccountsService`, `CustomersService`) and other key components like the `EventSourcingClient` (our [Command](02_command_.md) bus) – into a **single, unified point of access**. - -When an external part of the system, like the [API layer](05_api_handler_.md), needs to interact with the business logic, it simply talks to this `Core` facade. This keeps our overall architecture clean and makes it much easier to manage how different parts of the application interact. - -Essentially, the Core Facade is like a well-organized toolbox for our [API Handlers](05_api_handler_.md). Instead of the handler needing to know about and hold onto dozens of individual tools, it just gets this one `Core` toolbox, which contains everything it might need. - -## What Does the Core Facade Look Like? (The `Core` Struct) - -In our Go code, the Core Facade is represented by a struct named `Core`, defined in `api/core.go`. Let's look at a simplified version: - -```go -// From: api/core.go -// Core represents the entrypoint to call the business logic. -type Core struct { - HealthService HealthService - EventSourcingClient es.EventSourcingClient // To send Commands! - ParametersService ParametersService - AccountsService AccountsService // To read account data! - CustomersService CustomersService - ProductsService ProductsService - // ... and many other services and clients ... -} -``` -* You can see that `Core` is a struct that holds various fields. -* Each field represents a specific "department" or tool: - * `EventSourcingClient`: This is crucial for sending [Commands](02_command_.md) (like `CreateAccountCommand`) to our [Aggregates](03_aggregate_.md). We've seen this in action with [API Handlers](05_api_handler_.md). - * `AccountsService`: If an [API Handler](05_api_handler_.md) needs to fetch a list of accounts or details about a specific account (not through event sourcing, but from a read-friendly data store), it would use this [Service](07_service_.md). We'll learn more about [Services](07_service_.md) in the next chapter! - * `CustomersService`, `ProductsService`, etc.: These provide access to other areas of our banking logic. - -So, the `Core` struct acts as a container, a "bag" holding all these important pieces of our application's core logic. - -## How Do [API Handlers](05_api_handler_.md) Use the Core Facade? - -Remember from [Chapter 5](05_api_handler_.md) how our [API Handlers](05_api_handler_.md) are set up using a router? The router is often given access to this `Core` facade instance. When an [API Handler](05_api_handler_.md) function is defined, it can be passed the specific service or client it needs *from* the `Core` facade. - -Let's revisit a simplified example from `api/cmd/corebanking-api/routes/routes.go` where routes are defined: - -```go -// Simplified from: api/cmd/corebanking-api/routes/routes.go - -// 'core' here is an instance of our api.Core struct (the Facade!) -func getAPIRoutes(core *api.Core) http.Handler { - r := chi.NewRouter() // Our web router - - // For creating accounts, the handler needs the EventSourcingClient (CommandBus) - r.Post("/accounts", accounts.Create(core.EventSourcingClient)) - - // For getting account details, a handler might need the AccountsService - // (Example, actual handler for GET might be different) - // r.Get("/accounts/{accountId}", accounts.GetDetails(core.AccountsService)) - - // ... other routes using different parts of 'core' ... - return r -} -``` -* The `getAPIRoutes` function receives an argument `core *api.Core`. This `core` object is our fully assembled Core Facade! -* When setting up the route for `POST /accounts`, the `accounts.Create` handler function is given `core.EventSourcingClient`. This means the `CreateAccountHandler` doesn't need to find or create an `EventSourcingClient` itself; it gets it directly from the Core Facade. -* Similarly, if we had a handler to fetch account details (e.g., `accounts.GetDetails`), it could be given `core.AccountsService`. - -This way, the `Core` facade acts as a central provider. The [API Handlers](05_api_handler_.md) just ask the `Core` facade for the specific tool they need. - -## How is the Core Facade Built? (The Making of the Toolbox) - -You might be wondering: who creates this `Core` struct and fills it with all these services? - -This happens when our `corebanking` application starts up. There's a special part of our code, primarily in a function called `FromEnv` in `api/pkg/core/core.go`, whose job is to: -1. Initialize all the individual [Services](07_service_.md) (like `AccountsService`, `ProductsService`, etc.). -2. Initialize the `EventSourcingClient`. -3. Initialize [Repositories](04_repository_.md) and other necessary components. -4. Then, it packages all of these initialized components into a single `api.Core` struct instance. - -Let's look at a very simplified snippet from our application's main entry point (`api/cmd/corebanking-api/api.go`) to see where this happens: - -```go -// Simplified from: api/cmd/corebanking-api/api.go main() -func main() { - // ... (lots of initial setup: logging, configuration, etc.) ... - - // tenantsConfigManager is used for multi-tenancy configurations - tenantsConfigManager := wiggumauto.NewConfigManager() - tenantsConfigManager.MustInitialize(ctx) - - // Here! 'c' becomes our fully assembled Core Facade instance. - // core.FromEnv does all the hard work of creating and wiring up services. - c := core.FromEnv(tenantsConfigManager) - - // The Core Facade 'c' is then passed to the router setup. - r := routes.GetRouter(c, tenantsConfigManager, monitoring) - - // ... (starts the web server with these routes) ... - // ... (handles graceful shutdown, including c.EventSourcingClient.Close()) -} -``` -* `c := core.FromEnv(tenantsConfigManager)`: This is the key line. The `core.FromEnv()` function (from `api/pkg/core/core.go`) is called. It sets up everything needed for the core logic (databases, event buses, all services) and returns a pointer to a fully populated `api.Core` struct. This `c` is our Core Facade. -* `r := routes.GetRouter(c, ...)`: This `c` (our Core Facade) is then passed to the function that sets up all the API routes, as we saw in the previous section. - -The `FromEnv` function itself is quite complex because it initializes many parts of the system, connecting them together. For a beginner, the important takeaway is that *there is a dedicated place* where the `Core` facade is carefully assembled when the application starts. - -## The Flow: API Handler, Core Facade, and Beyond - -Let's visualize how an [API Handler](05_api_handler_.md) uses the Core Facade: - -```mermaid -sequenceDiagram - participant APIHandler as API Handler (e.g., Create Account) - participant CoreFacade as Core Facade (api.Core instance) - participant AccService as Accounts Service - participant ESClient as EventSourcingClient (CommandBus) - participant AccAgg as Account Aggregate - - APIHandler->>CoreFacade: I need to create an account. Here's the data (CreateAccountCommand). - Note over APIHandler: Handler gets ESClient from CoreFacade - CoreFacade->>ESClient: HandleCommand(CreateAccountCommand) - ESClient->>AccAgg: Process CreateAccountCommand - AccAgg-->>ESClient: Success (or Error) - ESClient-->>CoreFacade: Command processed! - CoreFacade-->>APIHandler: Account creation initiated (Result) - - APIHandler->>CoreFacade: I need details for account "acc-xyz". - Note over APIHandler: Handler gets AccountsService from CoreFacade - CoreFacade->>AccService: GetAccountByID("acc-xyz") - AccService-->>CoreFacade: Account Details - CoreFacade-->>APIHandler: Here are the Account Details. -``` -This diagram shows two scenarios: -1. **Creating an Account (a [Command](02_command_.md)):** The [API Handler](05_api_handler_.md) uses the `EventSourcingClient` *obtained from* the `CoreFacade` to send the `CreateAccountCommand`. The `CoreFacade` itself doesn't process the command but provides access to the component that does. -2. **Getting Account Details (a Query):** The [API Handler](05_api_handler_.md) uses the `AccountsService` *obtained from* the `CoreFacade` to request account information. The `CoreFacade` directs this to the actual `AccountsService`. - -## Benefits of the Core Facade - -Using this Core Facade pattern provides several advantages: - -1. **Organization:** All core business functionalities are neatly bundled and accessible from one place. -2. **Simplicity for Callers:** Components like [API Handlers](05_api_handler_.md) don't need to know about or manage dependencies on dozens of individual services. They just need the `Core` object. -3. **Centralized Dependency Management:** The creation and wiring of all major services and components happen in one central place (like `core.FromEnv`). This makes it easier to manage the application's startup and overall structure. -4. **Clear Entry Point:** It provides a clear and consistent entry point into the application's business logic for any external interface (like the API). - -## Conclusion - -The **Core Facade**, represented by our `api.Core` struct, acts as the central hub or main reception desk for our `corebanking` application. It doesn't do the business logic itself but holds references to all the specialized [Services](07_service_.md) and clients (like the `EventSourcingClient`) that do. - -For components like [API Handlers](05_api_handler_.md), the Core Facade simplifies interaction with the system's core functionalities by providing a single, organized point of access. This keeps the architecture clean and manageable. - -The Core Facade is filled with various "departments" or tools. One very common type of tool it holds is a "Service." In the next chapter, we'll dive deeper into what these [Services](07_service_.md) are and what role they play in our `corebanking` system. - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/07_service_.md b/nesto/corebanking/07_service_.md deleted file mode 100644 index a768a62d..00000000 --- a/nesto/corebanking/07_service_.md +++ /dev/null @@ -1,264 +0,0 @@ -# Chapter 7: Service - -Welcome to Chapter 7! In the [previous chapter](06_core_facade_.md), we learned about the [Core Facade](06_core_facade_.md), which acts as a central hub providing access to various parts of our `corebanking` system. One of the key components the [Core Facade](06_core_facade_.md) gives us access to are "Services." - -But what exactly is a Service? And why do we need them? - -## What's a Service? The Bank's Specialized Departments - -Imagine you're at a bank. There isn't just one giant room where everything happens. Instead, there are specialized departments: -* The "Account Opening" department helps new customers. -* The "Loans" department handles loan applications and processing. -* The "Customer Support" department answers your questions. - -In our `corebanking` system, a **Service** is like one of these specialized departments. It's a component that groups together related business operations and logic for a specific area of the bank. - -For example, we might have an `AccountsService`. This service would be responsible for all things related to bank accounts, such as: -* Fetching the details of a specific account. -* Searching for accounts based on certain criteria. -* Potentially, initiating the creation of a new account (though, as we'll see, this is often done by sending a [Command](02_command_.md)). - -Services help us organize the application's capabilities, making the system easier to understand, manage, and extend. They often act as a "facade" or an intermediary, receiving requests, interacting with [Repositories](04_repository_.md) to fetch or save data, or dispatching [Commands](02_command_.md) to [Aggregates](03_aggregate_.md) to perform actions. - -## Key Ideas About Services - -1. **Specialized Focus:** Each Service handles a specific domain or business area (e.g., accounts, customers, product definitions). -2. **Groups Business Operations:** It bundles related functions. For instance, an `AccountsService` might have functions like `GetAccountByID()`, `SearchAccounts()`, etc. -3. **Orchestrates Logic:** A Service can coordinate multiple steps to fulfill a request. It might talk to a [Repository](04_repository_.md) to get data, perform some calculations or checks, and then return a result. -4. **Interacts with Other Components:** Services often use [Repositories](04_repository_.md) to access data and can dispatch [Commands](02_command_.md) to [Aggregates](03_aggregate_.md) if changes to the system's state are needed. -5. **Simplifies Interaction:** For other parts of the system (like [API Handlers](05_api_handler_.md)), Services provide a clean and focused interface to the business logic, hiding the underlying complexity. - -## Using a Service: Fetching Account Details - -Let's say an [API Handler](05_api_handler_.md) needs to display Alice's account details. It wouldn't directly query the database. Instead, it would use the `AccountsService`. - -### The Contract: The `AccountsService` Interface - -First, there's an interface that defines what operations the `AccountsService` offers. This is like the list of services a bank department provides. - -```go -// From: api/accounts.go - -// AccountsService manages accounts -type AccountsService interface { - // ... (methods for creating/updating accounts - more on this later) - Search(ctx context.Context, params *AccountSearchParameters) (Accounts, error) - GetByID(ctx context.Context, accountID uuid.UUID) (*Account, error) - GetByIDs(ctx context.Context, accountIDs []uuid.UUID) ([]*Account, error) - // ... other methods ... -} -``` -* This interface declares methods like `Search`, `GetByID`, and `GetByIDs`. -* Notice the `GetByID` method: it takes a `context` and an `accountID` and returns an `*Account` (a struct with account details) and an `error`. - -### Calling the Service - -An [API Handler](05_api_handler_.md) (or another part of the system) that has access to an `AccountsService` instance (perhaps via the [Core Facade](06_core_facade_.md)) can then call its methods: - -```go -// Somewhere in an API Handler (conceptual) -var accountsSvc api.AccountsService // This would be provided by the CoreFacade - -// ... -accountID := uuid.FromString("acc-alice-123") // Alice's account ID -accountDetails, err := accountsSvc.GetByID(context.Background(), accountID) - -if err != nil { - // Handle error (e.g., account not found) - fmt.Println("Error fetching account:", err) - return -} - -// Now we can use accountDetails -fmt.Println("Account Status:", accountDetails.Status) -fmt.Println("Account Currency:", accountDetails.AvailableCurrencies[0]) -``` -* **Input:** The `accountID` for Alice's account. -* **Output:** The `accountDetails` struct (if found) or an `err`. - -The [API Handler](05_api_handler_.md) doesn't need to know *how* the `AccountsService` gets this information. That's the Service's job! - -## Under the Hood: How `AccountsServiceDefault.GetByID` Works - -Our project has a concrete implementation of the `AccountsService` interface called `AccountsServiceDefault` (in `api/accounts.go`). Let's see how it's structured and how its `GetByID` method works. - -First, the `AccountsServiceDefault` struct holds a reference to an `AccountsRepository`: - -```go -// From: api/accounts.go -// AccountsServiceDefault represents the instance of a AccountsService. -type AccountsServiceDefault struct { - connections repository.ConnectionsProvider // Manages DB connections - repository AccountsRepository // The tool to talk to the database -} -``` -When an `AccountsServiceDefault` is created, it's given an `AccountsRepository`: - -```go -// From: api/accounts.go -// NewAccountsService return a new instance of the accounts service. -func NewAccountsService( - connections repository.ConnectionsProvider, - repository AccountsRepository, -) AccountsService { - // ... (nil checks omitted for brevity) ... - return &AccountsServiceDefault{ - connections: connections, - repository: repository, - } -} -``` -This means the service has the necessary tool ([Repository](04_repository_.md)) to fetch account data. - -Now, let's look at a simplified version of its `GetByID` method: - -```go -// Simplified from: api/accounts.go AccountsServiceDefault.GetByID -func (s *AccountsServiceDefault) GetByID( - ctx context.Context, - accountID uuid.UUID, -) (*Account, error) { - // 1. Potentially start a database transaction (simplified) - ctx, err := s.connections.NewTx(ctx) - if err != nil { - return nil, err - } - defer s.connections.Rollback(ctx) // Ensure transaction is rolled back if not committed - - // 2. Use the repository to fetch the account data - account, err := s.repository.GetByID(ctx, accountID) - if err != nil { - return nil, err // Could be "not found" or other DB error - } - - // 3. (A real service might do more here, like checking permissions - // or enriching the data, before returning it) - - return account, nil -} -``` -Here's what happens: -1. **Manage Transaction (Simplified):** It might start a database transaction. This is often good practice for read operations too, to ensure consistency, though for a single `GetByID` it might be optional depending on the database setup. -2. **Use Repository:** The crucial step! It calls `s.repository.GetByID(ctx, accountID)`. The service delegates the actual data fetching to its [Repository](04_repository_.md). -3. **Return Data:** It returns the `account` (or an error) obtained from the [Repository](04_repository_.md). - -Here's a sequence diagram illustrating this flow: - -```mermaid -sequenceDiagram - participant APIHandler as API Handler - participant AccSvc as AccountsService - participant AccRepo as AccountsRepository - participant DB as Database - - APIHandler->>AccSvc: GetByID("acc-alice-123") - AccSvc->>AccSvc: Start DB Transaction (conceptual) - AccSvc->>AccRepo: GetByID(ctx, "acc-alice-123") - AccRepo->>DB: SELECT * FROM accounts WHERE id = "acc-alice-123" - DB-->>AccRepo: Account Row Data - AccRepo-->>AccSvc: *Account (struct) - AccSvc->>AccSvc: Commit/Rollback Transaction (conceptual) - AccSvc-->>APIHandler: *Account (details for Alice) -``` - -## Services and Write Operations (Changes to Data) - -So far, we've focused on how Services help read data. What about when we want to *change* data, like creating an account or updating its status? - -The `AccountsService` interface in `api/accounts.go` actually defines methods like `Create`, `Activate`, etc.: -```go -// From: api/accounts.go -type AccountsService interface { - Create(ctx context.Context, command CreateAccountCommand) error - Activate(ctx context.Context, command ActivateAccountCommand) error - // ... other methods like Close, LinkPaymentDevice ... - // ... and the read methods we saw earlier ... -} -``` -However, if you look at the `AccountsServiceDefault` implementation for these methods, you'll find something like this: -```go -// From: api/accounts.go AccountsServiceDefault.Create -func (s *AccountsServiceDefault) Create( - _ context.Context, - _ CreateAccountCommand, -) error { - // writes should be done through the commandBus - return e.NewNotImplemented() -} -``` -It returns `e.NewNotImplemented()`! This tells us that, in this specific design, the `AccountsServiceDefault` *itself* doesn't directly handle account creation logic. Instead, such "write" operations are intended to be performed by sending a [Command](02_command_.md) (like `CreateAccountCommand`) directly to the `EventSourcingClient` (our Command Bus), which then routes it to the appropriate [Aggregate](03_aggregate_.md) (e.g., `AccountAggregate`). This is often done by the [API Handler](05_api_handler_.md) using the `EventSourcingClient` from the [Core Facade](06_core_facade_.md). - -**Why this separation?** -* **Centralized Command Processing:** Keeps all state-changing logic within [Aggregates](03_aggregate_.md), ensuring business rules are consistently enforced. -* **Event Sourcing Purity:** [Aggregates](03_aggregate_.md) are the source of [Events](01_event_.md), and [Commands](02_command_.md) are the standard way to trigger changes in them. - -### When Services *Do* Orchestrate Writes - -However, this doesn't mean Services *never* handle logic related to writes. A Service might: -1. **Perform preliminary checks or gather data** before a [Command](02_command_.md) is dispatched. -2. **Dispatch a [Command](02_command_.md) itself** if the operation is more complex or involves coordination. -3. **Orchestrate multiple [Commands](02_command_.md)** or interactions with different [Aggregates](03_aggregate_.md) or [Repositories](04_repository_.md). - -For example, consider the `PostingsTransactionsServiceDefault` in `api/postings_transactions.go`. Its `RevertChildTransactions` method shows a service performing more complex orchestration: - -```go -// Simplified from: api/postings_transactions.go -// PostingsTransactionsServiceDefault holds a commandBus -type PostingsTransactionsServiceDefault struct { - // ... other fields ... - commandBus es.CommandBus -} - -// RevertChildTransactions reverts child transactions for a specific transaction. -func (s *PostingsTransactionsServiceDefault) RevertChildTransactions( - ctx context.Context, - transactionID, rootTransactionID, parentTransactionID uuid.UUID, - /* ... other params ... */ -) error { - // 1. Use repository to get child transactions - postingsTransactions, err := s.repository.GetPostingsTransactionByParentID(ctx, transactionID) - // ... error handling ... - - // 2. For each child, dispatch a RevertSettledPostingsTransactionCommand - for _, postingsTransaction := range postingsTransactions { - err := s.commandBus.HandleCommand(ctx, &RevertSettledPostingsTransactionCommand{ - BaseCommand: es.BaseCommand{AggregateID: postingsTransaction.ID}, - // ... populate command details ... - }) - // ... error handling ... - } - return nil -} -``` -In this case, the `PostingsTransactionsService` reads data using its [Repository](04_repository_.md) and then dispatches multiple [Commands](02_command_.md) using its `commandBus`. This is a perfect example of a Service acting as an orchestrator for a business process. - -So, a Service can: -* Directly use [Repositories](04_repository_.md) for reading data (common). -* Dispatch [Commands](02_command_.md) to [Aggregates](03_aggregate_.md) for writing data (also common, especially for complex orchestrations). -* Sometimes, for simpler writes, an [API Handler](05_api_handler_.md) might dispatch a [Command](02_command_.md) directly using the `EventSourcingClient` from the [Core Facade](06_core_facade_.md), bypassing a specific write method on a service if the service doesn't add extra orchestration. - -## Why are Services Important? - -1. **Clear Business API:** They offer a higher-level, business-oriented set of operations compared to raw [Repositories](04_repository_.md) or direct [Command](02_command_.md) dispatching. -2. **Encapsulation:** They group related business logic, making it easier to find and manage. If you need to know how account searching works, you look in the `AccountsService`. -3. **Decoupling:** [API Handlers](05_api_handler_.md) or other clients depend on the Service interface, not on the nitty-gritty details of data storage or [Command](02_command_.md) handling for every single operation. -4. **Orchestration Point:** They are the natural place to put logic that coordinates multiple steps or involves several other components to achieve a business goal (like the `RevertChildTransactions` example). -5. **Testability:** Services can be tested in isolation. For example, when testing `AccountsServiceDefault`, you can provide a "mock" `AccountsRepository` to simulate database interactions. - -## Conclusion - -Services are the specialized "departments" within our `corebanking` application. They: -* Group related business operations for a specific domain (like accounts or transactions). -* Often use [Repositories](04_repository_.md) to fetch data needed for read operations. -* Can orchestrate more complex business processes, sometimes by dispatching [Commands](02_command_.md) to [Aggregates](03_aggregate_.md). -* Provide a clean and organized way for other parts of the system, like [API Handlers](05_api_handler_.md), to interact with the core business logic. - -They help keep our application well-structured and maintainable by ensuring that responsibilities are clearly defined. - -So far, we've seen how requests come in ([API Handler](05_api_handler_.md)), how they might use a [Core Facade](06_core_facade_.md) to access Services, and how Services can interact with [Repositories](04_repository_.md) or dispatch [Commands](02_command_.md) that affect [Aggregates](03_aggregate_.md) and generate [Events](01_event_.md). - -But what happens *after* an [Event](01_event_.md) is generated? Sometimes, other parts of the system need to react to these [Events](01_event_.md) automatically. For example, when an `AccountCreatedEvent` occurs, maybe we need to update a search index or send a welcome email. How does that happen? In the next chapter, we'll explore the [Consumer](08_consumer_.md). - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/08_consumer_.md b/nesto/corebanking/08_consumer_.md deleted file mode 100644 index 530c6987..00000000 --- a/nesto/corebanking/08_consumer_.md +++ /dev/null @@ -1,204 +0,0 @@ -# Chapter 8: Consumer - -Welcome to Chapter 8! In our [previous chapter on Services](07_service_.md), we learned how specialized "departments" in our bank help organize business logic, often by reading data using [Repositories](04_repository_.md) or orchestrating actions, sometimes by sending [Commands](02_command_.md). These [Commands](02_command_.md), when processed by [Aggregates](03_aggregate_.md), result in [Events](01_event_.md) – records of things that have happened. - -But what happens *after* an [Event](01_event_.md) is generated? For instance, if a `PostingCommittedEvent` (signaling a financial transaction leg) occurs, we probably need to update the customer's account balance so they can see it. Or, if an `AccountCreatedEvent` happens, maybe we need to send a welcome email. How do these follow-up actions happen automatically without slowing down the original process? - -This is where the **Consumer** steps in! - -## What is a Consumer? The Bank's Automated Event Watcher - -Imagine in our bank, every time an important announcement is made (an [Event](01_event_.md) happens), there are dedicated staff members who are always listening for specific types of announcements. -* One staff member listens for "New Account Opened" announcements. When they hear one, they immediately prepare and send a welcome pack to the new customer. -* Another staff member listens for "Transaction Completed" announcements. When they hear one, they quickly update the account ledger display. - -A **Consumer** in our `corebanking` system is like one ofthese automated staff members. It's a component that **listens for specific types of [Events](01_event_.md)** that occur within the system. When it "hears" an [Event](01_event_.md) it's interested in, it performs a predefined action. - -For example, when a `PostingCommittedEvent` occurs (meaning a part of a financial transaction has been finalized), a `BalancesConsumer` might listen for this. Upon receiving it, the consumer will update the account's balance that is stored for quick display (this stored balance is often called a "read model" or "projection"). - -Consumers are crucial because they allow different parts of the system to react to business happenings **asynchronously**. This means the original action (like processing the transaction) can finish quickly, and the follow-up tasks (like updating the balance display or sending a notification) happen separately, a moment later. - -## Key Ideas About Consumers - -1. **Event Listener:** A Consumer "subscribes" to a stream of [Events](01_event_.md) and pays attention only to the types of [Events](01_event_.md) it cares about. -2. **Asynchronous Operation:** Consumers typically run in the background, separate from the process that originally created the [Event](01_event_.md). This keeps the main operations fast and responsive. The part of the system that created the event doesn't have to wait for the Consumer to finish its job. -3. **Performs Actions:** When a relevant [Event](01_event_.md) is received, the Consumer executes specific logic. Common actions include: - * **Updating Read Models:** Calculating and storing data in a format optimized for fast reading. For example, updating an account balance summary after a transaction. - * **Triggering Follow-up Processes:** Starting new workflows, sending notifications (like emails or SMS), or calling other [Services](07_service_.md). -4. **Decoupling:** The component that produces an [Event](01_event_.md) (e.g., an `AccountAggregate` producing `AccountCreatedEvent`) doesn't need to know anything about the Consumers that might be listening to that [Event](01_event_.md). This makes the system more flexible and easier to change. -5. **Data Consistency (Eventually):** Consumers help maintain consistency across different views of data. For example, after a transaction, the balance read model will *eventually* reflect the change. - -## A Consumer in Action: Updating Account Balances - -Let's say a customer makes a deposit. This might result in one or more `PostingCommittedEvent`s. We need to update the account balance that is shown to the customer on their app. - -This is a perfect job for a `BalancesConsumer` (like the `BalancesTimeSeriesConsumerDefault` found in `api/balances_consumer.go`): - -1. **[Event](01_event_.md) Happens:** An [Aggregate](03_aggregate_.md) processes a transaction, and a `PostingCommittedEvent` is generated and saved. This event contains details about the amount, currency, and account involved. -2. **Consumer is Notified:** The `BalancesConsumer` is subscribed to listen for `PostingCommittedEvent`s. An "event bus" (a system for distributing events) delivers this event to the consumer. -3. **Consumer Processes the [Event](01_event_.md):** - * The `BalancesConsumer` receives the `PostingCommittedEvent`. - * It extracts the necessary information (account ID, amount, type of posting - debit/credit). - * It might fetch the current stored balance for that account using a `BalancesRepository` (a type of [Repository](04_repository_.md)). - * It calculates the new balance. -4. **Consumer Updates Read Model:** The consumer then saves this new, updated balance back using the `BalancesRepository`. This updated balance is now ready to be quickly displayed to the user. -5. **Optional: Further [Events](01_event_.md):** The consumer might even publish a new [Event](01_event_.md) itself, like a `BalanceUpdatedEvent`, to signal that the read model balance has changed. - -This entire process happens *after* the original deposit transaction was confirmed. The customer got a quick confirmation of their deposit, and in the background, the consumer ensures their displayed balance is updated. - -## What Does a Consumer Look Like? (A Peek at the Code) - -Consumers are often structs that hold dependencies like [Repositories](04_repository_.md) or other [Services](07_service_.md). They typically have a `Start()` method to begin listening and a processing method that gets called when relevant [Events](01_event_.md) arrive. - -Let's look at a simplified structure for `BalancesTimeSeriesConsumerDefault` from `api/balances_consumer.go`: - -```go -// Simplified from: api/balances_consumer.go -type BalancesTimeSeriesConsumerDefault struct { - repository BalancesRepository // To read/write balance data - events es.EventPublisher // To publish new events (optional) - eventConsumer es.EventConsumer // The mechanism to receive events - // ... other dependencies ... -} - -// NewBalancesTimeSeriesConsumer creates the consumer -func NewBalancesTimeSeriesConsumer( - repository BalancesRepository, - events es.EventPublisher, - eventConsumer es.EventConsumer, - // ... other args ... -) BalancesTimeSeriesConsumer { - // ... (nil checks) ... - return &BalancesTimeSeriesConsumerDefault{ - repository: repository, - events: events, - eventConsumer: eventConsumer, - // ... - } -} -``` -* `repository`: This is a `BalancesRepository`, used to get the current balance and save the new balance. -* `events`: An `EventPublisher` which this consumer might use to send out new [Events](01_event_.md) (like `BalanceUpdatedEvent`). -* `eventConsumer`: This is the component from our event sourcing library (`es`) that actually delivers [Events](01_event_.md) from the event bus to this consumer. - -### Starting the Consumer and Processing Events - -The consumer needs to be "turned on" to start listening. This is usually done by its `Start()` method: - -```go -// Simplified from: api/balances_consumer.go -func (c *BalancesTimeSeriesConsumerDefault) Start() error { - // Tell the eventConsumer to start sending batches of events - // to our 'processBalancesTimeSeriesEvents' method. - err := c.eventConsumer.ConsumeTenantMultiAggregateBatch( - context.Background(), - c.processBalancesTimeSeriesEvents, // Our callback function - ) - return err -} -``` -* `c.eventConsumer.ConsumeTenantMultiAggregateBatch(...)`: This line registers the `processBalancesTimeSeriesEvents` method with the underlying event consumption mechanism. Now, whenever new [Events](01_event_.md) (that this consumer is configured to listen to) arrive, this method will be called with a batch of those [Events](01_event_.md). - -The actual work happens in the callback method, `processBalancesTimeSeriesEvents`. Here's a highly simplified conceptual view of what it does when it receives event data related to postings: - -```go -// Highly simplified concept of processBalancesTimeSeriesEvents -func (c *BalancesTimeSeriesConsumerDefault) processBalancesTimeSeriesEvents( - ctx context.Context, tenantID int, events []*es.Event, -) error { - // For each relevant event (e.g., indirectly from a PostingCommittedEvent): - for _, event := range events { - // 1. Extract posting details from the event - // (The real code uses a helper 'orderedPostingsFromEvents' - // to get 'Posting' data from events like PostingCommittedEvent) - postingData := extractPostingFrom(event) // Conceptual - accountID := postingData.AccountID - - // 2. Get current balances for the account (simplified) - accountBalances, _ := c.repository.GetBalancesByAccountIDs(ctx, []uuid.UUID{accountID}) - // (Real code handles 'not found' and groups by account) - currentBalanceInfo := accountBalances[0] // Simplified - - // 3. Apply the posting to update the balance - // (The real code uses 'ApplyPostingToAccountBalances') - newBalance := calculateNewBalance(currentBalanceInfo, postingData) // Conceptual - - // 4. Save the updated current balance - // (The real code accumulates 'updatedCurrentBalances' and saves in a batch) - err := c.repository.SaveCurrent(ctx, []*Balance{newBalance}) - if err != nil { - // Handle error - return err - } - - // 5. Optionally, publish a new event - balanceUpdatedEvt := createBalanceUpdatedEvent(newBalance) // Conceptual - c.events.PublishBatch(ctx, []es.Event{balanceUpdatedEvt}) - } - return nil -} -``` -Let's break down this conceptual flow: -1. **Extract Posting Details:** The consumer gets the necessary details from the incoming [Event](01_event_.md) (like account ID, amount, currency, debit/credit). In the actual `corebanking` code, `PostingCommittedEvent`s are processed to extract `Posting` objects. -2. **Get Current Balance:** It uses its `BalancesRepository` to fetch the most recent balance record for the affected account. -3. **Apply Posting:** It performs the calculation to update the balance. The actual function `ApplyPostingToAccountBalances` in `api/balances_consumer.go` handles the logic of adding or subtracting based on posting type and account category. -4. **Save Updated Balance:** The new balance is saved back to the read model storage using the `BalancesRepository`'s `SaveCurrent` method. -5. **Publish New Event (Optional):** The `BalancesConsumer` in our project also creates and publishes a `BalanceUpdatedEvent`. This allows other parts of the system to know that a balance read model has been updated. - -The actual `processBalancesTimeSeriesEvents` is more complex because it handles batches of events, deals with time-series data for historical balances, and manages database transactions. But the core idea is: **receive event -> process it -> update read model.** - -## How a Consumer Gets Triggered: The Event Flow - -Here’s a simplified sequence diagram showing how a `BalancesConsumer` might react to a `PostingCommittedEvent`: - -```mermaid -sequenceDiagram - participant TxnProcessor as Transaction Processor (e.g., Aggregate) - participant EventBus as Event Bus / Message Queue - participant BalConsumer as Balances Consumer - participant BalRepo as Balances Repository - participant ReadModelDB as Read Model Database - - TxnProcessor->>EventBus: Publish PostingCommittedEvent (for Account X, Amount $10) - EventBus->>BalConsumer: Deliver PostingCommittedEvent - BalConsumer->>BalRepo: Get Current Balance (Account X) - BalRepo->>ReadModelDB: Fetch balance for Account X - ReadModelDB-->>BalRepo: Current Balance: $50 - BalRepo-->>BalConsumer: Current Balance: $50 - BalConsumer-->>BalConsumer: Calculate New Balance ($50 + $10 = $60) - BalConsumer->>BalRepo: Save Balance (Account X, New Balance $60) - BalRepo->>ReadModelDB: Update balance for Account X to $60 - ReadModelDB-->>BalRepo: Success - BalRepo-->>BalConsumer: Success - BalConsumer->>EventBus: (Optional) Publish BalanceUpdatedEvent -``` -This shows the asynchronous nature: the `TxnProcessor` fires the event and moves on. The `BalConsumer` picks it up later and does its work. - -## Other Types of Consumers - -Our `corebanking` system has other consumers too: - -* **`PostingsTransactionsConsumer`** (from `api/postings_transactions_consumer.go`): - This consumer listens to events related to financial transactions (like `PostingsTransactionCreatedEvent`, `PostingsTransactionSettledEvent`). It updates a read model that stores details about these transactions, making it easy to query their status, amount, etc. - -* **`ProductEnginesConsumer`** (from `api/product_engines_consumer.go`): - This is an interesting one! It listens for events like `ScheduledEvent` (e.g., "it's end of day for account X") or even `PostingCommittedEvent`. When it receives such an event, it triggers the [Product Engine](09_product_engine_.md) (via `productEngines.HandleEvents()`). The [Product Engine](09_product_engine_.md) might then apply interest, charge fees, or perform other account-specific logic defined by the banking product. This shows a consumer triggering a more complex follow-up process. - -## Why Are Consumers So Useful? - -1. **Improved Performance & Responsiveness:** The system part that creates an [Event](01_event_.md) (e.g., an [Aggregate](03_aggregate_.md) handling a [Command](02_command_.md)) can finish its job quickly without waiting for all side effects (like updating multiple read models or sending emails) to complete. -2. **Decoupling & Modularity:** Event producers don't need to know about event consumers. You can add new consumers or change existing ones without affecting the code that generates the events. This makes the system very flexible. -3. **Resilience:** If a consumer temporarily fails while processing an event (e.g., a network issue while saving to a database), the event can often be re-processed later without losing the original data, as the [Event](01_event_.md) itself is already safely stored. -4. **Scalability:** Different consumers can often be scaled independently. If updating balances becomes a bottleneck, you might be able to run more instances of the `BalancesConsumer`. - -## Conclusion - -Consumers are the unsung heroes working diligently in the background of our `corebanking` system. They are **automated listeners** that subscribe to specific types of [Events](01_event_.md). When they receive an [Event](01_event_.md) they're interested in, they swing into action – perhaps updating a read-model for fast data display (like account balances) or triggering follow-up processes (like complex product calculations). - -By reacting to business [Events](01_event_.md) asynchronously, Consumers help keep our system responsive, flexible, and robust. They play a vital role in maintaining data consistency across different views and enabling complex workflows. - -One example we saw was the `ProductEnginesConsumer`, which listens for events and then invokes specific business logic defined by a banking product. What is this "Product Engine" that it calls? Let's explore that in our next chapter: the [Product Engine](09_product_engine_.md). - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/09_product_engine_.md b/nesto/corebanking/09_product_engine_.md deleted file mode 100644 index 11c45b49..00000000 --- a/nesto/corebanking/09_product_engine_.md +++ /dev/null @@ -1,265 +0,0 @@ -# Chapter 9: Product Engine - -Welcome to Chapter 9! In the [previous chapter](08_consumer_.md), we learned about [Consumers](08_consumer_.md) and how they listen for [Events](01_event_.md) to trigger follow-up actions. For example, a [Consumer](08_consumer_.md) might notice that an account needs its monthly interest calculated. But how does the system know *how* to calculate that interest, especially if different types of accounts (like a "Youth Savings Account" vs. a "Standard Mortgage") have vastly different rules? - -This is where the **Product Engine** comes in. - -## What's a Product Engine? The Bank's Specialized Rulebook - -Imagine a bank offers many different financial products: -* A "Basic Savings Account" with simple interest. -* A "Youth Savings Account" that gets bonus interest if no money is withdrawn during the month. -* A "Standard Mortgage" with complex payment schedules and fee structures. -* A "HELOC Remittance" account with its own unique remittance rules. - -Each of these products behaves differently. They have their own specific features, ways to calculate interest, fee rules, and reactions to events like an account being activated or a transaction occurring. - -A **Product Engine** is like a **detailed rulebook and automated process manager** specifically designed for *one particular financial product*. It defines exactly how that product, like "Standard Mortgage" or "Youth Savings Account," should work. - -When a customer opens an account (say, a "Youth Savings Account"), that account is linked to its specific product. From then on, whenever something relevant happens to that account, the system consults the "Youth Savings Account" Product Engine to: -* Check if an operation is allowed (validation). -* Determine any product-specific actions or outcomes (like calculating that special bonus interest). - -## Key Ideas About Product Engines - -1. **Product-Specific Logic:** Each Product Engine contains the unique rules and behaviors for one type of financial product. -2. **Defines Product Behavior:** It specifies features, fee structures, interest calculations, and how the product responds to lifecycle events (e.g., account activation, transaction posting, scheduled events like "end of month"). -3. **Consulted for Operations:** When an action needs to be performed on an account, or an event occurs related to it, the corresponding Product Engine is invoked. -4. **Automates Processes:** It automates tasks like applying interest, charging fees, or managing scheduled product-specific activities. -5. **Produces Actions:** Based on its rules and the current situation (e.g., an incoming [Event](01_event_.md) or [Command](02_command_.md)), a Product Engine can decide that certain actions need to happen. These actions are often new [Commands](02_command_.md) (e.g., "Create a transaction to credit bonus interest"). - -## The Product Engine in Action: Youth Savings Bonus Interest - -Let's use our "Youth Savings Account" example. This account type offers bonus interest if no withdrawals are made in a month. - -1. **Account Setup:** Alice has a "Youth Savings Account." This account is linked to the "Youth Savings Product Engine." -2. **End of Month [Event](01_event_.md):** A `ScheduledEvent` (like "EndOfMonthProcessing") occurs for Alice's account. -3. **[Consumer](08_consumer_.md) Routes to Service:** A [Consumer](08_consumer_.md) (like the `ProductEnginesConsumer` we touched upon in the [previous chapter](08_consumer_.md)) picks up this `ScheduledEvent`. It knows this event might require product-specific logic, so it passes the event to the `ProductEnginesService`. -4. **Service Invokes Product Engine:** The `ProductEnginesService`: - * Looks at Alice's account and sees it's a "Youth Savings Account." - * Retrieves the specific "Youth Savings Product Engine." - * Prepares a `ProductEngineContext` for Alice's account. This context contains information the engine might need, like Alice's current account details, balances, and the product's configuration. - * Calls a method on the "Youth Savings Product Engine," like `HandleScheduledEvent(context, scheduledEvent)`. -5. **Product Engine Applies Rules:** The "Youth Savings Product Engine" executes its logic for the "EndOfMonthProcessing" event: - * It checks Alice's transaction history for the past month (perhaps by looking at data in the `ProductEngineContext` or by querying for recent postings). - * It determines if any withdrawals were made. - * If no withdrawals, it calculates the bonus interest. -6. **Product Engine Returns Actions:** The engine decides that bonus interest should be applied. It returns a list of `ProductEngineAction`s. One of these actions might be a `HandleCommandAction` containing a `CreatePostingsTransactionCommand` to credit the bonus interest to Alice's account. -7. **Service Executes Actions:** The `ProductEnginesService` takes these actions and processes them. For a `HandleCommandAction`, it would send the `CreatePostingsTransactionCommand` to the [Command Bus](02_command_.md), which then ensures the transaction is created and Alice's balance is updated. - -This way, the specific bonus interest logic for the "Youth Savings Account" is neatly contained within its own Product Engine. - -## A Peek at the Code: Interfaces and Services - -Let's look at some of the key pieces involved. - -### 1. The `ProductEngine` Interface - -At its core, every product engine implements the `ProductEngine` interface (from `api/product_engines.go`). This interface defines the "hooks" or methods that can be called by the system for different situations. - -```go -// Simplified from: api/product_engines.go -type ProductEngine interface { - ProductEngineID() string // E.g., "YOUTH_SAVINGS_ACCOUNT" - ProductEngineVersion() string // E.g., "1.0.0" - - // Called when an account is activated - ActivateAccount(ctx context.Context, engineCtx ProductEngineContext) ([]ProductEngineAction, error) - - // Called for scheduled events (like our "EndOfMonthProcessing") - HandleScheduledEvent(ctx context.Context, engineCtx ProductEngineContext, scheduledEvent *ScheduledEvent) ([]ProductEngineAction, error) - - // Called when a posting (transaction leg) is accepted for an account - AcceptPosting(ctx context.Context, engineCtx ProductEngineContext, posting *Posting) ([]ProductEngineAction, error) - - // ... other methods for validating postings, updating parameters, closing accounts etc. -} -``` -* `ProductEngineID()` and `ProductEngineVersion()`: Identify the specific engine. -* `ActivateAccount`, `HandleScheduledEvent`, `AcceptPosting`: These are examples of methods that get called at different points in an account's lifecycle or when certain [Events](01_event_.md) occur. -* `ProductEngineContext`: This argument provides all the necessary information about the current account, its product settings, balances, etc., that the engine needs to make decisions. -* `[]ProductEngineAction`: The method returns a list of actions the system should take as a result of the engine's logic. - -### 2. The `ProductEngineContext` - -This struct (also from `api/product_engines.go`) is passed to the Product Engine methods. It's like a data package giving the engine all the relevant details for the current operation. - -```go -// Simplified from: api/product_engines.go -type ProductEngineContext struct { - ReferenceDate *date.Date // The current date for processing - Account *Account // Details of the specific account - Product *ResolvedProduct // Details & parameters of the banking product - CurrentBalances *BalancesReader // Access to the account's current balances - // ... and other fields like PostingsByPartition, BalancesByPartition for historical data -} -``` -* `Account`: The specific account being processed. -* `Product`: The resolved product definition, containing parameters like interest rates, fee schedules, etc. -* `CurrentBalances`: A way to read the account's current balances. -The engine uses this information to apply its rules. - -### 3. `ProductEngineAction` - -When a Product Engine finishes its logic, it needs to tell the system what to do next. It does this by returning a slice of `ProductEngineAction` (defined in `api/product_engines.go`). - -A common action is to issue a [Command](02_command_.md): -```go -// Simplified from: api/product_engines.go - -// ProductEngineAction represents a product engine action. -type ProductEngineAction interface { - Actuate() // A marker method -} - -// HandleCommandAction implements product engine action to handle command. -type HandleCommandAction struct { - command es.Command // The command to be executed -} - -// NewHandleCommandAction returns a new HandleCommandAction. -func NewHandleCommandAction(command es.Command) HandleCommandAction { - return HandleCommandAction{command} -} - -// GetCommand returns the underlying command -func (a HandleCommandAction) GetCommand() es.Command { - return a.command -} -``` -So, if our "Youth Savings Product Engine" decides to apply bonus interest, it would create a `CreatePostingsTransactionCommand` and wrap it in a `NewHandleCommandAction`. - -### 4. The `ProductEnginesService` - -The `ProductEnginesServiceDefault` (from `api/product_engines.go`) is the orchestrator. It's responsible for receiving [Events](01_event_.md) (often from a [Consumer](08_consumer_.md)), finding the right Product Engine, and invoking it. - -```go -// Conceptual flow within ProductEnginesServiceDefault.HandleEvents -// (Actual code is more complex, handling batches and various event types) - -func (s *ProductEnginesServiceDefault) HandleEvents(ctx context.Context, events []*es.Event) error { - for _, event := range events { - // 1. Determine which account this event is for (from event.AggregateID) - accountID := event.AggregateID - account, _ := s.accounts.GetByID(ctx, accountID) // Fetch account details - - // 2. Get the Product Engine for this account's product - productEngine, _ := s.GetProductEngine(ctx, account.Product) - if productEngine == nil { - continue // No engine for this product, or product not set - } - - // 3. Create an Event Handler wrapper (e.g., for a ScheduledEvent) - // The hook file (api/product_engines_hooks.go) has different handlers for different events. - var eventHandler ProductEngineEventHandler // Defined in api/product_engines_hooks.go - if se, ok := event.Data.(*ScheduledEvent); ok && eventHandlerNameFromAction(se.Action) == "HandleScheduledEvent" { - eventHandler = NewProductEngineEventHandlerHandleScheduledEvent(event.ID, se, /*...location...*/) - } - // ... (else if for AccountActivatedEvent, PostingCommittedEvent, etc.) ... - - if eventHandler == nil { - continue - } - - // 4. Build the ProductEngineContext (simplified) - engineCtx, _ := s.buildProductEngineContext(ctx, /*...details...*/, account, /*balances*/, /*config*/) - - // 5. Call the specific hook on the event handler, which calls the engine - actions, err := eventHandler.Hook(ctx, *engineCtx, productEngine) - // ... error handling ... - - // 6. Run the actions returned by the Product Engine - s.runActions(ctx, actions) // This would dispatch commands, schedule things, etc. - } - return nil -} -``` -* **Get Account & Engine:** It fetches the account and determines the correct Product Engine based on the account's product information (`account.Product.EngineID`, `account.Product.EngineVersion`). -* **Event Handler Wrapper:** An `ProductEngineEventHandler` (from `api/product_engines_hooks.go`) is chosen based on the type of incoming [Event](01_event_.md). This wrapper standardizes how different events are passed to the Product Engine. For example, `ProductEngineEventHandlerHandleScheduledEvent` has a `Hook` method that will ultimately call `engine.HandleScheduledEvent(...)`. -* **Build Context:** It prepares the `ProductEngineContext` with all necessary data. -* **Invoke Engine (via Hook):** It calls the `Hook` method on the event handler, which in turn calls the relevant method on the `ProductEngine` instance (e.g., `HandleScheduledEvent`). -* **Run Actions:** It takes the `ProductEngineAction`s returned by the engine and executes them (e.g., sends [Commands](02_command_.md) to the [Command Bus](02_command_.md)). - -### Example: A Simple Product Engine's `HandleScheduledEvent` - -Let's imagine a very basic "MonthlyFeeProductEngine": -```go -// Hypothetical MonthlyFeeProductEngine -type MonthlyFeeProductEngine struct { /* ... */ } - -func (pe *MonthlyFeeProductEngine) HandleScheduledEvent( - ctx context.Context, - engineCtx api.ProductEngineContext, - scheduledEvent *api.ScheduledEvent, -) ([]api.ProductEngineAction, error) { - - if scheduledEvent.Action == "APPLY_MONTHLY_FEE" { - feeAmountStr, _ := engineCtx.Product.Parameters["monthlyFeeAmount"] // Get fee from product config - feeAmount, _ := decimal.NewFromString(feeAmountStr) - - if feeAmount.IsPositive() { - // Create a command to charge the fee - feeTxCmd := &api.CreatePostingsTransactionCommand{ - BaseCommand: es.BaseCommand{AggregateID: engineCtx.Account.ID}, - Action: api.PostingActionTransfer, - Code: "MONTHLY_FEE", - Amount: feeAmount, - Debtor: &api.PostingDestination{AccountID: engineCtx.Account.ID}, // Debit customer - Creditor: &api.PostingDestination{AccountID: uuid.FromString("bank-fee-income-account-id")}, // Credit bank - } - return []api.ProductEngineAction{api.NewHandleCommandAction(feeTxCmd)}, nil - } - } - return nil, nil // No action -} - -// ... other required methods of ProductEngine interface ... -``` -When a `ScheduledEvent` with action "APPLY_MONTHLY_FEE" comes in for an account using this product: -1. The engine checks if the action matches. -2. It reads the `monthlyFeeAmount` from the product's parameters (available in `engineCtx.Product.Parameters`). -3. If a fee is due, it creates a `CreatePostingsTransactionCommand` to transfer the fee from the customer's account to the bank's fee income account. -4. It returns this command wrapped in a `HandleCommandAction`. - -## Visualizing the Flow - -Here's how an event for Alice's "Youth Savings Account" (YSA) might trigger its Product Engine to apply bonus interest: - -```mermaid -sequenceDiagram - participant EventBus - participant YSAConsumer as YSA Product Consumer - participant PEService as ProductEnginesService - participant YSAPEngine as YouthSavingsProductEngine - participant CmdBus as CommandBus - - EventBus->>YSAConsumer: ScheduledEvent (EndOfMonth, Account: AliceYSA) - YSAConsumer->>PEService: HandleEvents([ScheduledEvent]) - PEService->>YSAPEngine: GetEngine("YouthSavingsProductEngine", "v1") - Note over PEService: Builds ProductEngineContext for AliceYSA - PEService->>YSAPEngine: HandleScheduledEvent(ctx, engineCtx, ScheduledEvent) - YSAPEngine-->>YSAPEngine: Logic: Check no withdrawals, calculate bonus. - YSAPEngine-->>PEService: Return [HandleCommandAction(CreateBonusTxCmd)] - PEService->>CmdBus: HandleCommand(CreateBonusTxCmd) - CmdBus-->>PEService: Command processed (Alice gets bonus) -``` - -## Why Product Engines are Powerful - -* **Flexibility:** Easily define unique behaviors for many different banking products without cluttering the core system. -* **Modularity:** Product-specific logic is isolated. Changes to one product's rules don't affect others. -* **Maintainability:** Easier to understand and update the logic for a specific product. -* **Testability:** Individual Product Engines can be tested in isolation with a mocked `ProductEngineContext`. -* **Rapid Development:** New financial products can often be introduced by creating a new Product Engine and its configuration, rather than changing deep parts of the core system. - -## Conclusion - -Product Engines are the specialized "brains" behind each financial product offered by the bank. They act as detailed rulebooks and automated process managers, defining how a product like a "Standard Mortgage" or a "Youth Savings Account" behaves – its features, fees, interest calculations, and responses to lifecycle events. - -When an [Event](01_event_.md) occurs for an account, the `ProductEnginesService` (often triggered by a [Consumer](08_consumer_.md)) identifies the account's product and invokes the corresponding Product Engine. The engine then applies its specific rules and can return a list of actions, such as [Commands](02_command_.md) to create transactions, for the system to execute. This powerful mechanism allows `corebanking` to support a diverse range of financial products with complex, individualized behaviors. - -With all these intricate product behaviors and event-driven interactions, how can we be sure everything works as expected? How can we test different scenarios, perhaps even simulating future events? In our next and final chapter, we'll look into [Simulation Services and Repositories](10_simulation_services_and_repositories_.md). - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/10_simulation_services_and_repositories_.md b/nesto/corebanking/10_simulation_services_and_repositories_.md deleted file mode 100644 index dd0db4cd..00000000 --- a/nesto/corebanking/10_simulation_services_and_repositories_.md +++ /dev/null @@ -1,205 +0,0 @@ -# Chapter 10: Simulation Services and Repositories - -Welcome to the final chapter of our conceptual overview! In [Chapter 9: Product Engine](09_product_engine_.md), we saw how our `corebanking` system can support diverse financial products, each with its own complex set of rules for things like interest calculation, fee application, and behavior over time. - -Now, imagine we've just designed a sophisticated 30-year mortgage product using our [Product Engine](09_product_engine_.md). How can we be absolutely sure it works correctly under all sorts of conditions? How do we test its behavior over the full 30 years, perhaps with fluctuating interest rates, without actually waiting for three decades or messing with real customer data? This is where **Simulation Services and Repositories** come to the rescue! - -## What are Simulation Services and Repositories? Your Bank's Flight Simulator! - -Think of Simulation Services and Repositories as your bank's very own **flight simulator** or **practice mode**. They are special "test-double" or "fake" versions of the regular [Services](07_service_.md) and [Repositories](04_repository_.md) we've learned about. - -Instead of connecting to real databases and affecting real money or customer data, these simulators: -* **Work "in-memory":** They store their data temporarily, right in the computer's memory, for the duration of a test or simulation. When the simulation ends, this temporary data is gone. This is great because it's fast and completely isolated from the live system. -* **Mimic real behavior:** They are designed to act just like their real counterparts, but give us much more control. -* **Allow controlled testing:** We can use them to test complex business logic, like our 30-year mortgage product, under various scenarios (e.g., "what if interest rates double next year?") in a safe and repeatable way. - -Essentially, these simulators let us "play out" complex financial scenarios, fast-forward through time, and see what happens, all without any real-world consequences. - -## Key Ideas Behind Simulation - -How do these simulators achieve this magic? There are a few key ingredients: - -1. **In-Memory Storage:** - * Simulation [Repositories](04_repository_.md) (like `AccountRepositorySimulator` or `PostingsTransactionsRepositorySimulator`) don't save data to a permanent database. Instead, they use simple Go data structures like maps and slices to hold information temporarily. - * This makes them very fast and ensures that each simulation run starts fresh. - -2. **Controlled Time (The "Time Machine"):** - * A special `ClockServiceSimulator` lets us control the "current date" within the simulation. We can tell it to jump forward day by day, month by month, or even set it to a specific date in the future. This is crucial for testing products that behave differently over long periods. - -3. **Deterministic Behavior:** - * For a given starting point and set of actions, a simulation should ideally produce the exact same results every time it's run. This makes testing reliable and helps us pinpoint issues if something changes unexpectedly. - -4. **Isolation:** - * Simulations run completely separately from the live banking system. There's no risk of accidentally creating a real account or moving real money. - -## Using the Simulators: Testing Our Mortgage Product - -Let's go back to our 30-year mortgage product. How would we use the simulation tools to test it? The main entry point for complex product simulations is often the `ProductEnginesServiceSimulatorDefault` (found in `api/pkg/productengines/simulator/simulator.go`). - -Here's a conceptual walkthrough: - -1. **Setup the Simulation:** - * First, we'd create an instance of `ProductEnginesServiceSimulatorDefault`. This master simulator internally sets up other necessary simulator components like `AccountsServiceSimulator`, `BalancesServiceSimulator`, `PostingsServiceSimulator`, a `ClockServiceSimulator`, and a `CommandBusSimulator`. - * We'd tell the simulator about our "Mortgage Product" definition (its rules, interest rates, fee structures, etc.) by registering it. - * We'd then create a "Simulated Account" – let's say, "Alice's Mortgage Account" – and link it to our mortgage product. We might also set its initial loan amount and other starting conditions. - * We'd define a time period for our simulation, for example, from "2024-01-01" to "2025-01-01" to see how it behaves over one year. - -2. **Run the Simulation:** - * We'd call the `Simulate` method on our `ProductEnginesServiceSimulatorDefault`, giving it Alice's simulated account and our desired start and end dates. - -3. **What Happens Inside the `Simulate` Method (Simplified):** - * The simulator enters a loop, advancing day by day from the start date to the end date using its internal `ClockServiceSimulator`. - * On each simulated day: - * It checks if any scheduled events are due for Alice's account (e.g., "end of month interest calculation," "payment due date"). - * If so, it triggers these events. These events are then passed to the (real) [Product Engine](09_product_engine_.md) logic for our mortgage product. - * The mortgage [Product Engine](09_product_engine_.md) runs its rules based on the current simulated date and account state. It might decide to: - * Calculate interest due. - * Generate a [Command](02_command_.md) like `CreatePostingsTransactionCommand` to add this interest to Alice's loan balance. - * This `CreatePostingsTransactionCommand` is then handled by the `CommandBusSimulator`. The `CommandBusSimulator` doesn't send it to the real system; instead, it directs it to other *simulated* services. - * For example, the `PostingsServiceSimulator` would "commit" these interest postings to its in-memory store. - * The `BalancesServiceSimulator` would update Alice's simulated loan balance in its in-memory store. - * This loop continues until the simulation end date is reached. - -4. **Get the Results:** - * Once the simulation is complete, the `Simulate` method returns a `SimulationResult` struct. - * This `SimulationResult` contains a snapshot of Alice's mortgage account at the end of the simulated year: her final loan balance, a list of all transactions (postings) that occurred (like interest charges, payments), any scheduled items, and potentially any errors that happened during the simulation. - -5. **Check the Results:** - * We can then examine this `SimulationResult` to verify if our mortgage product behaved as expected. Did the interest calculate correctly? Were payments applied properly? Is the final balance what we predicted? - -## A Peek at Some Simulator Code - -Let's look at tiny, simplified snippets to get a feel for how these simulators are built. - -### 1. The `AccountRepositorySimulator` (Storing Data In-Memory) - -This simulator (from `api/pkg/productengines/simulator/accounts_repository.go`) fakes a [Repository](04_repository_.md) for accounts. - -```go -// Simplified from: api/pkg/productengines/simulator/accounts_repository.go -type AccountRepositorySimulator struct { - // Accounts are stored in a map, with account ID as the key. - accounts map[uuid.UUID]*api.Account - // ... other fields for simulation ... -} - -// GetByID tries to find an account in its in-memory map. -func (r *AccountRepositorySimulator) GetByID( - ctx context.Context, accountID uuid.UUID, -) (*api.Account, error) { - account, found := r.accounts[accountID] - if !found { - // Return an error if not found (simplified error) - return nil, errors.New("simulated account not found") - } - return account, nil // Return the account from memory -} -``` -* Instead of database queries, it uses a simple Go `map` (`r.accounts`) to store account data. -* The `GetByID` method just looks up the account in this map. Very fast! - -### 2. The `ClockServiceSimulator` (Controlling Time) - -The `ProductEnginesServiceSimulatorDefault` uses a clock service (like `api.ClockServiceDefault`) and controls it. A key method for the simulation loop is `SetDate`: - -```go -// Conceptual use of ClockService's SetDate within the simulation loop -// (s.clock is an instance of a clock service) -// s.clock.SetDate(currentSimulatedDate, timeZone) -``` -* Inside the `Simulate` method's loop, this `SetDate` is called repeatedly to advance the simulation's "current time" one day at a time. This makes the whole system believe it's that specific day. - -### 3. The `ProductEnginesServiceSimulatorDefault`'s `Simulate` Method - -This is the heart of the product simulation (from `api/pkg/productengines/simulator/simulator.go`). - -```go -// Highly simplified concept of Simulate method's loop -// (s is *ProductEnginesServiceSimulatorDefault) -func (s *ProductEnginesServiceSimulatorDefault) Simulate(/*...params...*/) (*SimulationResult, error) { - // ... (lots of initial setup: create account in simulator, set initial balances) ... - - // Loop from start date to end date - for refDate := startDate; refDate.Before(endDate); refDate = refDate.AddDays(1) { - s.clock.SetDate(refDate, s.c.ReferenceLocation) // Advance simulated time! - - // 1. Trigger scheduled events for this 'refDate' - // This involves calling s.HandleEvents(), which eventually - // invokes the Product Engine for the account. - // (Simplified representation) - s.processScheduledItemsForDate(ctx, account.ID, refDate) - - // 2. Settle any simulated transactions due on this 'refDate' - // (e.g., if a payment was scheduled to clear today) - s.settleTransactions(ctx, simulationConfig, account.ID, refDate) - } - - // ... (gather all simulated data: final account state, postings, balances) ... - return &SimulationResult{ /* ... collected data ... */ }, nil -} -``` -* The loop iterates day by day. -* `s.clock.SetDate(...)` tells the entire simulation what "today" is. -* `processScheduledItemsForDate` (a conceptual helper representing logic within `Simulate` and `GetSchedules/HandleEvents` from `simulator.go`) finds any scheduled tasks for the account on this simulated day (like "calculate monthly interest"). It then uses `s.HandleEvents()` to pass these to the [Product Engine](09_product_engine_.md). The Product Engine might then generate [Commands](02_command_.md). -* These [Commands](02_command_.md) are handled by the `CommandBusSimulator` (also part of `ProductEnginesServiceSimulatorDefault`), which ensures they are processed by *other simulated services and repositories*, updating the in-memory state. -* `s.settleTransactions` handles any pre-registered transaction settlements for that day. - -## How it All Connects: A Simulation Day - -Here's a simplified sequence of what might happen on one simulated day for our mortgage account: - -```mermaid -sequenceDiagram - participant SimLoop as Simulation Loop (in Simulate method) - participant ClockSim as ClockServiceSimulator - participant PESimService as ProductEnginesServiceSimulator - participant MortgagePE as Mortgage Product Engine - participant CmdBusSim as CommandBusSimulator - participant PostingRepoSim as PostingRepositorySimulator (In-Memory) - participant BalanceRepoSim as BalanceRepositorySimulator (In-Memory) - - SimLoop->>ClockSim: SetDate(currentSimulatedDay) - SimLoop->>PESimService: ProcessScheduledEvents(account, currentSimulatedDay) - PESimService->>MortgagePE: HandleScheduledEvent(context, scheduledEvent) - MortgagePE-->>MortgagePE: Calculate interest... - MortgagePE-->>PESimService: Return [Action: CreateInterestTxCommand] - PESimService->>CmdBusSim: HandleCommand(CreateInterestTxCommand) - CmdBusSim->>PostingRepoSim: Save new interest postings (in-memory) - CmdBusSim->>BalanceRepoSim: Update loan balance (in-memory) -``` -1. The simulation loop sets the "current day" using the `ClockSim`. -2. It asks the `PESimService` (our main simulator) to process any events for that day. -3. The `PESimService` invokes the actual `MortgagePE` (Product Engine) logic. -4. The `MortgagePE` decides interest needs to be charged and returns an action to create a transaction command. -5. The `PESimService` uses the `CmdBusSim` to handle this command. -6. The `CmdBusSim` ensures that the resulting postings and balance updates are stored in the *in-memory* repositories (`PostingRepoSim`, `BalanceRepoSim`). - -This cycle repeats for every day in the simulation period. - -## Why are these Simulators So Valuable? - -* **Safety First:** We can test the riskiest financial logic without any danger to real operations or data. -* **Blazing Speed:** Simulating years of financial activity can take just seconds or minutes, not actual years. -* **Perfect Repetition:** If a test fails, we can run the exact same simulation again to understand and fix the problem. This is called deterministic testing. -* **"What If?" Scenarios:** We can easily explore complex situations: What if interest rates soar? What if a customer misses payments? How does our product react? -* **Debugging Superpowers:** When something goes wrong in a simulation, it's often easier to trace the problem step-by-step through the in-memory state changes. -* **Building Confidence:** Thorough simulation gives us much higher confidence that our banking products will work correctly in the real world. - -## Conclusion - -Simulation Services and Repositories are powerful, specialized tools within the `corebanking` project. They create a safe, controlled, and fast "practice environment" where we can: -* Test complex financial products (especially those managed by [Product Engines](09_product_engine_.md)) over long periods. -* Simulate various market conditions and customer behaviors. -* Verify that our business logic is correct and robust. - -By using in-memory data storage, a controllable clock, and simulated versions of core components, we can gain deep insights into how our system will behave, all before it ever touches real customer information or funds. This is essential for building a reliable and trustworthy core banking system. - ---- - -Congratulations on completing this tour of the core concepts in the `corebanking` project! From the fundamental [Events](01_event_.md) and [Commands](02_command_.md) that drive the system, through [Aggregates](03_aggregate_.md) that protect business rules, [Repositories](04_repository_.md) that manage data, to the [API Handlers](05_api_handler_.md) that welcome external requests, the coordinating [Core Facade](06_core_facade_.md), specialized [Services](07_service_.md), reactive [Consumers](08_consumer_.md), intelligent [Product Engines](09_product_engine_.md), and finally, these invaluable Simulation tools, you now have a solid foundation. - -While this marks the end of our conceptual overview, your journey of exploration and contribution is just beginning. You can now revisit earlier chapters with a deeper understanding or start exploring the codebase to see these concepts in full action! - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/index.md b/nesto/corebanking/index.md deleted file mode 100644 index 162cbff4..00000000 --- a/nesto/corebanking/index.md +++ /dev/null @@ -1,78 +0,0 @@ -# Tutorial: corebanking - -The project is a **core banking system** designed to manage financial operations such as *account creation, transaction processing (postings), and financial product lifecycle management*. -It heavily relies on an *Event Sourcing* architecture, where system state changes are captured as a sequence of immutable **Events**. -*Aggregates* (like `AccountAggregate` or `PostingsTransactionAggregate`) are central entities that process **Commands** (requests for change) and emit these Events. -*Services* (e.g., `AccountsService`, `ProductEnginesService`) encapsulate business logic and often orchestrate interactions between Aggregates, Repositories, and Product Engines. -*Repositories* provide an abstraction layer for data persistence. -*Product Engines* define the specific rules, behaviors, and automated processes for different financial products (e.g., mortgages, HELOCs). - - -**Source Repository:** [None](None) - -```mermaid -flowchart TD - A0["Aggregate -"] - A1["Command -"] - A2["Event -"] - A3["Service -"] - A4["Repository -"] - A5["Product Engine -"] - A6["Consumer -"] - A7["API Handler -"] - A8["Core Facade -"] - A9["Simulation Services and Repositories -"] - A0 -- "Processes" --> A1 - A0 -- "Produces" --> A2 - A0 -- "Uses for persistence" --> A4 - A0 -- "Uses dependent" --> A3 - A7 -- "Uses" --> A8 - A8 -- "Dispatches (via EventSourci..." --> A1 - A8 -- "Provides access to" --> A3 - A3 -- "Uses for data access" --> A4 - A3 -- "Invokes/Manages" --> A5 - A6 -- "Consumes" --> A2 - A6 -- "Updates read-models via" --> A4 - A6 -- "Uses (e.g., ProductEnginesS..." --> A3 - A5 -- "Reacts to (via ProductEngin..." --> A2 - A9 -- "Simulates" --> A3 - A9 -- "Simulates" --> A4 -``` - -## Chapters - -1. [Event -](01_event_.md) -2. [Command -](02_command_.md) -3. [Aggregate -](03_aggregate_.md) -4. [Repository -](04_repository_.md) -5. [API Handler -](05_api_handler_.md) -6. [Core Facade -](06_core_facade_.md) -7. [Service -](07_service_.md) -8. [Consumer -](08_consumer_.md) -9. [Product Engine -](09_product_engine_.md) -10. [Simulation Services and Repositories -](10_simulation_services_and_repositories_.md) - - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/nesto/corebanking/review.md b/nesto/corebanking/review.md deleted file mode 100644 index d70a0d95..00000000 --- a/nesto/corebanking/review.md +++ /dev/null @@ -1,36 +0,0 @@ -# Core Banking Documentation Review - -## Section-by-Section Analysis - -- **Event**: Not bad overall, except the diagram which is really off -- **Command**: Not too bad -- **Aggregate**: Generally good, but hallucinates a command that doesn't exist (DepositFundsCommand) and from there it goes off the rails, claiming that the account manages transactions and balances... -- **Repository**: Not too bad, but the 2nd diagram is really off -- **API Handler**: Not too bad -- **Core Facade**: Not too bad -- **Service**: There are some impressive parts in this section, like the explanation of why services don't implement writes and an exception where the service does implement the write -- **Consumer**: Same thing - impressive analysis of the balances consumer. Some weird naming though: TxnProcessor/BalConsumer. -- **Product Engine**: Really good too (I'm curious if it will be as good with the latest changes I made), diagram is off again... -- **Simulation Services and Repos**: Weird product example, didn't catch that each product engine must implement its own simulator and allows offering features specific to the product, for example lump sum on a mortgage, drawdown request on a HELOC, etc. But overall conceptually it explains what can be done by implementing the simulator for a specific product engine. - -## Overall Assessment - -It's a really weird feeling because overall it's really not bad - there are even places where it surpasses the analysis I would have thought possible by an LLM. But it seems like there are enough small errors everywhere that it just feels like work done by someone who "doesn't care"... I'm not sure how to explain it. - -## Specific Issues - -For example, it generates code that's a bit more than pseudo-code, but outputs something like `uuid.FromString("acc-alice-123")`. The naming doesn't follow the conventions of the analyzed code like `accountsSvc` or `accountsRepo`. The diagrams invent layers that don't exist, and there are comments in the repo struct that are wrong. - -Otherwise, there are subtleties that I gloss over without thinking too much when reading because they make more or less sense, but not enough to really derail the reading. But I don't know if it would harm someone new to the codebase or not. - -## Major Concerns - -But there are some more major issues, like the invention of the DepositFundsCommand command which would really harm someone new to the codebase in understanding why transaction commands are sent directly to the account aggregate. - -## Missing Sections - -Otherwise, there would probably need to be a section for products, parameters, and customers. - -## Final Thoughts - -Overall, if there's a "parental advisory" sign on the documentation that it's auto-generated and to take certain details with a grain of salt, I think it covers the different concepts in corebanking and can probably help someone who doesn't know the project. It hammers the concepts of CQRS/event sourcing quite a bit. \ No newline at end of file From 558a71e116e4bd29991e2e28b6b336f456331eb7 Mon Sep 17 00:00:00 2001 From: Louis-David Coulombe Date: Mon, 27 Oct 2025 21:12:41 -0400 Subject: [PATCH 14/15] Removed tmp --- tmp/WorkflowDocumentation.md | 109 ------- tmp/configs/corebanking.yaml | 91 ------ tmp/configs/documents-extractor.yaml | 91 ------ tmp/configs/example_config.yaml | 72 ----- tmp/nesto/corebanking/01_event_.md | 179 ------------ tmp/nesto/corebanking/02_command_.md | 179 ------------ tmp/nesto/corebanking/03_aggregate_.md | 241 ---------------- tmp/nesto/corebanking/04_repository_.md | 229 --------------- tmp/nesto/corebanking/05_api_handler_.md | 225 --------------- tmp/nesto/corebanking/06_core_facade_.md | 160 ----------- tmp/nesto/corebanking/07_service_.md | 264 ----------------- tmp/nesto/corebanking/08_consumer_.md | 204 -------------- tmp/nesto/corebanking/09_product_engine_.md | 265 ------------------ ...0_simulation_services_and_repositories_.md | 205 -------------- tmp/nesto/corebanking/index.md | 78 ------ tmp/nesto/corebanking/review.md | 36 --- tmp/prompt.md | 177 ------------ tmp/run_analysis.sh | 21 -- 18 files changed, 2826 deletions(-) delete mode 100644 tmp/WorkflowDocumentation.md delete mode 100644 tmp/configs/corebanking.yaml delete mode 100644 tmp/configs/documents-extractor.yaml delete mode 100644 tmp/configs/example_config.yaml delete mode 100644 tmp/nesto/corebanking/01_event_.md delete mode 100644 tmp/nesto/corebanking/02_command_.md delete mode 100644 tmp/nesto/corebanking/03_aggregate_.md delete mode 100644 tmp/nesto/corebanking/04_repository_.md delete mode 100644 tmp/nesto/corebanking/05_api_handler_.md delete mode 100644 tmp/nesto/corebanking/06_core_facade_.md delete mode 100644 tmp/nesto/corebanking/07_service_.md delete mode 100644 tmp/nesto/corebanking/08_consumer_.md delete mode 100644 tmp/nesto/corebanking/09_product_engine_.md delete mode 100644 tmp/nesto/corebanking/10_simulation_services_and_repositories_.md delete mode 100644 tmp/nesto/corebanking/index.md delete mode 100644 tmp/nesto/corebanking/review.md delete mode 100644 tmp/prompt.md delete mode 100755 tmp/run_analysis.sh diff --git a/tmp/WorkflowDocumentation.md b/tmp/WorkflowDocumentation.md deleted file mode 100644 index 2d5bb9a7..00000000 --- a/tmp/WorkflowDocumentation.md +++ /dev/null @@ -1,109 +0,0 @@ - # Documentation Generation Workflow - -This document explains the workflow of the documentation generation process, focusing on how the Large Language Model (LLM) prompts chain together to create a comprehensive tutorial for a given codebase. - -## Overview - -The process is a pipeline of six sequential steps. Four of these steps involve interactions with an LLM to analyze the code, structure the content, and write the documentation. - -```mermaid -flowchart TD - A[1. Fetch Repository] --> B[2. Identify Abstractions] - B --> C[3. Analyze Relationships] - C --> D[4. Order Chapters] - D --> E[5. Write Chapters] - E --> F[6. Combine Tutorial] -``` - -## Detailed Steps & Prompt Chaining - -Here’s a detailed breakdown of each step, focusing on the inputs and outputs of the LLM prompts. - -### 1. Fetch Repository (`FetchRepo`) -This initial step does not use an LLM. It's responsible for gathering the source code. - -- **Input**: A repository URL or a local directory path. -- **Output**: A list of all relevant code files and their content. This list becomes the foundational input for the entire workflow. -- **Chaining**: The raw codebase content is passed to the next step. - ---- - -### 2. Identify Abstractions (`IdentifyAbstractions`) -This is the first interaction with the LLM. Its goal is to identify the most important, high-level concepts in the codebase. - -- **Input to Prompt**: The entire codebase content fetched in the previous step. -- **LLM Prompt Goal**: The LLM is asked to analyze the code and identify the top 5-10 core abstractions. For each abstraction, it must provide: - 1. A concise `name`. - 2. A beginner-friendly `description` with an analogy. - 3. A list of relevant `file_indices` that implement or define the abstraction. -- **Output of Prompt**: The LLM returns a YAML-formatted string. - ```yaml - - name: | - Core Concept A - description: | - An explanation of what this concept does, like a central controller. - file_indices: - - 0 # path/to/file1.py - - 3 # path/to/file2.py - ``` -- **Chaining**: The validated list of abstractions (name, description, file indices) is passed to the next step. - ---- - -### 3. Analyze Relationships (`AnalyzeRelationships`) -The second LLM interaction focuses on understanding how the identified abstractions interact. - -- **Input to Prompt**: The list of abstractions (names and descriptions) and the code snippets from their relevant files. -- **LLM Prompt Goal**: The LLM is prompted to: - 1. Create a high-level `summary` of the project's purpose. - 2. Define the `relationships` between the abstractions, describing how they interact (e.g., "Manages", "Inherits from", "Uses"). -- **Output of Prompt**: A YAML object containing the summary and a list of relationships. - ```yaml - summary: | - A brief, simple explanation of the project's purpose. - relationships: - - from_abstraction: 0 # Core Concept A - to_abstraction: 1 # Core Concept B - label: "Manages" - ``` -- **Chaining**: The project summary and the list of relationships are passed to the next step. - ---- - -### 4. Order Chapters (`OrderChapters`) -The third LLM interaction determines the best pedagogical order to present the concepts. - -- **Input to Prompt**: The project summary, the list of abstractions, and their relationships. -- **LLM Prompt Goal**: The LLM is asked to determine the optimal sequence for a tutorial. It's instructed to start with foundational or user-facing concepts and then move to lower-level implementation details, respecting dependencies revealed in the relationships. -- **Output of Prompt**: A YAML list of the abstraction indices, sorted in the recommended chapter order. - ```yaml - - 2 # FoundationalConcept - - 0 # CoreClassA - - 1 # CoreClassB (uses CoreClassA) - ``` -- **Chaining**: This ordered list of indices dictates the structure of the final tutorial and is passed to the chapter writing step. - ---- - -### 5. Write Chapters (`WriteChapters`) -This is the most intensive LLM step, where the actual tutorial content is generated for each abstraction, one by one. - -- **Input to Prompt (for each chapter)**: - - The specific abstraction's details (name, description). - - The full tutorial structure (for linking to other chapters). - - The content of previously written chapters (to ensure smooth transitions). - - Relevant code snippets for the current abstraction. -- **LLM Prompt Goal**: The LLM is given a detailed set of instructions to write a beginner-friendly Markdown chapter. This includes creating a heading, explaining the concept with analogies, providing simplified code examples (under 10 lines), using Mermaid diagrams for illustration, and writing transitions to the previous and next chapters. -- **Output of Prompt**: A Markdown-formatted string for each chapter. -- **Chaining**: The list of all generated Markdown chapter strings is passed to the final step. - ---- - -### 6. Combine Tutorial (`CombineTutorial`) -This final step does not use an LLM. It assembles all the generated pieces into the final documentation. - -- **Input**: The project summary, relationship graph, chapter order, and the content of all chapters. -- **Output**: A directory containing: - - `index.md`: An overview page with the project summary, a Mermaid diagram of the relationships, and a linked table of contents. - - `01_concept.md`, `02_another.md`, etc.: Individual chapter files in Markdown format. -- **Chaining**: This is the final step, and the output is the completed tutorial saved to the filesystem. \ No newline at end of file diff --git a/tmp/configs/corebanking.yaml b/tmp/configs/corebanking.yaml deleted file mode 100644 index a93a1063..00000000 --- a/tmp/configs/corebanking.yaml +++ /dev/null @@ -1,91 +0,0 @@ -# Corebanking Project Analysis Configuration - -# Source configuration - specify either repo or local_dir -source: - local_dir: "/Users/louis-davidcoulombe/github/corebanking" - # repo: "https://github.com/example/corebanking" # Use this instead of local_dir for GitHub repos - -# Project settings -project: - name: "corebanking" - language: "english" - output_dir: "nesto/corebanking/analysis_output" - -# File processing settings -file_processing: - max_file_size: 150000 # Maximum file size in bytes - include_patterns: - - "*.py" - - "*.js" - - "*.jsx" - - "*.ts" - - "*.tsx" - - "*.go" - - "*.java" - - "*.pyi" - - "*.pyx" - - "*.c" - - "*.cc" - - "*.cpp" - - "*.h" - - "*.md" - - "*.rst" - - "*Dockerfile" - - "*Makefile" - - "*.yaml" - - "*.yml" - exclude_patterns: - - "assets/*" - - "data/*" - - "images/*" - - "public/*" - - "static/*" - - "temp/*" - - "*docs/*" - - "*venv/*" - - "*.venv/*" - - "*test*" - - "*tests/*" - - "*examples/*" - - "v1/*" - - "*dist/*" - - "*build/*" - - "*experimental/*" - - "*deprecated/*" - - "*misc/*" - - "*legacy/*" - - ".git/*" - - ".github/*" - - ".next/*" - - ".vscode/*" - - "*obj/*" - - "*bin/*" - - "*node_modules/*" - - "*.log" - -# Analysis settings -analysis: - max_abstractions: 13 # Based on number of abstraction hints provided - abstractions_hints: - - "Event" - - "Command" - - "Aggregate" - - "Repository" - - "API Handler" - - "Core Facade" - - "Service" - - "Consumer" - - "Product Engine" - - "Simulation Services and Repositories" - - "products" - - "parameters" - - "customers" - feedback_file: "nesto/corebanking/review.md" - -# LLM settings -llm: - use_cache: true # Set to false to disable caching - -# GitHub settings (only needed if using repo instead of local_dir) -github: - # token: "your_github_token_here" # Or set GITHUB_TOKEN environment variable diff --git a/tmp/configs/documents-extractor.yaml b/tmp/configs/documents-extractor.yaml deleted file mode 100644 index 3fafdcf5..00000000 --- a/tmp/configs/documents-extractor.yaml +++ /dev/null @@ -1,91 +0,0 @@ -# Corebanking Project Analysis Configuration - -# Source configuration - specify either repo or local_dir -source: - local_dir: "/Users/louis-davidcoulombe/src/github.com/nestoca/documents-extractor" - # repo: "https://github.com/example/corebanking" # Use this instead of local_dir for GitHub repos - -# Project settings -project: - name: "documents-extractor" - language: "english" - output_dir: "nesto/corebanking/documents-extractor" - -# File processing settings -file_processing: - max_file_size: 150000 # Maximum file size in bytes - include_patterns: - - "*.py" - - "*.js" - - "*.jsx" - - "*.ts" - - "*.tsx" - - "*.go" - - "*.java" - - "*.pyi" - - "*.pyx" - - "*.c" - - "*.cc" - - "*.cpp" - - "*.h" - - "*.md" - - "*.rst" - - "*Dockerfile" - - "*Makefile" - - "*.yaml" - - "*.yml" - exclude_patterns: - - "assets/*" - - "data/*" - - "images/*" - - "public/*" - - "static/*" - - "temp/*" - - "*docs/*" - - "*venv/*" - - "*.venv/*" - - "*test*" - - "*tests/*" - - "*examples/*" - - "v1/*" - - "*dist/*" - - "*build/*" - - "*experimental/*" - - "*deprecated/*" - - "*misc/*" - - "*legacy/*" - - ".git/*" - - ".github/*" - - ".next/*" - - ".vscode/*" - - "*obj/*" - - "*bin/*" - - "*node_modules/*" - - "*.log" - -# Analysis settings -# analysis: -# max_abstractions: 13 # Based on number of abstraction hints provided -# abstractions_hints: -# - "Event" -# - "Command" -# - "Aggregate" -# - "Repository" -# - "API Handler" -# - "Core Facade" -# - "Service" -# - "Consumer" -# - "Product Engine" -# - "Simulation Services and Repositories" -# - "products" -# - "parameters" -# - "customers" -# feedback_file: "nesto/corebanking/review.md" - -# LLM settings -llm: - use_cache: false # Set to false to disable caching - -# GitHub settings (only needed if using repo instead of local_dir) -github: - # token: "your_github_token_here" # Or set GITHUB_TOKEN environment variable diff --git a/tmp/configs/example_config.yaml b/tmp/configs/example_config.yaml deleted file mode 100644 index e9c8d04c..00000000 --- a/tmp/configs/example_config.yaml +++ /dev/null @@ -1,72 +0,0 @@ -# Example Configuration File for Tutorial Generation -# Copy this file and modify for your specific project - -# Source configuration - specify either repo or local_dir (not both) -source: - # For local directories: - local_dir: "/path/to/your/local/project" - - # For GitHub repositories (comment out local_dir if using this): - # repo: "https://github.com/username/repository" - -# Project settings -project: - name: "your_project_name" - language: "english" # or "french", "spanish", etc. - output_dir: "output/your_project" # where to save the generated tutorial - -# File processing settings -file_processing: - max_file_size: 100000 # Maximum file size in bytes (default: 100KB) - - # File patterns to include in analysis - include_patterns: - - "*.py" - - "*.js" - - "*.jsx" - - "*.ts" - - "*.tsx" - - "*.go" - - "*.java" - - "*.md" - - "*.rst" - - "*.yaml" - - "*.yml" - - # File patterns to exclude from analysis - exclude_patterns: - - "*test*" - - "*tests/*" - - "*node_modules/*" - - "*venv/*" - - "*.venv/*" - - "*dist/*" - - "*build/*" - - ".git/*" - - "*.log" - -# Analysis settings -analysis: - max_abstractions: 10 # Maximum number of abstractions to identify - - # Specific abstractions to look for (optional) - abstractions_hints: - - "Class" - - "Interface" - - "Service" - - "Controller" - - "Repository" - - "Model" - - "Component" - - "Module" - - # Path to feedback file from previous runs (optional) - # feedback_file: "path/to/feedback.md" - -# LLM settings -llm: - use_cache: true # Set to false to disable response caching - -# GitHub settings (only needed if using repo instead of local_dir) -github: - # token: "your_github_token_here" # Or set GITHUB_TOKEN environment variable diff --git a/tmp/nesto/corebanking/01_event_.md b/tmp/nesto/corebanking/01_event_.md deleted file mode 100644 index 8d1f7c5c..00000000 --- a/tmp/nesto/corebanking/01_event_.md +++ /dev/null @@ -1,179 +0,0 @@ -# Chapter 1: Event - -Welcome to the `corebanking` project tutorial! We're thrilled to have you on board. In this first chapter, we'll dive into one of the most fundamental concepts in our system: the **Event**. - -## What's the Big Deal with Events? - -Imagine you're building a brand new digital bank. A customer, let's call her Alice, wants to open a new savings account. She fills out a form, clicks "Submit," and voilà, her account is created. - -But how does the bank *remember* that this account was created? How does it keep a permanent, trustworthy record of this action? What if we need to look back and see exactly when Alice's account was opened or what its initial details were? - -This is where **Events** come into play. - -In our `corebanking` system, an **Event** is an **immutable record of something significant that has already happened**. Think of it like an official entry in a historical logbook or a diary. Once an entry is written, it's not changed. - -When something important occurs in the system, like Alice's account being created, we record this fact as an Event. For example, after successfully processing Alice's request to create an account (which we'll later learn is a [Command](02_command_.md)), an `AccountCreatedEvent` is generated. - -## Key Characteristics of an Event - -Events have a few defining features: - -1. **Immutable Record:** Once an Event is recorded, it **cannot be changed**. It's a statement of fact about something that happened in the past. Like carving something in stone. -2. **Significant Happenings:** We don't create Events for every tiny operation. They are reserved for actions that have a meaningful impact on the system's state. Examples: `AccountCreatedEvent`, `FundsDepositedEvent`, `CustomerAddressUpdatedEvent`. -3. **Past Tense:** Event names usually reflect that the action has already completed. Notice the "ed" or "d" at the end: `AccountCreatedEvent`, `PaymentDeviceLinkedEvent`. -4. **Source of Truth:** Events are the ultimate, undeniable truth for what has occurred. If we ever need to understand the current state of something (like an account's balance or status), we can, in theory, replay all the Events related to it from the very beginning. It's like re-reading a ship's log to know its entire journey. -5. **Result of a [Command](02_command_.md):** Typically, an Event is produced after the system processes a [Command](02_command_.md). A [Command](02_command_.md) is a request to do something (e.g., "Create Account"). If the command is successful and changes the system's state, one or more Events are generated as a result. - -## Events in Action: Creating Alice's Account - -Let's revisit Alice creating her savings account: - -1. Alice submits her request (this is a [Command](02_command_.md)). -2. The `corebanking` system processes this request. It checks if Alice can open an account, if she provided all necessary details, etc. -3. If everything is okay, the system changes its state: a new account for Alice now exists! -4. To record this fact, the system generates an `AccountCreatedEvent`. This Event will contain all the crucial information: Alice's customer ID, the new account ID, the type of account, the currency, the date and time of creation, etc. -5. This `AccountCreatedEvent` is then durably stored, perhaps in a special database. It becomes part of the bank's permanent history. - -##What Does an Event Look Like? (A Peek at the Code) - -In Go, an Event is often represented as a struct. There's a general structure for all events, and then specific data for each type of event. - -Here's a simplified general `Event` structure from our system: - -```go -// From: api/events.go - -// Event represents an event -type Event struct { - ID uuid.UUID // Unique ID for this specific event instance - AggregateID uuid.UUID // ID of the entity this event pertains to (e.g., Account ID) - AggregateType string // Type of the entity (e.g., "Account") - Type string // Specific type of event (e.g., "AccountCreatedEvent") - Data interface{} // The actual data specific to this event type - Created time.Time // When the event was created - // ... other general fields like Actor, TenantID ... -} -``` - -* `ID`: Every single event occurrence gets its own unique identifier. -* `AggregateID`: This tells us which specific entity the event is about. For an `AccountCreatedEvent`, this would be the ID of the newly created account. We'll learn more about [Aggregates](03_aggregate_.md) later. -* `AggregateType`: The kind of entity, like "Account" or "Customer". -* `Type`: A string that clearly states what kind of event this is, e.g., `com.nesto.corebanking.accounts.events.AccountCreatedEvent`. -* `Data`: This is the payload, containing the specific details of *what happened*. For an `AccountCreatedEvent`, this would hold the account category, currency, etc. -* `Created`: A timestamp indicating when the event occurred. - -The `Data` field itself would be another struct, specific to the event type. For example, an `AccountCreatedEvent` might have data like this (simplified): - -```go -// From: api/account_aggregate.go - -// AccountCreatedEvent represents a created account. -type AccountCreatedEvent struct { - Category string `avro:"category"` - Product *AccountProduct `avro:"product"` - AvailableCurrencies []string `avro:"availableCurrencies"` - CustomerIDs []uuid.UUID `avro:"customerIds"` - // ... other fields like Parameters, Metadata ... -} -``` -This `AccountCreatedEvent` struct holds all the specific details that are relevant when a new account is made. - -## How an Event is "Born" - The Internals - -Let's look under the hood to see how an `AccountCreatedEvent` comes to life. - -1. **A Request Arrives:** Alice's request to create an account comes in as a [Command](02_command_.md) (e.g., `CreateAccountCommand`). -2. **Processing by an [Aggregate](03_aggregate_.md):** An [Aggregate](03_aggregate_.md) is responsible for handling commands related to a specific entity (like an `AccountAggregate` for accounts). It takes the `CreateAccountCommand`. -3. **Validation and State Change:** The `AccountAggregate` checks if the command is valid (e.g., are all required fields present?). If valid, it determines what state changes are needed. For a new account, this means setting its initial properties. -4. **Event Generation:** Because the state changed, the `AccountAggregate` now creates an `AccountCreatedEvent`, filling it with details from the command and any system-generated data (like the creation timestamp). -5. **Event Storage:** This newly minted event is then passed to an "Event Store" (a specialized database or system component) to be saved permanently. - -Here's a simplified sequence diagram: - -```mermaid -sequenceDiagram - participant Alice - participant CoreBankingAPI as API - participant AccountAgg as Account Aggregate - participant EventStore - - Alice->>API: Submit Create Account Form - API->>AccountAgg: Process CreateAccountCommand - AccountAgg-->>AccountAgg: Validate, decide to create account - AccountAgg->>EventStore: New AccountCreatedEvent(details...) - EventStore-->>AccountAgg: Event Saved! - AccountAgg-->>API: Account Creation Successful - API-->>Alice: Your account is created! -``` - -Let's look at a snippet from `api/account_aggregate.go` where an event is created within the `HandleCommand` method. This method is part of the [Aggregate](03_aggregate_.md) (which we'll cover in detail later). - -```go -// Simplified from api/account_aggregate.go -// Inside AccountAggregate's HandleCommand method: - -case *CreateAccountCommand: // This is the request to create an account - // ... (some validation and setup logic) ... - - // Prepare the specific data for our AccountCreatedEvent - data := &AccountCreatedEvent{ - Category: c.Category, - Product: c.Product, - AvailableCurrencies: c.AvailableCurrencies, - CustomerIDs: c.CustomerIDs, - // ... other relevant details from the command ... - } - - // This is the magic moment! An Event is created. - // a.NewEvent() wraps 'data' with general event info (ID, timestamp, etc.) - event = a.NewEvent(command, data) - - // ... (logic to apply this event to the aggregate's state and save it) ... - a.ApplyChangeHelper(a, event, true) -``` -In this code: -* We receive a `CreateAccountCommand` (aliased as `c`). -* We gather the necessary details into an `AccountCreatedEvent` struct (`data`). -* `a.NewEvent(command, data)` is a helper method that takes the original command and the event-specific `data` to construct the full `Event` object, including its unique ID, timestamp, and type. -* `a.ApplyChangeHelper` is another crucial step. After an event is "born," the [Aggregate](03_aggregate_.md) itself uses this event to update its own internal state. This ensures the [Aggregate's](03_aggregate_.md) in-memory representation reflects the new reality. - -The `ApplyChange` method (or a helper it calls) looks at the type of event and updates the [Aggregate's](03_aggregate_.md) fields accordingly: - -```go -// Simplified from api/account_aggregate.go -// Inside AccountAggregate's ApplyChange method: - -func (a *AccountAggregate) ApplyChange(event es.Event) { - switch e := event.Data.(type) { // e is the specific event data - case *AccountCreatedEvent: - a.ID = event.AggregateID // The aggregate now knows its ID - a.Category = e.Category - a.Status = AccountStatusCreated // Set initial status - a.Created = event.Created // Record creation time - // ... update other fields based on AccountCreatedEvent ... - case *AccountActivatedEvent: - a.Status = AccountStatusActivated - // ... - // ... other event types ... - } -} -``` -This ensures that the `AccountAggregate`'s state is consistent with the events it has produced and processed. - -## Why Immutability and "Facts" Matter So Much - -The fact that Events are immutable historical records is incredibly powerful: - -* **Reliability & Auditability:** You have a perfect, trustworthy audit trail of everything significant that has happened. Need to know why an account is in a particular state? Replay its events! This is invaluable for debugging, compliance, and understanding system behavior. -* **State Reconstruction:** If, for some reason, the current "snapshot" of an account's data gets corrupted, you can rebuild it by replaying all its historical Events in order. -* **Decoupling:** Other parts of the system can subscribe to events and react to them independently, without needing to know the intricate details of how the event was produced. For example, a notification service could listen for `AccountCreatedEvent` and send a welcome email to Alice. - -## Conclusion - -Events are the bedrock of our `corebanking` system's memory. They are **immutable facts** representing **significant past occurrences**. Each Event tells a small part of a larger story, like the story of Alice's bank account. By recording and storing these Events, we build a reliable, auditable, and resilient system. - -We've seen that Events are often born from processing requests. In the next chapter, we'll take a closer look at these requests themselves. Get ready to learn about the [Command](02_command_.md)! - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/02_command_.md b/tmp/nesto/corebanking/02_command_.md deleted file mode 100644 index d8113c5b..00000000 --- a/tmp/nesto/corebanking/02_command_.md +++ /dev/null @@ -1,179 +0,0 @@ -# Chapter 2: Command - -In our [previous chapter](01_event_.md), we learned about [Events](01_event_.md) – the immutable records of things that have already happened in our `corebanking` system. For example, when Alice successfully created her bank account, an `AccountCreatedEvent` was generated and stored. - -But how did Alice *tell* the bank she wanted to create an account in the first place? How does she ask the bank to do something, like deposit money or change her address? This is where **Commands** come in. - -## What's a Command? The Bank's Instruction Slip - -Imagine you walk into a physical bank. To make a deposit, you fill out a deposit slip. To open an account, you fill out an application form. These forms are your formal instructions to the bank. - -In the `corebanking` system, a **Command** is very similar. It's like an **instruction or a formal request** you submit to the system. It represents an **intent** to change something. - -Think of it this way: -* An [Event](01_event_.md) says: "This happened!" (e.g., `AccountCreatedEvent`) -* A **Command** says: "I want this to happen!" (e.g., `CreateAccountCommand`) - -Commands are specific, named operations that carry all the necessary data to perform an action. They are the primary way external users or other systems interact with the core logic of our bank to request changes. - -## Key Characteristics of a Command - -1. **Intent to Act:** A Command expresses a desire for the system to perform an action. It's a request, not a statement of fact. -2. **Specific Operation:** Each Command has a clear name that describes the intended action, like `CreateAccountCommand`, `DepositFundsCommand`, or `UpdateCustomerAddressCommand`. -3. **Carries Data:** A Command bundles all the information needed to execute the desired action. For instance, a `CreateAccountCommand` would include details like who the customer is, what type of account they want, and in what currency. -4. **Sent for Processing:** Commands are typically sent to a specific part of our system called an [Aggregate](03_aggregate_.md) (we'll learn about Aggregates in the next chapter!). The [Aggregate](03_aggregate_.md) is responsible for deciding if the Command is valid and then carrying out the action. -5. **May Result in [Events](01_event_.md):** If a Command is successfully processed and leads to a change in the system's state, one or more [Events](01_event_.md) are usually generated to record that change. If the Command is invalid (e.g., trying to deposit into a non-existent account), it might be rejected, and no [Events](01_event_.md) related to the intended change would be created. - -## Commands in Action: Alice's Account Creation (The "How") - -Let's go back to Alice wanting to open her savings account: - -1. **Alice Expresses Her Intent:** Alice, through a web form or a mobile app, indicates she wants a new account. She provides her details (customer ID, desired currency, etc.). -2. **The `CreateAccountCommand` is Born:** The application she's using takes this information and packages it into a `CreateAccountCommand`. This command now holds all the data needed to open the account. -3. **Command is Sent:** This `CreateAccountCommand` is sent to the `corebanking` system. -4. **Processing by an [Aggregate](03_aggregate_.md):** An `AccountAggregate` (which is responsible for managing account-related operations) receives the `CreateAccountCommand`. -5. **Validation & Logic:** The `AccountAggregate` checks: - * Is the customer ID valid? - * Is the currency supported? - * Are all required fields present? - * Any other business rules? -6. **Success and [Event](01_event_.md) Generation:** If everything is valid, the `AccountAggregate` proceeds to create the account. As a result of this successful operation, it generates an `AccountCreatedEvent` (which we saw in Chapter 1!). -7. **Failure:** If the command is invalid (e.g., Alice provides an unsupported currency), the `AccountAggregate` rejects the command, and no `AccountCreatedEvent` is generated. An error message would typically be returned to Alice. - -## What Does a Command Look Like? (A Peek at the Code) - -In Go, just like [Events](01_event_.md), Commands are often represented as structs. They carry the data necessary for the operation. - -Here's a simplified look at the `CreateAccountCommand` from our `corebanking` project (`api/account_aggregate.go`): - -```go -// From: api/account_aggregate.go - -// CreateAccountCommand represents the creation of an account. -type CreateAccountCommand struct { - es.BaseCommand // Provides common fields for all commands - Category string `json:"category"` - Product *AccountProduct `json:"product"` // Details about the specific account product - AvailableCurrencies []string `json:"availableCurrencies"` - CustomerIDs []uuid.UUID `json:"customerIds"` - // ... other fields like Parameters, Metadata ... -} -``` - -Let's break this down: -* `es.BaseCommand`: This is a standard part of many commands in our system. It usually includes an `AggregateID`. For `CreateAccountCommand`, this `AggregateID` will be the ID of the *new* account we want to create. For a command like `DepositFundsCommand`, the `AggregateID` would be the ID of the *existing* account to deposit into. -* `Category`: What kind of account is it (e.g., "ASSET", "LIABILITY"). -* `Product`: Specifies the type of banking product (e.g., "Savings Account Basic", "Current Account Premium"). -* `AvailableCurrencies`: Which currencies can this account hold (e.g., `["USD", "EUR"]`). -* `CustomerIDs`: A list of customer IDs associated with this account. - -Each field in the command provides a piece of information that the system needs to fulfill the request. - -Another example, a `CloseAccountCommand`, would be simpler: - -```go -// From: api/account_aggregate.go - -// CloseAccountCommand represents the closing of an account. -type CloseAccountCommand struct { - es.BaseCommand // Contains the ID of the account to close - Reason string `json:"reason"` // Why is the account being closed? -} -``` -This command needs to know *which* account to close (via `BaseCommand`'s `AggregateID`) and optionally, a reason for closing it. - -## How a Command is Processed: Under the Hood - -When a command arrives, how does the system handle it? - -1. **Dispatch:** The command is typically sent from an API layer (like a web server handling Alice's request) to the core business logic. -2. **Targeting an [Aggregate](03_aggregate_.md):** The command is routed to the correct [Aggregate](03_aggregate_.md). An [Aggregate](03_aggregate_.md) is like a guardian for a specific piece of data (e.g., an `AccountAggregate` guards a specific bank account). If it's a `CreateAccountCommand`, a new `AccountAggregate` instance might be effectively created to handle it. If it's `DepositFundsCommand`, it's sent to the existing `AccountAggregate` for that account. -3. **Validation:** The [Aggregate](03_aggregate_.md) first validates the command. Can this action be performed? Is all the data correct and complete? -4. **Business Logic:** If valid, the [Aggregate](03_aggregate_.md) executes the business rules associated with the command. -5. **[Event](01_event_.md) Generation:** If the business logic results in a state change, the [Aggregate](03_aggregate_.md) creates one or more [Events](01_event_.md) to describe what happened. - -Here's a simplified sequence diagram illustrating the flow for creating an account: - -```mermaid -sequenceDiagram - participant AliceApp as Alice's App - participant CoreAPI as Core Banking API - participant AccountAgg as Account Aggregate - participant EventStore - - AliceApp->>CoreAPI: I want to create an account (details...) - CoreAPI->>AccountAgg: CreateAccountCommand{details...} - AccountAgg-->>AccountAgg: Validate command data (e.g., currency OK?) - AccountAgg-->>AccountAgg: Business logic: Okay, let's create it! - AccountAgg->>EventStore: Store this new AccountCreatedEvent(...) - EventStore-->>AccountAgg: Event saved! - AccountAgg-->>CoreAPI: Success! Account created. - CoreAPI-->>AliceApp: Your account is ready! -``` - -Let's look at a snippet from `api/account_aggregate.go` where an `AccountAggregate` handles a `CreateAccountCommand`. This happens inside a method called `HandleCommand`: - -```go -// Simplified from AccountAggregate.HandleCommand in api/account_aggregate.go - -// ... (inside HandleCommand method) ... -switch c := command.(type) { // 'c' is the incoming command -case *CreateAccountCommand: - // 1. Check if a product is specified and valid (simplified) - if c.Product != nil { - // ... (logic to validate product exists) ... - } - - // 2. The command carries the intended ID for the new account - a.ID = command.GetAggregateID() // 'a' is the AccountAggregate - - // 3. Prepare the data for the Event that will be created - eventData := &AccountCreatedEvent{ - Category: c.Category, - Product: c.Product, - CustomerIDs: c.CustomerIDs, - AvailableCurrencies: c.AvailableCurrencies, - // ... other details from the command 'c' ... - } - - // 4. Create the actual Event (as we saw in Chapter 1) - event := a.NewEvent(command, eventData) - - // 5. Apply the event to change the aggregate's state and record it - a.ApplyChangeHelper(a, event, true) // This stores the event for saving -``` - -In this snippet: -1. The `HandleCommand` method receives the `CreateAccountCommand` (aliased as `c`). -2. It performs some initial validation (like checking the product). -3. It uses the data *from the command* (`c.Category`, `c.Product`, etc.) to populate an `AccountCreatedEvent`. -4. The `a.NewEvent(...)` function (which we touched on in Chapter 1) creates the full [Event](01_event_.md) object, adding things like a unique event ID and timestamp. -5. `a.ApplyChangeHelper(...)` is a crucial step where the [Aggregate](03_aggregate_.md) updates its own state based on this new event and adds the event to a list of changes to be saved. - -So, the **Command** provides the *intent* and the *data*, and the [Aggregate's](03_aggregate_.md) `HandleCommand` method uses that to *do the work* and produce [Events](01_event_.md). - -## Commands vs. [Events](01_event_.md): A Quick Reminder - -It's vital to distinguish between Commands and [Events](01_event_.md): - -| Feature | Command | [Event](01_event_.md) | -| :-------------- | :-------------------------------------------- | :------------------------------------------------------ | -| **Purpose** | Request an action (intent) | Record a fact (something that happened) | -| **Tense** | Present/Future (e.g., "Create Account") | Past (e.g., "Account Created") | -| **Outcome** | May succeed or fail | Is a record of a successful state change | -| **Result of** | User input, system process | Successful command processing | -| **Analogy** | Filling out a request form | An entry in a historical logbook | - -You send a `CreateAccountCommand` (your request). If it's successful, an `AccountCreatedEvent` (the historical record) is generated. - -## Conclusion - -Commands are the way we tell our `corebanking` system what we want to do. They are formal, data-rich instructions representing an **intent to change the system's state**. They are named operations like `CreateAccountCommand` or `DepositFundsCommand`. - -When a Command is received, it's typically handled by an [Aggregate](03_aggregate_.md), which validates it, executes business logic, and, if successful, produces [Events](01_event_.md) to record the changes. - -Now that we understand how we *ask* the system to do things (Commands) and how the system *remembers* what happened ([Events](01_event_.md)), it's time to meet the component responsible for processing these Commands and generating those Events. Let's dive into the world of the [Aggregate](03_aggregate_.md)! - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/03_aggregate_.md b/tmp/nesto/corebanking/03_aggregate_.md deleted file mode 100644 index 4d605b58..00000000 --- a/tmp/nesto/corebanking/03_aggregate_.md +++ /dev/null @@ -1,241 +0,0 @@ -# Chapter 3: Aggregate - -Welcome to Chapter 3! In our [previous chapter](02_command_.md), we learned about [Commands](02_command_.md) – how we tell the `corebanking` system what we want to do, like "Create an Account" or "Deposit Funds." We also know from [Chapter 1](01_event_.md) that when these [Commands](02_command_.md) are successful, [Events](01_event_.md) are generated to record what happened. - -But who is in charge of receiving a [Command](02_command_.md), deciding if it's okay, and then creating the [Event](01_event_.md)? If Alice wants to deposit money into *her specific account*, how does the system make sure the change happens only to *her* account and follows all the rules? This is where the **Aggregate** steps in. - -## Meet the Main Character: The Aggregate - -Think of an **Aggregate** as the main character in a story, like a specific bank account (Account #12345) or a particular customer (Alice Wonderland). It's a self-contained unit that looks after its own information (its "state") and its own rules. - -When a [Command](02_command_.md) arrives—like a request to deposit money into Account #12345—it's the `AccountAggregate` for Account #12345 that handles it. This Aggregate will: -1. Check if the command is valid (e.g., "Is the account open?", "Is the deposit amount positive?"). -2. If everything is okay, it will change its state (e.g., increase its balance). -3. Then, it will record this change as a fact by producing an [Event](01_event_.md) (e.g., `FundsDepositedEvent`). - -This way, the Aggregate ensures that all changes to *its* data are consistent and follow the rules. It also provides a historical record of everything that has happened to it through the [Events](01_event_.md) it produces. Aggregates are a cornerstone of the Event Sourcing pattern used in our `corebanking` system. - -## Key Ideas About Aggregates - -1. **A Guardian of Data:** An Aggregate is responsible for a specific piece of data or a small group of related data. For example, an `AccountAggregate` is responsible for *one specific bank account*. It manages the account's balance, status, customer links, etc. -2. **Unique Identity:** Each Aggregate instance has a unique ID. So, `AccountAggregate` with ID `acc-123` is different from `AccountAggregate` with ID `acc-456`. -3. **Manages Its Own State:** The "state" is the current information about the entity. For an account, this includes its balance, status (CREATED, ACTIVATED, CLOSED), currency, etc. -4. **Enforces Business Rules:** Aggregates contain the logic to check if a requested action (a [Command](02_command_.md)) is allowed. Can you withdraw more money than you have? Can you close an account that has a pending transaction? The Aggregate knows! -5. **Processes [Commands](02_command_.md):** It takes a [Command](02_command_.md) as input. -6. **Produces [Events](01_event_.md):** If a [Command](02_command_.md) is valid and changes the state, the Aggregate creates one or more [Events](01_event_.md) to describe what happened. -7. **Consistency Boundary:** All changes within a single Aggregate happen together (atomically). Either all changes are applied, or none are. This keeps the data for that specific entity consistent. - -## Example: Alice Deposits Money - -Let's imagine Alice (customer `cust-001`) has an account (`acc-123`) and wants to deposit $50. - -1. **A [Command](02_command_.md) is Sent:** Alice's banking app creates a `DepositFundsCommand` with details: `AccountID: "acc-123"`, `Amount: 50.00`. -2. **The Right Aggregate is Found:** The system directs this command to the `AccountAggregate` responsible for account `acc-123`. -3. **The Aggregate Works:** - * The `AccountAggregate` for `acc-123` loads its current state (perhaps its current balance is $100 and its status is "ACTIVATED"). - * It checks business rules: Is the account "ACTIVATED"? Yes. Is $50 a valid amount? Yes. - * Since the command is valid, the Aggregate decides an `AccountCreditedEvent` (or similar) needs to happen. -4. **An [Event](01_event_.md) is Born:** It creates an `AccountCreditedEvent` with details: `AmountCredited: 50.00`, `NewBalance: 150.00` (or the event might just state the amount credited, and the balance is recalculated). -5. **State is Updated:** The Aggregate updates its own internal state. Its balance is now $150. -6. **[Event](01_event_.md) is Stored:** The `AccountCreditedEvent` is saved permanently. - -Now, account `acc-123` correctly reflects the deposit, and there's a permanent record of it. - -## What Does an Aggregate Look Like? (A Peek at the Code) - -In Go, an Aggregate is typically a struct. It often embeds a base struct that provides common functionalities and includes fields to hold its state. - -Here's a very simplified view of an `AccountAggregate` from `api/account_aggregate.go`: - -```go -// From: api/account_aggregate.go -// Simplified AccountAggregate structure -type AccountAggregate struct { - es.BaseAggregate // Provides common aggregate features (like ID, event helpers) - - // State of the account (the actual data for *this* account instance) - *AccountAggregateState - - // (Dependencies like repositories are omitted for simplicity here) -} -``` -* `es.BaseAggregate`: This is a helper struct from our event sourcing library (`es`). It provides common things an aggregate needs, like a place to store its ID, version, and methods for creating new [Events](01_event_.md). -* `*AccountAggregateState`: This pointer field is where the actual data specific to *this* account instance is stored. - -The `AccountAggregateState` itself is another struct: - -```go -// From: api/account_aggregate.go -// AccountAggregateState holds the data for an account -type AccountAggregateState struct { - Category string // e.g., "ASSET", "LIABILITY" - Status string // e.g., "CREATED", "ACTIVATED", "CLOSED" - CustomerIDs []uuid.UUID // Which customers own this account - // ... other fields like product info, currency, parameters ... - Created time.Time // When the account was created - // (The actual balance is often managed by postings and calculated, - // but for conceptual understanding, imagine it's part of the state) -} -``` -This `AccountAggregateState` holds all the important details about one specific account. When we talk about an Aggregate's "state," we're talking about the values in these fields. - -## How an Aggregate Handles a [Command](02_command_.md) - -There are two main methods in an Aggregate that work together: -1. `HandleCommand`: This is the entry point. It receives a [Command](02_command_.md), applies business rules, and if successful, creates [Event(s)](01_event_.md). -2. `ApplyChange`: This method takes an [Event](01_event_.md) and uses it to update the Aggregate's internal state. - -Let's look at simplified versions: - -### 1. `HandleCommand` - -When a [Command](02_command_.md) like `CreateAccountCommand` comes in: - -```go -// Simplified from AccountAggregate.HandleCommand in api/account_aggregate.go -// This method decides what to do when a command arrives. -func (a *AccountAggregate) HandleCommand(ctx context.Context, command es.Command) error { - var event es.Event // This will hold the event we generate - - switch c := command.(type) { // 'c' is the specific command (e.g., CreateAccountCommand) - case *CreateAccountCommand: - // 1. Validations (e.g., is product specified? are customers valid?) - // (Detailed validation logic omitted for brevity) - if c.Product == nil { - return errors.New("product is required") // Simplified error - } - - // 2. Set the ID for this new account aggregate from the command - // The command carries the intended ID for the new entity. - a.ID = command.GetAggregateID() - - // 3. Prepare data for the AccountCreatedEvent using info from the command - eventData := &AccountCreatedEvent{ - Category: c.Category, - Product: c.Product, // Details of the account type - CustomerIDs: c.CustomerIDs, - // ... other details from the CreateAccountCommand 'c' ... - } - - // 4. Create the actual Event object using a helper from BaseAggregate - event = a.NewEvent(command, eventData) // 'command' provides context - - // 5. Tell the aggregate to apply this change to itself - // and mark the event to be saved. - a.ApplyChangeHelper(a, event, true) - - // case *DepositFundsCommand: - // // ... logic to handle deposit, create FundsDepositedEvent ... - // case *ActivateAccountCommand: - // // ... logic to handle activation, create AccountActivatedEvent ... - } - return nil // If successful -} -``` -* The `switch` statement checks the type of [Command](02_command_.md). -* For `CreateAccountCommand`, it performs validations. -* It then prepares the data for an `AccountCreatedEvent` using information from the command. -* `a.NewEvent(...)` is a helper (from `es.BaseAggregate`) that wraps `eventData` with standard event fields like a unique Event ID, timestamp, Aggregate ID, etc. -* `a.ApplyChangeHelper(...)` is another crucial helper. It does two things: - 1. Calls `a.ApplyChange(event)` (see below) to update the aggregate's in-memory state. - 2. Adds the `event` to a list of "changes" that will be saved to the Event Store later. - -### 2. `ApplyChange` - -After an [Event](01_event_.md) is created by `HandleCommand`, the `ApplyChange` method is called to update the Aggregate's state. **The state of an Aggregate is *always* a result of applying its [Events](01_event_.md).** - -```go -// Simplified from AccountAggregate.ApplyChange in api/account_aggregate.go -// This method updates the aggregate's state based on an event. -func (a *AccountAggregate) ApplyChange(event es.Event) { - // 'e' is the specific event data (like AccountCreatedEvent details) - switch e := event.Data.(type) { - case *AccountCreatedEvent: - // The aggregate's ID is usually set by BaseAggregate from the event - a.ID = event.AggregateID - // Now, update the AccountAggregateState fields - a.Category = e.Category - a.Product = e.Product // Store product info - a.Status = AccountStatusCreated // Set initial status - a.CustomerIDs = e.CustomerIDs - a.Created = event.Created // Record creation time - // ... update other AccountAggregateState fields based on the event ... - - case *AccountActivatedEvent: // If an AccountActivatedEvent occurred - a.Status = AccountStatusActivated // Update the status - a.Updated = event.Created // Update modification time - - // case *FundsDepositedEvent: - // a.Balance = a.Balance + e.AmountDeposited // Hypothetical balance update - // a.Updated = event.Created - } -} -``` -* This method also uses a `switch` on the *type* of [Event](01_event_.md). -* For an `AccountCreatedEvent`, it sets the initial properties of the `AccountAggregateState` (like `Category`, `Status`, `Created` timestamp). -* For an `AccountActivatedEvent`, it would just update the `Status` field. -* This ensures that the Aggregate's in-memory state accurately reflects all the [Events](01_event_.md) that have happened to it. - -## The Life Cycle: Loading an Existing Aggregate - -What if Alice wants to deposit into an *existing* account? The system doesn't just create a new `AccountAggregate`. Instead: -1. It knows the `AccountID` (e.g., `acc-123`) from the `DepositFundsCommand`. -2. It goes to the Event Store (where all [Events](01_event_.md) are saved). -3. It fetches *all* the past [Events](01_event_.md) for `acc-123` in the order they happened. -4. It creates a fresh `AccountAggregate` instance. -5. It then "replays" each historical [Event](01_event_.md) by calling `ApplyChange` for each one. - * `ApplyChange(AccountCreatedEvent{...})` -> sets initial state. - * `ApplyChange(AccountActivatedEvent{...})` -> sets status to "ACTIVATED". - * `ApplyChange(SomeOtherEvent{...})` -> updates state further. -6. After replaying all its history, the `AccountAggregate` is now in its correct, current state, ready to handle the new `DepositFundsCommand`. - -This process of rebuilding state from [Events](01_event_.md) is fundamental to Event Sourcing. - -## Why Aggregates are So Important - -* **Encapsulation:** They bundle data (state) and behavior (rules) together, making the system easier to understand and manage. -* **Consistency:** They ensure that the data for a specific entity (like an account) is always valid and consistent according to business rules. -* **Clear Responsibilities:** It's clear which Aggregate is responsible for which [Commands](02_command_.md) and which part of the system's data. -* **Event Sourcing Enabler:** They are the primary producers of [Events](01_event_.md) in an Event Sourced system. The history of [Events](01_event_.md) an Aggregate produces *is* the history of that entity. - -## The Big Picture: Processing a Command - -Let's visualize how a command flows through an Aggregate to produce an Event. - -```mermaid -sequenceDiagram - participant ClientApp as Alice's App - participant CoreAPI as Core Banking API - participant AccountAgg as Account Aggregate (for acc-123) - participant EventStore - - ClientApp->>CoreAPI: Request: Deposit $50 into Account 123 - CoreAPI->>AccountAgg: Process DepositFundsCommand{AccountID: 123, Amount: 50} - Note over AccountAgg: 1. Load/Rehydrate Account 123's state
by replaying its past events from EventStore. - AccountAgg-->>AccountAgg: 2. HandleCommand: Validate Command (e.g., account active?) - AccountAgg-->>AccountAgg: 3. HandleCommand: Business Logic: Okay to deposit. - AccountAgg->>AccountAgg: 4. HandleCommand: Create FundsDepositedEvent{Amount: 50} - AccountAgg->>AccountAgg: 5. HandleCommand calls ApplyChange(FundsDepositedEvent)
--> Update internal state (e.g., balance) - Note over AccountAgg: 6. Event is marked for saving. - AccountAgg->>EventStore: 7. System saves FundsDepositedEvent - EventStore-->>AccountAgg: Event Saved! - AccountAgg-->>CoreAPI: Success! Deposit processed. - CoreAPI-->>ClientApp: Your deposit is successful! -``` -This diagram shows that the Aggregate is the central processor. It loads its state, validates the [Command](02_command_.md), applies logic, creates an [Event](01_event_.md), updates itself based on that [Event](01_event_.md), and then the [Event](01_event_.md) gets stored. - -## Conclusion - -Aggregates are like mini-managers for specific entities in our system, such as individual bank accounts or customers. They are responsible for: -* Guarding their own data (state). -* Enforcing business rules. -* Processing incoming [Commands](02_command_.md). -* Producing [Events](01_event_.md) as a record of successful changes. - -By doing this, Aggregates ensure data consistency and are the heart of how our `corebanking` system uses Event Sourcing. They take the "intent" from a [Command](02_command_.md) and, if valid, turn it into a historical "fact" as an [Event](01_event_.md). - -Now that we understand how Aggregates work and how their state can be built from [Events](01_event_.md), you might be wondering: how are these Aggregates (and their [Events](01_event_.md)) actually loaded from and saved to storage? That's where our next topic comes in. Get ready to learn about the [Repository](04_repository_.md)! - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/04_repository_.md b/tmp/nesto/corebanking/04_repository_.md deleted file mode 100644 index 1c6b2e27..00000000 --- a/tmp/nesto/corebanking/04_repository_.md +++ /dev/null @@ -1,229 +0,0 @@ -# Chapter 4: Repository - -Welcome to Chapter 4! In the [previous chapter](03_aggregate_.md), we learned about [Aggregates](03_aggregate_.md), like our `AccountAggregate`. We saw how they manage their own state (like an account's balance or status) by processing [Commands](02_command_.md) and then producing [Events](01_event_.md) to record what happened. - -Now, a couple of important questions arise: -1. When an `AccountAggregate` needs to process a [Command](02_command_.md) for an *existing* account, how does it get its current state? Remember, its state is built by replaying its past [Events](01_event_.md). Where do these [Events](01_event_.md) come from? -2. After an `AccountAggregate` processes a [Command](02_command_.md) and creates new [Events](01_event_.md), where do these new [Events](01_event_.md) get stored so they are not forgotten? -3. Beyond [Aggregates](03_aggregate_.md), what if other parts of our bank system simply need to *read* account information – say, to display a list of Alice's accounts on a web page? How do they get this data without needing to understand all the [Events](01_event_.md)? - -This is where the **Repository** pattern comes to our rescue! - -## What is a Repository? The Bank's Diligent Archivist - -Imagine a bank's large, old-fashioned archive room, filled with rows and rows of filing cabinets. This room has a team of dedicated archivists. - -* If a bank teller (let's say, an [Aggregate](03_aggregate_.md)) needs the history of a specific account to understand its current situation, they don't go rummaging through the cabinets themselves. They fill out a request slip and give it to an archivist. The archivist knows exactly where to find the files (the [Events](01_event_.md) or account data) and brings them back. -* If the teller completes a transaction and needs to file away the record (a new [Event](01_event_.md) or updated account information), they hand it to the archivist, who ensures it's stored in the right place, safely and correctly. - -A **Repository** in our `corebanking` system works just like this team of archivists. It's a component responsible for **persisting (saving) and retrieving data**. This data could be account details, customer information, transaction histories, or the [Events](01_event_.md) that make up an [Aggregate's](03_aggregate_.md) history. - -The crucial part is that the Repository provides an **abstraction** over the actual data storage mechanism. This means other parts of the system (like [Aggregates](03_aggregate_.md) or [Services](07_service_.md) which we'll see later) don't need to know if the data is stored in a MySQL database, a different type of database, or even text files. They just talk to the Repository using simple, clear methods. - -## Key Ideas About Repositories - -1. **Hides Storage Details:** The rest of the application doesn't care *how* or *where* data is stored. Is it SQL? Is it a NoSQL database? The Repository handles those details. This is like not needing to know the archivist's specific filing system, just that they can get you the file you need. -2. **Defined Contract (Interface):** A Repository offers a clear set of operations, like `GetByID()`, `Save()`, or `Search()`. In Go, this "contract" is usually defined by an `interface`. -3. **One Repository per Data Type (Usually):** You'll often have a specific repository for each main type of data you're managing. For example: - * An `AccountsRepository` to manage `Account` data. - * A `CustomersRepository` to manage `Customer` data. -4. **Different Repositories for Different Needs:** - * **For [Aggregates](03_aggregate_.md) (Event Sourcing):** In an Event Sourced system like ours, [Aggregates](03_aggregate_.md) (like `AccountAggregate`) are special. They are not stored directly as a single row in a database. Instead, their state is derived from their history of [Events](01_event_.md). A specialized type of repository (often called an `EventSourcedRepository` or similar) is used: - * To **load** an [Aggregate](03_aggregate_.md): It fetches all its past [Events](01_event_.md) from an "Event Store" (a database optimized for storing events). - * To **save** an [Aggregate](03_aggregate_.md): It takes any new [Events](01_event_.md) the [Aggregate](03_aggregate_.md) has produced and stores them in the Event Store. - * **For Read Data (Projections/Views):** Sometimes, we need to query data that's already nicely formatted for display or reporting – this is often called a "read model" or "projection." For example, when displaying account details on a screen, we want the current status, balance, etc., directly, not a list of [Events](01_event_.md). Repositories are also used to fetch this kind of data. Our `AccountsRepository` in `api/accounts.go` is an example of this type. - -## Using a Repository: Let's Look at Account Data - -Let's focus on how we might get information about an account that's easy to read and display, using the `AccountsRepository` from our project. Imagine a [Service](07_service_.md) (a component we'll discuss later) needs to fetch Alice's account details. - -### The Contract: The `AccountsRepository` Interface - -First, there's an interface that defines what operations can be performed for accounts. This is like the list of services the archivists offer. - -```go -// From: api/accounts.go - -// AccountsRepository defines methods for account data storage -type AccountsRepository interface { - Save(ctx context.Context, account *Account) error - // UpdateStatus updates the status of an account - UpdateStatus(ctx context.Context, accountID uuid.UUID, status string, updated time.Time) error - GetByID(ctx context.Context, accountID uuid.UUID) (*Account, error) - // ... other methods like Search, GetByIDs, SavePaymentDeviceLink etc. -} -``` -* This interface declares methods like `Save`, `UpdateStatus`, and `GetByID`. -* It uses `*Account`, which is a struct (`api.Account`) representing the "read model" of an account – a snapshot of its current, easily readable state. This is different from the `AccountAggregate` which is focused on processing [Commands](02_command_.md) and [Events](01_event_.md). - -### Getting Account Data - -If a part of our system needs to get details for account `acc-123`, it would use an implementation of `AccountsRepository`: - -```go -// Somewhere in our application (e.g., inside an Account Service) -var accountsRepo api.AccountsRepository // This would be a concrete implementation - -// ... -accountID := uuid.FromString("acc-123") // The ID of the account we want -account, err := accountsRepo.GetByID(context.Background(), accountID) -if err != nil { - // Handle error, maybe the account wasn't found - fmt.Println("Error fetching account:", err) - return -} - -fmt.Println("Fetched Account Category:", account.Category) -fmt.Println("Fetched Account Status:", account.Status) -``` -* **Input:** The `GetByID` method takes a `context` and the `accountID`. -* **Output:** It returns an `*api.Account` struct (containing the account's details like category, status, etc.) and an error (which will be `nil` if successful). - -The code calling `accountsRepo.GetByID` doesn't know or care if the data came from MySQL, a different database, or even a text file. That's the beauty of the repository abstraction! - -## Under the Hood: The `MySQLRepository` for Accounts - -Our project has a concrete implementation of the `AccountsRepository` interface that uses a MySQL database. It's located in `api/pkg/accounts/mysql_repository.go`. - -Let's see a simplified version of how its `GetByID` method might work: - -```go -// Simplified from: api/pkg/accounts/mysql_repository.go - -// MySQLRepository implements AccountsRepository using MySQL. -type MySQLRepository struct { - // (Internal details, like how it gets a database connection, are hidden here) -} - -// GetByID retrieves an account's projection (read model) by its ID from MySQL. -func (r *MySQLRepository) GetByID(ctx context.Context, accountID uuid.UUID) (*api.Account, error) { - // 1. Get a database connection (simplified) - conn, _ := repository.SQLConnection(ctx) - - // 2. This struct holds the raw data fetched from the 'account' table - var dbAccountData struct { // In real code, this is 'accountDbEntry' - ID uuid.UUID `db:"id"` - Category string `db:"category"` - Status string `db:"status"` - // ... other fields matching the database table columns ... - } - - // 3. Build the SQL query to select account data - sqlQuery := "SELECT id, category, status /*, ...other columns... */ FROM account WHERE id = ?" - - // 4. Execute the query against the MySQL database - err := conn.GetContext(ctx, &dbAccountData, sqlQuery, accountID) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, e.NewEntityIDNotFound("account", accountID.String()) // Specific error for "not found" - } - // Some other database error occurred - return nil, e.NewInternal(err) - } - - // 5. Convert the raw database data (dbAccountData) into the application's api.Account struct - apiAccount := &api.Account{ - ID: dbAccountData.ID, - Category: dbAccountData.Category, - Status: dbAccountData.Status, - // ... populate other fields ... - } - - // 6. In the real code, it also fetches related data like customer links, parameters, - // payment device links and populates them into apiAccount. - // Example (highly simplified): - // apiAccount.CustomerIDs, _ = r.getCustomersForAccount(ctx, accountID) - // apiAccount.Parameters, _ = r.getParametersForAccount(ctx, accountID) - - return apiAccount, nil -} -``` -Let's break this down step-by-step: -1. **Get Connection:** It first obtains a connection to the MySQL database. -2. **Data Holder:** `dbAccountData` is a temporary struct to hold the data exactly as it comes from the database table. -3. **SQL Query:** It defines the SQL `SELECT` statement to fetch data from the `account` table based on the `id`. -4. **Execute Query:** The query is executed. If no account is found, it returns a "not found" error. Other database errors are also handled. -5. **Map Data:** If data is found, the raw `dbAccountData` is converted into an `*api.Account` struct. This `api.Account` struct is what the rest of our application understands and expects. -6. **Fetch Related Data:** An account might have associated customers, specific parameters, or linked payment devices. The full repository method would also query other tables to fetch this related information and populate the `api.Account` struct completely. - -Here's a sequence diagram showing this: - -```mermaid -sequenceDiagram - participant AppCode as Application Code - participant AccRepo as AccountsRepository (MySQL) - participant MySQLDB as MySQL Database - - AppCode->>AccRepo: GetByID("acc-123") - AccRepo->>MySQLDB: SELECT id, category, status FROM account WHERE id = "acc-123" - MySQLDB-->>AccRepo: Row data (ID, Category, Status) - AccRepo-->>AccRepo: Map row to api.Account struct - AccRepo->>MySQLDB: SELECT customer_id FROM account_customer WHERE account_id = "acc-123" (etc. for other related data) - MySQLDB-->>AccRepo: Related data - AccRepo-->>AccRepo: Add related data to api.Account struct - AccRepo-->>AppCode: *api.Account (fully populated) -``` - -Similarly, a `Save` method in `MySQLRepository` would take an `*api.Account` struct, convert its fields into a format suitable for the database, and then execute an `INSERT` or `UPDATE` SQL statement. - -## What About [Aggregates](03_aggregate_.md) and Their [Events](01_event_.md)? - -As mentioned earlier, [Aggregates](03_aggregate_.md) like `AccountAggregate` are handled a bit differently in an Event Sourced system. They also use a Repository, but it's a specialized one. - -* **Loading an [Aggregate](03_aggregate_.md):** - When the system needs to load, say, `AccountAggregate` for `acc-123`: - 1. It asks an `EventSourcedRepository` to `Load("acc-123")`. - 2. This repository connects to an **Event Store** (a special database designed to store sequences of [Events](01_event_.md)). - 3. It fetches all [Events](01_event_.md) ever recorded for `acc-123`. - 4. It creates a new, empty `AccountAggregate` instance. - 5. It then "replays" each historical [Event](01_event_.md) on this instance by calling its `ApplyChange(event)` method. This rebuilds the [Aggregate's](03_aggregate_.md) current state. - 6. The fully rehydrated `AccountAggregate` is returned. - -* **Saving an [Aggregate](03_aggregate_.md):** - After `AccountAggregate` processes a [Command](02_command_.md) and generates new [Events](01_event_.md): - 1. The system calls `Save(accountAggregate)` on the `EventSourcedRepository`. - 2. The repository gets the list of newly generated (uncommitted) [Events](01_event_.md) from the `accountAggregate`. - 3. It appends these new [Events](01_event_.md) to the Event Store, associated with `acc-123`. - 4. It then clears the list of uncommitted [Events](01_event_.md) from the `accountAggregate`. - -Here's how loading an [Aggregate](03_aggregate_.md) via an EventSourced Repository looks: -```mermaid -sequenceDiagram - participant AppLogic as Application Logic - participant ESRepo as EventSourced Repository - participant EventStoreDB as Event Store Database - participant AccAgg as Account Aggregate - - AppLogic->>ESRepo: Load AccountAggregate (ID: "acc-123") - ESRepo->>EventStoreDB: GetEvents("acc-123") - EventStoreDB-->>ESRepo: [Event1, Event2, Event3...] - ESRepo->>AccAgg: New AccountAggregate() - loop For each Event in List - ESRepo->>AccAgg: ApplyChange(CurrentEvent) - end - ESRepo-->>AppLogic: AccountAggregate (state rehydrated) -``` -This shows that the Repository pattern is flexible. We can have different repository implementations for different kinds of data (read models like `api.Account`) and different storage strategies (like an Event Store for [Aggregates](03_aggregate_.md)). - -## Why is This Abstraction So Useful? - -Using the Repository pattern brings several benefits: - -1. **Testability:** When testing parts of your application that need data, you don't always want to connect to a real database. It can be slow and complicated to set up. With repositories, you can create a "mock" or "fake" repository for your tests. This fake repository can pretend to be a database, returning predefined data or checking if `Save` was called correctly, all without any actual database interaction. -2. **Flexibility (Change Your Database Later):** Imagine your bank starts with MySQL but later decides to switch to a different database, say PostgreSQL. If your application code directly uses MySQL-specific queries everywhere, changing the database would be a nightmare! With repositories, you only need to write a *new implementation* of the `AccountsRepository` interface (e.g., `PostgreSQLRepository`). The rest of your application code that uses the `AccountsRepository` interface doesn't need to change at all because it was only depending on the interface, not the specific MySQL details. -3. **Clear Separation of Concerns:** Business logic (what the bank *does*) is kept separate from data access logic (how data is saved and retrieved). This makes the code cleaner, easier to understand, and maintain. [Aggregates](03_aggregate_.md) and [Services](07_service_.md) focus on their tasks without being cluttered by SQL queries. - -## Conclusion - -Repositories are like the organized and efficient archivists of our `corebanking` system. They handle all the details of storing and retrieving data, whether it's: -* Fetching historical [Events](01_event_.md) to rebuild an [Aggregate's](03_aggregate_.md) state. -* Saving new [Events](01_event_.md) produced by an [Aggregate](03_aggregate_.md). -* Querying user-friendly "read models" (like an `api.Account`) from a database for display or reporting. - -By providing a clean abstraction layer over data storage, Repositories make our system more maintainable, testable, and flexible. They ensure that other parts of the system can request or save data using simple methods, without needing to know the nitty-gritty details of how or where that data is actually stored. - -Now that we know how data can be persisted and retrieved, how do external requests (like Alice trying to create an account through a mobile app) actually reach our core system and make use of these components? In the next chapter, we'll look at the entry points for such requests: the [API Handler](05_api_handler_.md). - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/05_api_handler_.md b/tmp/nesto/corebanking/05_api_handler_.md deleted file mode 100644 index 756e8914..00000000 --- a/tmp/nesto/corebanking/05_api_handler_.md +++ /dev/null @@ -1,225 +0,0 @@ -# Chapter 5: API Handler - -Welcome to Chapter 5! In the [previous chapter](04_repository_.md), we explored [Repositories](04_repository_.md) and how they help us save and retrieve data, like account information or the history of [Events](01_event_.md) for an [Aggregate](03_aggregate_.md). We now know how our system manages its data. - -But how does a request from the outside world, like Alice trying to open a new bank account using her mobile app, actually reach our `corebanking` system and start this whole process? How does Alice's app "talk" to our bank's brain? This is where the **API Handler** comes into play. - -## What's an API Handler? The Bank's Welcome Desk - -Imagine walking into a bank. You don't just wander into the vault or start talking to random employees. You usually go to a specific counter or desk: -* "Account Opening" desk. -* "Deposits & Withdrawals" window. -* A specific online form for "Loan Application." - -An **API Handler** in our `corebanking` system is just like that specific desk or form. It's the **official entry point** for any external request that wants to interact with our system. "API" stands for Application Programming Interface, which is a way for different software programs to communicate with each other. - -Each API Handler is responsible for: -1. A specific **URL path** (like a web address, e.g., `/api/accounts`). -2. A specific **HTTP method** (like the type of action, e.g., `POST` for creating something, `GET` for fetching something). - -When a request arrives matching its designated URL and method, the API Handler takes charge. Its job is to: -* Understand the incoming request (e.g., read the data Alice submitted). -* Translate this request into an instruction our system understands, like a [Command](02_command_.md) (e.g., "Please create an account with these details"). -* Or, it might ask a [Service](07_service_.md) to fetch some information. -* Finally, it sends back an HTTP response to tell the requester what happened (e.g., "Account created successfully!" or "Sorry, there was an error."). - -Essentially, API Handlers are the bridge connecting the "web world" (HTTP requests and responses) to the internal business logic of our bank. - -## Key Responsibilities of an API Handler - -Let's break down what an API Handler typically does: - -1. **Listens at a Specific Address:** Each handler is tied to a unique combination of a URL path and an HTTP method. For example: - * `POST /api/accounts`: Handler for creating new accounts. - * `GET /api/accounts/{accountId}`: Handler for fetching details of a specific account. - * `PUT /api/accounts/{accountId}/close`: Handler for closing an account. -2. **Parses the Incoming Request:** When Alice submits her account application form, her details (name, desired currency, etc.) are often sent as a JSON payload in the body of an HTTP request. The API Handler needs to read this data and convert it into a format our Go code can understand (like a Go struct). -3. **Basic Validation (Sometimes):** The handler might do some very basic checks, like "Is the JSON data correctly formatted?" or "Are essential fields like 'currency' present?". However, deep business rule validation (e.g., "Is this currency supported by our bank?") is usually done further inside the system by [Aggregates](03_aggregate_.md) or [Services](07_service_.md). -4. **Translates to Internal Actions:** This is a crucial step. - * **For actions that change data (Writes):** The handler often creates a [Command](02_command_.md) object (like `CreateAccountCommand`) and populates it with the data from the request. It then sends this [Command](02_command_.md) to a "Command Bus," which is like a central dispatcher that routes the [Command](02_command_.md) to the correct [Aggregate](03_aggregate_.md) (e.g., `AccountAggregate`) for processing. - * **For actions that only read data (Reads):** The handler might directly call a [Service](07_service_.md) (e.g., an `AccountsService`) to fetch the requested information (like account details from a [Repository](04_repository_.md)). -5. **Formats and Sends the Response:** After the [Command](02_command_.md) is processed or the [Service](07_service_.md) returns data, the API Handler takes the result and crafts an HTTP response. This includes: - * An **HTTP Status Code** (e.g., `200 OK` for success, `201 Created` if something new was made, `400 Bad Request` if the input was wrong, `500 Internal Server Error` if something went wrong on our side). - * A **Response Body** (often JSON data, like the details of the created account, or an error message). - -## Alice Creates an Account: The API Handler's Role - -Let's see how an API Handler helps Alice open her account: - -1. **Alice's App Sends a Request:** Alice fills out the "New Account" form on her banking app and taps "Submit." Her app sends an HTTP `POST` request to the URL `/api/accounts`. The body of this request contains her details in JSON format, like: - ```json - { - "category": "ASSET", - "product": { "id": "prod-savings-basic", "name": "Basic Savings" }, - "availableCurrencies": ["USD"], - "customerIds": ["cust-alice-123"] - } - ``` -2. **The Correct Handler is Activated:** Our `corebanking` system's web server receives this request. It looks at the method (`POST`) and the path (`/api/accounts`) and finds the specific API Handler responsible for this combination. -3. **Handler Parses the Request:** This API Handler (let's call it the `CreateAccountHandler`) takes the JSON data from the request's body. It uses a JSON parser to convert this data into a Go struct that matches the structure of a `CreateAccountCommand`. -4. **Handler Creates and Dispatches a [Command](02_command_.md):** The handler now creates an instance of `api.CreateAccountCommand` and fills it with the details parsed from Alice's request. It then passes this command to something called a `CommandBus` (part of `core.EventSourcingClient` in our project). - ```go - // Inside the handler (conceptual) - var cmd api.CreateAccountCommand - // ... (parse JSON from request into cmd) ... - err := commandBus.HandleCommand(context, &cmd) - ``` -5. **Handler Waits and Responds:** The Command Bus ensures the `CreateAccountCommand` is processed (by an `AccountAggregate`, which generates an `AccountCreatedEvent`, as we saw in earlier chapters). - * If the command is processed successfully, the handler sends back an HTTP response like `204 No Content` (meaning "I did what you asked, and there's no specific content to return") or `201 Created` (if it were to return details of the created account). - * If there's an error (e.g., Alice provided an invalid customer ID), the [Aggregate](03_aggregate_.md) would signal an error, and the handler would send back an HTTP response like `400 Bad Request` with a JSON body describing the error. - -## A Peek at the Code: An Account Creation Handler - -Our project has API handlers defined in packages under `api/cmd/corebanking-api/`. For creating accounts, the relevant file is `api/cmd/corebanking-api/accounts/create.go`. - -Let's look at a simplified version of the `Create` function, which acts as our API Handler: - -```go -// Simplified from: api/cmd/corebanking-api/accounts/create.go - -// 'cb' is a CommandBus, which knows how to send commands -// to the right part of our system. -func Create(cb es.CommandBus) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - // 1. Parse the request to get command data - command, err := parseCreateAccountCommand(r) - if err != nil { - // If parsing fails, send an error response - e.MarshallHTTP(err, w) // Helper to send formatted error - return - } - - // 2. Send the command for processing - err = cb.HandleCommand(r.Context(), &command) - if err != nil { - // If command processing fails, send an error response - e.MarshallHTTP(err, w) - return - } - - // 3. If everything is OK, send a success response - w.WriteHeader(http.StatusNoContent) // "204 No Content" - } -} -``` -Let's break this down: -* `func Create(cb es.CommandBus) http.HandlerFunc`: This function `Create` takes a `CommandBus` (our `es.EventSourcingClient`) as an argument and *returns* another function. This returned function is the actual handler that Go's HTTP server will call for each incoming request. - * `w http.ResponseWriter`: Used to write the HTTP response back to Alice's app. - * `r *http.Request`: Represents the incoming HTTP request from Alice's app. - -Inside the returned function: -1. **Parse Request:** `command, err := parseCreateAccountCommand(r)` - This calls a helper function (we'll see it next) to read Alice's data from the request (`r`) and turn it into our `api.CreateAccountCommand` struct. If there's an error during parsing (e.g., malformed JSON), it sends an error response using `e.MarshallHTTP(err, w)`. -2. **Send Command:** `err = cb.HandleCommand(r.Context(), &command)` - This is where the magic happens! The parsed `command` is handed off to the `CommandBus` (`cb`). The `CommandBus` will make sure it gets to the right [Aggregate](03_aggregate_.md) for processing. If the [Aggregate](03_aggregate_.md) rejects the command (e.g., business rule violation), `HandleCommand` returns an error, which is then sent back as an HTTP error. -3. **Success Response:** `w.WriteHeader(http.StatusNoContent)` - If both parsing and command processing are successful, this line sends an HTTP status code `204 No Content` back to Alice's app, indicating success. - -### Parsing the Request Data - -The `parseCreateAccountCommand` helper function looks something like this (simplified): - -```go -// Simplified from: api/cmd/corebanking-api/accounts/create.go -func parseCreateAccountCommand(r *http.Request) (api.CreateAccountCommand, error) { - var command api.CreateAccountCommand // Prepare an empty command struct - - // Decode the JSON data from the request body into our 'command' struct - if err := json.NewDecoder(r.Body).Decode(&command); err != nil { - // If decoding fails, log it and return a "bad format" error - log.G(r.Context()).WithError(err).Error("could not decode json") - return command, e.NewBadFormat(err.Error()) - } - - return command, nil // Return the populated command and no error -} -``` -* `var command api.CreateAccountCommand`: An empty `CreateAccountCommand` struct is created. -* `json.NewDecoder(r.Body).Decode(&command)`: This is standard Go for reading JSON. It takes the request's body (`r.Body`), creates a JSON decoder, and tries to fill the `command` struct with the data from the JSON. -* If `Decode` fails (e.g., the JSON is broken or doesn't match the command structure), an error is logged and a specific "bad format" error is returned. - -So, the handler first uses this parsing logic to understand the request, then dispatches the [Command](02_command_.md). - -## How Does the System Know Which Handler to Call? Routing! - -You might be wondering: if Alice's app sends a `POST` request to `/api/accounts`, how does our `corebanking` application know to call the `accounts.Create` handler function we just looked at? - -This is done by a **router**. A router is like a traffic controller for web requests. It looks at the URL and HTTP method of an incoming request and directs it to the correct handler function. Our project uses a popular Go router called `chi`. - -The setup for these routes is typically done in a file like `api/cmd/corebanking-api/routes/routes.go`. Here's a tiny snippet showing how the account creation route is defined: - -```go -// Simplified from: api/cmd/corebanking-api/routes/routes.go - -// 'core' contains things like our CommandBus (EventSourcingClient) -// and Services. -func getAPIRoutes(core *api.Core) http.Handler { - r := chi.NewRouter() // Create a new router - - // ... other routes ... - - // This line tells the router: - // If a POST request comes to "/accounts", - // call the handler returned by accounts.Create(), - // passing it the EventSourcingClient from 'core'. - r.Post("/accounts", accounts.Create(core.EventSourcingClient)) - - // Example of a GET request handler: - // r.Get("/accounts/{id}", accounts.Get(core.AccountsService)) - - // ... many other routes ... - - return r -} -``` -* `r := chi.NewRouter()`: Initializes a new router. -* `r.Post("/accounts", accounts.Create(core.EventSourcingClient))`: This is the key line. - * `r.Post` means this rule applies to HTTP `POST` requests. - * `"/accounts"` is the URL path. - * `accounts.Create(core.EventSourcingClient)` is the handler function we want to execute. Notice that we *call* `accounts.Create(...)` here. This function, as we saw, *returns* the actual `http.HandlerFunc` that `chi` will use. We pass `core.EventSourcingClient` (which is our `CommandBus`) to it so the handler can dispatch commands. - -For requests that read data, like getting account details, you'd see something like `r.Get("/accounts/{id}", accounts.Get(core.AccountsService))`. Here, `accounts.Get` would be another handler function, likely taking an `AccountsService` to fetch data. - -## The Journey of a Request: From App to Response - -Let's visualize the entire flow for Alice creating an account: - -```mermaid -sequenceDiagram - participant AliceApp as Alice's App - participant WebRouter as API Router (Chi) - participant CreateAccHandler as accounts.Create Handler - participant CmdBus as CommandBus (EventSourcingClient) - participant AccAgg as AccountAggregate - - AliceApp->>WebRouter: POST /api/accounts (with JSON data) - WebRouter->>CreateAccHandler: Execute handler for this route - CreateAccHandler-->>CreateAccHandler: parseCreateAccountCommand(request) - CreateAccHandler->>CmdBus: HandleCommand(CreateAccountCommand) - CmdBus->>AccAgg: Process CreateAccountCommand - Note over AccAgg: Validates, creates AccountCreatedEvent,
updates state, saves event (via Repository). - AccAgg-->>CmdBus: Success (or Error) - CmdBus-->>CreateAccHandler: Result of command processing - CreateAccHandler-->>CreateAccHandler: Prepare HTTP Response (e.g., 204) - CreateAccHandler->>AliceApp: HTTP 204 No Content (or error response) -``` -1. Alice's app sends the `POST` request. -2. The Web Router (Chi) finds the `accounts.Create` handler. -3. The `CreateAccountHandler` parses the JSON into a `CreateAccountCommand`. -4. It sends the command to the `CommandBus`. -5. The `CommandBus` delivers it to the `AccountAggregate`. -6. The `AccountAggregate` processes the command (validates, generates [Events](01_event_.md), updates its state, and ensures the [Events](01_event_.md) are saved via a [Repository](04_repository_.md) - this part was covered in detail in Chapters 1-4). -7. The result (success/error) bubbles back up to the handler. -8. The handler sends the appropriate HTTP response back to Alice's app. - -## Conclusion - -API Handlers are the vital "front doors" of our `corebanking` system. They listen for specific web requests (URLs and HTTP methods), parse the incoming data, and then translate these external requests into actions our internal system understands – usually by creating and dispatching [Commands](02_command_.md) for write operations or calling [Services](07_service_.md) for read operations. They then take the result of these actions and formulate an HTTP response to send back to the original requester. - -By bridging the gap between the web world (HTTP) and our core business logic, API Handlers make our system accessible to external clients like mobile apps, websites, or even other banking systems. - -Now that we've seen how requests enter our system, you might wonder: is there a central place that coordinates these actions, especially when a single request might involve multiple steps or components? That's where our next topic comes in. Get ready to learn about the [Core Facade](06_core_facade_.md)! - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/06_core_facade_.md b/tmp/nesto/corebanking/06_core_facade_.md deleted file mode 100644 index 30f191b3..00000000 --- a/tmp/nesto/corebanking/06_core_facade_.md +++ /dev/null @@ -1,160 +0,0 @@ -# Chapter 6: Core Facade - -Welcome to Chapter 6! In our [previous chapter on API Handlers](05_api_handler_.md), we saw how external requests, like Alice wanting to create an account, enter our `corebanking` system through specific API endpoints. These [API Handlers](05_api_handler_.md) then need to interact with various parts of our bank's brain – maybe to send a [Command](02_command_.md) or to fetch some data. - -But imagine an [API Handler](05_api_handler_.md) trying to juggle many different tools: one tool for sending [Commands](02_command_.md), another for looking up customer details, yet another for checking product information. It could get messy quickly! How do we keep this organized and simple for the [API Handler](05_api_handler_.md)? - -This is where the **Core Facade** comes in. - -## What is the Core Facade? The Bank's Main Office - -Think of the `Core` struct (our Core Facade) as the **main reception desk or central switchboard** for our entire `corebanking` application. It's like a central office in a large company. If you (an [API Handler](05_api_handler_.md), for instance) need something from any department, you don't run around the building looking for them. You go to the main office, and they connect you to the right place. - -The Core Facade brings together all the different specialized "departments" – which in our system are various [Services](07_service_.md) (like `AccountsService`, `CustomersService`) and other key components like the `EventSourcingClient` (our [Command](02_command_.md) bus) – into a **single, unified point of access**. - -When an external part of the system, like the [API layer](05_api_handler_.md), needs to interact with the business logic, it simply talks to this `Core` facade. This keeps our overall architecture clean and makes it much easier to manage how different parts of the application interact. - -Essentially, the Core Facade is like a well-organized toolbox for our [API Handlers](05_api_handler_.md). Instead of the handler needing to know about and hold onto dozens of individual tools, it just gets this one `Core` toolbox, which contains everything it might need. - -## What Does the Core Facade Look Like? (The `Core` Struct) - -In our Go code, the Core Facade is represented by a struct named `Core`, defined in `api/core.go`. Let's look at a simplified version: - -```go -// From: api/core.go -// Core represents the entrypoint to call the business logic. -type Core struct { - HealthService HealthService - EventSourcingClient es.EventSourcingClient // To send Commands! - ParametersService ParametersService - AccountsService AccountsService // To read account data! - CustomersService CustomersService - ProductsService ProductsService - // ... and many other services and clients ... -} -``` -* You can see that `Core` is a struct that holds various fields. -* Each field represents a specific "department" or tool: - * `EventSourcingClient`: This is crucial for sending [Commands](02_command_.md) (like `CreateAccountCommand`) to our [Aggregates](03_aggregate_.md). We've seen this in action with [API Handlers](05_api_handler_.md). - * `AccountsService`: If an [API Handler](05_api_handler_.md) needs to fetch a list of accounts or details about a specific account (not through event sourcing, but from a read-friendly data store), it would use this [Service](07_service_.md). We'll learn more about [Services](07_service_.md) in the next chapter! - * `CustomersService`, `ProductsService`, etc.: These provide access to other areas of our banking logic. - -So, the `Core` struct acts as a container, a "bag" holding all these important pieces of our application's core logic. - -## How Do [API Handlers](05_api_handler_.md) Use the Core Facade? - -Remember from [Chapter 5](05_api_handler_.md) how our [API Handlers](05_api_handler_.md) are set up using a router? The router is often given access to this `Core` facade instance. When an [API Handler](05_api_handler_.md) function is defined, it can be passed the specific service or client it needs *from* the `Core` facade. - -Let's revisit a simplified example from `api/cmd/corebanking-api/routes/routes.go` where routes are defined: - -```go -// Simplified from: api/cmd/corebanking-api/routes/routes.go - -// 'core' here is an instance of our api.Core struct (the Facade!) -func getAPIRoutes(core *api.Core) http.Handler { - r := chi.NewRouter() // Our web router - - // For creating accounts, the handler needs the EventSourcingClient (CommandBus) - r.Post("/accounts", accounts.Create(core.EventSourcingClient)) - - // For getting account details, a handler might need the AccountsService - // (Example, actual handler for GET might be different) - // r.Get("/accounts/{accountId}", accounts.GetDetails(core.AccountsService)) - - // ... other routes using different parts of 'core' ... - return r -} -``` -* The `getAPIRoutes` function receives an argument `core *api.Core`. This `core` object is our fully assembled Core Facade! -* When setting up the route for `POST /accounts`, the `accounts.Create` handler function is given `core.EventSourcingClient`. This means the `CreateAccountHandler` doesn't need to find or create an `EventSourcingClient` itself; it gets it directly from the Core Facade. -* Similarly, if we had a handler to fetch account details (e.g., `accounts.GetDetails`), it could be given `core.AccountsService`. - -This way, the `Core` facade acts as a central provider. The [API Handlers](05_api_handler_.md) just ask the `Core` facade for the specific tool they need. - -## How is the Core Facade Built? (The Making of the Toolbox) - -You might be wondering: who creates this `Core` struct and fills it with all these services? - -This happens when our `corebanking` application starts up. There's a special part of our code, primarily in a function called `FromEnv` in `api/pkg/core/core.go`, whose job is to: -1. Initialize all the individual [Services](07_service_.md) (like `AccountsService`, `ProductsService`, etc.). -2. Initialize the `EventSourcingClient`. -3. Initialize [Repositories](04_repository_.md) and other necessary components. -4. Then, it packages all of these initialized components into a single `api.Core` struct instance. - -Let's look at a very simplified snippet from our application's main entry point (`api/cmd/corebanking-api/api.go`) to see where this happens: - -```go -// Simplified from: api/cmd/corebanking-api/api.go main() -func main() { - // ... (lots of initial setup: logging, configuration, etc.) ... - - // tenantsConfigManager is used for multi-tenancy configurations - tenantsConfigManager := wiggumauto.NewConfigManager() - tenantsConfigManager.MustInitialize(ctx) - - // Here! 'c' becomes our fully assembled Core Facade instance. - // core.FromEnv does all the hard work of creating and wiring up services. - c := core.FromEnv(tenantsConfigManager) - - // The Core Facade 'c' is then passed to the router setup. - r := routes.GetRouter(c, tenantsConfigManager, monitoring) - - // ... (starts the web server with these routes) ... - // ... (handles graceful shutdown, including c.EventSourcingClient.Close()) -} -``` -* `c := core.FromEnv(tenantsConfigManager)`: This is the key line. The `core.FromEnv()` function (from `api/pkg/core/core.go`) is called. It sets up everything needed for the core logic (databases, event buses, all services) and returns a pointer to a fully populated `api.Core` struct. This `c` is our Core Facade. -* `r := routes.GetRouter(c, ...)`: This `c` (our Core Facade) is then passed to the function that sets up all the API routes, as we saw in the previous section. - -The `FromEnv` function itself is quite complex because it initializes many parts of the system, connecting them together. For a beginner, the important takeaway is that *there is a dedicated place* where the `Core` facade is carefully assembled when the application starts. - -## The Flow: API Handler, Core Facade, and Beyond - -Let's visualize how an [API Handler](05_api_handler_.md) uses the Core Facade: - -```mermaid -sequenceDiagram - participant APIHandler as API Handler (e.g., Create Account) - participant CoreFacade as Core Facade (api.Core instance) - participant AccService as Accounts Service - participant ESClient as EventSourcingClient (CommandBus) - participant AccAgg as Account Aggregate - - APIHandler->>CoreFacade: I need to create an account. Here's the data (CreateAccountCommand). - Note over APIHandler: Handler gets ESClient from CoreFacade - CoreFacade->>ESClient: HandleCommand(CreateAccountCommand) - ESClient->>AccAgg: Process CreateAccountCommand - AccAgg-->>ESClient: Success (or Error) - ESClient-->>CoreFacade: Command processed! - CoreFacade-->>APIHandler: Account creation initiated (Result) - - APIHandler->>CoreFacade: I need details for account "acc-xyz". - Note over APIHandler: Handler gets AccountsService from CoreFacade - CoreFacade->>AccService: GetAccountByID("acc-xyz") - AccService-->>CoreFacade: Account Details - CoreFacade-->>APIHandler: Here are the Account Details. -``` -This diagram shows two scenarios: -1. **Creating an Account (a [Command](02_command_.md)):** The [API Handler](05_api_handler_.md) uses the `EventSourcingClient` *obtained from* the `CoreFacade` to send the `CreateAccountCommand`. The `CoreFacade` itself doesn't process the command but provides access to the component that does. -2. **Getting Account Details (a Query):** The [API Handler](05_api_handler_.md) uses the `AccountsService` *obtained from* the `CoreFacade` to request account information. The `CoreFacade` directs this to the actual `AccountsService`. - -## Benefits of the Core Facade - -Using this Core Facade pattern provides several advantages: - -1. **Organization:** All core business functionalities are neatly bundled and accessible from one place. -2. **Simplicity for Callers:** Components like [API Handlers](05_api_handler_.md) don't need to know about or manage dependencies on dozens of individual services. They just need the `Core` object. -3. **Centralized Dependency Management:** The creation and wiring of all major services and components happen in one central place (like `core.FromEnv`). This makes it easier to manage the application's startup and overall structure. -4. **Clear Entry Point:** It provides a clear and consistent entry point into the application's business logic for any external interface (like the API). - -## Conclusion - -The **Core Facade**, represented by our `api.Core` struct, acts as the central hub or main reception desk for our `corebanking` application. It doesn't do the business logic itself but holds references to all the specialized [Services](07_service_.md) and clients (like the `EventSourcingClient`) that do. - -For components like [API Handlers](05_api_handler_.md), the Core Facade simplifies interaction with the system's core functionalities by providing a single, organized point of access. This keeps the architecture clean and manageable. - -The Core Facade is filled with various "departments" or tools. One very common type of tool it holds is a "Service." In the next chapter, we'll dive deeper into what these [Services](07_service_.md) are and what role they play in our `corebanking` system. - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/07_service_.md b/tmp/nesto/corebanking/07_service_.md deleted file mode 100644 index a768a62d..00000000 --- a/tmp/nesto/corebanking/07_service_.md +++ /dev/null @@ -1,264 +0,0 @@ -# Chapter 7: Service - -Welcome to Chapter 7! In the [previous chapter](06_core_facade_.md), we learned about the [Core Facade](06_core_facade_.md), which acts as a central hub providing access to various parts of our `corebanking` system. One of the key components the [Core Facade](06_core_facade_.md) gives us access to are "Services." - -But what exactly is a Service? And why do we need them? - -## What's a Service? The Bank's Specialized Departments - -Imagine you're at a bank. There isn't just one giant room where everything happens. Instead, there are specialized departments: -* The "Account Opening" department helps new customers. -* The "Loans" department handles loan applications and processing. -* The "Customer Support" department answers your questions. - -In our `corebanking` system, a **Service** is like one of these specialized departments. It's a component that groups together related business operations and logic for a specific area of the bank. - -For example, we might have an `AccountsService`. This service would be responsible for all things related to bank accounts, such as: -* Fetching the details of a specific account. -* Searching for accounts based on certain criteria. -* Potentially, initiating the creation of a new account (though, as we'll see, this is often done by sending a [Command](02_command_.md)). - -Services help us organize the application's capabilities, making the system easier to understand, manage, and extend. They often act as a "facade" or an intermediary, receiving requests, interacting with [Repositories](04_repository_.md) to fetch or save data, or dispatching [Commands](02_command_.md) to [Aggregates](03_aggregate_.md) to perform actions. - -## Key Ideas About Services - -1. **Specialized Focus:** Each Service handles a specific domain or business area (e.g., accounts, customers, product definitions). -2. **Groups Business Operations:** It bundles related functions. For instance, an `AccountsService` might have functions like `GetAccountByID()`, `SearchAccounts()`, etc. -3. **Orchestrates Logic:** A Service can coordinate multiple steps to fulfill a request. It might talk to a [Repository](04_repository_.md) to get data, perform some calculations or checks, and then return a result. -4. **Interacts with Other Components:** Services often use [Repositories](04_repository_.md) to access data and can dispatch [Commands](02_command_.md) to [Aggregates](03_aggregate_.md) if changes to the system's state are needed. -5. **Simplifies Interaction:** For other parts of the system (like [API Handlers](05_api_handler_.md)), Services provide a clean and focused interface to the business logic, hiding the underlying complexity. - -## Using a Service: Fetching Account Details - -Let's say an [API Handler](05_api_handler_.md) needs to display Alice's account details. It wouldn't directly query the database. Instead, it would use the `AccountsService`. - -### The Contract: The `AccountsService` Interface - -First, there's an interface that defines what operations the `AccountsService` offers. This is like the list of services a bank department provides. - -```go -// From: api/accounts.go - -// AccountsService manages accounts -type AccountsService interface { - // ... (methods for creating/updating accounts - more on this later) - Search(ctx context.Context, params *AccountSearchParameters) (Accounts, error) - GetByID(ctx context.Context, accountID uuid.UUID) (*Account, error) - GetByIDs(ctx context.Context, accountIDs []uuid.UUID) ([]*Account, error) - // ... other methods ... -} -``` -* This interface declares methods like `Search`, `GetByID`, and `GetByIDs`. -* Notice the `GetByID` method: it takes a `context` and an `accountID` and returns an `*Account` (a struct with account details) and an `error`. - -### Calling the Service - -An [API Handler](05_api_handler_.md) (or another part of the system) that has access to an `AccountsService` instance (perhaps via the [Core Facade](06_core_facade_.md)) can then call its methods: - -```go -// Somewhere in an API Handler (conceptual) -var accountsSvc api.AccountsService // This would be provided by the CoreFacade - -// ... -accountID := uuid.FromString("acc-alice-123") // Alice's account ID -accountDetails, err := accountsSvc.GetByID(context.Background(), accountID) - -if err != nil { - // Handle error (e.g., account not found) - fmt.Println("Error fetching account:", err) - return -} - -// Now we can use accountDetails -fmt.Println("Account Status:", accountDetails.Status) -fmt.Println("Account Currency:", accountDetails.AvailableCurrencies[0]) -``` -* **Input:** The `accountID` for Alice's account. -* **Output:** The `accountDetails` struct (if found) or an `err`. - -The [API Handler](05_api_handler_.md) doesn't need to know *how* the `AccountsService` gets this information. That's the Service's job! - -## Under the Hood: How `AccountsServiceDefault.GetByID` Works - -Our project has a concrete implementation of the `AccountsService` interface called `AccountsServiceDefault` (in `api/accounts.go`). Let's see how it's structured and how its `GetByID` method works. - -First, the `AccountsServiceDefault` struct holds a reference to an `AccountsRepository`: - -```go -// From: api/accounts.go -// AccountsServiceDefault represents the instance of a AccountsService. -type AccountsServiceDefault struct { - connections repository.ConnectionsProvider // Manages DB connections - repository AccountsRepository // The tool to talk to the database -} -``` -When an `AccountsServiceDefault` is created, it's given an `AccountsRepository`: - -```go -// From: api/accounts.go -// NewAccountsService return a new instance of the accounts service. -func NewAccountsService( - connections repository.ConnectionsProvider, - repository AccountsRepository, -) AccountsService { - // ... (nil checks omitted for brevity) ... - return &AccountsServiceDefault{ - connections: connections, - repository: repository, - } -} -``` -This means the service has the necessary tool ([Repository](04_repository_.md)) to fetch account data. - -Now, let's look at a simplified version of its `GetByID` method: - -```go -// Simplified from: api/accounts.go AccountsServiceDefault.GetByID -func (s *AccountsServiceDefault) GetByID( - ctx context.Context, - accountID uuid.UUID, -) (*Account, error) { - // 1. Potentially start a database transaction (simplified) - ctx, err := s.connections.NewTx(ctx) - if err != nil { - return nil, err - } - defer s.connections.Rollback(ctx) // Ensure transaction is rolled back if not committed - - // 2. Use the repository to fetch the account data - account, err := s.repository.GetByID(ctx, accountID) - if err != nil { - return nil, err // Could be "not found" or other DB error - } - - // 3. (A real service might do more here, like checking permissions - // or enriching the data, before returning it) - - return account, nil -} -``` -Here's what happens: -1. **Manage Transaction (Simplified):** It might start a database transaction. This is often good practice for read operations too, to ensure consistency, though for a single `GetByID` it might be optional depending on the database setup. -2. **Use Repository:** The crucial step! It calls `s.repository.GetByID(ctx, accountID)`. The service delegates the actual data fetching to its [Repository](04_repository_.md). -3. **Return Data:** It returns the `account` (or an error) obtained from the [Repository](04_repository_.md). - -Here's a sequence diagram illustrating this flow: - -```mermaid -sequenceDiagram - participant APIHandler as API Handler - participant AccSvc as AccountsService - participant AccRepo as AccountsRepository - participant DB as Database - - APIHandler->>AccSvc: GetByID("acc-alice-123") - AccSvc->>AccSvc: Start DB Transaction (conceptual) - AccSvc->>AccRepo: GetByID(ctx, "acc-alice-123") - AccRepo->>DB: SELECT * FROM accounts WHERE id = "acc-alice-123" - DB-->>AccRepo: Account Row Data - AccRepo-->>AccSvc: *Account (struct) - AccSvc->>AccSvc: Commit/Rollback Transaction (conceptual) - AccSvc-->>APIHandler: *Account (details for Alice) -``` - -## Services and Write Operations (Changes to Data) - -So far, we've focused on how Services help read data. What about when we want to *change* data, like creating an account or updating its status? - -The `AccountsService` interface in `api/accounts.go` actually defines methods like `Create`, `Activate`, etc.: -```go -// From: api/accounts.go -type AccountsService interface { - Create(ctx context.Context, command CreateAccountCommand) error - Activate(ctx context.Context, command ActivateAccountCommand) error - // ... other methods like Close, LinkPaymentDevice ... - // ... and the read methods we saw earlier ... -} -``` -However, if you look at the `AccountsServiceDefault` implementation for these methods, you'll find something like this: -```go -// From: api/accounts.go AccountsServiceDefault.Create -func (s *AccountsServiceDefault) Create( - _ context.Context, - _ CreateAccountCommand, -) error { - // writes should be done through the commandBus - return e.NewNotImplemented() -} -``` -It returns `e.NewNotImplemented()`! This tells us that, in this specific design, the `AccountsServiceDefault` *itself* doesn't directly handle account creation logic. Instead, such "write" operations are intended to be performed by sending a [Command](02_command_.md) (like `CreateAccountCommand`) directly to the `EventSourcingClient` (our Command Bus), which then routes it to the appropriate [Aggregate](03_aggregate_.md) (e.g., `AccountAggregate`). This is often done by the [API Handler](05_api_handler_.md) using the `EventSourcingClient` from the [Core Facade](06_core_facade_.md). - -**Why this separation?** -* **Centralized Command Processing:** Keeps all state-changing logic within [Aggregates](03_aggregate_.md), ensuring business rules are consistently enforced. -* **Event Sourcing Purity:** [Aggregates](03_aggregate_.md) are the source of [Events](01_event_.md), and [Commands](02_command_.md) are the standard way to trigger changes in them. - -### When Services *Do* Orchestrate Writes - -However, this doesn't mean Services *never* handle logic related to writes. A Service might: -1. **Perform preliminary checks or gather data** before a [Command](02_command_.md) is dispatched. -2. **Dispatch a [Command](02_command_.md) itself** if the operation is more complex or involves coordination. -3. **Orchestrate multiple [Commands](02_command_.md)** or interactions with different [Aggregates](03_aggregate_.md) or [Repositories](04_repository_.md). - -For example, consider the `PostingsTransactionsServiceDefault` in `api/postings_transactions.go`. Its `RevertChildTransactions` method shows a service performing more complex orchestration: - -```go -// Simplified from: api/postings_transactions.go -// PostingsTransactionsServiceDefault holds a commandBus -type PostingsTransactionsServiceDefault struct { - // ... other fields ... - commandBus es.CommandBus -} - -// RevertChildTransactions reverts child transactions for a specific transaction. -func (s *PostingsTransactionsServiceDefault) RevertChildTransactions( - ctx context.Context, - transactionID, rootTransactionID, parentTransactionID uuid.UUID, - /* ... other params ... */ -) error { - // 1. Use repository to get child transactions - postingsTransactions, err := s.repository.GetPostingsTransactionByParentID(ctx, transactionID) - // ... error handling ... - - // 2. For each child, dispatch a RevertSettledPostingsTransactionCommand - for _, postingsTransaction := range postingsTransactions { - err := s.commandBus.HandleCommand(ctx, &RevertSettledPostingsTransactionCommand{ - BaseCommand: es.BaseCommand{AggregateID: postingsTransaction.ID}, - // ... populate command details ... - }) - // ... error handling ... - } - return nil -} -``` -In this case, the `PostingsTransactionsService` reads data using its [Repository](04_repository_.md) and then dispatches multiple [Commands](02_command_.md) using its `commandBus`. This is a perfect example of a Service acting as an orchestrator for a business process. - -So, a Service can: -* Directly use [Repositories](04_repository_.md) for reading data (common). -* Dispatch [Commands](02_command_.md) to [Aggregates](03_aggregate_.md) for writing data (also common, especially for complex orchestrations). -* Sometimes, for simpler writes, an [API Handler](05_api_handler_.md) might dispatch a [Command](02_command_.md) directly using the `EventSourcingClient` from the [Core Facade](06_core_facade_.md), bypassing a specific write method on a service if the service doesn't add extra orchestration. - -## Why are Services Important? - -1. **Clear Business API:** They offer a higher-level, business-oriented set of operations compared to raw [Repositories](04_repository_.md) or direct [Command](02_command_.md) dispatching. -2. **Encapsulation:** They group related business logic, making it easier to find and manage. If you need to know how account searching works, you look in the `AccountsService`. -3. **Decoupling:** [API Handlers](05_api_handler_.md) or other clients depend on the Service interface, not on the nitty-gritty details of data storage or [Command](02_command_.md) handling for every single operation. -4. **Orchestration Point:** They are the natural place to put logic that coordinates multiple steps or involves several other components to achieve a business goal (like the `RevertChildTransactions` example). -5. **Testability:** Services can be tested in isolation. For example, when testing `AccountsServiceDefault`, you can provide a "mock" `AccountsRepository` to simulate database interactions. - -## Conclusion - -Services are the specialized "departments" within our `corebanking` application. They: -* Group related business operations for a specific domain (like accounts or transactions). -* Often use [Repositories](04_repository_.md) to fetch data needed for read operations. -* Can orchestrate more complex business processes, sometimes by dispatching [Commands](02_command_.md) to [Aggregates](03_aggregate_.md). -* Provide a clean and organized way for other parts of the system, like [API Handlers](05_api_handler_.md), to interact with the core business logic. - -They help keep our application well-structured and maintainable by ensuring that responsibilities are clearly defined. - -So far, we've seen how requests come in ([API Handler](05_api_handler_.md)), how they might use a [Core Facade](06_core_facade_.md) to access Services, and how Services can interact with [Repositories](04_repository_.md) or dispatch [Commands](02_command_.md) that affect [Aggregates](03_aggregate_.md) and generate [Events](01_event_.md). - -But what happens *after* an [Event](01_event_.md) is generated? Sometimes, other parts of the system need to react to these [Events](01_event_.md) automatically. For example, when an `AccountCreatedEvent` occurs, maybe we need to update a search index or send a welcome email. How does that happen? In the next chapter, we'll explore the [Consumer](08_consumer_.md). - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/08_consumer_.md b/tmp/nesto/corebanking/08_consumer_.md deleted file mode 100644 index 530c6987..00000000 --- a/tmp/nesto/corebanking/08_consumer_.md +++ /dev/null @@ -1,204 +0,0 @@ -# Chapter 8: Consumer - -Welcome to Chapter 8! In our [previous chapter on Services](07_service_.md), we learned how specialized "departments" in our bank help organize business logic, often by reading data using [Repositories](04_repository_.md) or orchestrating actions, sometimes by sending [Commands](02_command_.md). These [Commands](02_command_.md), when processed by [Aggregates](03_aggregate_.md), result in [Events](01_event_.md) – records of things that have happened. - -But what happens *after* an [Event](01_event_.md) is generated? For instance, if a `PostingCommittedEvent` (signaling a financial transaction leg) occurs, we probably need to update the customer's account balance so they can see it. Or, if an `AccountCreatedEvent` happens, maybe we need to send a welcome email. How do these follow-up actions happen automatically without slowing down the original process? - -This is where the **Consumer** steps in! - -## What is a Consumer? The Bank's Automated Event Watcher - -Imagine in our bank, every time an important announcement is made (an [Event](01_event_.md) happens), there are dedicated staff members who are always listening for specific types of announcements. -* One staff member listens for "New Account Opened" announcements. When they hear one, they immediately prepare and send a welcome pack to the new customer. -* Another staff member listens for "Transaction Completed" announcements. When they hear one, they quickly update the account ledger display. - -A **Consumer** in our `corebanking` system is like one ofthese automated staff members. It's a component that **listens for specific types of [Events](01_event_.md)** that occur within the system. When it "hears" an [Event](01_event_.md) it's interested in, it performs a predefined action. - -For example, when a `PostingCommittedEvent` occurs (meaning a part of a financial transaction has been finalized), a `BalancesConsumer` might listen for this. Upon receiving it, the consumer will update the account's balance that is stored for quick display (this stored balance is often called a "read model" or "projection"). - -Consumers are crucial because they allow different parts of the system to react to business happenings **asynchronously**. This means the original action (like processing the transaction) can finish quickly, and the follow-up tasks (like updating the balance display or sending a notification) happen separately, a moment later. - -## Key Ideas About Consumers - -1. **Event Listener:** A Consumer "subscribes" to a stream of [Events](01_event_.md) and pays attention only to the types of [Events](01_event_.md) it cares about. -2. **Asynchronous Operation:** Consumers typically run in the background, separate from the process that originally created the [Event](01_event_.md). This keeps the main operations fast and responsive. The part of the system that created the event doesn't have to wait for the Consumer to finish its job. -3. **Performs Actions:** When a relevant [Event](01_event_.md) is received, the Consumer executes specific logic. Common actions include: - * **Updating Read Models:** Calculating and storing data in a format optimized for fast reading. For example, updating an account balance summary after a transaction. - * **Triggering Follow-up Processes:** Starting new workflows, sending notifications (like emails or SMS), or calling other [Services](07_service_.md). -4. **Decoupling:** The component that produces an [Event](01_event_.md) (e.g., an `AccountAggregate` producing `AccountCreatedEvent`) doesn't need to know anything about the Consumers that might be listening to that [Event](01_event_.md). This makes the system more flexible and easier to change. -5. **Data Consistency (Eventually):** Consumers help maintain consistency across different views of data. For example, after a transaction, the balance read model will *eventually* reflect the change. - -## A Consumer in Action: Updating Account Balances - -Let's say a customer makes a deposit. This might result in one or more `PostingCommittedEvent`s. We need to update the account balance that is shown to the customer on their app. - -This is a perfect job for a `BalancesConsumer` (like the `BalancesTimeSeriesConsumerDefault` found in `api/balances_consumer.go`): - -1. **[Event](01_event_.md) Happens:** An [Aggregate](03_aggregate_.md) processes a transaction, and a `PostingCommittedEvent` is generated and saved. This event contains details about the amount, currency, and account involved. -2. **Consumer is Notified:** The `BalancesConsumer` is subscribed to listen for `PostingCommittedEvent`s. An "event bus" (a system for distributing events) delivers this event to the consumer. -3. **Consumer Processes the [Event](01_event_.md):** - * The `BalancesConsumer` receives the `PostingCommittedEvent`. - * It extracts the necessary information (account ID, amount, type of posting - debit/credit). - * It might fetch the current stored balance for that account using a `BalancesRepository` (a type of [Repository](04_repository_.md)). - * It calculates the new balance. -4. **Consumer Updates Read Model:** The consumer then saves this new, updated balance back using the `BalancesRepository`. This updated balance is now ready to be quickly displayed to the user. -5. **Optional: Further [Events](01_event_.md):** The consumer might even publish a new [Event](01_event_.md) itself, like a `BalanceUpdatedEvent`, to signal that the read model balance has changed. - -This entire process happens *after* the original deposit transaction was confirmed. The customer got a quick confirmation of their deposit, and in the background, the consumer ensures their displayed balance is updated. - -## What Does a Consumer Look Like? (A Peek at the Code) - -Consumers are often structs that hold dependencies like [Repositories](04_repository_.md) or other [Services](07_service_.md). They typically have a `Start()` method to begin listening and a processing method that gets called when relevant [Events](01_event_.md) arrive. - -Let's look at a simplified structure for `BalancesTimeSeriesConsumerDefault` from `api/balances_consumer.go`: - -```go -// Simplified from: api/balances_consumer.go -type BalancesTimeSeriesConsumerDefault struct { - repository BalancesRepository // To read/write balance data - events es.EventPublisher // To publish new events (optional) - eventConsumer es.EventConsumer // The mechanism to receive events - // ... other dependencies ... -} - -// NewBalancesTimeSeriesConsumer creates the consumer -func NewBalancesTimeSeriesConsumer( - repository BalancesRepository, - events es.EventPublisher, - eventConsumer es.EventConsumer, - // ... other args ... -) BalancesTimeSeriesConsumer { - // ... (nil checks) ... - return &BalancesTimeSeriesConsumerDefault{ - repository: repository, - events: events, - eventConsumer: eventConsumer, - // ... - } -} -``` -* `repository`: This is a `BalancesRepository`, used to get the current balance and save the new balance. -* `events`: An `EventPublisher` which this consumer might use to send out new [Events](01_event_.md) (like `BalanceUpdatedEvent`). -* `eventConsumer`: This is the component from our event sourcing library (`es`) that actually delivers [Events](01_event_.md) from the event bus to this consumer. - -### Starting the Consumer and Processing Events - -The consumer needs to be "turned on" to start listening. This is usually done by its `Start()` method: - -```go -// Simplified from: api/balances_consumer.go -func (c *BalancesTimeSeriesConsumerDefault) Start() error { - // Tell the eventConsumer to start sending batches of events - // to our 'processBalancesTimeSeriesEvents' method. - err := c.eventConsumer.ConsumeTenantMultiAggregateBatch( - context.Background(), - c.processBalancesTimeSeriesEvents, // Our callback function - ) - return err -} -``` -* `c.eventConsumer.ConsumeTenantMultiAggregateBatch(...)`: This line registers the `processBalancesTimeSeriesEvents` method with the underlying event consumption mechanism. Now, whenever new [Events](01_event_.md) (that this consumer is configured to listen to) arrive, this method will be called with a batch of those [Events](01_event_.md). - -The actual work happens in the callback method, `processBalancesTimeSeriesEvents`. Here's a highly simplified conceptual view of what it does when it receives event data related to postings: - -```go -// Highly simplified concept of processBalancesTimeSeriesEvents -func (c *BalancesTimeSeriesConsumerDefault) processBalancesTimeSeriesEvents( - ctx context.Context, tenantID int, events []*es.Event, -) error { - // For each relevant event (e.g., indirectly from a PostingCommittedEvent): - for _, event := range events { - // 1. Extract posting details from the event - // (The real code uses a helper 'orderedPostingsFromEvents' - // to get 'Posting' data from events like PostingCommittedEvent) - postingData := extractPostingFrom(event) // Conceptual - accountID := postingData.AccountID - - // 2. Get current balances for the account (simplified) - accountBalances, _ := c.repository.GetBalancesByAccountIDs(ctx, []uuid.UUID{accountID}) - // (Real code handles 'not found' and groups by account) - currentBalanceInfo := accountBalances[0] // Simplified - - // 3. Apply the posting to update the balance - // (The real code uses 'ApplyPostingToAccountBalances') - newBalance := calculateNewBalance(currentBalanceInfo, postingData) // Conceptual - - // 4. Save the updated current balance - // (The real code accumulates 'updatedCurrentBalances' and saves in a batch) - err := c.repository.SaveCurrent(ctx, []*Balance{newBalance}) - if err != nil { - // Handle error - return err - } - - // 5. Optionally, publish a new event - balanceUpdatedEvt := createBalanceUpdatedEvent(newBalance) // Conceptual - c.events.PublishBatch(ctx, []es.Event{balanceUpdatedEvt}) - } - return nil -} -``` -Let's break down this conceptual flow: -1. **Extract Posting Details:** The consumer gets the necessary details from the incoming [Event](01_event_.md) (like account ID, amount, currency, debit/credit). In the actual `corebanking` code, `PostingCommittedEvent`s are processed to extract `Posting` objects. -2. **Get Current Balance:** It uses its `BalancesRepository` to fetch the most recent balance record for the affected account. -3. **Apply Posting:** It performs the calculation to update the balance. The actual function `ApplyPostingToAccountBalances` in `api/balances_consumer.go` handles the logic of adding or subtracting based on posting type and account category. -4. **Save Updated Balance:** The new balance is saved back to the read model storage using the `BalancesRepository`'s `SaveCurrent` method. -5. **Publish New Event (Optional):** The `BalancesConsumer` in our project also creates and publishes a `BalanceUpdatedEvent`. This allows other parts of the system to know that a balance read model has been updated. - -The actual `processBalancesTimeSeriesEvents` is more complex because it handles batches of events, deals with time-series data for historical balances, and manages database transactions. But the core idea is: **receive event -> process it -> update read model.** - -## How a Consumer Gets Triggered: The Event Flow - -Here’s a simplified sequence diagram showing how a `BalancesConsumer` might react to a `PostingCommittedEvent`: - -```mermaid -sequenceDiagram - participant TxnProcessor as Transaction Processor (e.g., Aggregate) - participant EventBus as Event Bus / Message Queue - participant BalConsumer as Balances Consumer - participant BalRepo as Balances Repository - participant ReadModelDB as Read Model Database - - TxnProcessor->>EventBus: Publish PostingCommittedEvent (for Account X, Amount $10) - EventBus->>BalConsumer: Deliver PostingCommittedEvent - BalConsumer->>BalRepo: Get Current Balance (Account X) - BalRepo->>ReadModelDB: Fetch balance for Account X - ReadModelDB-->>BalRepo: Current Balance: $50 - BalRepo-->>BalConsumer: Current Balance: $50 - BalConsumer-->>BalConsumer: Calculate New Balance ($50 + $10 = $60) - BalConsumer->>BalRepo: Save Balance (Account X, New Balance $60) - BalRepo->>ReadModelDB: Update balance for Account X to $60 - ReadModelDB-->>BalRepo: Success - BalRepo-->>BalConsumer: Success - BalConsumer->>EventBus: (Optional) Publish BalanceUpdatedEvent -``` -This shows the asynchronous nature: the `TxnProcessor` fires the event and moves on. The `BalConsumer` picks it up later and does its work. - -## Other Types of Consumers - -Our `corebanking` system has other consumers too: - -* **`PostingsTransactionsConsumer`** (from `api/postings_transactions_consumer.go`): - This consumer listens to events related to financial transactions (like `PostingsTransactionCreatedEvent`, `PostingsTransactionSettledEvent`). It updates a read model that stores details about these transactions, making it easy to query their status, amount, etc. - -* **`ProductEnginesConsumer`** (from `api/product_engines_consumer.go`): - This is an interesting one! It listens for events like `ScheduledEvent` (e.g., "it's end of day for account X") or even `PostingCommittedEvent`. When it receives such an event, it triggers the [Product Engine](09_product_engine_.md) (via `productEngines.HandleEvents()`). The [Product Engine](09_product_engine_.md) might then apply interest, charge fees, or perform other account-specific logic defined by the banking product. This shows a consumer triggering a more complex follow-up process. - -## Why Are Consumers So Useful? - -1. **Improved Performance & Responsiveness:** The system part that creates an [Event](01_event_.md) (e.g., an [Aggregate](03_aggregate_.md) handling a [Command](02_command_.md)) can finish its job quickly without waiting for all side effects (like updating multiple read models or sending emails) to complete. -2. **Decoupling & Modularity:** Event producers don't need to know about event consumers. You can add new consumers or change existing ones without affecting the code that generates the events. This makes the system very flexible. -3. **Resilience:** If a consumer temporarily fails while processing an event (e.g., a network issue while saving to a database), the event can often be re-processed later without losing the original data, as the [Event](01_event_.md) itself is already safely stored. -4. **Scalability:** Different consumers can often be scaled independently. If updating balances becomes a bottleneck, you might be able to run more instances of the `BalancesConsumer`. - -## Conclusion - -Consumers are the unsung heroes working diligently in the background of our `corebanking` system. They are **automated listeners** that subscribe to specific types of [Events](01_event_.md). When they receive an [Event](01_event_.md) they're interested in, they swing into action – perhaps updating a read-model for fast data display (like account balances) or triggering follow-up processes (like complex product calculations). - -By reacting to business [Events](01_event_.md) asynchronously, Consumers help keep our system responsive, flexible, and robust. They play a vital role in maintaining data consistency across different views and enabling complex workflows. - -One example we saw was the `ProductEnginesConsumer`, which listens for events and then invokes specific business logic defined by a banking product. What is this "Product Engine" that it calls? Let's explore that in our next chapter: the [Product Engine](09_product_engine_.md). - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/09_product_engine_.md b/tmp/nesto/corebanking/09_product_engine_.md deleted file mode 100644 index 11c45b49..00000000 --- a/tmp/nesto/corebanking/09_product_engine_.md +++ /dev/null @@ -1,265 +0,0 @@ -# Chapter 9: Product Engine - -Welcome to Chapter 9! In the [previous chapter](08_consumer_.md), we learned about [Consumers](08_consumer_.md) and how they listen for [Events](01_event_.md) to trigger follow-up actions. For example, a [Consumer](08_consumer_.md) might notice that an account needs its monthly interest calculated. But how does the system know *how* to calculate that interest, especially if different types of accounts (like a "Youth Savings Account" vs. a "Standard Mortgage") have vastly different rules? - -This is where the **Product Engine** comes in. - -## What's a Product Engine? The Bank's Specialized Rulebook - -Imagine a bank offers many different financial products: -* A "Basic Savings Account" with simple interest. -* A "Youth Savings Account" that gets bonus interest if no money is withdrawn during the month. -* A "Standard Mortgage" with complex payment schedules and fee structures. -* A "HELOC Remittance" account with its own unique remittance rules. - -Each of these products behaves differently. They have their own specific features, ways to calculate interest, fee rules, and reactions to events like an account being activated or a transaction occurring. - -A **Product Engine** is like a **detailed rulebook and automated process manager** specifically designed for *one particular financial product*. It defines exactly how that product, like "Standard Mortgage" or "Youth Savings Account," should work. - -When a customer opens an account (say, a "Youth Savings Account"), that account is linked to its specific product. From then on, whenever something relevant happens to that account, the system consults the "Youth Savings Account" Product Engine to: -* Check if an operation is allowed (validation). -* Determine any product-specific actions or outcomes (like calculating that special bonus interest). - -## Key Ideas About Product Engines - -1. **Product-Specific Logic:** Each Product Engine contains the unique rules and behaviors for one type of financial product. -2. **Defines Product Behavior:** It specifies features, fee structures, interest calculations, and how the product responds to lifecycle events (e.g., account activation, transaction posting, scheduled events like "end of month"). -3. **Consulted for Operations:** When an action needs to be performed on an account, or an event occurs related to it, the corresponding Product Engine is invoked. -4. **Automates Processes:** It automates tasks like applying interest, charging fees, or managing scheduled product-specific activities. -5. **Produces Actions:** Based on its rules and the current situation (e.g., an incoming [Event](01_event_.md) or [Command](02_command_.md)), a Product Engine can decide that certain actions need to happen. These actions are often new [Commands](02_command_.md) (e.g., "Create a transaction to credit bonus interest"). - -## The Product Engine in Action: Youth Savings Bonus Interest - -Let's use our "Youth Savings Account" example. This account type offers bonus interest if no withdrawals are made in a month. - -1. **Account Setup:** Alice has a "Youth Savings Account." This account is linked to the "Youth Savings Product Engine." -2. **End of Month [Event](01_event_.md):** A `ScheduledEvent` (like "EndOfMonthProcessing") occurs for Alice's account. -3. **[Consumer](08_consumer_.md) Routes to Service:** A [Consumer](08_consumer_.md) (like the `ProductEnginesConsumer` we touched upon in the [previous chapter](08_consumer_.md)) picks up this `ScheduledEvent`. It knows this event might require product-specific logic, so it passes the event to the `ProductEnginesService`. -4. **Service Invokes Product Engine:** The `ProductEnginesService`: - * Looks at Alice's account and sees it's a "Youth Savings Account." - * Retrieves the specific "Youth Savings Product Engine." - * Prepares a `ProductEngineContext` for Alice's account. This context contains information the engine might need, like Alice's current account details, balances, and the product's configuration. - * Calls a method on the "Youth Savings Product Engine," like `HandleScheduledEvent(context, scheduledEvent)`. -5. **Product Engine Applies Rules:** The "Youth Savings Product Engine" executes its logic for the "EndOfMonthProcessing" event: - * It checks Alice's transaction history for the past month (perhaps by looking at data in the `ProductEngineContext` or by querying for recent postings). - * It determines if any withdrawals were made. - * If no withdrawals, it calculates the bonus interest. -6. **Product Engine Returns Actions:** The engine decides that bonus interest should be applied. It returns a list of `ProductEngineAction`s. One of these actions might be a `HandleCommandAction` containing a `CreatePostingsTransactionCommand` to credit the bonus interest to Alice's account. -7. **Service Executes Actions:** The `ProductEnginesService` takes these actions and processes them. For a `HandleCommandAction`, it would send the `CreatePostingsTransactionCommand` to the [Command Bus](02_command_.md), which then ensures the transaction is created and Alice's balance is updated. - -This way, the specific bonus interest logic for the "Youth Savings Account" is neatly contained within its own Product Engine. - -## A Peek at the Code: Interfaces and Services - -Let's look at some of the key pieces involved. - -### 1. The `ProductEngine` Interface - -At its core, every product engine implements the `ProductEngine` interface (from `api/product_engines.go`). This interface defines the "hooks" or methods that can be called by the system for different situations. - -```go -// Simplified from: api/product_engines.go -type ProductEngine interface { - ProductEngineID() string // E.g., "YOUTH_SAVINGS_ACCOUNT" - ProductEngineVersion() string // E.g., "1.0.0" - - // Called when an account is activated - ActivateAccount(ctx context.Context, engineCtx ProductEngineContext) ([]ProductEngineAction, error) - - // Called for scheduled events (like our "EndOfMonthProcessing") - HandleScheduledEvent(ctx context.Context, engineCtx ProductEngineContext, scheduledEvent *ScheduledEvent) ([]ProductEngineAction, error) - - // Called when a posting (transaction leg) is accepted for an account - AcceptPosting(ctx context.Context, engineCtx ProductEngineContext, posting *Posting) ([]ProductEngineAction, error) - - // ... other methods for validating postings, updating parameters, closing accounts etc. -} -``` -* `ProductEngineID()` and `ProductEngineVersion()`: Identify the specific engine. -* `ActivateAccount`, `HandleScheduledEvent`, `AcceptPosting`: These are examples of methods that get called at different points in an account's lifecycle or when certain [Events](01_event_.md) occur. -* `ProductEngineContext`: This argument provides all the necessary information about the current account, its product settings, balances, etc., that the engine needs to make decisions. -* `[]ProductEngineAction`: The method returns a list of actions the system should take as a result of the engine's logic. - -### 2. The `ProductEngineContext` - -This struct (also from `api/product_engines.go`) is passed to the Product Engine methods. It's like a data package giving the engine all the relevant details for the current operation. - -```go -// Simplified from: api/product_engines.go -type ProductEngineContext struct { - ReferenceDate *date.Date // The current date for processing - Account *Account // Details of the specific account - Product *ResolvedProduct // Details & parameters of the banking product - CurrentBalances *BalancesReader // Access to the account's current balances - // ... and other fields like PostingsByPartition, BalancesByPartition for historical data -} -``` -* `Account`: The specific account being processed. -* `Product`: The resolved product definition, containing parameters like interest rates, fee schedules, etc. -* `CurrentBalances`: A way to read the account's current balances. -The engine uses this information to apply its rules. - -### 3. `ProductEngineAction` - -When a Product Engine finishes its logic, it needs to tell the system what to do next. It does this by returning a slice of `ProductEngineAction` (defined in `api/product_engines.go`). - -A common action is to issue a [Command](02_command_.md): -```go -// Simplified from: api/product_engines.go - -// ProductEngineAction represents a product engine action. -type ProductEngineAction interface { - Actuate() // A marker method -} - -// HandleCommandAction implements product engine action to handle command. -type HandleCommandAction struct { - command es.Command // The command to be executed -} - -// NewHandleCommandAction returns a new HandleCommandAction. -func NewHandleCommandAction(command es.Command) HandleCommandAction { - return HandleCommandAction{command} -} - -// GetCommand returns the underlying command -func (a HandleCommandAction) GetCommand() es.Command { - return a.command -} -``` -So, if our "Youth Savings Product Engine" decides to apply bonus interest, it would create a `CreatePostingsTransactionCommand` and wrap it in a `NewHandleCommandAction`. - -### 4. The `ProductEnginesService` - -The `ProductEnginesServiceDefault` (from `api/product_engines.go`) is the orchestrator. It's responsible for receiving [Events](01_event_.md) (often from a [Consumer](08_consumer_.md)), finding the right Product Engine, and invoking it. - -```go -// Conceptual flow within ProductEnginesServiceDefault.HandleEvents -// (Actual code is more complex, handling batches and various event types) - -func (s *ProductEnginesServiceDefault) HandleEvents(ctx context.Context, events []*es.Event) error { - for _, event := range events { - // 1. Determine which account this event is for (from event.AggregateID) - accountID := event.AggregateID - account, _ := s.accounts.GetByID(ctx, accountID) // Fetch account details - - // 2. Get the Product Engine for this account's product - productEngine, _ := s.GetProductEngine(ctx, account.Product) - if productEngine == nil { - continue // No engine for this product, or product not set - } - - // 3. Create an Event Handler wrapper (e.g., for a ScheduledEvent) - // The hook file (api/product_engines_hooks.go) has different handlers for different events. - var eventHandler ProductEngineEventHandler // Defined in api/product_engines_hooks.go - if se, ok := event.Data.(*ScheduledEvent); ok && eventHandlerNameFromAction(se.Action) == "HandleScheduledEvent" { - eventHandler = NewProductEngineEventHandlerHandleScheduledEvent(event.ID, se, /*...location...*/) - } - // ... (else if for AccountActivatedEvent, PostingCommittedEvent, etc.) ... - - if eventHandler == nil { - continue - } - - // 4. Build the ProductEngineContext (simplified) - engineCtx, _ := s.buildProductEngineContext(ctx, /*...details...*/, account, /*balances*/, /*config*/) - - // 5. Call the specific hook on the event handler, which calls the engine - actions, err := eventHandler.Hook(ctx, *engineCtx, productEngine) - // ... error handling ... - - // 6. Run the actions returned by the Product Engine - s.runActions(ctx, actions) // This would dispatch commands, schedule things, etc. - } - return nil -} -``` -* **Get Account & Engine:** It fetches the account and determines the correct Product Engine based on the account's product information (`account.Product.EngineID`, `account.Product.EngineVersion`). -* **Event Handler Wrapper:** An `ProductEngineEventHandler` (from `api/product_engines_hooks.go`) is chosen based on the type of incoming [Event](01_event_.md). This wrapper standardizes how different events are passed to the Product Engine. For example, `ProductEngineEventHandlerHandleScheduledEvent` has a `Hook` method that will ultimately call `engine.HandleScheduledEvent(...)`. -* **Build Context:** It prepares the `ProductEngineContext` with all necessary data. -* **Invoke Engine (via Hook):** It calls the `Hook` method on the event handler, which in turn calls the relevant method on the `ProductEngine` instance (e.g., `HandleScheduledEvent`). -* **Run Actions:** It takes the `ProductEngineAction`s returned by the engine and executes them (e.g., sends [Commands](02_command_.md) to the [Command Bus](02_command_.md)). - -### Example: A Simple Product Engine's `HandleScheduledEvent` - -Let's imagine a very basic "MonthlyFeeProductEngine": -```go -// Hypothetical MonthlyFeeProductEngine -type MonthlyFeeProductEngine struct { /* ... */ } - -func (pe *MonthlyFeeProductEngine) HandleScheduledEvent( - ctx context.Context, - engineCtx api.ProductEngineContext, - scheduledEvent *api.ScheduledEvent, -) ([]api.ProductEngineAction, error) { - - if scheduledEvent.Action == "APPLY_MONTHLY_FEE" { - feeAmountStr, _ := engineCtx.Product.Parameters["monthlyFeeAmount"] // Get fee from product config - feeAmount, _ := decimal.NewFromString(feeAmountStr) - - if feeAmount.IsPositive() { - // Create a command to charge the fee - feeTxCmd := &api.CreatePostingsTransactionCommand{ - BaseCommand: es.BaseCommand{AggregateID: engineCtx.Account.ID}, - Action: api.PostingActionTransfer, - Code: "MONTHLY_FEE", - Amount: feeAmount, - Debtor: &api.PostingDestination{AccountID: engineCtx.Account.ID}, // Debit customer - Creditor: &api.PostingDestination{AccountID: uuid.FromString("bank-fee-income-account-id")}, // Credit bank - } - return []api.ProductEngineAction{api.NewHandleCommandAction(feeTxCmd)}, nil - } - } - return nil, nil // No action -} - -// ... other required methods of ProductEngine interface ... -``` -When a `ScheduledEvent` with action "APPLY_MONTHLY_FEE" comes in for an account using this product: -1. The engine checks if the action matches. -2. It reads the `monthlyFeeAmount` from the product's parameters (available in `engineCtx.Product.Parameters`). -3. If a fee is due, it creates a `CreatePostingsTransactionCommand` to transfer the fee from the customer's account to the bank's fee income account. -4. It returns this command wrapped in a `HandleCommandAction`. - -## Visualizing the Flow - -Here's how an event for Alice's "Youth Savings Account" (YSA) might trigger its Product Engine to apply bonus interest: - -```mermaid -sequenceDiagram - participant EventBus - participant YSAConsumer as YSA Product Consumer - participant PEService as ProductEnginesService - participant YSAPEngine as YouthSavingsProductEngine - participant CmdBus as CommandBus - - EventBus->>YSAConsumer: ScheduledEvent (EndOfMonth, Account: AliceYSA) - YSAConsumer->>PEService: HandleEvents([ScheduledEvent]) - PEService->>YSAPEngine: GetEngine("YouthSavingsProductEngine", "v1") - Note over PEService: Builds ProductEngineContext for AliceYSA - PEService->>YSAPEngine: HandleScheduledEvent(ctx, engineCtx, ScheduledEvent) - YSAPEngine-->>YSAPEngine: Logic: Check no withdrawals, calculate bonus. - YSAPEngine-->>PEService: Return [HandleCommandAction(CreateBonusTxCmd)] - PEService->>CmdBus: HandleCommand(CreateBonusTxCmd) - CmdBus-->>PEService: Command processed (Alice gets bonus) -``` - -## Why Product Engines are Powerful - -* **Flexibility:** Easily define unique behaviors for many different banking products without cluttering the core system. -* **Modularity:** Product-specific logic is isolated. Changes to one product's rules don't affect others. -* **Maintainability:** Easier to understand and update the logic for a specific product. -* **Testability:** Individual Product Engines can be tested in isolation with a mocked `ProductEngineContext`. -* **Rapid Development:** New financial products can often be introduced by creating a new Product Engine and its configuration, rather than changing deep parts of the core system. - -## Conclusion - -Product Engines are the specialized "brains" behind each financial product offered by the bank. They act as detailed rulebooks and automated process managers, defining how a product like a "Standard Mortgage" or a "Youth Savings Account" behaves – its features, fees, interest calculations, and responses to lifecycle events. - -When an [Event](01_event_.md) occurs for an account, the `ProductEnginesService` (often triggered by a [Consumer](08_consumer_.md)) identifies the account's product and invokes the corresponding Product Engine. The engine then applies its specific rules and can return a list of actions, such as [Commands](02_command_.md) to create transactions, for the system to execute. This powerful mechanism allows `corebanking` to support a diverse range of financial products with complex, individualized behaviors. - -With all these intricate product behaviors and event-driven interactions, how can we be sure everything works as expected? How can we test different scenarios, perhaps even simulating future events? In our next and final chapter, we'll look into [Simulation Services and Repositories](10_simulation_services_and_repositories_.md). - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/10_simulation_services_and_repositories_.md b/tmp/nesto/corebanking/10_simulation_services_and_repositories_.md deleted file mode 100644 index dd0db4cd..00000000 --- a/tmp/nesto/corebanking/10_simulation_services_and_repositories_.md +++ /dev/null @@ -1,205 +0,0 @@ -# Chapter 10: Simulation Services and Repositories - -Welcome to the final chapter of our conceptual overview! In [Chapter 9: Product Engine](09_product_engine_.md), we saw how our `corebanking` system can support diverse financial products, each with its own complex set of rules for things like interest calculation, fee application, and behavior over time. - -Now, imagine we've just designed a sophisticated 30-year mortgage product using our [Product Engine](09_product_engine_.md). How can we be absolutely sure it works correctly under all sorts of conditions? How do we test its behavior over the full 30 years, perhaps with fluctuating interest rates, without actually waiting for three decades or messing with real customer data? This is where **Simulation Services and Repositories** come to the rescue! - -## What are Simulation Services and Repositories? Your Bank's Flight Simulator! - -Think of Simulation Services and Repositories as your bank's very own **flight simulator** or **practice mode**. They are special "test-double" or "fake" versions of the regular [Services](07_service_.md) and [Repositories](04_repository_.md) we've learned about. - -Instead of connecting to real databases and affecting real money or customer data, these simulators: -* **Work "in-memory":** They store their data temporarily, right in the computer's memory, for the duration of a test or simulation. When the simulation ends, this temporary data is gone. This is great because it's fast and completely isolated from the live system. -* **Mimic real behavior:** They are designed to act just like their real counterparts, but give us much more control. -* **Allow controlled testing:** We can use them to test complex business logic, like our 30-year mortgage product, under various scenarios (e.g., "what if interest rates double next year?") in a safe and repeatable way. - -Essentially, these simulators let us "play out" complex financial scenarios, fast-forward through time, and see what happens, all without any real-world consequences. - -## Key Ideas Behind Simulation - -How do these simulators achieve this magic? There are a few key ingredients: - -1. **In-Memory Storage:** - * Simulation [Repositories](04_repository_.md) (like `AccountRepositorySimulator` or `PostingsTransactionsRepositorySimulator`) don't save data to a permanent database. Instead, they use simple Go data structures like maps and slices to hold information temporarily. - * This makes them very fast and ensures that each simulation run starts fresh. - -2. **Controlled Time (The "Time Machine"):** - * A special `ClockServiceSimulator` lets us control the "current date" within the simulation. We can tell it to jump forward day by day, month by month, or even set it to a specific date in the future. This is crucial for testing products that behave differently over long periods. - -3. **Deterministic Behavior:** - * For a given starting point and set of actions, a simulation should ideally produce the exact same results every time it's run. This makes testing reliable and helps us pinpoint issues if something changes unexpectedly. - -4. **Isolation:** - * Simulations run completely separately from the live banking system. There's no risk of accidentally creating a real account or moving real money. - -## Using the Simulators: Testing Our Mortgage Product - -Let's go back to our 30-year mortgage product. How would we use the simulation tools to test it? The main entry point for complex product simulations is often the `ProductEnginesServiceSimulatorDefault` (found in `api/pkg/productengines/simulator/simulator.go`). - -Here's a conceptual walkthrough: - -1. **Setup the Simulation:** - * First, we'd create an instance of `ProductEnginesServiceSimulatorDefault`. This master simulator internally sets up other necessary simulator components like `AccountsServiceSimulator`, `BalancesServiceSimulator`, `PostingsServiceSimulator`, a `ClockServiceSimulator`, and a `CommandBusSimulator`. - * We'd tell the simulator about our "Mortgage Product" definition (its rules, interest rates, fee structures, etc.) by registering it. - * We'd then create a "Simulated Account" – let's say, "Alice's Mortgage Account" – and link it to our mortgage product. We might also set its initial loan amount and other starting conditions. - * We'd define a time period for our simulation, for example, from "2024-01-01" to "2025-01-01" to see how it behaves over one year. - -2. **Run the Simulation:** - * We'd call the `Simulate` method on our `ProductEnginesServiceSimulatorDefault`, giving it Alice's simulated account and our desired start and end dates. - -3. **What Happens Inside the `Simulate` Method (Simplified):** - * The simulator enters a loop, advancing day by day from the start date to the end date using its internal `ClockServiceSimulator`. - * On each simulated day: - * It checks if any scheduled events are due for Alice's account (e.g., "end of month interest calculation," "payment due date"). - * If so, it triggers these events. These events are then passed to the (real) [Product Engine](09_product_engine_.md) logic for our mortgage product. - * The mortgage [Product Engine](09_product_engine_.md) runs its rules based on the current simulated date and account state. It might decide to: - * Calculate interest due. - * Generate a [Command](02_command_.md) like `CreatePostingsTransactionCommand` to add this interest to Alice's loan balance. - * This `CreatePostingsTransactionCommand` is then handled by the `CommandBusSimulator`. The `CommandBusSimulator` doesn't send it to the real system; instead, it directs it to other *simulated* services. - * For example, the `PostingsServiceSimulator` would "commit" these interest postings to its in-memory store. - * The `BalancesServiceSimulator` would update Alice's simulated loan balance in its in-memory store. - * This loop continues until the simulation end date is reached. - -4. **Get the Results:** - * Once the simulation is complete, the `Simulate` method returns a `SimulationResult` struct. - * This `SimulationResult` contains a snapshot of Alice's mortgage account at the end of the simulated year: her final loan balance, a list of all transactions (postings) that occurred (like interest charges, payments), any scheduled items, and potentially any errors that happened during the simulation. - -5. **Check the Results:** - * We can then examine this `SimulationResult` to verify if our mortgage product behaved as expected. Did the interest calculate correctly? Were payments applied properly? Is the final balance what we predicted? - -## A Peek at Some Simulator Code - -Let's look at tiny, simplified snippets to get a feel for how these simulators are built. - -### 1. The `AccountRepositorySimulator` (Storing Data In-Memory) - -This simulator (from `api/pkg/productengines/simulator/accounts_repository.go`) fakes a [Repository](04_repository_.md) for accounts. - -```go -// Simplified from: api/pkg/productengines/simulator/accounts_repository.go -type AccountRepositorySimulator struct { - // Accounts are stored in a map, with account ID as the key. - accounts map[uuid.UUID]*api.Account - // ... other fields for simulation ... -} - -// GetByID tries to find an account in its in-memory map. -func (r *AccountRepositorySimulator) GetByID( - ctx context.Context, accountID uuid.UUID, -) (*api.Account, error) { - account, found := r.accounts[accountID] - if !found { - // Return an error if not found (simplified error) - return nil, errors.New("simulated account not found") - } - return account, nil // Return the account from memory -} -``` -* Instead of database queries, it uses a simple Go `map` (`r.accounts`) to store account data. -* The `GetByID` method just looks up the account in this map. Very fast! - -### 2. The `ClockServiceSimulator` (Controlling Time) - -The `ProductEnginesServiceSimulatorDefault` uses a clock service (like `api.ClockServiceDefault`) and controls it. A key method for the simulation loop is `SetDate`: - -```go -// Conceptual use of ClockService's SetDate within the simulation loop -// (s.clock is an instance of a clock service) -// s.clock.SetDate(currentSimulatedDate, timeZone) -``` -* Inside the `Simulate` method's loop, this `SetDate` is called repeatedly to advance the simulation's "current time" one day at a time. This makes the whole system believe it's that specific day. - -### 3. The `ProductEnginesServiceSimulatorDefault`'s `Simulate` Method - -This is the heart of the product simulation (from `api/pkg/productengines/simulator/simulator.go`). - -```go -// Highly simplified concept of Simulate method's loop -// (s is *ProductEnginesServiceSimulatorDefault) -func (s *ProductEnginesServiceSimulatorDefault) Simulate(/*...params...*/) (*SimulationResult, error) { - // ... (lots of initial setup: create account in simulator, set initial balances) ... - - // Loop from start date to end date - for refDate := startDate; refDate.Before(endDate); refDate = refDate.AddDays(1) { - s.clock.SetDate(refDate, s.c.ReferenceLocation) // Advance simulated time! - - // 1. Trigger scheduled events for this 'refDate' - // This involves calling s.HandleEvents(), which eventually - // invokes the Product Engine for the account. - // (Simplified representation) - s.processScheduledItemsForDate(ctx, account.ID, refDate) - - // 2. Settle any simulated transactions due on this 'refDate' - // (e.g., if a payment was scheduled to clear today) - s.settleTransactions(ctx, simulationConfig, account.ID, refDate) - } - - // ... (gather all simulated data: final account state, postings, balances) ... - return &SimulationResult{ /* ... collected data ... */ }, nil -} -``` -* The loop iterates day by day. -* `s.clock.SetDate(...)` tells the entire simulation what "today" is. -* `processScheduledItemsForDate` (a conceptual helper representing logic within `Simulate` and `GetSchedules/HandleEvents` from `simulator.go`) finds any scheduled tasks for the account on this simulated day (like "calculate monthly interest"). It then uses `s.HandleEvents()` to pass these to the [Product Engine](09_product_engine_.md). The Product Engine might then generate [Commands](02_command_.md). -* These [Commands](02_command_.md) are handled by the `CommandBusSimulator` (also part of `ProductEnginesServiceSimulatorDefault`), which ensures they are processed by *other simulated services and repositories*, updating the in-memory state. -* `s.settleTransactions` handles any pre-registered transaction settlements for that day. - -## How it All Connects: A Simulation Day - -Here's a simplified sequence of what might happen on one simulated day for our mortgage account: - -```mermaid -sequenceDiagram - participant SimLoop as Simulation Loop (in Simulate method) - participant ClockSim as ClockServiceSimulator - participant PESimService as ProductEnginesServiceSimulator - participant MortgagePE as Mortgage Product Engine - participant CmdBusSim as CommandBusSimulator - participant PostingRepoSim as PostingRepositorySimulator (In-Memory) - participant BalanceRepoSim as BalanceRepositorySimulator (In-Memory) - - SimLoop->>ClockSim: SetDate(currentSimulatedDay) - SimLoop->>PESimService: ProcessScheduledEvents(account, currentSimulatedDay) - PESimService->>MortgagePE: HandleScheduledEvent(context, scheduledEvent) - MortgagePE-->>MortgagePE: Calculate interest... - MortgagePE-->>PESimService: Return [Action: CreateInterestTxCommand] - PESimService->>CmdBusSim: HandleCommand(CreateInterestTxCommand) - CmdBusSim->>PostingRepoSim: Save new interest postings (in-memory) - CmdBusSim->>BalanceRepoSim: Update loan balance (in-memory) -``` -1. The simulation loop sets the "current day" using the `ClockSim`. -2. It asks the `PESimService` (our main simulator) to process any events for that day. -3. The `PESimService` invokes the actual `MortgagePE` (Product Engine) logic. -4. The `MortgagePE` decides interest needs to be charged and returns an action to create a transaction command. -5. The `PESimService` uses the `CmdBusSim` to handle this command. -6. The `CmdBusSim` ensures that the resulting postings and balance updates are stored in the *in-memory* repositories (`PostingRepoSim`, `BalanceRepoSim`). - -This cycle repeats for every day in the simulation period. - -## Why are these Simulators So Valuable? - -* **Safety First:** We can test the riskiest financial logic without any danger to real operations or data. -* **Blazing Speed:** Simulating years of financial activity can take just seconds or minutes, not actual years. -* **Perfect Repetition:** If a test fails, we can run the exact same simulation again to understand and fix the problem. This is called deterministic testing. -* **"What If?" Scenarios:** We can easily explore complex situations: What if interest rates soar? What if a customer misses payments? How does our product react? -* **Debugging Superpowers:** When something goes wrong in a simulation, it's often easier to trace the problem step-by-step through the in-memory state changes. -* **Building Confidence:** Thorough simulation gives us much higher confidence that our banking products will work correctly in the real world. - -## Conclusion - -Simulation Services and Repositories are powerful, specialized tools within the `corebanking` project. They create a safe, controlled, and fast "practice environment" where we can: -* Test complex financial products (especially those managed by [Product Engines](09_product_engine_.md)) over long periods. -* Simulate various market conditions and customer behaviors. -* Verify that our business logic is correct and robust. - -By using in-memory data storage, a controllable clock, and simulated versions of core components, we can gain deep insights into how our system will behave, all before it ever touches real customer information or funds. This is essential for building a reliable and trustworthy core banking system. - ---- - -Congratulations on completing this tour of the core concepts in the `corebanking` project! From the fundamental [Events](01_event_.md) and [Commands](02_command_.md) that drive the system, through [Aggregates](03_aggregate_.md) that protect business rules, [Repositories](04_repository_.md) that manage data, to the [API Handlers](05_api_handler_.md) that welcome external requests, the coordinating [Core Facade](06_core_facade_.md), specialized [Services](07_service_.md), reactive [Consumers](08_consumer_.md), intelligent [Product Engines](09_product_engine_.md), and finally, these invaluable Simulation tools, you now have a solid foundation. - -While this marks the end of our conceptual overview, your journey of exploration and contribution is just beginning. You can now revisit earlier chapters with a deeper understanding or start exploring the codebase to see these concepts in full action! - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/index.md b/tmp/nesto/corebanking/index.md deleted file mode 100644 index 162cbff4..00000000 --- a/tmp/nesto/corebanking/index.md +++ /dev/null @@ -1,78 +0,0 @@ -# Tutorial: corebanking - -The project is a **core banking system** designed to manage financial operations such as *account creation, transaction processing (postings), and financial product lifecycle management*. -It heavily relies on an *Event Sourcing* architecture, where system state changes are captured as a sequence of immutable **Events**. -*Aggregates* (like `AccountAggregate` or `PostingsTransactionAggregate`) are central entities that process **Commands** (requests for change) and emit these Events. -*Services* (e.g., `AccountsService`, `ProductEnginesService`) encapsulate business logic and often orchestrate interactions between Aggregates, Repositories, and Product Engines. -*Repositories* provide an abstraction layer for data persistence. -*Product Engines* define the specific rules, behaviors, and automated processes for different financial products (e.g., mortgages, HELOCs). - - -**Source Repository:** [None](None) - -```mermaid -flowchart TD - A0["Aggregate -"] - A1["Command -"] - A2["Event -"] - A3["Service -"] - A4["Repository -"] - A5["Product Engine -"] - A6["Consumer -"] - A7["API Handler -"] - A8["Core Facade -"] - A9["Simulation Services and Repositories -"] - A0 -- "Processes" --> A1 - A0 -- "Produces" --> A2 - A0 -- "Uses for persistence" --> A4 - A0 -- "Uses dependent" --> A3 - A7 -- "Uses" --> A8 - A8 -- "Dispatches (via EventSourci..." --> A1 - A8 -- "Provides access to" --> A3 - A3 -- "Uses for data access" --> A4 - A3 -- "Invokes/Manages" --> A5 - A6 -- "Consumes" --> A2 - A6 -- "Updates read-models via" --> A4 - A6 -- "Uses (e.g., ProductEnginesS..." --> A3 - A5 -- "Reacts to (via ProductEngin..." --> A2 - A9 -- "Simulates" --> A3 - A9 -- "Simulates" --> A4 -``` - -## Chapters - -1. [Event -](01_event_.md) -2. [Command -](02_command_.md) -3. [Aggregate -](03_aggregate_.md) -4. [Repository -](04_repository_.md) -5. [API Handler -](05_api_handler_.md) -6. [Core Facade -](06_core_facade_.md) -7. [Service -](07_service_.md) -8. [Consumer -](08_consumer_.md) -9. [Product Engine -](09_product_engine_.md) -10. [Simulation Services and Repositories -](10_simulation_services_and_repositories_.md) - - ---- - -Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) \ No newline at end of file diff --git a/tmp/nesto/corebanking/review.md b/tmp/nesto/corebanking/review.md deleted file mode 100644 index d70a0d95..00000000 --- a/tmp/nesto/corebanking/review.md +++ /dev/null @@ -1,36 +0,0 @@ -# Core Banking Documentation Review - -## Section-by-Section Analysis - -- **Event**: Not bad overall, except the diagram which is really off -- **Command**: Not too bad -- **Aggregate**: Generally good, but hallucinates a command that doesn't exist (DepositFundsCommand) and from there it goes off the rails, claiming that the account manages transactions and balances... -- **Repository**: Not too bad, but the 2nd diagram is really off -- **API Handler**: Not too bad -- **Core Facade**: Not too bad -- **Service**: There are some impressive parts in this section, like the explanation of why services don't implement writes and an exception where the service does implement the write -- **Consumer**: Same thing - impressive analysis of the balances consumer. Some weird naming though: TxnProcessor/BalConsumer. -- **Product Engine**: Really good too (I'm curious if it will be as good with the latest changes I made), diagram is off again... -- **Simulation Services and Repos**: Weird product example, didn't catch that each product engine must implement its own simulator and allows offering features specific to the product, for example lump sum on a mortgage, drawdown request on a HELOC, etc. But overall conceptually it explains what can be done by implementing the simulator for a specific product engine. - -## Overall Assessment - -It's a really weird feeling because overall it's really not bad - there are even places where it surpasses the analysis I would have thought possible by an LLM. But it seems like there are enough small errors everywhere that it just feels like work done by someone who "doesn't care"... I'm not sure how to explain it. - -## Specific Issues - -For example, it generates code that's a bit more than pseudo-code, but outputs something like `uuid.FromString("acc-alice-123")`. The naming doesn't follow the conventions of the analyzed code like `accountsSvc` or `accountsRepo`. The diagrams invent layers that don't exist, and there are comments in the repo struct that are wrong. - -Otherwise, there are subtleties that I gloss over without thinking too much when reading because they make more or less sense, but not enough to really derail the reading. But I don't know if it would harm someone new to the codebase or not. - -## Major Concerns - -But there are some more major issues, like the invention of the DepositFundsCommand command which would really harm someone new to the codebase in understanding why transaction commands are sent directly to the account aggregate. - -## Missing Sections - -Otherwise, there would probably need to be a section for products, parameters, and customers. - -## Final Thoughts - -Overall, if there's a "parental advisory" sign on the documentation that it's auto-generated and to take certain details with a grain of salt, I think it covers the different concepts in corebanking and can probably help someone who doesn't know the project. It hammers the concepts of CQRS/event sourcing quite a bit. \ No newline at end of file diff --git a/tmp/prompt.md b/tmp/prompt.md deleted file mode 100644 index af69e582..00000000 --- a/tmp/prompt.md +++ /dev/null @@ -1,177 +0,0 @@ -Of course. I will extract all the LLM prompts from the `nodes.py` file and present them in a markdown format. First, I need to read the `nodes.py` file. -Read file: nodes.py -Okay, I have the content of `nodes.py`. Now I will extract the prompts and format them into a markdown file named `prompts.md`. -My apologies, it seems I tried to use a tool that isn't available to me. I will now output the extracted prompts directly into our conversation. - -# LLM Prompts from `nodes.py` - -This file contains all the LLM prompts used in the documentation generation workflow, extracted from the `nodes.py` file. - -## 1. `IdentifyAbstractions` Node - -This prompt analyzes the entire codebase to identify the most important core abstractions. - -```python -f""" -For the project `{project_name}`: - -Codebase Context: -{context} - -{language_instruction}Analyze the codebase context. -Identify the top 5-{max_abstraction_num} core most important abstractions to help those new to the codebase. - -For each abstraction, provide: -1. A concise `name`{name_lang_hint}. -2. A beginner-friendly `description` explaining what it is with a simple analogy, in around 100 words{desc_lang_hint}. -3. A list of relevant `file_indices` (integers) using the format `idx # path/comment`. - -List of file indices and paths present in the context: -{file_listing_for_prompt} - -Format the output as a YAML list of dictionaries: - -```yaml -- name: | - Query Processing{name_lang_hint} - description: | - Explains what the abstraction does. - It's like a central dispatcher routing requests.{desc_lang_hint} - file_indices: - - 0 # path/to/file1.py - - 3 # path/to/related.py -- name: | - Query Optimization{name_lang_hint} - description: | - Another core concept, similar to a blueprint for objects.{desc_lang_hint} - file_indices: - - 5 # path/to/another.js -# ... up to {max_abstraction_num} abstractions -```""" -``` - -## 2. `AnalyzeRelationships` Node - -This prompt takes the identified abstractions and analyzes their relationships to build a high-level overview of the project. - -```python -f""" -Based on the following abstractions and relevant code snippets from the project `{project_name}`: - -List of Abstraction Indices and Names{list_lang_note}: -{abstraction_listing} - -Context (Abstractions, Descriptions, Code): -{context} - -{language_instruction}Please provide: -1. A high-level `summary` of the project's main purpose and functionality in a few beginner-friendly sentences{lang_hint}. Use markdown formatting with **bold** and *italic* text to highlight important concepts. -2. A list (`relationships`) describing the key interactions between these abstractions. For each relationship, specify: - - `from_abstraction`: Index of the source abstraction (e.g., `0 # AbstractionName1`) - - `to_abstraction`: Index of the target abstraction (e.g., `1 # AbstractionName2`) - - `label`: A brief label for the interaction **in just a few words**{lang_hint} (e.g., "Manages", "Inherits", "Uses"). - Ideally the relationship should be backed by one abstraction calling or passing parameters to another. - Simplify the relationship and exclude those non-important ones. - -IMPORTANT: Make sure EVERY abstraction is involved in at least ONE relationship (either as source or target). Each abstraction index must appear at least once across all relationships. - -Format the output as YAML: - -```yaml -summary: | - A brief, simple explanation of the project{lang_hint}. - Can span multiple lines with **bold** and *italic* for emphasis. -relationships: - - from_abstraction: 0 # AbstractionName1 - to_abstraction: 1 # AbstractionName2 - label: "Manages"{lang_hint} - - from_abstraction: 2 # AbstractionName3 - to_abstraction: 0 # AbstractionName1 - label: "Provides config"{lang_hint} - # ... other relationships -``` - - -## 3. `OrderChapters` Node - -This prompt determines the best pedagogical order to explain the identified abstractions to a newcomer. - -```python -f""" -Given the following project abstractions and their relationships for the project `{project_name}`: - -Abstractions (Index # Name){list_lang_note}: -{abstraction_listing} - -Context about relationships and project summary: -{context} - -If you are going to make a tutorial for ```` {project_name} ````, what is the best order to explain these abstractions, from first to last? -Ideally, first explain those that are the most important or foundational, perhaps user-facing concepts or entry points. Then move to more detailed, lower-level implementation details or supporting concepts. - -Output the ordered list of abstraction indices, including the name in a comment for clarity. Use the format `idx # AbstractionName`. - -```yaml -- 2 # FoundationalConcept -- 0 # CoreClassA -- 1 # CoreClassB (uses CoreClassA) -- ... -``` - - - - -## 4. `WriteChapters` Node - -This prompt is executed for each abstraction to generate a detailed, beginner-friendly tutorial chapter in Markdown. - -```python -f""" -{language_instruction}Write a very beginner-friendly tutorial chapter (in Markdown format) for the project `{project_name}` about the concept: "{abstraction_name}". This is Chapter {chapter_num}. - -Concept Details{concept_details_note}: -- Name: {abstraction_name} -- Description: -{abstraction_description} - -Complete Tutorial Structure{structure_note}: -{item["full_chapter_listing"]} - -Context from previous chapters{prev_summary_note}: -{previous_chapters_summary if previous_chapters_summary else "This is the first chapter."} - -Relevant Code Snippets (Code itself remains unchanged): -{file_context_str if file_context_str else "No specific code snippets provided for this abstraction."} - -Instructions for the chapter (Generate content in {language.capitalize()} unless specified otherwise): -- Start with a clear heading (e.g., `# Chapter {chapter_num}: {abstraction_name}`). Use the provided concept name. - -- If this is not the first chapter, begin with a brief transition from the previous chapter{instruction_lang_note}, referencing it with a proper Markdown link using its name{link_lang_note}. - -- Begin with a high-level motivation explaining what problem this abstraction solves{instruction_lang_note}. Start with a central use case as a concrete example. The whole chapter should guide the reader to understand how to solve this use case. Make it very minimal and friendly to beginners. - -- If the abstraction is complex, break it down into key concepts. Explain each concept one-by-one in a very beginner-friendly way{instruction_lang_note}. - -- Explain how to use this abstraction to solve the use case{instruction_lang_note}. Give example inputs and outputs for code snippets (if the output isn't values, describe at a high level what will happen{instruction_lang_note}). - -- Each code block should be BELOW 10 lines! If longer code blocks are needed, break them down into smaller pieces and walk through them one-by-one. Aggresively simplify the code to make it minimal. Use comments{code_comment_note} to skip non-important implementation details. Each code block should have a beginner friendly explanation right after it{instruction_lang_note}. - -- Describe the internal implementation to help understand what's under the hood{instruction_lang_note}. First provide a non-code or code-light walkthrough on what happens step-by-step when the abstraction is called{instruction_lang_note}. It's recommended to use a simple sequenceDiagram with a dummy example - keep it minimal with at most 5 participants to ensure clarity. If participant name has space, use: `participant QP as Query Processing`. {mermaid_lang_note}. - -- Then dive deeper into code for the internal implementation with references to files. Provide example code blocks, but make them similarly simple and beginner-friendly. Explain{instruction_lang_note}. - -- IMPORTANT: When you need to refer to other core abstractions covered in other chapters, ALWAYS use proper Markdown links like this: [Chapter Title](filename.md). Use the Complete Tutorial Structure above to find the correct filename and the chapter title{link_lang_note}. Translate the surrounding text. - -- Use mermaid diagrams to illustrate complex concepts (```mermaid``` format). {mermaid_lang_note}. - -- Heavily use analogies and examples throughout{instruction_lang_note} to help beginners understand. - -- End the chapter with a brief conclusion that summarizes what was learned{instruction_lang_note} and provides a transition to the next chapter{instruction_lang_note}. If there is a next chapter, use a proper Markdown link: [Next Chapter Title](next_chapter_filename){link_lang_note}. - -- Ensure the tone is welcoming and easy for a newcomer to understand{tone_note}. - -- Output *only* the Markdown content for this chapter. - -Now, directly provide a super beginner-friendly Markdown output (DON'T need ```markdown``` tags): -""" -``` \ No newline at end of file diff --git a/tmp/run_analysis.sh b/tmp/run_analysis.sh deleted file mode 100755 index 3e5b3b22..00000000 --- a/tmp/run_analysis.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# Shell script to analyze the local corebanking repository -# using YAML configuration file - -echo "Starting corebanking tutorial generation..." -source .venv/bin/activate - -# Validate configuration before running -echo "Validating configuration..." -python main.py "$1" --validate-only - -if [ $? -eq 0 ]; then - echo "Configuration is valid. Running analysis..." - python main.py "$1" -else - echo "Configuration validation failed. Please check the config file." - exit 1 -fi - -echo "Done" \ No newline at end of file From 7900f46fd9550ae5631a193412e2dca70b6c14bf Mon Sep 17 00:00:00 2001 From: Louis-David Coulombe Date: Mon, 27 Oct 2025 21:23:43 -0400 Subject: [PATCH 15/15] Added Makefile and updated readme --- Makefile | 208 +++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 228 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 436 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..d9121c8b --- /dev/null +++ b/Makefile @@ -0,0 +1,208 @@ +# Makefile for PocketFlow Tutorial Codebase Knowledge + +# Variables - can be overridden from command line +IMAGE_NAME ?= documentation-generator +IMAGE_TAG ?= latest +REGISTRY ?= docker.io +REGISTRY_USER ?= your-username +FULL_IMAGE_NAME = $(REGISTRY)/$(REGISTRY_USER)/$(IMAGE_NAME):$(IMAGE_TAG) + +# Docker build arguments +DOCKER_BUILD_ARGS ?= --no-cache + +# Default paths for local development +CONFIG_FILE ?= configs/example_config.yaml +OUTPUT_DIR ?= ./output +MOUNT_DIR ?= ./mount + +# Colors for output +GREEN = \033[0;32m +YELLOW = \033[1;33m +RED = \033[0;31m +NC = \033[0m # No Color + +.PHONY: help build push run run-local clean validate lint test all + +# Default target +all: build + +## Help - Display available targets +help: + @echo "$(GREEN)PocketFlow Tutorial Docker Management$(NC)" + @echo "" + @echo "$(YELLOW)Available targets:$(NC)" + @echo " $(GREEN)build$(NC) - Build the Docker image" + @echo " $(GREEN)push$(NC) - Push the Docker image to registry" + @echo " $(GREEN)run$(NC) - Run the Docker container with file input" + @echo " $(GREEN)run-local$(NC) - Run container with local directory mounting" + @echo " $(GREEN)validate$(NC) - Validate configuration file without running" + @echo " $(GREEN)clean$(NC) - Clean up Docker images and containers" + @echo " $(GREEN)lint$(NC) - Run linting and code quality checks" + @echo " $(GREEN)test$(NC) - Run tests" + @echo " $(GREEN)shell$(NC) - Open an interactive shell in the container" + @echo " $(GREEN)logs$(NC) - Show logs from the last container run" + @echo " $(GREEN)help$(NC) - Show this help message" + @echo "" + @echo "$(YELLOW)Configuration:$(NC)" + @echo " IMAGE_NAME=$(IMAGE_NAME)" + @echo " IMAGE_TAG=$(IMAGE_TAG)" + @echo " REGISTRY=$(REGISTRY)" + @echo " REGISTRY_USER=$(REGISTRY_USER)" + @echo " FULL_IMAGE_NAME=$(FULL_IMAGE_NAME)" + @echo "" + @echo "$(YELLOW)Usage examples:$(NC)" + @echo " make build" + @echo " make push REGISTRY_USER=myusername" + @echo " make run CONFIG_FILE=configs/my_config.yaml" + @echo " make run-local CONFIG_FILE=configs/my_config.yaml MOUNT_DIR=/path/to/source" + +## Build - Build the Docker image +build: + @echo "$(GREEN)Building Docker image: $(FULL_IMAGE_NAME)$(NC)" + docker build $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAME):$(IMAGE_TAG) -t $(FULL_IMAGE_NAME) . + @echo "$(GREEN)✅ Build completed successfully!$(NC)" + +## Push - Push the Docker image to registry +push: build + @echo "$(GREEN)Pushing Docker image: $(FULL_IMAGE_NAME)$(NC)" + docker push $(FULL_IMAGE_NAME) + @echo "$(GREEN)✅ Push completed successfully!$(NC)" + +## Run - Run the Docker container with file input support +run: build + @echo "$(GREEN)Running Docker container with config: $(CONFIG_FILE)$(NC)" + @if [ ! -f "$(CONFIG_FILE)" ]; then \ + echo "$(RED)❌ Configuration file not found: $(CONFIG_FILE)$(NC)"; \ + echo "$(YELLOW)Please specify a valid config file with: make run CONFIG_FILE=path/to/config.yaml$(NC)"; \ + exit 1; \ + fi + @mkdir -p $(OUTPUT_DIR) + docker run --rm \ + -v "$(PWD)/$(CONFIG_FILE):/app/config.yaml:ro" \ + -v "$(PWD)/$(OUTPUT_DIR):/app/output" \ + -v "$(PWD)/.env:/app/.env:ro" \ + --name $(IMAGE_NAME)-run \ + $(FULL_IMAGE_NAME) config.yaml + @echo "$(GREEN)✅ Container run completed! Check output in: $(OUTPUT_DIR)$(NC)" + +## Run with local directory mounting +run-local: build + @echo "$(GREEN)Running Docker container with local directory mounting$(NC)" + @if [ ! -f "$(CONFIG_FILE)" ]; then \ + echo "$(RED)❌ Configuration file not found: $(CONFIG_FILE)$(NC)"; \ + exit 1; \ + fi + @mkdir -p $(OUTPUT_DIR) + @if [ -d "$(MOUNT_DIR)" ]; then \ + echo "$(YELLOW)Mounting local directory: $(MOUNT_DIR)$(NC)"; \ + docker run --rm \ + -v "$(PWD)/$(CONFIG_FILE):/app/config.yaml:ro" \ + -v "$(PWD)/$(OUTPUT_DIR):/app/output" \ + -v "$(PWD)/$(MOUNT_DIR):/app/mount:ro" \ + -v "$(PWD)/.env:/app/.env:ro" \ + --name $(IMAGE_NAME)-run \ + $(FULL_IMAGE_NAME) config.yaml; \ + else \ + echo "$(YELLOW)Mount directory not found, running without local mount$(NC)"; \ + $(MAKE) run CONFIG_FILE=$(CONFIG_FILE); \ + fi + +## Validate - Validate configuration file without running analysis +validate: build + @echo "$(GREEN)Validating configuration: $(CONFIG_FILE)$(NC)" + @if [ ! -f "$(CONFIG_FILE)" ]; then \ + echo "$(RED)❌ Configuration file not found: $(CONFIG_FILE)$(NC)"; \ + exit 1; \ + fi + docker run --rm \ + -v "$(PWD)/$(CONFIG_FILE):/app/config.yaml:ro" \ + --name $(IMAGE_NAME)-validate \ + $(FULL_IMAGE_NAME) config.yaml --validate-only + @echo "$(GREEN)✅ Configuration validation completed!$(NC)" + +## Shell - Open an interactive shell in the container +shell: build + @echo "$(GREEN)Opening interactive shell in container$(NC)" + docker run --rm -it \ + -v "$(PWD):/app/workspace:ro" \ + -v "$(PWD)/$(OUTPUT_DIR):/app/output" \ + --name $(IMAGE_NAME)-shell \ + --entrypoint /bin/bash \ + $(FULL_IMAGE_NAME) + +## Logs - Show logs from the last container run +logs: + @echo "$(GREEN)Showing logs from last container run$(NC)" + docker logs $(IMAGE_NAME)-run 2>/dev/null || echo "$(YELLOW)No logs found for $(IMAGE_NAME)-run$(NC)" + +## Clean - Clean up Docker images and containers +clean: + @echo "$(GREEN)Cleaning up Docker resources$(NC)" + @docker rm -f $(IMAGE_NAME)-run 2>/dev/null || true + @docker rm -f $(IMAGE_NAME)-validate 2>/dev/null || true + @docker rm -f $(IMAGE_NAME)-shell 2>/dev/null || true + @docker rmi $(IMAGE_NAME):$(IMAGE_TAG) 2>/dev/null || true + @docker rmi $(FULL_IMAGE_NAME) 2>/dev/null || true + @docker system prune -f + @echo "$(GREEN)✅ Cleanup completed!$(NC)" + +## Lint - Run linting and code quality checks +lint: + @echo "$(GREEN)Running linting checks$(NC)" + @if command -v python3 >/dev/null 2>&1; then \ + python3 -m py_compile main.py flow.py nodes.py utils/*.py; \ + echo "$(GREEN)✅ Python syntax check passed$(NC)"; \ + else \ + echo "$(YELLOW)⚠️ Python3 not found, skipping syntax check$(NC)"; \ + fi + @if command -v yamllint >/dev/null 2>&1; then \ + find configs -name "*.yaml" -o -name "*.yml" | xargs yamllint; \ + echo "$(GREEN)✅ YAML lint check passed$(NC)"; \ + else \ + echo "$(YELLOW)⚠️ yamllint not found, skipping YAML check$(NC)"; \ + fi + +## Test - Run tests +test: build + @echo "$(GREEN)Running tests$(NC)" + docker run --rm \ + -v "$(PWD)/configs/example_config.yaml:/app/config.yaml:ro" \ + --name $(IMAGE_NAME)-test \ + $(FULL_IMAGE_NAME) config.yaml --validate-only + @echo "$(GREEN)✅ Tests completed!$(NC)" + +## Quick start for development +dev: build validate + @echo "$(GREEN)Development environment ready!$(NC)" + @echo "$(YELLOW)Try: make run CONFIG_FILE=configs/example_config.yaml$(NC)" + +# Advanced targets + +## Build with custom args +build-dev: + $(MAKE) build DOCKER_BUILD_ARGS="--target development" + +## Run with debug mode +run-debug: build + @echo "$(GREEN)Running in debug mode$(NC)" + docker run --rm -it \ + -v "$(PWD)/$(CONFIG_FILE):/app/config.yaml:ro" \ + -v "$(PWD)/$(OUTPUT_DIR):/app/output" \ + -v "$(PWD)/.env:/app/.env:ro" \ + --name $(IMAGE_NAME)-debug \ + --entrypoint /bin/bash \ + $(FULL_IMAGE_NAME) + +## Tag and push with version +tag-and-push: + @if [ -z "$(VERSION)" ]; then \ + echo "$(RED)❌ VERSION is required. Usage: make tag-and-push VERSION=1.0.0$(NC)"; \ + exit 1; \ + fi + docker tag $(IMAGE_NAME):$(IMAGE_TAG) $(REGISTRY)/$(REGISTRY_USER)/$(IMAGE_NAME):$(VERSION) + docker push $(REGISTRY)/$(REGISTRY_USER)/$(IMAGE_NAME):$(VERSION) + @echo "$(GREEN)✅ Tagged and pushed version $(VERSION)$(NC)" + +# Make sure output directory exists +$(OUTPUT_DIR): + mkdir -p $(OUTPUT_DIR) diff --git a/README.md b/README.md index cc8ad4e8..41366d65 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,231 @@ +# Docker Usage Guide + +This guide explains how to build, push, and run the PocketFlow Tutorial application using Docker and the provided Makefile. + +## Quick Start + +1. **Build the Docker image:** + ```bash + make build + ``` + +2. **Run with a configuration file:** + ```bash + make run CONFIG_FILE=configs/example_config.yaml + ``` + +3. **Validate configuration without running:** + ```bash + make validate CONFIG_FILE=configs/my_config.yaml + ``` + +## Configuration + +The Makefile supports several configurable variables: + +| Variable | Default | Description | +|----------|---------|-------------| +| `IMAGE_NAME` | `pocketflow-tutorial` | Docker image name | +| `IMAGE_TAG` | `latest` | Docker image tag | +| `REGISTRY` | `docker.io` | Docker registry URL | +| `REGISTRY_USER` | `your-username` | Registry username | +| `CONFIG_FILE` | `configs/example_config.yaml` | Path to YAML config file | +| `OUTPUT_DIR` | `./output` | Local output directory | +| `MOUNT_DIR` | `./mount` | Local directory to mount for source code | + +## Available Commands + +### Build Commands +- `make build` - Build the Docker image +- `make build-dev` - Build with development target +- `make push` - Build and push to registry +- `make tag-and-push VERSION=1.0.0` - Tag with version and push + +### Run Commands +- `make run` - Run with config file +- `make run-local` - Run with local directory mounting +- `make run-debug` - Run in debug mode with interactive shell +- `make shell` - Open interactive shell in container + +### Utility Commands +- `make validate` - Validate configuration file +- `make clean` - Clean up Docker resources +- `make lint` - Run code quality checks +- `make test` - Run tests +- `make logs` - Show container logs +- `make help` - Show help message + +## Usage Examples + +### Basic Usage +```bash +# Build and run with default config +make build +make run + +# Use custom configuration +make run CONFIG_FILE=configs/corebanking.yaml + +# Validate configuration before running +make validate CONFIG_FILE=configs/my_config.yaml +``` + +### Working with Local Directories +```bash +# Mount a local source directory +make run-local CONFIG_FILE=configs/my_config.yaml MOUNT_DIR=/path/to/source/code + +# The mounted directory will be available at /app/mount inside the container +``` + +### Registry Operations +```bash +# Push to Docker Hub +make push REGISTRY_USER=myusername + +# Push to a different registry +make push REGISTRY=ghcr.io REGISTRY_USER=myusername + +# Tag and push a specific version +make tag-and-push VERSION=1.2.3 REGISTRY_USER=myusername +``` + +### Development Workflow +```bash +# Development setup +make dev + +# Debug a configuration issue +make run-debug CONFIG_FILE=configs/problematic.yaml + +# Check logs from the last run +make logs + +# Clean up when done +make clean +``` + +## File Mounting + +The Makefile automatically mounts several directories: + +1. **Configuration file**: Mounted as read-only at `/app/config.yaml` +2. **Output directory**: Mounted at `/app/output` for results +3. **Environment file**: `.env` mounted as read-only at `/app/.env` +4. **Source directory** (when using `run-local`): Mounted at `/app/mount` + +## Configuration File Format + +Your YAML configuration file should follow this structure: + +```yaml +source: + repo: "https://github.com/user/repo" # OR + local_dir: "/app/mount" # for local directories + +project: + name: "My Project" + output_dir: "output" + language: "english" + +file_processing: + max_file_size: 100000 + include_patterns: + - "*.py" + - "*.js" + - "*.md" + exclude_patterns: + - "*test*" + - "node_modules/*" + +analysis: + max_abstractions: 10 + abstractions_hints: + - "authentication" + - "data processing" + +llm: + use_cache: true + +github: + token: "${GITHUB_TOKEN}" # Will use environment variable +``` + +## Environment Variables + +Create a `.env` file in your project root: + +```bash +# .env +GITHUB_TOKEN=your_github_token_here +OPENAI_API_KEY=your_openai_api_key_here +# Add other required environment variables +``` + +## Troubleshooting + +### Common Issues + +1. **Configuration file not found** + ```bash + make run CONFIG_FILE=path/to/your/config.yaml + ``` + +2. **Permission denied for output directory** + ```bash + sudo chown -R $USER:$USER ./output + ``` + +3. **Docker build fails** + ```bash + make clean + make build DOCKER_BUILD_ARGS="--no-cache --pull" + ``` + +4. **Registry push fails** + ```bash + docker login + make push REGISTRY_USER=your_actual_username + ``` + +### Debug Mode + +Use debug mode to troubleshoot issues: + +```bash +make run-debug CONFIG_FILE=configs/my_config.yaml + +# Inside the container: +python main.py config.yaml --validate-only +python main.py config.yaml +``` + +## Advanced Usage + +### Custom Docker Build Args +```bash +make build DOCKER_BUILD_ARGS="--build-arg PYTHON_VERSION=3.11" +``` + +### Multiple Configurations +```bash +# Process multiple projects +for config in configs/*.yaml; do + echo "Processing $config" + make run CONFIG_FILE="$config" +done +``` + +### CI/CD Integration +```bash +# In your CI pipeline +make build +make test +make push REGISTRY_USER=$CI_REGISTRY_USER +``` + + +

Turns Codebase into Easy Tutorial with AI

![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)