From cf34dba6c9f7322bb491f56d894622c7b670ed27 Mon Sep 17 00:00:00 2001 From: Jamieson Walker Date: Fri, 17 Oct 2025 11:01:25 -0500 Subject: [PATCH 1/6] Added support for AI Auto mapper lambda function --- LAMBDA_IMPLEMENTATION_GUIDE.md | 619 +++++++++++++++++++++++++++++++ claude.md | 353 ++++++++++++++++++ src/lib/feed-generate.ts | 22 +- src/utils/field-mapper-client.ts | 181 +++++++++ src/utils/options.ts | 13 +- 5 files changed, 1182 insertions(+), 6 deletions(-) create mode 100644 LAMBDA_IMPLEMENTATION_GUIDE.md create mode 100644 claude.md create mode 100644 src/utils/field-mapper-client.ts diff --git a/LAMBDA_IMPLEMENTATION_GUIDE.md b/LAMBDA_IMPLEMENTATION_GUIDE.md new file mode 100644 index 0000000..cd2e985 --- /dev/null +++ b/LAMBDA_IMPLEMENTATION_GUIDE.md @@ -0,0 +1,619 @@ +# AWS Lambda Field Mapper Implementation Guide + +## Overview + +This guide provides specifications for implementing the AWS Lambda function that enhances AsyncAPI feedrules with intelligent field mappings, including **Topic Parameter Mapping** - the ability to map payload fields to topic variables. + +## Feature: Topic Parameter Mapping + +### Purpose + +When AsyncAPI topics contain variable placeholders (e.g., `acme/orders/{orderId}/{region}`), and the message payload contains fields with matching names (e.g., `orderId`, `region`), the Lambda function should automatically create mappings to inject the payload-generated values into the topic address. + +### Benefits + +1. **Realistic Topics**: Topic addresses will contain actual correlated data (e.g., order ID in topic matches order ID in payload) +2. **Data Consistency**: Ensures the same value appears in both topic and payload +3. **Type Conversion**: Handles type conversions (e.g., payload number → topic string) +4. **Reduced Configuration**: Automatic detection eliminates manual mapping configuration + +## Lambda Function Specification + +### Input Format + +```json +{ + "feedrules": [ + { + "topic": "acmeRental/orders/created/v1/{region}/{orderId}", + "eventName": "Order Created", + "eventVersion": "1.0.0", + "messageName": "Order_Created", + "topicParameters": { + "region": { + "schema": { + "type": "string", + "enum": ["us-east", "us-west", "eu-central"] + }, + "rule": { + "name": "region", + "type": "string", + "group": "StringRules", + "rule": "enum", + "enum": ["us-east", "us-west", "eu-central"] + } + }, + "orderId": { + "schema": { + "type": "string" + }, + "rule": { + "name": "orderId", + "type": "string", + "group": "StringRules", + "rule": "alpha", + "casing": "mixed", + "minLength": 10, + "maxLength": 10 + } + } + }, + "payload": { + "orderId": { + "type": "number", + "minimum": 1000, + "maximum": 999999 + }, + "customerName": { + "type": "string" + }, + "amount": { + "type": "number", + "minimum": 0, + "maximum": 10000 + } + }, + "publishSettings": { + "count": 0, + "interval": 1000, + "delay": 0 + } + } + ], + "asyncApiSpec": { /* Full AsyncAPI document */ } +} +``` + +### Expected Output Format + +```json +{ + "success": true, + "enhancedFeedrules": [ + { + "topic": "acmeRental/orders/created/v1/{region}/{orderId}", + "eventName": "Order Created", + "eventVersion": "1.0.0", + "messageName": "Order_Created", + "topicParameters": { /* Same as input */ }, + "payload": { + "orderId": { + "type": "number", + "minimum": 1000, + "maximum": 999999, + "rule": { + "name": "orderId", + "type": "number", + "group": "NumberRules", + "rule": "int", + "minimum": 1000, + "maximum": 999999 + } + }, + "customerName": { + "type": "string", + "rule": { + "name": "customerName", + "type": "string", + "group": "PersonRules", + "rule": "fullName" + } + }, + "amount": { + "type": "number", + "minimum": 0, + "maximum": 10000, + "rule": { + "name": "amount", + "type": "number", + "group": "FinanceRules", + "rule": "amount", + "minimum": 0, + "maximum": 10000 + } + } + }, + "publishSettings": { /* Same as input */ }, + "mappings": [ + { + "type": "Topic Parameter", + "source": { + "type": "Payload Parameter", + "name": "orderId", + "fieldName": "orderId", + "fieldType": "number" + }, + "target": { + "type": "Topic Parameter", + "name": "orderId", + "fieldName": "orderId", + "fieldType": "string" + } + } + ] + } + ], + "summary": { + "totalFields": 3, + "improved": 3, + "unchanged": 0, + "improvementPercentage": 100 + }, + "context": { + "eventName": "Order Created", + "topic": "acmeRental/orders/created/v1/{region}/{orderId}", + "domain": "Retail" + } +} +``` + +## Implementation Algorithm + +### Step 1: Parse Topic for Variables + +```python +import re + +def extract_topic_variables(topic: str) -> list[str]: + """ + Extract variable names from topic string. + + Example: "acme/{region}/orders/{orderId}" -> ["region", "orderId"] + """ + pattern = r'\{([^}]+)\}' + return re.findall(pattern, topic) +``` + +### Step 2: Build Field Inventories + +```python +def build_field_inventories(feedrule: dict) -> tuple[dict, dict]: + """ + Create inventories of topic parameters and payload fields. + + Returns: + (topic_fields, payload_fields) + """ + # Extract topic parameter definitions + topic_fields = {} + if 'topicParameters' in feedrule: + for name, config in feedrule['topicParameters'].items(): + topic_fields[name] = { + 'name': name, + 'type': config.get('schema', {}).get('type', 'string'), + 'config': config + } + + # Extract payload field definitions + payload_fields = {} + if 'payload' in feedrule: + for name, config in feedrule['payload'].items(): + payload_fields[name] = { + 'name': name, + 'type': config.get('type', 'string'), + 'config': config + } + + return topic_fields, payload_fields +``` + +### Step 3: Identify Matching Fields + +```python +def find_matching_fields(topic_vars: list[str], topic_fields: dict, payload_fields: dict) -> list[dict]: + """ + Find payload fields that match topic variable names. + + Returns list of matches with source and target information. + """ + matches = [] + + for var_name in topic_vars: + # Check if this variable exists in both topic parameters and payload + if var_name in payload_fields: + payload_field = payload_fields[var_name] + topic_field = topic_fields.get(var_name, {}) + + matches.append({ + 'variable_name': var_name, + 'source': { + 'type': 'Payload Parameter', + 'name': var_name, + 'fieldName': var_name, + 'fieldType': payload_field['type'] + }, + 'target': { + 'type': 'Topic Parameter', + 'name': var_name, + 'fieldName': var_name, + 'fieldType': topic_field.get('type', 'string') # Topics typically use strings + } + }) + + return matches +``` + +### Step 4: Generate Mapping Entries + +```python +def generate_topic_mappings(feedrule: dict) -> list[dict]: + """ + Generate Topic Parameter mappings for a feedrule. + + Returns list of mapping objects. + """ + topic = feedrule.get('topic', '') + topic_vars = extract_topic_variables(topic) + + if not topic_vars: + return [] + + topic_fields, payload_fields = build_field_inventories(feedrule) + matches = find_matching_fields(topic_vars, topic_fields, payload_fields) + + mappings = [] + for match in matches: + mappings.append({ + 'type': 'Topic Parameter', + 'source': match['source'], + 'target': match['target'] + }) + + return mappings +``` + +### Step 5: Enhance Payload Fields + +```python +def enhance_payload_fields(feedrule: dict) -> dict: + """ + Add Faker.js rules to payload fields based on field names and types. + + This is your existing functionality - enhance it to recognize common patterns: + - "name", "firstName", "lastName" -> PersonRules + - "email" -> InternetRules (email) + - "phone" -> StringRules (phoneNumber) + - "amount", "price" -> FinanceRules (amount) + - "date", "timestamp" -> DateRules + - etc. + """ + enhanced_payload = {} + + for field_name, field_config in feedrule.get('payload', {}).items(): + enhanced_field = dict(field_config) + + # Generate appropriate Faker rule based on field name and type + faker_rule = infer_faker_rule(field_name, field_config.get('type')) + enhanced_field['rule'] = faker_rule + + enhanced_payload[field_name] = enhanced_field + + return enhanced_payload +``` + +### Step 6: Complete Enhancement Pipeline + +```python +def enhance_feedrule(feedrule: dict, asyncapi_spec: dict) -> dict: + """ + Complete enhancement pipeline for a single feedrule. + """ + enhanced = dict(feedrule) + + # 1. Enhance payload fields with Faker rules + enhanced['payload'] = enhance_payload_fields(feedrule) + + # 2. Generate topic parameter mappings + topic_mappings = generate_topic_mappings(feedrule) + + # 3. Add or merge mappings array + if 'mappings' not in enhanced: + enhanced['mappings'] = [] + + enhanced['mappings'].extend(topic_mappings) + + return enhanced + +def lambda_handler(event, context): + """ + Main Lambda handler. + """ + try: + feedrules = event['feedrules'] + asyncapi_spec = event['asyncApiSpec'] + + enhanced_feedrules = [] + total_fields = 0 + improved_fields = 0 + + for feedrule in feedrules: + enhanced = enhance_feedrule(feedrule, asyncapi_spec) + enhanced_feedrules.append(enhanced) + + # Count statistics + payload_count = len(feedrule.get('payload', {})) + total_fields += payload_count + + # Check how many fields got rules + enhanced_payload_count = sum( + 1 for field in enhanced.get('payload', {}).values() + if 'rule' in field + ) + improved_fields += enhanced_payload_count + + improvement_percentage = ( + int((improved_fields / total_fields) * 100) if total_fields > 0 else 0 + ) + + return { + 'success': True, + 'enhancedFeedrules': enhanced_feedrules, + 'summary': { + 'totalFields': total_fields, + 'improved': improved_fields, + 'unchanged': total_fields - improved_fields, + 'improvementPercentage': improvement_percentage + }, + 'context': { + 'eventName': feedrules[0].get('eventName', 'Unknown'), + 'topic': feedrules[0].get('topic', ''), + 'domain': asyncapi_spec.get('info', {}).get('x-application-domain', 'Unknown') + } + } + + except Exception as e: + return { + 'success': False, + 'error': str(e), + 'errorType': type(e).__name__ + } +``` + +## Example Test Cases + +### Test Case 1: Simple Topic Variable Match + +**Input:** +```json +{ + "topic": "orders/{orderId}", + "topicParameters": { + "orderId": { "schema": { "type": "string" } } + }, + "payload": { + "orderId": { "type": "number", "minimum": 1000, "maximum": 9999 } + } +} +``` + +**Expected Mapping:** +```json +{ + "type": "Topic Parameter", + "source": { + "type": "Payload Parameter", + "name": "orderId", + "fieldName": "orderId", + "fieldType": "number" + }, + "target": { + "type": "Topic Parameter", + "name": "orderId", + "fieldName": "orderId", + "fieldType": "string" + } +} +``` + +### Test Case 2: Multiple Matching Variables + +**Input:** +```json +{ + "topic": "acme/{region}/{storeId}/sales/{transactionId}", + "topicParameters": { + "region": { "schema": { "type": "string" } }, + "storeId": { "schema": { "type": "string" } }, + "transactionId": { "schema": { "type": "string" } } + }, + "payload": { + "transactionId": { "type": "number" }, + "storeId": { "type": "number" }, + "amount": { "type": "number" }, + "region": { "type": "string" } + } +} +``` + +**Expected Mappings:** 3 mappings (region, storeId, transactionId) + +### Test Case 3: No Matching Fields + +**Input:** +```json +{ + "topic": "orders/{orderId}", + "topicParameters": { + "orderId": { "schema": { "type": "string" } } + }, + "payload": { + "customerName": { "type": "string" }, + "amount": { "type": "number" } + } +} +``` + +**Expected Mappings:** Empty array (no matches) + +### Test Case 4: Partial Match + +**Input:** +```json +{ + "topic": "store/{storeId}/employee/{employeeId}", + "topicParameters": { + "storeId": { "schema": { "type": "string" } }, + "employeeId": { "schema": { "type": "string" } } + }, + "payload": { + "employeeId": { "type": "number" }, + "shiftHours": { "type": "number" } + } +} +``` + +**Expected Mappings:** 1 mapping (employeeId only) + +## Deployment Configuration + +### Environment Variables + +- `LOG_LEVEL`: Set to `DEBUG` for detailed logging +- `ENABLE_TOPIC_MAPPING`: Feature flag (default: `true`) +- `CASE_SENSITIVE_MATCHING`: Whether field matching is case-sensitive (default: `false`) + +### Lambda Configuration + +- **Runtime**: Python 3.11+ or Node.js 18+ +- **Memory**: 512 MB (adjust based on AsyncAPI document size) +- **Timeout**: 30 seconds +- **Concurrency**: 10 (adjust based on expected load) + +### API Gateway Setup + +```yaml +Path: /field-mapper +Method: POST +Authorization: API Key or IAM +CORS: Enabled +Content-Type: application/json +``` + +## Error Handling + +```python +class FieldMapperError(Exception): + """Base exception for field mapper errors.""" + pass + +class InvalidInputError(FieldMapperError): + """Invalid input format.""" + pass + +class ProcessingError(FieldMapperError): + """Error during field processing.""" + pass + +def lambda_handler(event, context): + try: + # Validate input + if 'feedrules' not in event: + raise InvalidInputError('Missing required field: feedrules') + + if 'asyncApiSpec' not in event: + raise InvalidInputError('Missing required field: asyncApiSpec') + + # Process feedrules + # ... + + except InvalidInputError as e: + return { + 'success': False, + 'error': str(e), + 'errorType': 'INVALID_INPUT', + 'details': ['feedrules and asyncApiSpec are required'] + } + + except ProcessingError as e: + return { + 'success': False, + 'error': str(e), + 'errorType': 'PROCESSING_ERROR', + 'details': [str(e)] + } + + except Exception as e: + return { + 'success': False, + 'error': 'Internal server error', + 'errorType': 'INTERNAL_ERROR', + 'details': [str(e)] + } +``` + +## Testing + +### Unit Tests + +```python +import unittest + +class TestTopicVariableExtraction(unittest.TestCase): + def test_single_variable(self): + topic = "orders/{orderId}" + result = extract_topic_variables(topic) + self.assertEqual(result, ["orderId"]) + + def test_multiple_variables(self): + topic = "acme/{region}/store/{storeId}/orders/{orderId}" + result = extract_topic_variables(topic) + self.assertEqual(result, ["region", "storeId", "orderId"]) + + def test_no_variables(self): + topic = "orders/created/v1" + result = extract_topic_variables(topic) + self.assertEqual(result, []) + +class TestFieldMatching(unittest.TestCase): + def test_exact_match(self): + topic_vars = ["orderId"] + topic_fields = {"orderId": {"name": "orderId", "type": "string"}} + payload_fields = {"orderId": {"name": "orderId", "type": "number"}} + + matches = find_matching_fields(topic_vars, topic_fields, payload_fields) + self.assertEqual(len(matches), 1) + self.assertEqual(matches[0]['variable_name'], 'orderId') +``` + +### Integration Test + +```bash +curl -X POST https://your-lambda-url.amazonaws.com/field-mapper \ + -H "Content-Type: application/json" \ + -d @test-payload.json +``` + +## Performance Considerations + +1. **Caching**: Cache AsyncAPI parsing results for repeated calls +2. **Batch Processing**: Process multiple feedrules in parallel +3. **Lazy Loading**: Only parse relevant sections of large AsyncAPI documents +4. **Memory Management**: Stream process very large payload definitions + +## Future Enhancements + +1. **Fuzzy Matching**: Match fields with similar names (e.g., "order_id" vs "orderId") +2. **Custom Mapping Rules**: Allow users to define custom mapping preferences +3. **Type Coercion Warnings**: Warn about potential data loss in type conversions +4. **Nested Field Support**: Support nested payload fields (e.g., "order.id") +5. **Array Field Handling**: Map array elements to topic variables diff --git a/claude.md b/claude.md new file mode 100644 index 0000000..2009bb1 --- /dev/null +++ b/claude.md @@ -0,0 +1,353 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Development Commands + +### Build and Development +- `npm run build` - Compile TypeScript to JavaScript (outputs to `./dist`) +- `npm run dev` - Watch mode compilation for development +- `tsc` - Direct TypeScript compilation +- `./bin/index.js` - Run the local development version directly + +### Local Testing +- `stm --help` - Test CLI help (if globally installed) +- `./bin/index.js --help` - Test local build version +- `stm --version` - Check installed version + +### Package Management +- `npm install` - Install dependencies after changes +- `npm run package` - Create distributable packages using pkg +- `npm run publish` - Build and publish to npm + +## Architecture Overview + +### Core Components + +**CLI Entry Point**: `src/index.ts` contains the `Commander` class that sets up all CLI commands using commander.js. Each command (send, receive, request, reply, config, manage, feed) is configured with its options and maps to corresponding library functions. + +**Command Structure**: +- **Messaging Commands**: `send`, `receive`, `request`, `reply` - Handle Solace pub/sub operations +- **Management Commands**: `manage` - Broker resource management (queues, profiles, connections) +- **Configuration Commands**: `config` - Persistent CLI configuration management +- **Feed Commands**: `feed` - AsyncAPI-based event feed generation and management + +### Key Directories + +- **`src/lib/`**: Core business logic for each command type (publish.ts, receive.ts, etc.) +- **`src/common/`**: Reusable client implementations for different message patterns +- **`src/utils/`**: Shared utilities (config management, logging, validation, option parsing) +- **`src/types/`**: TypeScript type definitions +- **`public/`**: Static assets and feed portal UI components +- **`documentation/`**: Comprehensive docs for messaging, feeds, and configuration + +### Client Architecture + +The application uses a layered client architecture: + +1. **Command Layer** (`src/lib/`): High-level command implementations +2. **Client Layer** (`src/common/`): Protocol-specific client implementations + - `publish-client.ts` - Publishing messages + - `receive-client.ts` - Subscribing and receiving + - `request-client.ts` / `reply-client.ts` - Request/reply patterns + - `feed-publish-client.ts` - AsyncAPI feed publishing +3. **Solace Layer**: Uses `solclientjs` SDK for broker connectivity + +### Configuration System + +Commands support persistent configuration through: +- **Config Storage**: `~/.stm/` directory (configurable via `STM_HOME`) +- **Command Configs**: Saved parameter sets for reuse across command invocations +- **Connection Profiles**: Reusable broker connection settings + +### Feed System Architecture + +The feed system generates realistic event streams from AsyncAPI documents: +- **Feed Generation**: Parses AsyncAPI specs and creates data generation rules +- **Data Generation**: Uses Faker.js with custom rules for realistic mock data +- **Feed Management**: Import/export, validation, community contribution workflows +- **UI Portal**: Browser-based feed management interface + +### Message Flow Patterns + +- **Direct Messaging**: Point-to-point via topics +- **Queued Messaging**: Persistent delivery via Solace queues +- **Request/Reply**: Synchronous messaging patterns +- **Event Streaming**: AsyncAPI-driven continuous event generation + +## Key Dependencies + +- **solclientjs**: Solace PubSub+ JavaScript client library +- **commander**: CLI framework for option parsing and command structure +- **@faker-js/faker**: Mock data generation for feeds +- **@asyncapi/parser**: AsyncAPI document parsing and validation +- **chalk**: Terminal output coloring and formatting + +## Important Notes + +- The CLI checks for version updates on startup via GitHub API +- Configuration files are stored in `~/.stm/` by default (overrideable with `STM_HOME`) +- Event feeds must follow community contribution guidelines in `documentation/CONTRIBUTION_GUIDELINES.md` +- The application supports both direct broker connections and SEMP (management API) connections +- Visualization features are available when `SHOW_VISUALIZATION` environment variable is set + +## AI-Powered Field Mapping + +### Overview + +The `stm feed generate` command supports AI-enhanced field mapping via the `--ai-enhance` flag. This feature uses an external AWS Lambda function to intelligently map fields in AsyncAPI specifications. + +### Usage + +```bash +# Generate feed with AI field mapping enhancement +stm feed generate \ + --file-name asyncapi.yaml \ + --feed-name "MyFeed" \ + --ai-enhance \ + --ai-mapper-endpoint "https://your-lambda-url.amazonaws.com/field-mapper" + +# Or set endpoint via environment variable +export STM_FIELD_MAPPER_ENDPOINT="https://your-lambda-url.amazonaws.com/field-mapper" +stm feed generate --file-name asyncapi.yaml --ai-enhance +``` + +### What It Does + +The AI field mapper performs two types of enhancements: + +1. **Payload Field Enhancement**: Analyzes payload field types and names to generate realistic Faker.js data generation rules +2. **Topic Parameter Mapping**: Detects topic variables (e.g., `{employeeId}`, `{amount}`) that match payload field names and creates mappings to inject payload values into the topic address + +### Topic Parameter Mapping Example + +**AsyncAPI Topic with Variables:** +```yaml +channels: + acmeRental/orders/created/v1/{region}/{orderId}: + parameters: + region: + schema: + type: string + orderId: + schema: + type: string + message: + payload: + type: object + properties: + orderId: + type: number + customerName: + type: string + amount: + type: number +``` + +**Generated Mapping:** +```json +{ + "mappings": [ + { + "type": "Topic Parameter", + "source": { + "type": "Payload Parameter", + "name": "orderId", + "fieldName": "orderId", + "fieldType": "number" + }, + "target": { + "type": "Topic Parameter", + "name": "orderId", + "fieldName": "orderId", + "fieldType": "string" + } + } + ] +} +``` + +**Result**: When events are published, the `orderId` value generated for the payload (e.g., `12345`) will be automatically used in the topic address: `acmeRental/orders/created/v1/us-east/12345` + +### Field Mapper Lambda Requirements + +The Lambda function endpoint should: + +1. Accept POST requests with JSON body: + ```json + { + "feedrules": [...], + "asyncApiSpec": {...} + } + ``` + +2. Return response format: + ```json + { + "success": true, + "enhancedFeedrules": [...], + "summary": { + "totalFields": 10, + "improved": 7, + "unchanged": 3, + "improvementPercentage": 70 + }, + "context": { + "eventName": "Order Created", + "topic": "acmeRental/orders/created/v1/{region}/{orderId}", + "domain": "Retail" + } + } + ``` + +3. Process each feedrule to: + - Parse topic string to extract variable placeholders (regex: `/\{([^}]+)\}/g`) + - Identify matches where topic variable names equal payload field names + - Generate Faker.js rules for payload fields based on field names/types + - Create Topic Parameter mappings for matched fields + - Include all mappings in the `mappings` array of each feedrule + +### Implementation Reference + +See `src/utils/field-mapper-client.ts` for the client implementation and detailed JSDoc documentation. + +## STM Feed Commands (Non-Interactive) + +### Feed Event Name Bug & Workaround + +**Issue**: There's a bug in `src/lib/feed-run.ts` where event names for `--event-names` must include both the message name AND topic with exactly 6 spaces between them, due to this line: +```typescript +name: event.name + ' ' + event.topic +``` + +### How to Run Feeds Non-Interactively + +**Method 1: Get Event Names from Interactive Mode** +```bash +# Run interactive mode once to see available events +stm feed run --feed-name "YourFeedName" +# Copy the exact event text shown in the selection prompt +``` + +**Method 2: Use Full Event Identifier Format** +```bash +# Format: "message_name topic_name" (exactly 6 spaces) +stm feed run \ + --feed-name "DynamicPricingEngine-0" \ + --event-names "subscribe.message acmeRental/pricingAvailability/pricing/updated/v1/{vehicleType}/{location}" \ + --count 10 +``` + +**Method 3: Multiple Events** +```bash +stm feed run \ + --feed-name "DynamicPricingEngine-0" \ + --event-names \ + "subscribe.message acmeRental/pricingAvailability/pricing/updated/v1/{vehicleType}/{location}" \ + "subscribe.message acmeRental/pricingAvailability/promotion/applied/v1/{promotionID}/{vehicleType}" \ + --count 5 +``` + +### Common Feed Commands + +```bash +# List available feeds +stm feed list --local-only +stm feed list --community-only + +# Preview feed structure +stm feed preview --feed-name "YourFeedName" +stm feed preview --feed-name "YourFeedName" --community-feed + +# Run community feed +stm feed run --feed-name "Point_of_Sale_System" --community-feed --event-names "EventName" --count 10 + +# Continuous streaming (count=0) +stm feed run --feed-name "YourFeed" --event-names "YourEvent" --count 0 + +# Custom intervals and delays +stm feed run --feed-name "YourFeed" --event-names "YourEvent" --count 5 --interval 2000 --initial-delay 1000 +``` + +**Key Parameters:** +- `--feed-name` - Exact feed name (required to avoid feed selection prompt) +- `--event-names` - Full event identifiers (required to avoid event selection prompt) +- `--community-feed` - Use for community feeds vs local feeds +- `--count` - Number of events (0 = continuous streaming) +- `--interval` - Milliseconds between events +- `--config` - Custom configuration file path + +# Using Gemini CLI for Large Codebase Analysis + +When analyzing large codebases or multiple files that might exceed context limits, use the Gemini CLI with its massive +context window. Use `gemini -p` to leverage Google Gemini's large context capacity. + +## File and Directory Inclusion Syntax + +Use the `@` syntax to include files and directories in your Gemini prompts. The paths should be relative to WHERE you run the + gemini command: + +### Examples: + +**Single file analysis:** +gemini -p "@src/main.py Explain this file's purpose and structure" + +Multiple files: +gemini -p "@package.json @src/index.js Analyze the dependencies used in the code" + +Entire directory: +gemini -p "@src/ Summarize the architecture of this codebase" + +Multiple directories: +gemini -p "@src/ @tests/ Analyze test coverage for the source code" + +Current directory and subdirectories: +gemini -p "@./ Give me an overview of this entire project" + +# Or use --all_files flag: +gemini --all_files -p "Analyze the project structure and dependencies" + +Implementation Verification Examples + +Check if a feature is implemented: +gemini -p "@src/ @lib/ Has dark mode been implemented in this codebase? Show me the relevant files and functions" + +Verify authentication implementation: +gemini -p "@src/ @middleware/ Is JWT authentication implemented? List all auth-related endpoints and middleware" + +Check for specific patterns: +gemini -p "@src/ Are there any React hooks that handle WebSocket connections? List them with file paths" + +Verify error handling: +gemini -p "@src/ @api/ Is proper error handling implemented for all API endpoints? Show examples of try-catch blocks" + +Check for rate limiting: +gemini -p "@backend/ @middleware/ Is rate limiting implemented for the API? Show the implementation details" + +Verify caching strategy: +gemini -p "@src/ @lib/ @services/ Is Redis caching implemented? List all cache-related functions and their usage" + +Check for specific security measures: +gemini -p "@src/ @api/ Are SQL injection protections implemented? Show how user inputs are sanitized" + +Verify test coverage for features: +gemini -p "@src/payment/ @tests/ Is the payment processing module fully tested? List all test cases" + +When to Use Gemini CLI + +Use gemini -p when: +- Analyzing entire codebases or large directories +- Comparing multiple large files +- Need to understand project-wide patterns or architecture +- Current context window is insufficient for the task +- Working with files totaling more than 100KB +- Verifying if specific features, patterns, or security measures are implemented +- Checking for the presence of certain coding patterns across the entire codebase + +Important Notes + +- Paths in @ syntax are relative to your current working directory when invoking gemini +- The CLI will include file contents directly in the context +- No need for --yolo flag for read-only analysis +- Gemini's context window can handle entire codebases that would overflow Claude's context +- When checking implementations, be specific about what you're looking for to get accurate results \ No newline at end of file diff --git a/src/lib/feed-generate.ts b/src/lib/feed-generate.ts index a897c1d..1781438 100644 --- a/src/lib/feed-generate.ts +++ b/src/lib/feed-generate.ts @@ -10,6 +10,7 @@ import { analyze, analyzeV2, analyzeEP, formulateRules, formulateSchemas, load } import { AsyncAPIDocumentInterface } from '@asyncapi/parser'; import { checkFeedGenerateOptions, getPotentialFeedName, getPotentialTopicFromFeedName } from '../utils/checkparams'; import { sessionPropertiesJson } from '../utils/sessionprops'; +import { enhanceFeedrulesWithAI } from '../utils/field-mapper-client'; const generate = async (options: ManageFeedClientOptions, optionsSource: any) => { var { fileName, feedName, feedType, feedView } = options; @@ -133,9 +134,28 @@ const generate = async (options: ManageFeedClientOptions, optionsSource: any) => await analyzeV2(documentCopy, reverseView) : await analyze(documentCopy, reverseView); data.fileName = fileName.lastIndexOf('/') >= 0 ? fileName.substring(fileName.lastIndexOf('/')+1) : fileName; - const rules = await formulateRules(document, reverseView); // Uses original unmodified document + let rules = await formulateRules(document, reverseView); // Uses original unmodified document const schemas = await formulateSchemas(document); // Uses original unmodified document + // AI Enhancement: Optionally enhance feedrules with intelligent field mappings + if (options.aiEnhance) { + Logger.info('AI enhancement enabled - enhancing field mappings...'); + // Parse the AsyncAPI schema string to an object for the Lambda + const asyncApiObject = typeof asyncApiSchema === 'string' ? JSON.parse(asyncApiSchema) : asyncApiSchema; + const enhancedRules = await enhanceFeedrulesWithAI( + rules, + asyncApiObject, + options.aiMapperEndpoint + ); + + if (enhancedRules && enhancedRules.length > 0) { + Logger.logSuccess('Successfully enhanced feedrules with AI mappings'); + rules = enhancedRules; + } else { + Logger.logWarn('AI enhancement failed or returned no results, using original rules'); + } + } + await checkEventValidity(data); if (options.useDefaults) { diff --git a/src/utils/field-mapper-client.ts b/src/utils/field-mapper-client.ts new file mode 100644 index 0000000..732e7d8 --- /dev/null +++ b/src/utils/field-mapper-client.ts @@ -0,0 +1,181 @@ +import { Logger } from './logger'; +import chalk from 'chalk'; + +// Default endpoint - can be overridden via environment variable +const DEFAULT_FIELD_MAPPER_ENDPOINT = process.env.STM_FIELD_MAPPER_ENDPOINT || ''; + +/** + * Mapping between topic parameters and payload parameters + */ +export interface FieldMapping { + type: 'Topic Parameter'; + source: { + type: 'Payload Parameter' | 'Topic Parameter'; + name: string; + fieldName: string; + fieldType: string; + }; + target: { + type: 'Topic Parameter' | 'Payload Parameter'; + name: string; + fieldName: string; + fieldType: string; + }; +} + +/** + * Response from the AI Field Mapper Lambda function + */ +export interface FieldMapperResponse { + success: boolean; + enhancedFeedrules: any[]; + summary: { + totalFields: number; + improved: number; + unchanged: number; + improvementPercentage: number; + }; + context: { + eventName: string; + topic: string; + domain: string; + }; +} + +export interface FieldMapperError { + error: string; + details?: string[]; + errorType?: string; +} + +/** + * Call the AI Field Mapper Lambda function to enhance feedrules with intelligent field mappings. + * + * This function sends feedrules and AsyncAPI specification to an AI-powered Lambda function that: + * 1. Analyzes payload field types and names to generate realistic Faker.js rules + * 2. Identifies topic variables (e.g., {amount}, {employeeId}) that match payload field names + * 3. Creates mappings to use payload-generated values in topic addresses + * + * Example: If the topic is "orders/{orderId}" and payload has an "orderId" field, + * the Lambda will generate a mapping to inject the payload's orderId value into the topic. + * + * @param feedrules - The vanilla/generic feedrules to enhance with mappings array + * @param asyncApiSpec - The AsyncAPI specification containing topic and payload schemas + * @param endpoint - Optional custom endpoint URL (defaults to STM_FIELD_MAPPER_ENDPOINT env var) + * @returns Enhanced feedrules with mappings array populated, or null on failure + * + * @example + * const enhanced = await enhanceFeedrulesWithAI( + * [{ topic: 'acme/{region}/{employeeId}', topicParameters: {...}, payload: {...} }], + * asyncApiSpec, + * 'https://my-lambda-url.com' + * ); + * // Returns feedrules with mappings like: + * // mappings: [ + * // { + * // type: 'Topic Parameter', + * // source: { type: 'Payload Parameter', name: 'employeeId', fieldType: 'number' }, + * // target: { type: 'Topic Parameter', name: 'employeeId', fieldType: 'string' } + * // } + * // ] + */ +export async function enhanceFeedrulesWithAI( + feedrules: any[], + asyncApiSpec: any, + endpoint?: string +): Promise { + const apiEndpoint = endpoint || DEFAULT_FIELD_MAPPER_ENDPOINT; + + if (!apiEndpoint) { + Logger.logError('No field mapper endpoint configured'); + Logger.logWarn('Set STM_FIELD_MAPPER_ENDPOINT environment variable or use --ai-mapper-endpoint flag'); + return null; + } + + Logger.info('Enhancing feedrules using AI field mapper...'); + Logger.logInfo(`Endpoint: ${chalk.cyanBright(apiEndpoint)}`); + + const requestBody = { + feedrules, + asyncApiSpec + }; + + try { + const response = await fetch(apiEndpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json' + }, + body: JSON.stringify(requestBody) + }); + + if (!response.ok) { + const errorData: FieldMapperError = await response.json(); + Logger.logError(`Field mapper returned error (${response.status}): ${errorData.error}`); + if (errorData.details) { + errorData.details.forEach(detail => Logger.logWarn(` - ${detail}`)); + } + return null; + } + + const result: FieldMapperResponse = await response.json(); + + if (!result.success) { + Logger.logError('Field mapper did not succeed'); + return null; + } + + // Log summary + Logger.logSuccess('Field mapping completed successfully!'); + Logger.logInfo(`Context: ${chalk.greenBright(result.context.eventName)} in ${chalk.cyanBright(result.context.domain)}`); + Logger.logInfo(`Topic: ${chalk.yellowBright(result.context.topic)}`); + Logger.logInfo(`Fields analyzed: ${chalk.whiteBright(result.summary.totalFields)}`); + Logger.logInfo(`Improved mappings: ${chalk.greenBright(result.summary.improved)} (${result.summary.improvementPercentage}%)`); + Logger.logInfo(`Unchanged: ${chalk.gray(result.summary.unchanged)}`); + + return result.enhancedFeedrules; + + } catch (error: any) { + if (error.name === 'TypeError' && error.message.includes('fetch')) { + Logger.logError('Failed to connect to field mapper endpoint'); + Logger.logWarn('Please check that the endpoint is correct and accessible'); + } else if (error.name === 'AbortError') { + Logger.logError('Request to field mapper timed out'); + } else { + Logger.logError(`Error calling field mapper: ${error.message}`); + } + Logger.logDetailedError('Field mapper error', error.toString()); + return null; + } +} + +/** + * Validate that the field mapper endpoint is accessible + * @param endpoint - The endpoint URL to validate + * @returns true if accessible, false otherwise + */ +export async function validateFieldMapperEndpoint(endpoint: string): Promise { + if (!endpoint) { + return false; + } + + try { + // Try to parse as URL + new URL(endpoint); + + // Optionally, you could make a HEAD or OPTIONS request here + // For now, just validate the URL format + return true; + } catch (error) { + return false; + } +} + +/** + * Get the configured field mapper endpoint + * @returns The endpoint URL or null if not configured + */ +export function getFieldMapperEndpoint(): string | null { + return DEFAULT_FIELD_MAPPER_ENDPOINT || null; +} diff --git a/src/utils/options.ts b/src/utils/options.ts index 32be19a..1f1b4dd 100644 --- a/src/utils/options.ts +++ b/src/utils/options.ts @@ -692,13 +692,16 @@ export const addFeedGenerateOptions = (cmd: Command, advanced: boolean) => { cmd .addOption(new Option('-file, --file-name ', chalk.whiteBright('the asyncapi document')) ) .addOption(new Option('-feed, --feed-name ', chalk.whiteBright('the feed name')) ) - .addOption(new Option('-type, --feed-type ', chalk.whiteBright('the feed type')) + .addOption(new Option('-type, --feed-type ', chalk.whiteBright('the feed type')) .argParser(parseFeedType) .default(defaultFeedConfig.feedType) ) - .addOption(new Option('-view, --feed-view ', chalk.whiteBright('the feed view: publisher, provider;\n generates feed for subscribe operations and vice versa')) - .argParser(parseFeedView) - .default(defaultFeedConfig.feedView === 'default' ? 'publisher' : 'provider')) + .addOption(new Option('-view, --feed-view ', chalk.whiteBright('the feed view: publisher, provider;\n generates feed for subscribe operations and vice versa')) + .argParser(parseFeedView) + .default(defaultFeedConfig.feedView === 'default' ? 'publisher' : 'provider')) // .conflicts('feedName')) // INVALID CONFLICT CONDITION - // hidden option to use defaults + // AI enhancement options + .addOption(new Option('-ai, --ai-enhance [BOOLEAN]', chalk.whiteBright('use AI to intelligently enhance field mappings')) .argParser(parseBoolean) .default(false) .hideHelp(advanced)) + .addOption(new Option('--ai-mapper-endpoint ', chalk.whiteBright('[advanced] the AI field mapper endpoint URL')) .hideHelp(!advanced)) + // hidden option to use defaults .addOption(new Option('-defaults, --use-defaults', chalk.whiteBright('use defaults for feed name and feed type')) .hideHelp(true) .default(false)) // lint options - validate CLI arguments quitely (without executing the command) .addOption(new Option('--lint [BOOLEAN]', chalk.whiteBright('validate CLI arguments quietly (without executing the command)')) .argParser(parseBoolean) .default(false) .hideHelp(true)) From 48cfaa1cd6f417bc9cf7f91cbf6c4015bc8d2ba6 Mon Sep 17 00:00:00 2001 From: Jamieson Walker Date: Tue, 21 Oct 2025 14:32:30 -0500 Subject: [PATCH 2/6] Fix undefined feed directory bug and add default AI mapper endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit addresses two issues with feed generation: 1. Fixed bug where 'undefined' directory was created during feed generation - Set feed.name when feedName is provided via CLI - Moved lastUpdated timestamp before createFeed() call - Removed redundant updateAndLoadFeedInfo() call that wrote feedinfo.json twice - Removed unused updateAndLoadFeedInfo import 2. Added hard-coded default AI field mapper endpoint - Set default to https://b0hv9uf5m8.execute-api.us-east-2.amazonaws.com/Prod/fieldmap - Priority order: CLI flag > env var > hard-coded default - Updated documentation with usage examples and configuration priority 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- claude.md | 15 +++++++++++++-- src/lib/feed-generate.ts | 8 ++++---- src/utils/field-mapper-client.ts | 8 +++++--- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/claude.md b/claude.md index 2009bb1..cc88ac0 100644 --- a/claude.md +++ b/claude.md @@ -100,18 +100,29 @@ The `stm feed generate` command supports AI-enhanced field mapping via the `--ai ### Usage ```bash -# Generate feed with AI field mapping enhancement +# Generate feed with AI field mapping enhancement (uses built-in default endpoint) +stm feed generate \ + --file-name asyncapi.yaml \ + --feed-name "MyFeed" \ + --ai-enhance + +# Or override the default endpoint with custom Lambda stm feed generate \ --file-name asyncapi.yaml \ --feed-name "MyFeed" \ --ai-enhance \ --ai-mapper-endpoint "https://your-lambda-url.amazonaws.com/field-mapper" -# Or set endpoint via environment variable +# Or set endpoint via environment variable to override default export STM_FIELD_MAPPER_ENDPOINT="https://your-lambda-url.amazonaws.com/field-mapper" stm feed generate --file-name asyncapi.yaml --ai-enhance ``` +**Default Endpoint**: The CLI includes a built-in default endpoint (https://b0hv9uf5m8.execute-api.us-east-2.amazonaws.com/Prod/fieldmap) that is used when `--ai-enhance` is specified without additional configuration. You can override this with: +1. The `--ai-mapper-endpoint` flag (highest priority) +2. The `STM_FIELD_MAPPER_ENDPOINT` environment variable (medium priority) +3. Built-in default endpoint (fallback) + ### What It Does The AI field mapper performs two types of enhancements: diff --git a/src/lib/feed-generate.ts b/src/lib/feed-generate.ts index 1781438..f5ca3dc 100644 --- a/src/lib/feed-generate.ts +++ b/src/lib/feed-generate.ts @@ -1,6 +1,6 @@ import * as fs from 'fs' import { Logger } from '../utils/logger' -import { createFeed, fileExists, updateAndLoadFeedInfo, loadLocalFeedFile, processPlainPath, writeJsonFile, readAsyncAPIFile } from '../utils/config'; +import { createFeed, fileExists, loadLocalFeedFile, processPlainPath, writeJsonFile, readAsyncAPIFile } from '../utils/config'; import { prettyJSON } from '../utils/prettify'; import { defaultFakerRulesFile, defaultFeedApiEndpointFile, defaultFeedInfoFile, defaultFeedMajorVersion, defaultFeedMinorVersion, defaultFeedRulesFile, defaultFeedSessionFile, defaultStmFeedsHome } from '../utils/defaults'; import { chalkBoldLabel, chalkBoldVariable, chalkBoldWhite } from '../utils/chalkUtils'; @@ -85,7 +85,8 @@ const generate = async (options: ManageFeedClientOptions, optionsSource: any) => if (optionsSource.feedName === 'cli' || optionsSource.fileName === 'cli') { feedName = options.feedName; fileName = options.fileName; - } + feed.name = feedName; + } if (!feedName) { const pFeedName = new Input({ message: `${chalkBoldWhite('Enter feed name')}\n` + @@ -262,9 +263,8 @@ const generate = async (options: ManageFeedClientOptions, optionsSource: any) => }); } - createFeed(fileName, feedName, feed, data, rules, schemas, sessionPropertiesJson, options.useDefaults ? true : false); feed.lastUpdated = new Date().toISOString(); - updateAndLoadFeedInfo(feed); + createFeed(fileName, feedName, feed, data, rules, schemas, sessionPropertiesJson, options.useDefaults ? true : false); Logger.success(`Successfully created event feed ${feedName}`); diff --git a/src/utils/field-mapper-client.ts b/src/utils/field-mapper-client.ts index 732e7d8..a3f5b46 100644 --- a/src/utils/field-mapper-client.ts +++ b/src/utils/field-mapper-client.ts @@ -1,8 +1,9 @@ import { Logger } from './logger'; import chalk from 'chalk'; -// Default endpoint - can be overridden via environment variable -const DEFAULT_FIELD_MAPPER_ENDPOINT = process.env.STM_FIELD_MAPPER_ENDPOINT || ''; +// Default endpoint - can be overridden via environment variable or command line flag +const DEFAULT_FIELD_MAPPER_ENDPOINT = process.env.STM_FIELD_MAPPER_ENDPOINT || + 'https://b0hv9uf5m8.execute-api.us-east-2.amazonaws.com/Prod/fieldmap'; /** * Mapping between topic parameters and payload parameters @@ -61,7 +62,7 @@ export interface FieldMapperError { * * @param feedrules - The vanilla/generic feedrules to enhance with mappings array * @param asyncApiSpec - The AsyncAPI specification containing topic and payload schemas - * @param endpoint - Optional custom endpoint URL (defaults to STM_FIELD_MAPPER_ENDPOINT env var) + * @param endpoint - Optional custom endpoint URL (defaults to hard-coded endpoint or STM_FIELD_MAPPER_ENDPOINT env var) * @returns Enhanced feedrules with mappings array populated, or null on failure * * @example @@ -87,6 +88,7 @@ export async function enhanceFeedrulesWithAI( const apiEndpoint = endpoint || DEFAULT_FIELD_MAPPER_ENDPOINT; if (!apiEndpoint) { + // This should not happen since we have a hard-coded default, but keep as safety check Logger.logError('No field mapper endpoint configured'); Logger.logWarn('Set STM_FIELD_MAPPER_ENDPOINT environment variable or use --ai-mapper-endpoint flag'); return null; From 899a999efb510a9abd0685533a3136c84f21cf9a Mon Sep 17 00:00:00 2001 From: Jamieson Walker Date: Thu, 23 Oct 2025 13:38:58 -0500 Subject: [PATCH 3/6] Added ai-enhance to interactive flow + AI Authorization --- LAMBDA_IMPLEMENTATION_GUIDE.md | 619 --------------------------------- README.md | 6 +- documentation/EVENT_FEEDS.md | 31 ++ src/lib/feed-configure.ts | 85 ++++- src/lib/feed-generate.ts | 66 +++- src/types/global.d.ts | 4 + src/utils/ai-disclaimer.ts | 107 ++++++ src/utils/defaults.ts | 1 + src/utils/instances.ts | 5 + src/utils/options.ts | 3 + 10 files changed, 289 insertions(+), 638 deletions(-) delete mode 100644 LAMBDA_IMPLEMENTATION_GUIDE.md create mode 100644 src/utils/ai-disclaimer.ts diff --git a/LAMBDA_IMPLEMENTATION_GUIDE.md b/LAMBDA_IMPLEMENTATION_GUIDE.md deleted file mode 100644 index cd2e985..0000000 --- a/LAMBDA_IMPLEMENTATION_GUIDE.md +++ /dev/null @@ -1,619 +0,0 @@ -# AWS Lambda Field Mapper Implementation Guide - -## Overview - -This guide provides specifications for implementing the AWS Lambda function that enhances AsyncAPI feedrules with intelligent field mappings, including **Topic Parameter Mapping** - the ability to map payload fields to topic variables. - -## Feature: Topic Parameter Mapping - -### Purpose - -When AsyncAPI topics contain variable placeholders (e.g., `acme/orders/{orderId}/{region}`), and the message payload contains fields with matching names (e.g., `orderId`, `region`), the Lambda function should automatically create mappings to inject the payload-generated values into the topic address. - -### Benefits - -1. **Realistic Topics**: Topic addresses will contain actual correlated data (e.g., order ID in topic matches order ID in payload) -2. **Data Consistency**: Ensures the same value appears in both topic and payload -3. **Type Conversion**: Handles type conversions (e.g., payload number → topic string) -4. **Reduced Configuration**: Automatic detection eliminates manual mapping configuration - -## Lambda Function Specification - -### Input Format - -```json -{ - "feedrules": [ - { - "topic": "acmeRental/orders/created/v1/{region}/{orderId}", - "eventName": "Order Created", - "eventVersion": "1.0.0", - "messageName": "Order_Created", - "topicParameters": { - "region": { - "schema": { - "type": "string", - "enum": ["us-east", "us-west", "eu-central"] - }, - "rule": { - "name": "region", - "type": "string", - "group": "StringRules", - "rule": "enum", - "enum": ["us-east", "us-west", "eu-central"] - } - }, - "orderId": { - "schema": { - "type": "string" - }, - "rule": { - "name": "orderId", - "type": "string", - "group": "StringRules", - "rule": "alpha", - "casing": "mixed", - "minLength": 10, - "maxLength": 10 - } - } - }, - "payload": { - "orderId": { - "type": "number", - "minimum": 1000, - "maximum": 999999 - }, - "customerName": { - "type": "string" - }, - "amount": { - "type": "number", - "minimum": 0, - "maximum": 10000 - } - }, - "publishSettings": { - "count": 0, - "interval": 1000, - "delay": 0 - } - } - ], - "asyncApiSpec": { /* Full AsyncAPI document */ } -} -``` - -### Expected Output Format - -```json -{ - "success": true, - "enhancedFeedrules": [ - { - "topic": "acmeRental/orders/created/v1/{region}/{orderId}", - "eventName": "Order Created", - "eventVersion": "1.0.0", - "messageName": "Order_Created", - "topicParameters": { /* Same as input */ }, - "payload": { - "orderId": { - "type": "number", - "minimum": 1000, - "maximum": 999999, - "rule": { - "name": "orderId", - "type": "number", - "group": "NumberRules", - "rule": "int", - "minimum": 1000, - "maximum": 999999 - } - }, - "customerName": { - "type": "string", - "rule": { - "name": "customerName", - "type": "string", - "group": "PersonRules", - "rule": "fullName" - } - }, - "amount": { - "type": "number", - "minimum": 0, - "maximum": 10000, - "rule": { - "name": "amount", - "type": "number", - "group": "FinanceRules", - "rule": "amount", - "minimum": 0, - "maximum": 10000 - } - } - }, - "publishSettings": { /* Same as input */ }, - "mappings": [ - { - "type": "Topic Parameter", - "source": { - "type": "Payload Parameter", - "name": "orderId", - "fieldName": "orderId", - "fieldType": "number" - }, - "target": { - "type": "Topic Parameter", - "name": "orderId", - "fieldName": "orderId", - "fieldType": "string" - } - } - ] - } - ], - "summary": { - "totalFields": 3, - "improved": 3, - "unchanged": 0, - "improvementPercentage": 100 - }, - "context": { - "eventName": "Order Created", - "topic": "acmeRental/orders/created/v1/{region}/{orderId}", - "domain": "Retail" - } -} -``` - -## Implementation Algorithm - -### Step 1: Parse Topic for Variables - -```python -import re - -def extract_topic_variables(topic: str) -> list[str]: - """ - Extract variable names from topic string. - - Example: "acme/{region}/orders/{orderId}" -> ["region", "orderId"] - """ - pattern = r'\{([^}]+)\}' - return re.findall(pattern, topic) -``` - -### Step 2: Build Field Inventories - -```python -def build_field_inventories(feedrule: dict) -> tuple[dict, dict]: - """ - Create inventories of topic parameters and payload fields. - - Returns: - (topic_fields, payload_fields) - """ - # Extract topic parameter definitions - topic_fields = {} - if 'topicParameters' in feedrule: - for name, config in feedrule['topicParameters'].items(): - topic_fields[name] = { - 'name': name, - 'type': config.get('schema', {}).get('type', 'string'), - 'config': config - } - - # Extract payload field definitions - payload_fields = {} - if 'payload' in feedrule: - for name, config in feedrule['payload'].items(): - payload_fields[name] = { - 'name': name, - 'type': config.get('type', 'string'), - 'config': config - } - - return topic_fields, payload_fields -``` - -### Step 3: Identify Matching Fields - -```python -def find_matching_fields(topic_vars: list[str], topic_fields: dict, payload_fields: dict) -> list[dict]: - """ - Find payload fields that match topic variable names. - - Returns list of matches with source and target information. - """ - matches = [] - - for var_name in topic_vars: - # Check if this variable exists in both topic parameters and payload - if var_name in payload_fields: - payload_field = payload_fields[var_name] - topic_field = topic_fields.get(var_name, {}) - - matches.append({ - 'variable_name': var_name, - 'source': { - 'type': 'Payload Parameter', - 'name': var_name, - 'fieldName': var_name, - 'fieldType': payload_field['type'] - }, - 'target': { - 'type': 'Topic Parameter', - 'name': var_name, - 'fieldName': var_name, - 'fieldType': topic_field.get('type', 'string') # Topics typically use strings - } - }) - - return matches -``` - -### Step 4: Generate Mapping Entries - -```python -def generate_topic_mappings(feedrule: dict) -> list[dict]: - """ - Generate Topic Parameter mappings for a feedrule. - - Returns list of mapping objects. - """ - topic = feedrule.get('topic', '') - topic_vars = extract_topic_variables(topic) - - if not topic_vars: - return [] - - topic_fields, payload_fields = build_field_inventories(feedrule) - matches = find_matching_fields(topic_vars, topic_fields, payload_fields) - - mappings = [] - for match in matches: - mappings.append({ - 'type': 'Topic Parameter', - 'source': match['source'], - 'target': match['target'] - }) - - return mappings -``` - -### Step 5: Enhance Payload Fields - -```python -def enhance_payload_fields(feedrule: dict) -> dict: - """ - Add Faker.js rules to payload fields based on field names and types. - - This is your existing functionality - enhance it to recognize common patterns: - - "name", "firstName", "lastName" -> PersonRules - - "email" -> InternetRules (email) - - "phone" -> StringRules (phoneNumber) - - "amount", "price" -> FinanceRules (amount) - - "date", "timestamp" -> DateRules - - etc. - """ - enhanced_payload = {} - - for field_name, field_config in feedrule.get('payload', {}).items(): - enhanced_field = dict(field_config) - - # Generate appropriate Faker rule based on field name and type - faker_rule = infer_faker_rule(field_name, field_config.get('type')) - enhanced_field['rule'] = faker_rule - - enhanced_payload[field_name] = enhanced_field - - return enhanced_payload -``` - -### Step 6: Complete Enhancement Pipeline - -```python -def enhance_feedrule(feedrule: dict, asyncapi_spec: dict) -> dict: - """ - Complete enhancement pipeline for a single feedrule. - """ - enhanced = dict(feedrule) - - # 1. Enhance payload fields with Faker rules - enhanced['payload'] = enhance_payload_fields(feedrule) - - # 2. Generate topic parameter mappings - topic_mappings = generate_topic_mappings(feedrule) - - # 3. Add or merge mappings array - if 'mappings' not in enhanced: - enhanced['mappings'] = [] - - enhanced['mappings'].extend(topic_mappings) - - return enhanced - -def lambda_handler(event, context): - """ - Main Lambda handler. - """ - try: - feedrules = event['feedrules'] - asyncapi_spec = event['asyncApiSpec'] - - enhanced_feedrules = [] - total_fields = 0 - improved_fields = 0 - - for feedrule in feedrules: - enhanced = enhance_feedrule(feedrule, asyncapi_spec) - enhanced_feedrules.append(enhanced) - - # Count statistics - payload_count = len(feedrule.get('payload', {})) - total_fields += payload_count - - # Check how many fields got rules - enhanced_payload_count = sum( - 1 for field in enhanced.get('payload', {}).values() - if 'rule' in field - ) - improved_fields += enhanced_payload_count - - improvement_percentage = ( - int((improved_fields / total_fields) * 100) if total_fields > 0 else 0 - ) - - return { - 'success': True, - 'enhancedFeedrules': enhanced_feedrules, - 'summary': { - 'totalFields': total_fields, - 'improved': improved_fields, - 'unchanged': total_fields - improved_fields, - 'improvementPercentage': improvement_percentage - }, - 'context': { - 'eventName': feedrules[0].get('eventName', 'Unknown'), - 'topic': feedrules[0].get('topic', ''), - 'domain': asyncapi_spec.get('info', {}).get('x-application-domain', 'Unknown') - } - } - - except Exception as e: - return { - 'success': False, - 'error': str(e), - 'errorType': type(e).__name__ - } -``` - -## Example Test Cases - -### Test Case 1: Simple Topic Variable Match - -**Input:** -```json -{ - "topic": "orders/{orderId}", - "topicParameters": { - "orderId": { "schema": { "type": "string" } } - }, - "payload": { - "orderId": { "type": "number", "minimum": 1000, "maximum": 9999 } - } -} -``` - -**Expected Mapping:** -```json -{ - "type": "Topic Parameter", - "source": { - "type": "Payload Parameter", - "name": "orderId", - "fieldName": "orderId", - "fieldType": "number" - }, - "target": { - "type": "Topic Parameter", - "name": "orderId", - "fieldName": "orderId", - "fieldType": "string" - } -} -``` - -### Test Case 2: Multiple Matching Variables - -**Input:** -```json -{ - "topic": "acme/{region}/{storeId}/sales/{transactionId}", - "topicParameters": { - "region": { "schema": { "type": "string" } }, - "storeId": { "schema": { "type": "string" } }, - "transactionId": { "schema": { "type": "string" } } - }, - "payload": { - "transactionId": { "type": "number" }, - "storeId": { "type": "number" }, - "amount": { "type": "number" }, - "region": { "type": "string" } - } -} -``` - -**Expected Mappings:** 3 mappings (region, storeId, transactionId) - -### Test Case 3: No Matching Fields - -**Input:** -```json -{ - "topic": "orders/{orderId}", - "topicParameters": { - "orderId": { "schema": { "type": "string" } } - }, - "payload": { - "customerName": { "type": "string" }, - "amount": { "type": "number" } - } -} -``` - -**Expected Mappings:** Empty array (no matches) - -### Test Case 4: Partial Match - -**Input:** -```json -{ - "topic": "store/{storeId}/employee/{employeeId}", - "topicParameters": { - "storeId": { "schema": { "type": "string" } }, - "employeeId": { "schema": { "type": "string" } } - }, - "payload": { - "employeeId": { "type": "number" }, - "shiftHours": { "type": "number" } - } -} -``` - -**Expected Mappings:** 1 mapping (employeeId only) - -## Deployment Configuration - -### Environment Variables - -- `LOG_LEVEL`: Set to `DEBUG` for detailed logging -- `ENABLE_TOPIC_MAPPING`: Feature flag (default: `true`) -- `CASE_SENSITIVE_MATCHING`: Whether field matching is case-sensitive (default: `false`) - -### Lambda Configuration - -- **Runtime**: Python 3.11+ or Node.js 18+ -- **Memory**: 512 MB (adjust based on AsyncAPI document size) -- **Timeout**: 30 seconds -- **Concurrency**: 10 (adjust based on expected load) - -### API Gateway Setup - -```yaml -Path: /field-mapper -Method: POST -Authorization: API Key or IAM -CORS: Enabled -Content-Type: application/json -``` - -## Error Handling - -```python -class FieldMapperError(Exception): - """Base exception for field mapper errors.""" - pass - -class InvalidInputError(FieldMapperError): - """Invalid input format.""" - pass - -class ProcessingError(FieldMapperError): - """Error during field processing.""" - pass - -def lambda_handler(event, context): - try: - # Validate input - if 'feedrules' not in event: - raise InvalidInputError('Missing required field: feedrules') - - if 'asyncApiSpec' not in event: - raise InvalidInputError('Missing required field: asyncApiSpec') - - # Process feedrules - # ... - - except InvalidInputError as e: - return { - 'success': False, - 'error': str(e), - 'errorType': 'INVALID_INPUT', - 'details': ['feedrules and asyncApiSpec are required'] - } - - except ProcessingError as e: - return { - 'success': False, - 'error': str(e), - 'errorType': 'PROCESSING_ERROR', - 'details': [str(e)] - } - - except Exception as e: - return { - 'success': False, - 'error': 'Internal server error', - 'errorType': 'INTERNAL_ERROR', - 'details': [str(e)] - } -``` - -## Testing - -### Unit Tests - -```python -import unittest - -class TestTopicVariableExtraction(unittest.TestCase): - def test_single_variable(self): - topic = "orders/{orderId}" - result = extract_topic_variables(topic) - self.assertEqual(result, ["orderId"]) - - def test_multiple_variables(self): - topic = "acme/{region}/store/{storeId}/orders/{orderId}" - result = extract_topic_variables(topic) - self.assertEqual(result, ["region", "storeId", "orderId"]) - - def test_no_variables(self): - topic = "orders/created/v1" - result = extract_topic_variables(topic) - self.assertEqual(result, []) - -class TestFieldMatching(unittest.TestCase): - def test_exact_match(self): - topic_vars = ["orderId"] - topic_fields = {"orderId": {"name": "orderId", "type": "string"}} - payload_fields = {"orderId": {"name": "orderId", "type": "number"}} - - matches = find_matching_fields(topic_vars, topic_fields, payload_fields) - self.assertEqual(len(matches), 1) - self.assertEqual(matches[0]['variable_name'], 'orderId') -``` - -### Integration Test - -```bash -curl -X POST https://your-lambda-url.amazonaws.com/field-mapper \ - -H "Content-Type: application/json" \ - -d @test-payload.json -``` - -## Performance Considerations - -1. **Caching**: Cache AsyncAPI parsing results for repeated calls -2. **Batch Processing**: Process multiple feedrules in parallel -3. **Lazy Loading**: Only parse relevant sections of large AsyncAPI documents -4. **Memory Management**: Stream process very large payload definitions - -## Future Enhancements - -1. **Fuzzy Matching**: Match fields with similar names (e.g., "order_id" vs "orderId") -2. **Custom Mapping Rules**: Allow users to define custom mapping preferences -3. **Type Coercion Warnings**: Warn about potential data loss in type conversions -4. **Nested Field Support**: Support nested payload fields (e.g., "order.id") -5. **Array Field Handling**: Map array elements to topic variables diff --git a/README.md b/README.md index 13699d5..b2de4b9 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,11 @@ _**stm**_ provides a feature to generate event feeds directly from AsyncAPI docu --- ![](docrefs/eventfeeds.webp "stm - feeds") -`stm` supports a set of operations to generate, configure and run an event feed from an AsyncAPI document corresponding to an asynchronous application or an API (from Event Portal). The goal is to help you quickly set up an event feed on your local machine, configure data generation rules, and run to stream events to a broker. Additionally, you can contribute your feed for community use by following the contribution process (referred to as community or contributed feeds shared on the [Community Event Feeds site](https://github.com/solacecommunity/solace-event-feeds)). +`stm` supports a set of operations to generate, configure and run an event feed from an AsyncAPI document corresponding to an asynchronous application or an API (from Event Portal). The goal is to help you quickly set up an event feed on your local machine, configure data generation rules, and run to stream events to a broker. Additionally, you can contribute your feed for community use by following the contribution process (referred to as community or contributed feeds shared on the [Community Event Feeds site](https://github.com/solacecommunity/solace-event-feeds)). + +**New in this release:** Event feeds now support AI-powered field mapping to automatically generate realistic data rules and topic parameter mappings. Use the `--ai-enhance` flag with `stm feed generate` or `stm feed configure` commands. + +**⚠️ Important:** When using AI enhancement for the first time, you will be prompted to accept a disclaimer regarding data processing by AI models. Your AsyncAPI specification content will be sent to AWS Bedrock services (including Google Gemini and Anthropic Claude) for analysis. Do not include sensitive or proprietary information in AsyncAPI documents used with AI enhancement features. > **For more details on working with event feeds, please review the [EVENT_FEEDS](./documentation/EVENT_FEEDS.md) documentation.** diff --git a/documentation/EVENT_FEEDS.md b/documentation/EVENT_FEEDS.md index 914195e..2a61e9b 100644 --- a/documentation/EVENT_FEEDS.md +++ b/documentation/EVENT_FEEDS.md @@ -136,6 +136,24 @@ Generate a Feed| Validate the Feed --|-- ![](./docrefs/feed-generate.gif)|![](./docrefs/feed-generate-verify.gif)| +### AI-Enhanced Field Mapping (Optional) + +The `generate` command supports an optional `--ai-enhance` flag that uses AI to automatically create intelligent field mappings and data generation rules: + +```bash +# Generate feed with AI enhancement +stm feed generate --file-name asyncapi.yaml --feed-name "MyFeed" --ai-enhance +``` + +**Benefits:** +- Automatically generates realistic Faker.js rules based on field names and types +- Creates topic parameter mappings when topic variables match payload fields +- Reduces manual configuration time + +> **⚠️ First-Time Use:** You will be prompted to accept an AI disclaimer before using this feature. Your AsyncAPI specification will be processed by external AI services (AWS Bedrock with Google Gemini and Anthropic Claude). Do not include sensitive or proprietary information. + +For more details, see the [AI-Powered Field Mapping section in CLAUDE.md](../claude.md#ai-powered-field-mapping). + ## Configure Data Generation Rules for an Event Feed The `configure` sub-command enables setting up data generation rules for *topic* and *payload* parameters. It also allows optional mapping of attributes (e.g., `topic parameter → payload attribute, payload attribute → another attribute`). @@ -153,6 +171,19 @@ Options: -h, --help display help for command ``` +### AI-Enhanced Configuration (Optional) + +You can also use AI to automatically enhance an existing feed's configuration: + +```bash +# Auto-configure feed with AI +stm feed configure --feed-name "MyFeed" --ai-enhance +``` + +This skips the web UI and automatically applies intelligent field mappings to your feed. To review or modify the AI-generated mappings afterwards, run the configure command without the `--ai-enhance` flag. + +> **⚠️ First-Time Use:** You will be prompted to accept an AI disclaimer before using this feature. Your AsyncAPI specification will be processed by external AI services (AWS Bedrock with Google Gemini and Anthropic Claude). Do not include sensitive or proprietary information. + To learn about supported data generation rules, refer to the documentation: [Data Generation Rules](./DATAGENERATION_RULES.md) An Event Feed configuration exposes feed operations — send and receive events and their schemas. Under the *Messages* group, you can explore all exposed `messages` (events). Selecting a message reveals its details, including `name, schema`, and the `topics` it uses for send and receive operations. diff --git a/src/lib/feed-configure.ts b/src/lib/feed-configure.ts index 1d9c3d8..ebeb014 100644 --- a/src/lib/feed-configure.ts +++ b/src/lib/feed-configure.ts @@ -1,11 +1,16 @@ -import { getAllFeeds, getFeed, loadLocalFeedFile, loadLocalFeedSessionSettingsFile, processPlainPath, updateRules, updateSession } from '../utils/config'; -import { defaultFakerRulesFile, defaultFeedInfoFile, defaultFeedSessionFile, defaultProjectName, defaultStmFeedsHome } from '../utils/defaults'; +import { getAllFeeds, getFeed, loadLocalFeedFile, loadLocalFeedSessionSettingsFile, processPlainPath, updateRules, updateSession, loadRawLocalFeedFile, fileExists } from '../utils/config'; +import { defaultFakerRulesFile, defaultFeedInfoFile, defaultFeedSessionFile, defaultProjectName, defaultStmFeedsHome, defaultFeedRulesFile } from '../utils/defaults'; import { getLocalEventFeeds } from '../utils/listfeeds'; import { Logger } from '../utils/logger' import { fakeDataObjectGenerator, fakeDataValueGenerator } from './feed-datahelper'; import { chalkBoldLabel, chalkBoldVariable } from '../utils/chalkUtils'; import { messagePropertiesJson } from '../utils/msgprops'; import { sessionPropertiesJson } from '../utils/sessionprops'; +import { enhanceFeedrulesWithAI } from '../utils/field-mapper-client'; +import { hasAcceptedAiDisclaimer, showAiDisclaimer, recordAiDisclaimerAcceptance } from '../utils/ai-disclaimer'; +import chalk from 'chalk'; +import * as fs from 'fs'; +import * as path from 'path'; // @ts-ignore import { generateEvent } from '@solace-labs/solace-data-generator'; @@ -61,7 +66,81 @@ const manage = async (options: ManageFeedClientOptions, optionsSource: any) => { if (feedName === 'All Event Feeds') { feedName = undefined; Logger.success(`will explore all available feeds`) - } + } + } + + // AI Enhancement: If --ai-enhance flag is provided, automatically enhance feedrules with AI + if (options.aiEnhance) { + if (!feedName) { + Logger.logError('--feed-name is required when using --ai-enhance'); + Logger.logError('exiting...'); + process.exit(1); + } + + // Check if user has accepted AI disclaimer + if (!hasAcceptedAiDisclaimer()) { + const accepted = await showAiDisclaimer(); + if (!accepted) { + Logger.logWarn('AI enhancement declined. Feed configuration was not modified.'); + Logger.success('exiting...'); + process.exit(0); + } else { + recordAiDisclaimerAcceptance(); + Logger.logSuccess('AI disclaimer accepted. Proceeding with AI enhancement.'); + } + } + + Logger.info('AI enhancement enabled - automatically configuring feed with intelligent field mappings...'); + + // 1. Load existing feedrules + const feedrules = loadLocalFeedFile(feedName, defaultFeedRulesFile); + + // 2. Find and load the AsyncAPI spec file from the feed directory + const feedPath = processPlainPath(`${defaultStmFeedsHome}/${feedName}`); + const files = fs.readdirSync(feedPath); + + // Filter to find AsyncAPI spec file (exclude known feed files) + const excludedFiles = ['feedrules.json', 'analysis.json', 'fakerrules.json', 'feedinfo.json', 'feedschemas.json', 'feedsession.json']; + const asyncApiFile = files.find(file => { + const ext = path.extname(file).toLowerCase(); + return (ext === '.json' || ext === '.yaml' || ext === '.yml') && !excludedFiles.includes(file); + }); + + if (!asyncApiFile) { + Logger.logError('Could not find AsyncAPI specification file in feed directory'); + Logger.logError('exiting...'); + process.exit(1); + } + + Logger.logInfo(`Found AsyncAPI spec: ${chalk.cyanBright(asyncApiFile)}`); + + // 3. Load the AsyncAPI spec + const asyncApiSpec = loadRawLocalFeedFile(feedName, asyncApiFile); + + // 4. Parse AsyncAPI to object if it's a string + const asyncApiObject = typeof asyncApiSpec === 'string' ? JSON.parse(asyncApiSpec) : asyncApiSpec; + + // 5. Call AI enhancement + const enhancedRules = await enhanceFeedrulesWithAI( + feedrules, + asyncApiObject, + options.aiMapperEndpoint + ); + + // 6. Save enhanced rules back if successful + if (enhancedRules && enhancedRules.length > 0) { + await updateRules(feedName, enhancedRules); + Logger.logSuccess(`Feed configuration for '${chalk.greenBright(feedName)}' automatically enhanced with AI!`); + Logger.hint(`\nWhat's next?\n` + + ` To explore or modify the AI-generated field mappings, run:\n` + + ` ${chalk.italic.cyanBright(`stm feed configure --feed-name ${feedName}`)}`); + } else { + Logger.logError('AI enhancement failed - no enhanced rules returned'); + Logger.logWarn('Feed configuration was not modified'); + } + + Logger.success('exiting...'); + process.exit(0); } const express = require('express'); diff --git a/src/lib/feed-generate.ts b/src/lib/feed-generate.ts index f5ca3dc..03b6cc0 100644 --- a/src/lib/feed-generate.ts +++ b/src/lib/feed-generate.ts @@ -11,6 +11,7 @@ import { AsyncAPIDocumentInterface } from '@asyncapi/parser'; import { checkFeedGenerateOptions, getPotentialFeedName, getPotentialTopicFromFeedName } from '../utils/checkparams'; import { sessionPropertiesJson } from '../utils/sessionprops'; import { enhanceFeedrulesWithAI } from '../utils/field-mapper-client'; +import { hasAcceptedAiDisclaimer, showAiDisclaimer, recordAiDisclaimerAcceptance } from '../utils/ai-disclaimer'; const generate = async (options: ManageFeedClientOptions, optionsSource: any) => { var { fileName, feedName, feedType, feedView } = options; @@ -106,7 +107,27 @@ const generate = async (options: ManageFeedClientOptions, optionsSource: any) => Logger.logDetailedError('interrupted...', error) process.exit(1); }); - } + } + + // AI Enhancement prompt (only in interactive mode when not specified via CLI) + if (optionsSource.aiEnhance !== 'cli') { + const { Confirm } = require('enquirer'); + const pAiEnhance = new Confirm({ + message: `${chalkBoldWhite('Use AI to enhance field mappings?')}\n` + + `${chalkBoldLabel('Hint')}: AI can automatically generate realistic data rules and topic mappings\n`, + initial: false + }); + + await pAiEnhance.run() + .then((answer: boolean) => { + options.aiEnhance = answer; + optionsSource.aiEnhance = 'interactive'; + }) + .catch((error: any) => { + Logger.logDetailedError('interrupted...', error) + process.exit(1); + }); + } if (options.lint) { Logger.logSuccess('linting successful...') @@ -140,20 +161,35 @@ const generate = async (options: ManageFeedClientOptions, optionsSource: any) => // AI Enhancement: Optionally enhance feedrules with intelligent field mappings if (options.aiEnhance) { - Logger.info('AI enhancement enabled - enhancing field mappings...'); - // Parse the AsyncAPI schema string to an object for the Lambda - const asyncApiObject = typeof asyncApiSchema === 'string' ? JSON.parse(asyncApiSchema) : asyncApiSchema; - const enhancedRules = await enhanceFeedrulesWithAI( - rules, - asyncApiObject, - options.aiMapperEndpoint - ); - - if (enhancedRules && enhancedRules.length > 0) { - Logger.logSuccess('Successfully enhanced feedrules with AI mappings'); - rules = enhancedRules; - } else { - Logger.logWarn('AI enhancement failed or returned no results, using original rules'); + // Check if user has accepted AI disclaimer + if (!hasAcceptedAiDisclaimer()) { + const accepted = await showAiDisclaimer(); + if (!accepted) { + Logger.logWarn('AI enhancement declined. Continuing with standard field generation.'); + options.aiEnhance = false; + } else { + recordAiDisclaimerAcceptance(); + Logger.logSuccess('AI disclaimer accepted. Proceeding with AI enhancement.'); + } + } + + // Proceed with AI enhancement if still enabled + if (options.aiEnhance) { + Logger.info('AI enhancement enabled - enhancing field mappings...'); + // Parse the AsyncAPI schema string to an object for the Lambda + const asyncApiObject = typeof asyncApiSchema === 'string' ? JSON.parse(asyncApiSchema) : asyncApiSchema; + const enhancedRules = await enhanceFeedrulesWithAI( + rules, + asyncApiObject, + options.aiMapperEndpoint + ); + + if (enhancedRules && enhancedRules.length > 0) { + Logger.logSuccess('Successfully enhanced feedrules with AI mappings'); + rules = enhancedRules; + } else { + Logger.logWarn('AI enhancement failed or returned no results, using original rules'); + } } } diff --git a/src/types/global.d.ts b/src/types/global.d.ts index f99ea2c..a4d23e9 100644 --- a/src/types/global.d.ts +++ b/src/types/global.d.ts @@ -232,6 +232,10 @@ declare global { // manager port managePort: number | undefined + + // AI enhancement options + aiEnhance: boolean + aiMapperEndpoint: string | undefined } interface ManageFeedClientOptions extends StmFeedConfigOptions { diff --git a/src/utils/ai-disclaimer.ts b/src/utils/ai-disclaimer.ts new file mode 100644 index 0000000..5a933c8 --- /dev/null +++ b/src/utils/ai-disclaimer.ts @@ -0,0 +1,107 @@ +import * as fs from 'fs'; +import { defaultStmHome } from './defaults'; +import { Logger } from './logger'; +import chalk from 'chalk'; + +const DISCLAIMER_FILE_NAME = 'ai-disclaimer-accepted'; + +/** + * Get the path to the AI disclaimer acceptance file + */ +function getDisclaimerFilePath(): string { + return `${defaultStmHome}/${DISCLAIMER_FILE_NAME}`; +} + +/** + * Check if the user has previously accepted the AI enhancement disclaimer + * @returns true if disclaimer has been accepted, false otherwise + */ +export function hasAcceptedAiDisclaimer(): boolean { + return fs.existsSync(getDisclaimerFilePath()); +} + +/** + * Record that the user has accepted the AI enhancement disclaimer + */ +export function recordAiDisclaimerAcceptance(): void { + const timestamp = new Date().toISOString(); + try { + fs.writeFileSync(getDisclaimerFilePath(), timestamp, 'utf8'); + } catch (error: any) { + Logger.logWarn(`Could not save AI disclaimer acceptance: ${error.message}`); + } +} + +/** + * Reset the AI disclaimer acceptance (primarily for testing) + */ +export function resetAiDisclaimerAcceptance(): void { + try { + if (fs.existsSync(getDisclaimerFilePath())) { + fs.unlinkSync(getDisclaimerFilePath()); + } + } catch (error: any) { + Logger.logWarn(`Could not reset AI disclaimer acceptance: ${error.message}`); + } +} + +/** + * Display the AI enhancement disclaimer and prompt user for acceptance + * @returns Promise - true if user accepts, false if user declines + */ +export async function showAiDisclaimer(): Promise { + const { Confirm } = require('enquirer'); + + // Display disclaimer header + console.log('\n' + chalk.bold.yellowBright('═'.repeat(80))); + console.log(chalk.bold.yellowBright('AI ENHANCEMENT DISCLAIMER')); + console.log(chalk.bold.yellowBright('═'.repeat(80))); + console.log(); + + // Display disclaimer content + console.log(chalk.white('The AI Enhancement feature uses generative AI models (including ') + + chalk.cyanBright('Google Gemini') + chalk.white(' and')); + console.log(chalk.cyanBright('Anthropic Claude') + chalk.white(' via ') + + chalk.cyanBright('AWS Bedrock') + chalk.white(') to analyze your AsyncAPI specification and')); + console.log(chalk.white('automatically generate field mappings and data generation rules.')); + console.log(); + + console.log(chalk.bold.whiteBright('IMPORTANT INFORMATION:')); + console.log(chalk.white(' • Your AsyncAPI specification content will be sent to AWS Bedrock for processing')); + console.log(chalk.white(' • The AI models may process schema definitions, field names, topic structures, and')); + console.log(chalk.white(' message payload information from your AsyncAPI document')); + console.log(chalk.white(' • Information is processed "as is" without guaranteed accuracy or completeness')); + console.log(chalk.white(' • Generated mappings should be reviewed and validated before production use')); + console.log(); + + console.log(chalk.bold.whiteBright('PRIVACY & DATA PROTECTION:')); + console.log(chalk.yellow(' • Do NOT include sensitive information, credentials, or proprietary data in AsyncAPI')); + console.log(chalk.yellow(' specifications used with AI enhancement')); + console.log(chalk.white(' • Your AsyncAPI content may be processed by third-party AI services')); + console.log(chalk.white(' • Solace is not responsible for data processed by external AI services')); + console.log(); + + console.log(chalk.bold.whiteBright('USER RESPONSIBILITY:')); + console.log(chalk.white(' • You are solely responsible for any data shared with AI enhancement features')); + console.log(chalk.white(' • Review and validate all AI-generated mappings before use')); + console.log(chalk.white(' • Use AI-generated results as a starting point, not as production-ready configuration')); + console.log(); + + console.log(chalk.bold.yellowBright('═'.repeat(80))); + console.log(); + + // Prompt for acceptance + const prompt = new Confirm({ + message: chalk.bold.whiteBright('Do you accept these terms and wish to proceed with AI enhancement?'), + initial: false + }); + + try { + const answer = await prompt.run(); + return answer; + } catch (error: any) { + // User interrupted (Ctrl+C) + Logger.logDetailedError('interrupted...', error.toString()); + return false; + } +} diff --git a/src/utils/defaults.ts b/src/utils/defaults.ts index 61a9416..170988b 100644 --- a/src/utils/defaults.ts +++ b/src/utils/defaults.ts @@ -449,6 +449,7 @@ export const homedir = currentHomeDir; export const defaultStmHome = currentStmHome; export const defaultStmFeedsHome = `${defaultStmHome}/feeds` export const defaultStmTempFeedsHome = `${defaultStmHome}/feeds/tmp` +export const defaultAiDisclaimerFile = 'ai-disclaimer-accepted' export const defaultFeedAnalysisFile = 'analysis.json' export const defaultFakerRulesFile = 'fakerrules.json' export const defaultFeedMajorVersion = 1 diff --git a/src/utils/instances.ts b/src/utils/instances.ts index 1c30b1d..d529f89 100644 --- a/src/utils/instances.ts +++ b/src/utils/instances.ts @@ -286,6 +286,9 @@ export class ManageFeedClientOptionsEmpty implements ManageFeedClientOptions { managePort: number | undefined + aiEnhance: boolean + aiMapperEndpoint: string | undefined + constructor() { this.fileName = ""; this.outputPath = ""; @@ -303,5 +306,7 @@ export class ManageFeedClientOptionsEmpty implements ManageFeedClientOptions { this.verbose = false; this.uiPortal = false; this.managePort = 0; + this.aiEnhance = false; + this.aiMapperEndpoint = undefined; } } diff --git a/src/utils/options.ts b/src/utils/options.ts index 1f1b4dd..3f3a6a7 100644 --- a/src/utils/options.ts +++ b/src/utils/options.ts @@ -711,6 +711,9 @@ export const addFeedConfigureOptions = (cmd: Command, advanced: boolean) => { cmd .addOption(new Option('-feed, --feed-name ', chalk.whiteBright('the feed name')) ) .addOption(new Option('-port, --manage-port [PORT]', chalk.whiteBright('the port for the manager')) .argParser(parseManagePort) .default(0)) + // AI enhancement options + .addOption(new Option('-ai, --ai-enhance [BOOLEAN]', chalk.whiteBright('use AI to intelligently enhance field mappings')) .argParser(parseBoolean) .default(false) .hideHelp(advanced)) + .addOption(new Option('--ai-mapper-endpoint ', chalk.whiteBright('[advanced] the AI field mapper endpoint URL')) .hideHelp(!advanced)) // lint options - validate CLI arguments quitely (without executing the command) .addOption(new Option('--lint [BOOLEAN]', chalk.whiteBright('validate CLI arguments quietly (without executing the command)')) .argParser(parseBoolean) .default(false) .hideHelp(true)) } From ac133ed6fcde6914b25dd109013a5f7f98c5eced Mon Sep 17 00:00:00 2001 From: Jamieson Walker Date: Thu, 23 Oct 2025 14:03:38 -0500 Subject: [PATCH 4/6] Fixed bug with AI mapper in cli mode --- src/index.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/index.ts b/src/index.ts index e9a62a7..7443456 100755 --- a/src/index.ts +++ b/src/index.ts @@ -587,6 +587,9 @@ if (process.env.SHOW_VISUALIZATION) { for (var i=0; i Date: Tue, 11 Nov 2025 17:18:58 -0600 Subject: [PATCH 5/6] Fixed AI map removed need for async api spec file and updated the codeflow --- claude.md | 10 +++------ src/lib/feed-configure.ts | 36 ++++---------------------------- src/lib/feed-generate.ts | 3 --- src/utils/field-mapper-client.ts | 15 ++----------- 4 files changed, 9 insertions(+), 55 deletions(-) diff --git a/claude.md b/claude.md index cc88ac0..4822eaf 100644 --- a/claude.md +++ b/claude.md @@ -187,8 +187,7 @@ The Lambda function endpoint should: 1. Accept POST requests with JSON body: ```json { - "feedrules": [...], - "asyncApiSpec": {...} + "feedrules": [...] } ``` @@ -202,11 +201,6 @@ The Lambda function endpoint should: "improved": 7, "unchanged": 3, "improvementPercentage": 70 - }, - "context": { - "eventName": "Order Created", - "topic": "acmeRental/orders/created/v1/{region}/{orderId}", - "domain": "Retail" } } ``` @@ -218,6 +212,8 @@ The Lambda function endpoint should: - Create Topic Parameter mappings for matched fields - Include all mappings in the `mappings` array of each feedrule +**Note**: The AsyncAPI specification is no longer sent to the Lambda endpoint. All necessary information is contained within the feedrules structure itself. + ### Implementation Reference See `src/utils/field-mapper-client.ts` for the client implementation and detailed JSDoc documentation. diff --git a/src/lib/feed-configure.ts b/src/lib/feed-configure.ts index ebeb014..74f5d92 100644 --- a/src/lib/feed-configure.ts +++ b/src/lib/feed-configure.ts @@ -1,5 +1,5 @@ -import { getAllFeeds, getFeed, loadLocalFeedFile, loadLocalFeedSessionSettingsFile, processPlainPath, updateRules, updateSession, loadRawLocalFeedFile, fileExists } from '../utils/config'; -import { defaultFakerRulesFile, defaultFeedInfoFile, defaultFeedSessionFile, defaultProjectName, defaultStmFeedsHome, defaultFeedRulesFile } from '../utils/defaults'; +import { getAllFeeds, getFeed, loadLocalFeedFile, loadLocalFeedSessionSettingsFile, processPlainPath, updateRules, updateSession } from '../utils/config'; +import { defaultFakerRulesFile, defaultFeedInfoFile, defaultFeedSessionFile, defaultProjectName, defaultFeedRulesFile } from '../utils/defaults'; import { getLocalEventFeeds } from '../utils/listfeeds'; import { Logger } from '../utils/logger' import { fakeDataObjectGenerator, fakeDataValueGenerator } from './feed-datahelper'; @@ -9,8 +9,6 @@ import { sessionPropertiesJson } from '../utils/sessionprops'; import { enhanceFeedrulesWithAI } from '../utils/field-mapper-client'; import { hasAcceptedAiDisclaimer, showAiDisclaimer, recordAiDisclaimerAcceptance } from '../utils/ai-disclaimer'; import chalk from 'chalk'; -import * as fs from 'fs'; -import * as path from 'path'; // @ts-ignore import { generateEvent } from '@solace-labs/solace-data-generator'; @@ -95,39 +93,13 @@ const manage = async (options: ManageFeedClientOptions, optionsSource: any) => { // 1. Load existing feedrules const feedrules = loadLocalFeedFile(feedName, defaultFeedRulesFile); - // 2. Find and load the AsyncAPI spec file from the feed directory - const feedPath = processPlainPath(`${defaultStmFeedsHome}/${feedName}`); - const files = fs.readdirSync(feedPath); - - // Filter to find AsyncAPI spec file (exclude known feed files) - const excludedFiles = ['feedrules.json', 'analysis.json', 'fakerrules.json', 'feedinfo.json', 'feedschemas.json', 'feedsession.json']; - const asyncApiFile = files.find(file => { - const ext = path.extname(file).toLowerCase(); - return (ext === '.json' || ext === '.yaml' || ext === '.yml') && !excludedFiles.includes(file); - }); - - if (!asyncApiFile) { - Logger.logError('Could not find AsyncAPI specification file in feed directory'); - Logger.logError('exiting...'); - process.exit(1); - } - - Logger.logInfo(`Found AsyncAPI spec: ${chalk.cyanBright(asyncApiFile)}`); - - // 3. Load the AsyncAPI spec - const asyncApiSpec = loadRawLocalFeedFile(feedName, asyncApiFile); - - // 4. Parse AsyncAPI to object if it's a string - const asyncApiObject = typeof asyncApiSpec === 'string' ? JSON.parse(asyncApiSpec) : asyncApiSpec; - - // 5. Call AI enhancement + // 2. Call AI enhancement const enhancedRules = await enhanceFeedrulesWithAI( feedrules, - asyncApiObject, options.aiMapperEndpoint ); - // 6. Save enhanced rules back if successful + // 3. Save enhanced rules back if successful if (enhancedRules && enhancedRules.length > 0) { await updateRules(feedName, enhancedRules); Logger.logSuccess(`Feed configuration for '${chalk.greenBright(feedName)}' automatically enhanced with AI!`); diff --git a/src/lib/feed-generate.ts b/src/lib/feed-generate.ts index 03b6cc0..3f9388f 100644 --- a/src/lib/feed-generate.ts +++ b/src/lib/feed-generate.ts @@ -176,11 +176,8 @@ const generate = async (options: ManageFeedClientOptions, optionsSource: any) => // Proceed with AI enhancement if still enabled if (options.aiEnhance) { Logger.info('AI enhancement enabled - enhancing field mappings...'); - // Parse the AsyncAPI schema string to an object for the Lambda - const asyncApiObject = typeof asyncApiSchema === 'string' ? JSON.parse(asyncApiSchema) : asyncApiSchema; const enhancedRules = await enhanceFeedrulesWithAI( rules, - asyncApiObject, options.aiMapperEndpoint ); diff --git a/src/utils/field-mapper-client.ts b/src/utils/field-mapper-client.ts index a3f5b46..d87cc03 100644 --- a/src/utils/field-mapper-client.ts +++ b/src/utils/field-mapper-client.ts @@ -36,11 +36,6 @@ export interface FieldMapperResponse { unchanged: number; improvementPercentage: number; }; - context: { - eventName: string; - topic: string; - domain: string; - }; } export interface FieldMapperError { @@ -52,7 +47,7 @@ export interface FieldMapperError { /** * Call the AI Field Mapper Lambda function to enhance feedrules with intelligent field mappings. * - * This function sends feedrules and AsyncAPI specification to an AI-powered Lambda function that: + * This function sends feedrules to an AI-powered Lambda function that: * 1. Analyzes payload field types and names to generate realistic Faker.js rules * 2. Identifies topic variables (e.g., {amount}, {employeeId}) that match payload field names * 3. Creates mappings to use payload-generated values in topic addresses @@ -61,14 +56,12 @@ export interface FieldMapperError { * the Lambda will generate a mapping to inject the payload's orderId value into the topic. * * @param feedrules - The vanilla/generic feedrules to enhance with mappings array - * @param asyncApiSpec - The AsyncAPI specification containing topic and payload schemas * @param endpoint - Optional custom endpoint URL (defaults to hard-coded endpoint or STM_FIELD_MAPPER_ENDPOINT env var) * @returns Enhanced feedrules with mappings array populated, or null on failure * * @example * const enhanced = await enhanceFeedrulesWithAI( * [{ topic: 'acme/{region}/{employeeId}', topicParameters: {...}, payload: {...} }], - * asyncApiSpec, * 'https://my-lambda-url.com' * ); * // Returns feedrules with mappings like: @@ -82,7 +75,6 @@ export interface FieldMapperError { */ export async function enhanceFeedrulesWithAI( feedrules: any[], - asyncApiSpec: any, endpoint?: string ): Promise { const apiEndpoint = endpoint || DEFAULT_FIELD_MAPPER_ENDPOINT; @@ -98,8 +90,7 @@ export async function enhanceFeedrulesWithAI( Logger.logInfo(`Endpoint: ${chalk.cyanBright(apiEndpoint)}`); const requestBody = { - feedrules, - asyncApiSpec + feedrules }; try { @@ -130,8 +121,6 @@ export async function enhanceFeedrulesWithAI( // Log summary Logger.logSuccess('Field mapping completed successfully!'); - Logger.logInfo(`Context: ${chalk.greenBright(result.context.eventName)} in ${chalk.cyanBright(result.context.domain)}`); - Logger.logInfo(`Topic: ${chalk.yellowBright(result.context.topic)}`); Logger.logInfo(`Fields analyzed: ${chalk.whiteBright(result.summary.totalFields)}`); Logger.logInfo(`Improved mappings: ${chalk.greenBright(result.summary.improved)} (${result.summary.improvementPercentage}%)`); Logger.logInfo(`Unchanged: ${chalk.gray(result.summary.unchanged)}`); From 27f2d5e429679d3775dedf50d983c3c83166d161 Mon Sep 17 00:00:00 2001 From: Jamieson Walker Date: Fri, 21 Nov 2025 20:25:08 -0600 Subject: [PATCH 6/6] Fix stm manage queue create to respect queue name from positional argument MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The queue command was not respecting the queue name when provided as a positional argument (e.g., `stm manage queue create MyQueue`). The queue name from the config file was being used instead. Problem: When using positional arguments, the config file was being loaded BEFORE the positional arguments were processed. This meant: 1. Config loaded → sets options.queue from config file 2. Positional args processed → sets options.create to the queue name 3. checkparams.ts correctly sets options.queue from options.create 4. BUT the config loader uses cliOptions to determine what was set via CLI, and since positional args were processed AFTER config loading, cliOptions.create wasn't marked as 'cli' when the config was applied Solution: Moved the positional argument handling to BEFORE the config loading in src/index.ts. This ensures that when the config loader checks cliOptions, it sees that the operation was set via CLI, so it won't override the queue name with the config value. Changes: - src/index.ts: Added positional arguments [operation] and [queueName], moved positional arg processing before config loading - src/utils/checkparams.ts: Minor whitespace cleanup - src/common/queue-client.ts: Minor whitespace cleanup 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- src/common/queue-client.ts | 2 +- src/index.ts | 22 ++++++++++++++++++++-- src/utils/checkparams.ts | 2 +- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/common/queue-client.ts b/src/common/queue-client.ts index f8a02f5..a18b24d 100644 --- a/src/common/queue-client.ts +++ b/src/common/queue-client.ts @@ -33,7 +33,7 @@ export class SempClient { case 'DELETE': sempUrl += `${this.urlFixture}/msgVpns/${this.options.sempVpn}/queues/${encodeURIComponent(this.options.queue)}`; break; } - this.sempBody = { + this.sempBody = { msgVpnName: this.options?.sempVpn, queueName: this.options?.queue, owner: this.options?.owner, diff --git a/src/index.ts b/src/index.ts index e9a62a7..06557ad 100755 --- a/src/index.ts +++ b/src/index.ts @@ -442,23 +442,41 @@ if (process.env.SHOW_VISUALIZATION) { const manageQueueCmd = manageCmd .command('queue') .description(chalk.whiteBright('Manage a queue')) + .argument('[operation]', 'queue operation: create, update, delete, or list') + .argument('[queueName]', 'queue name') .allowUnknownOption(false) .addHelpText('after', this.newVersionCheck()) addManageQueueOptions(manageQueueCmd, this.advanced); - manageQueueCmd.action((options: ManageClientOptions) => { + manageQueueCmd.action((operation: string | undefined, queueName: string | undefined, options: ManageClientOptions) => { this.newVersionCheck(); const cliOptions:any = {}; const defaultKeys = Object.keys(new ManageClientOptionsEmpty('queue')); for (var i=0; i