From 988507ac8ef7438f5d09176c1ae637e2a80b49ab Mon Sep 17 00:00:00 2001 From: Ale Pouroullis Date: Tue, 20 May 2025 11:16:33 +0100 Subject: [PATCH 01/16] Merge custom code from old p_sync/pull branch with latest autogenerated SDK --- .fernignore | 5 + src/cache/LRUCache.ts | 48 ++ src/cache/index.ts | 1 + src/cli.ts | 69 +++ src/humanloop.client.ts | 47 ++ src/sync/SyncClient.ts | 311 +++++++++++ src/sync/index.ts | 7 + src/utils/Logger.ts | 106 ++++ src/utils/index.ts | 1 + tests/custom.test.ts | 13 - tests/custom/integration/decorators.test.ts | 502 +++++++++++++++++ tests/custom/integration/evals.test.ts | 577 ++++++++++++++++++++ tests/custom/integration/fixtures.ts | 246 +++++++++ tests/custom/unit/LRUCache.test.ts | 61 +++ tests/custom/unit/Logger.test.ts | 79 +++ 15 files changed, 2060 insertions(+), 13 deletions(-) create mode 100644 src/cache/LRUCache.ts create mode 100644 src/cache/index.ts create mode 100644 src/cli.ts create mode 100644 src/sync/SyncClient.ts create mode 100644 src/sync/index.ts create mode 100644 src/utils/Logger.ts create mode 100644 src/utils/index.ts delete mode 100644 tests/custom.test.ts create mode 100644 tests/custom/integration/decorators.test.ts create mode 100644 tests/custom/integration/evals.test.ts create mode 100644 tests/custom/integration/fixtures.ts create mode 100644 tests/custom/unit/LRUCache.test.ts create mode 100644 tests/custom/unit/Logger.test.ts diff --git a/.fernignore b/.fernignore index 19625584..c24f34a4 100644 --- a/.fernignore +++ b/.fernignore @@ -10,11 +10,16 @@ src/humanloop.client.ts src/overload.ts src/error.ts src/context.ts +src/cli.ts +src/cache +src/sync +src/utils # Tests # Modified due to issues with OTEL tests/unit/fetcher/stream-wrappers/webpack.test.ts +tests/custom/ # CI Action diff --git a/src/cache/LRUCache.ts b/src/cache/LRUCache.ts new file mode 100644 index 00000000..30d0877b --- /dev/null +++ b/src/cache/LRUCache.ts @@ -0,0 +1,48 @@ +/** + * LRU Cache implementation + */ +export default class LRUCache { + private cache: Map; + private readonly maxSize: number; + + constructor(maxSize: number) { + this.cache = new Map(); + this.maxSize = maxSize; + } + + get(key: K): V | undefined { + if (!this.cache.has(key)) { + return undefined; + } + + // Get the value + const value = this.cache.get(key); + + // Remove key and re-insert to mark as most recently used + this.cache.delete(key); + this.cache.set(key, value!); + + return value; + } + + set(key: K, value: V): void { + // If key already exists, refresh its position + if (this.cache.has(key)) { + this.cache.delete(key); + } + // If cache is full, remove the least recently used item (first item in the map) + else if (this.cache.size >= this.maxSize) { + const lruKey = this.cache.keys().next().value; + if (lruKey) { + this.cache.delete(lruKey); + } + } + + // Add new key-value pair + this.cache.set(key, value); + } + + clear(): void { + this.cache.clear(); + } +} diff --git a/src/cache/index.ts b/src/cache/index.ts new file mode 100644 index 00000000..d440ec99 --- /dev/null +++ b/src/cache/index.ts @@ -0,0 +1 @@ +export { default as LRUCache } from './LRUCache'; \ No newline at end of file diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 00000000..b80d1c94 --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,69 @@ +#!/usr/bin/env node +import * as dotenv from "dotenv"; +import { Command } from "commander"; + +import { HumanloopClient } from "./humanloop.client"; +import Logger from "./utils/Logger"; + +const { version } = require("../package.json"); + +// Load environment variables +dotenv.config(); + +const program = new Command(); +program + .name("humanloop") + .description("Humanloop CLI for managing sync operations") + .version(version); + +// Common auth options +const addAuthOptions = (command: Command) => + command + .option("--api-key ", "Humanloop API key") + .option("--env-file ", "Path to .env file") + .option("--base-url ", "Base URL for Humanloop API"); + +// Helper to get client +function getClient(options: { + envFile?: string; + apiKey?: string; + baseUrl?: string; + baseDir?: string; +}): HumanloopClient { + if (options.envFile) dotenv.config({ path: options.envFile }); + const apiKey = options.apiKey || process.env.HUMANLOOP_API_KEY; + if (!apiKey) { + Logger.error( + "No API key found. Set HUMANLOOP_API_KEY in .env file or use --api-key", + ); + process.exit(1); + } + return new HumanloopClient({ + apiKey, + baseUrl: options.baseUrl, + sync: { baseDir: options.baseDir }, + }); +} + +// Pull command +addAuthOptions( + program + .command("pull") + .description("Pull files from Humanloop to local filesystem") + .option("-p, --path ", "Path to pull (file or directory)") + .option("-e, --environment ", "Environment to pull from") + .option("--base-dir ", "Base directory for synced files", "humanloop"), +).action(async (options) => { + Logger.info("Pulling files from Humanloop..."); + // try { + // Logger.info("Pulling files from Humanloop..."); + // const client = getClient(options); + // const files = await client.pull(options.path, options.environment); + // Logger.success(`Successfully synced ${files.length} files`); + // } catch (error) { + // Logger.error(`Error: ${error}`); + // process.exit(1); + // } +}); + +program.parse(process.argv); diff --git a/src/humanloop.client.ts b/src/humanloop.client.ts index ac7cb197..28c3070e 100644 --- a/src/humanloop.client.ts +++ b/src/humanloop.client.ts @@ -29,6 +29,7 @@ import { import { HumanloopSpanExporter } from "./otel/exporter"; import { HumanloopSpanProcessor } from "./otel/processor"; import { overloadCall, overloadLog } from "./overload"; +import { SyncClient, SyncClientOptions } from "./sync"; import { SDK_VERSION } from "./version"; const RED = "\x1b[91m"; @@ -210,6 +211,7 @@ export class HumanloopClient extends BaseHumanloopClient { Anthropic?: any; CohereAI?: any; }; + protected readonly _syncClient: SyncClient; protected get opentelemetryTracer(): Tracer { return HumanloopTracerSingleton.getInstance({ @@ -250,10 +252,13 @@ export class HumanloopClient extends BaseHumanloopClient { Anthropic?: any; CohereAI?: any; }; + sync?: SyncClientOptions; }, ) { super(_options); + this._syncClient = new SyncClient(this, _options.sync); + this.instrumentProviders = _options.instrumentProviders || {}; this._prompts_overloaded = overloadLog(super.prompts); @@ -560,6 +565,48 @@ ${RESET}`, ); } + /** + * Pull Prompt and Agent files from Humanloop to local filesystem. + * + * This method will: + * 1. Fetch Prompt and Agent files from your Humanloop workspace + * 2. Save them to the local filesystem using the client's files_directory (set during initialization) + * 3. Maintain the same directory structure as in Humanloop + * 4. Add appropriate file extensions (.prompt or .agent) + * + * The path parameter can be used in two ways: + * - If it points to a specific file (e.g. "path/to/file.prompt" or "path/to/file.agent"), only that file will be pulled + * - If it points to a directory (e.g. "path/to/directory"), all Prompt and Agent files in that directory will be pulled + * - If no path is provided, all Prompt and Agent files will be pulled + * + * The operation will overwrite existing files with the latest version from Humanloop + * but will not delete local files that don't exist in the remote workspace. + * + * Currently only supports syncing prompt and agent files. Other file types will be skipped. + * + * The files will be saved with the following structure: + * ``` + * {files_directory}/ + * ├── prompts/ + * │ ├── my_prompt.prompt + * │ └── nested/ + * │ └── another_prompt.prompt + * └── agents/ + * └── my_agent.agent + * ``` + * + * @param path - Optional path to either a specific file (e.g. "path/to/file.prompt") or a directory (e.g. "path/to/directory"). + * If not provided, all Prompt and Agent files will be pulled. + * @param environment - The environment to pull the files from. + * @returns List of successfully processed file paths. + */ + public async pull( + path?: string, + environment?: string, + ): Promise { + return this._syncClient.pull(path, environment); + } + public get evaluations(): ExtendedEvaluations { return this._evaluations; } diff --git a/src/sync/SyncClient.ts b/src/sync/SyncClient.ts new file mode 100644 index 00000000..df9ec0f5 --- /dev/null +++ b/src/sync/SyncClient.ts @@ -0,0 +1,311 @@ +import { FileType } from "api"; +import fs from "fs"; +import path from "path"; + +import { HumanloopClient as BaseHumanloopClient } from "../Client"; +import LRUCache from "../cache/LRUCache"; +import { HumanloopRuntimeError } from "../error"; +import Logger, { LogLevel } from "../utils/Logger"; // Import your existing Logger + +// Default cache size for file content caching +const DEFAULT_CACHE_SIZE = 100; + +export interface SyncClientOptions { + baseDir?: string; + cacheSize?: number; + logLevel?: LogLevel; +} + +/** + * Format API error messages to be more user-friendly. + */ +function formatApiError(error: Error): string { + const errorMsg = error.toString(); + + // If the error doesn't look like an API error with status code and body + if (!errorMsg.includes("status_code") || !errorMsg.includes("body:")) { + return errorMsg; + } + + try { + // Extract the body part and parse as JSON + const bodyParts = errorMsg.split("body:"); + if (bodyParts.length < 2) return errorMsg; + + const bodyStr = bodyParts[1].trim(); + const body = JSON.parse(bodyStr); + + // Get the detail from the body + const detail = body.detail || {}; + + // Prefer description, fall back to msg + return detail.description || detail.msg || errorMsg; + } catch (e) { + Logger.debug(`Failed to parse error message: ${e}`); // Use debug level + return errorMsg; + } +} + +/** + * Client for managing synchronization between local filesystem and Humanloop. + * + * This client provides file synchronization between Humanloop and the local filesystem, + * with built-in caching for improved performance. + */ +export default class SyncClient { + private client: BaseHumanloopClient; + private baseDir: string; + private cacheSize: number; + private fileContentCache: LRUCache; + + constructor( + client: BaseHumanloopClient, + options: SyncClientOptions = {} + ) { + this.client = client; + this.baseDir = options.baseDir || "humanloop"; + this.cacheSize = options.cacheSize || DEFAULT_CACHE_SIZE; + this.fileContentCache = new LRUCache(this.cacheSize); + + // Set the log level using your Logger's setLevel method + Logger.setLevel(options.logLevel || 'warn'); + } + + /** + * Get the file content from cache or filesystem. + */ + public getFileContent(filePath: string, fileType: FileType): string { + const cacheKey = `${filePath}:${fileType}`; + + // Check if in cache + const cachedContent = this.fileContentCache.get(cacheKey); + if (cachedContent !== undefined) { + // Use debug level for cache hits + Logger.debug(`Using cached file content for ${filePath}.${fileType}`); + return cachedContent; + } + + // Not in cache, get from filesystem + const localPath = path.join(this.baseDir, `${filePath}.${fileType}`); + + if (!fs.existsSync(localPath)) { + throw new HumanloopRuntimeError(`Local file not found: ${localPath}`); + } + + try { + const fileContent = fs.readFileSync(localPath, 'utf8'); + Logger.debug(`Using local file content from ${localPath}`); + + // Add to cache + this.fileContentCache.set(cacheKey, fileContent); + + return fileContent; + } catch (error) { + throw new HumanloopRuntimeError( + `Error reading local file ${localPath}: ${error}` + ); + } + } + + /** + * Clear the cache. + */ + public clearCache(): void { + this.fileContentCache.clear(); + } + + /** + * Normalize the path by removing extensions, etc. + */ + private normalizePath(filePath: string): string { + if (!filePath) return ""; + + // Remove any file extensions + let normalizedPath = filePath.includes(".") + ? filePath.substring(0, filePath.lastIndexOf(".")) + : filePath; + + // Convert backslashes to forward slashes + normalizedPath = normalizedPath.replace(/\\/g, "/"); + + // Remove leading/trailing whitespace and slashes + normalizedPath = normalizedPath.trim().replace(/^\/+|\/+$/g, ""); + + // Normalize multiple consecutive slashes into a single forward slash + while (normalizedPath.includes("//")) { + normalizedPath = normalizedPath.replace(/\/\//g, "/"); + } + + return normalizedPath; + } + + /** + * Check if the path is a file by checking for .prompt or .agent extension. + */ + private isFile(path: string): boolean { + return path.trim().endsWith(".prompt") || path.trim().endsWith(".agent"); + } + + /** + * Save serialized file to local filesystem. + */ + private saveSerializedFile( + serializedContent: string, + filePath: string, + fileType: FileType + ): void { + try { + // Create full path including baseDir prefix + const fullPath = path.join(this.baseDir, filePath); + const directory = path.dirname(fullPath); + const fileName = path.basename(fullPath); + + // Create directory if it doesn't exist + fs.mkdirSync(directory, { recursive: true }); + + // Add file type extension + const newPath = path.join(directory, `${fileName}.${fileType}`); + + // Write raw file content to file + fs.writeFileSync(newPath, serializedContent); + + // Clear the cache for this file to ensure we get fresh content next time + this.clearCache(); + } catch (error) { + Logger.error(`Failed to sync ${fileType} ${filePath}: ${error}`); + throw error; + } + } + + /** + * Pull a specific file from Humanloop to local filesystem. + */ + private async pullFile(path: string, environment?: string): Promise { + const file = await this.client.files.retrieveByPath({ + path, + environment, + includeRawFileContent: true, + }); + + if (file.type !== "prompt" && file.type !== "agent") { + throw new Error(`Unsupported file type: ${file.type}`); + } + + this.saveSerializedFile(file.rawFileContent!, file.path, file.type); + } + + /** + * Pull all files from a directory in Humanloop to local filesystem. + */ + private async pullDirectory( + path?: string, + environment?: string, + ): Promise { + const successfulFiles: string[] = []; + const failedFiles: string[] = []; + let page = 1; + + Logger.debug(`Fetching files from directory: ${path || '(root)'} in environment: ${environment || '(default)'}`); + + while (true) { + try { + Logger.debug(`Requesting page ${page} of files`); + + const response = await this.client.files.listFiles({ + type: ["prompt", "agent"], + page, + includeRawFileContent: true, + environment, + path, + }); + + if (response.records.length === 0) { + Logger.debug("No more files found"); + break; + } + + Logger.debug(`Found ${response.records.length} files from page ${page}`); + + // Process each file + for (const file of response.records) { + // Skip if not a Prompt or Agent + if (file.type !== "prompt" && file.type !== "agent") { + Logger.warn(`Skipping unsupported file type: ${file.type}`); + continue; + } + + // Skip if no raw file content + if (!file.rawFileContent) { + Logger.warn(`No content found for ${file.type} ${file.id || ""}`); + continue; + } + + try { + Logger.debug(`Saving ${file.type} ${file.path}`); + + this.saveSerializedFile( + file.rawFileContent, + file.path, + file.type, + ); + successfulFiles.push(file.path); + } catch (error) { + failedFiles.push(file.path); + Logger.error(`Task failed for ${file.path}: ${error}`); + } + } + + page += 1; + } catch (error) { + const formattedError = formatApiError(error as Error); + throw new HumanloopRuntimeError( + `Failed to pull files: ${formattedError}` + ); + } + } + + if (successfulFiles.length > 0) { + Logger.info(`Successfully pulled ${successfulFiles.length} files`); + } + if (failedFiles.length > 0) { + Logger.warn(`Failed to pull ${failedFiles.length} files`); + } + + return successfulFiles; + } + + /** + * Pull files from Humanloop to local filesystem. + */ + public async pull(path?: string, environment?: string): Promise { + const startTime = Date.now(); + const normalizedPath = path ? this.normalizePath(path) : undefined; + + Logger.info(`Starting pull operation: path=${normalizedPath || '(root)'}, environment=${environment || '(default)'}`); + + let successfulFiles: string[] = []; + + if (!path) { + // Pull all files from the root + Logger.debug("Pulling all files from root"); + successfulFiles = await this.pullDirectory(undefined, environment); + } else { + if (this.isFile(path)) { + Logger.debug(`Pulling specific file: ${normalizedPath}`); + await this.pullFile(normalizedPath!, environment); + successfulFiles = [path]; + } else { + Logger.debug(`Pulling directory: ${normalizedPath}`); + successfulFiles = await this.pullDirectory( + normalizedPath, + environment, + ); + } + } + + const duration = Date.now() - startTime; + Logger.success(`Pull completed in ${duration}ms: ${successfulFiles.length} files succeeded`); + + return successfulFiles; + } +} \ No newline at end of file diff --git a/src/sync/index.ts b/src/sync/index.ts new file mode 100644 index 00000000..252b4ea8 --- /dev/null +++ b/src/sync/index.ts @@ -0,0 +1,7 @@ +/** + * File synchronization for Humanloop + * + * This module provides sync functionality between Humanloop and the local filesystem. + */ + +export { default as SyncClient, SyncClientOptions } from './SyncClient'; \ No newline at end of file diff --git a/src/utils/Logger.ts b/src/utils/Logger.ts new file mode 100644 index 00000000..3c74a47b --- /dev/null +++ b/src/utils/Logger.ts @@ -0,0 +1,106 @@ +/** + * Logger utility for consistent colored console output across the Humanloop SDK. + */ + +// ANSI escape codes for colors +export const Colors = { + YELLOW: "\x1b[93m", + CYAN: "\x1b[96m", + GREEN: "\x1b[92m", + RED: "\x1b[91m", + RESET: "\x1b[0m", +} as const; + +export type LogLevel = 'error' | 'warn' | 'info' | 'debug'; + +/** + * Helper class for colored console output with log level filtering + */ +export default class Logger { + private static currentLevel: number = 1; // Default to 'warn' + private static readonly levels: Record = { + 'error': 0, + 'warn': 1, + 'info': 2, + 'debug': 3 + }; + + /** + * Set the log level for filtering + */ + static setLevel(level: LogLevel): void { + this.currentLevel = this.levels[level] || 1; + } + + /** + * Safely converts any value to a string, handling undefined/null + */ + private static toString(value: any): string { + if (value === undefined) return "undefined"; + if (value === null) return "null"; + return String(value); + } + + /** + * Log a warning message in yellow + */ + static warn(message: any): void { + if (this.currentLevel >= 1) { + console.warn(`${Colors.YELLOW}${Logger.toString(message)}${Colors.RESET}`); + } + } + + /** + * Log an info message in cyan + */ + static info(message: any): void { + if (this.currentLevel >= 2) { + console.info(`${Colors.CYAN}${Logger.toString(message)}${Colors.RESET}`); + } + } + + /** + * Log a success message in green + */ + static success(message: any): void { + if (this.currentLevel >= 2) { // Success is info level + console.log(`${Colors.GREEN}${Logger.toString(message)}${Colors.RESET}`); + } + } + + /** + * Log an error message in red + */ + static error(message: any): void { + if (this.currentLevel >= 0) { + console.error(`${Colors.RED}${Logger.toString(message)}${Colors.RESET}`); + } + } + + /** + * Log a plain message without any color (at info level) + */ + static log(message: any): void { + if (this.currentLevel >= 2) { + console.log(Logger.toString(message)); + } + } + + /** + * Log a debug message (for detailed information) + */ + static debug(message: any): void { + if (this.currentLevel >= 3) { + console.debug(Logger.toString(message)); + } + } + + /** + * Log a message with custom color (at info level) + */ + static withColor(message: any, color: keyof typeof Colors): void { + if (this.currentLevel >= 2) { + console.log(`${Colors[color]}${Logger.toString(message)}${Colors.RESET}`); + } + } +} \ No newline at end of file diff --git a/src/utils/index.ts b/src/utils/index.ts new file mode 100644 index 00000000..a20b6187 --- /dev/null +++ b/src/utils/index.ts @@ -0,0 +1 @@ +export * from "./Logger"; \ No newline at end of file diff --git a/tests/custom.test.ts b/tests/custom.test.ts deleted file mode 100644 index 7f5e031c..00000000 --- a/tests/custom.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -/** - * This is a custom test file, if you wish to add more tests - * to your SDK. - * Be sure to mark this file in `.fernignore`. - * - * If you include example requests/responses in your fern definition, - * you will have tests automatically generated for you. - */ -describe("test", () => { - it("default", () => { - expect(true).toBe(true); - }); -}); diff --git a/tests/custom/integration/decorators.test.ts b/tests/custom/integration/decorators.test.ts new file mode 100644 index 00000000..0cddc948 --- /dev/null +++ b/tests/custom/integration/decorators.test.ts @@ -0,0 +1,502 @@ +import OpenAI from "openai"; + +import { PromptRequest } from "../../src/api"; +import { HumanloopRuntimeError } from "../../src/error"; +import { + CleanupResources, + TestPrompt, + TestSetup, + cleanupTestEnvironment, + setupTestEnvironment, +} from "./fixtures"; + +// Long timeout per test +jest.setTimeout(30 * 1000); + +// process.stdout.moveCursor is undefined in jest; mocking it since STDOUT is not relevant +if (typeof process.stdout.moveCursor !== "function") { + process.stdout.moveCursor = ( + dx: number, + dy: number, + callback?: () => void, + ): boolean => { + if (callback) callback(); + return true; + }; +} + +/** + * Creates a test prompt in the specified test environment + */ +async function createTestPrompt( + setup: TestSetup, + name: string = "test_prompt", + customConfig?: Partial, +): Promise { + const promptPath = `${setup.sdkTestDir.path}/${name}`; + const config = customConfig + ? { ...setup.testPromptConfig, ...customConfig } + : setup.testPromptConfig; + + const promptResponse = await setup.humanloopClient.prompts.upsert({ + path: promptPath, + ...config, + }); + + return { + id: promptResponse.id, + path: promptPath, + response: promptResponse, + }; +} + +/** + * Creates a base function for LLM calls that can be decorated + */ +function createBaseLLMFunction(setup: TestSetup, model: string = "gpt-4o-mini") { + return async (question: string): Promise => { + const openaiClient = new OpenAI({ apiKey: setup.openaiApiKey }); + + const response = await openaiClient.chat.completions.create({ + model: model, + messages: [{ role: "user", content: question }], + }); + + return response.choices[0].message.content || ""; + }; +} + +/** + * Applies the prompt decorator to a function and tests it + */ +async function testPromptDecorator( + setup: TestSetup, + prompt: TestPrompt, + input: string = "What is the capital of the France?", + expectedSubstring: string = "paris", +): Promise { + // Create the base function + const myPromptBase = createBaseLLMFunction(setup); + + // Apply the higher-order function instead of decorator + const myPrompt = setup.humanloopClient.prompt({ + path: prompt.path, + callable: myPromptBase, + }); + + // Call the decorated function + const result = await myPrompt(input); + if (result) { + expect(result.toLowerCase()).toContain(expectedSubstring.toLowerCase()); + } else { + throw new Error("Expected result to be defined"); + } + + // Wait for 5 seconds for the log to be created + await new Promise((resolve) => setTimeout(resolve, 5000)); +} + +describe("decorators", () => { + it("should create a prompt log when using the decorator", async () => { + let testSetup: TestSetup | undefined = undefined; + let testPrompt: TestPrompt | undefined = undefined; + + try { + testSetup = await setupTestEnvironment("test_prompt_call_decorator"); + // Create test prompt + testPrompt = await createTestPrompt(testSetup); + + // Check initial version count + const promptVersionsResponse = + await testSetup.humanloopClient.prompts.listVersions(testPrompt.id); + expect(promptVersionsResponse.records.length).toBe(1); + + // Test the prompt decorator + await testPromptDecorator(testSetup, testPrompt); + + // Verify a new version was created + const updatedPromptVersionsResponse = + await testSetup.humanloopClient.prompts.listVersions(testPrompt.id); + expect(updatedPromptVersionsResponse.records.length).toBe(2); + + // Verify logs were created + const logsResponse = await testSetup.humanloopClient.logs.list({ + fileId: testPrompt.id, + page: 1, + size: 50, + }); + expect(logsResponse.data.length).toBe(1); + } catch (error) { + // Make sure to clean up if the test fails + const cleanupResources: CleanupResources[] = []; + if (testPrompt) { + cleanupResources.push({ + type: "prompt", + id: testPrompt.id, + }); + } + if (testSetup) { + await cleanupTestEnvironment(testSetup, cleanupResources); + } + throw error; + } + }); + + it("should create logs with proper tracing when using prompt in flow decorator", async () => { + let testSetup: TestSetup | undefined = undefined; + let flowId: string | null = null; + let promptId: string | null = null; + + try { + // Create test flow and prompt paths + testSetup = await setupTestEnvironment("test_flow_decorator"); + const flowPath = `${testSetup.sdkTestDir.path}/test_flow`; + const promptPath = `${testSetup.sdkTestDir.path}/test_prompt`; + + // Create the prompt + const promptResponse = await testSetup.humanloopClient.prompts.upsert({ + path: promptPath, + provider: "openai", + model: "gpt-4o-mini", + temperature: 0, + }); + const promptId = promptResponse.id; + + // Define the flow callable function with the correct type signature + const flowCallable = async (question: { + question: string; + }): Promise => { + const response = await testSetup!.humanloopClient.prompts.call({ + path: promptPath, + messages: [{ role: "user", content: question.question }], + providerApiKeys: { openai: testSetup!.openaiApiKey }, + }); + + const output = response.logs?.[0]?.output; + expect(output).not.toBeNull(); + return output || ""; + }; + + // Apply the flow decorator + const myFlow = testSetup.humanloopClient.flow({ + path: flowPath, + callable: flowCallable, + }); + + // Call the flow with the expected input format + const result = await myFlow({ + question: "What is the capital of the France?", + }); + expect(result?.toLowerCase()).toContain("paris"); + + // Wait for logs to be created + await new Promise((resolve) => setTimeout(resolve, 5000)); + + // Verify prompt logs + const promptRetrieveResponse = + await testSetup.humanloopClient.files.retrieveByPath({ + path: promptPath, + }); + expect(promptRetrieveResponse).not.toBeNull(); + const promptLogsResponse = await testSetup.humanloopClient.logs.list({ + fileId: promptRetrieveResponse.id, + page: 1, + size: 50, + }); + expect(promptLogsResponse.data.length).toBe(1); + const promptLog = promptLogsResponse.data[0]; + + // Verify flow logs + const flowRetrieveResponse = + await testSetup.humanloopClient.files.retrieveByPath({ + path: flowPath, + }); + expect(flowRetrieveResponse).not.toBeNull(); + flowId = flowRetrieveResponse.id; + const flowLogsResponse = await testSetup.humanloopClient.logs.list({ + fileId: flowRetrieveResponse.id, + page: 1, + size: 50, + }); + expect(flowLogsResponse.data.length).toBe(1); + const flowLog = flowLogsResponse.data[0]; + + // Verify tracing between logs + expect(promptLog.traceParentId).toBe(flowLog.id); + } finally { + // Clean up resources + const cleanupResources: CleanupResources[] = []; + if (flowId) { + cleanupResources.push({ + type: "flow", + id: flowId, + }); + } + if (promptId) { + cleanupResources.push({ + type: "prompt", + id: promptId, + }); + } + if (testSetup) { + await cleanupTestEnvironment(testSetup, cleanupResources); + } + } + }); + + it("should log exceptions when using the flow decorator", async () => { + let testSetup: TestSetup | undefined = undefined; + let flowId: string | null = null; + + try { + // Create test flow path + testSetup = await setupTestEnvironment("test_flow_decorator"); + const flowPath = `${testSetup.sdkTestDir.path}/test_flow_log_error`; + + // Define a flow callable that throws an error + const flowCallable = async ({ + question, + }: { + question: string; + }): Promise => { + throw new Error("This is a test exception"); + }; + + // Apply the flow decorator + const myFlow = testSetup.humanloopClient.flow({ + path: flowPath, + callable: flowCallable, + }); + + // Call the flow and expect it to throw + try { + await myFlow({ question: "test" }); + // If we get here, the test should fail + throw new Error("Expected flow to throw an error but it didn't"); + } catch (error) { + // Expected error + expect(error).toBeDefined(); + } + + // Wait for logs to be created + await new Promise((resolve) => setTimeout(resolve, 5000)); + + // Verify flow logs + const flowRetrieveResponse = + await testSetup.humanloopClient.files.retrieveByPath({ + path: flowPath, + }); + expect(flowRetrieveResponse).not.toBeNull(); + flowId = flowRetrieveResponse.id; + + const flowLogsResponse = await testSetup.humanloopClient.logs.list({ + fileId: flowRetrieveResponse.id, + page: 1, + size: 50, + }); + expect(flowLogsResponse.data.length).toBe(1); + + const flowLog = flowLogsResponse.data[0]; + expect(flowLog.error).not.toBeUndefined(); + expect(flowLog.output).toBeUndefined(); + } finally { + if (testSetup) { + await cleanupTestEnvironment( + testSetup, + flowId + ? [ + { + type: "flow", + id: flowId, + }, + ] + : [], + ); + } + } + }); + + it("should populate outputMessage when flow returns chat message format", async () => { + let testSetup: TestSetup | undefined = undefined; + let flowId: string | null = null; + + try { + // Create test flow path + testSetup = await setupTestEnvironment("test_flow_decorator"); + const flowPath = `${testSetup.sdkTestDir.path}/test_flow_log_output_message`; + + // Define a flow callable that returns a chat message format + const flowCallable = async ({ question }: { question: string }) => { + return { + role: "user", + content: question, + }; + }; + + // Apply the flow decorator + const myFlow = testSetup.humanloopClient.flow({ + path: flowPath, + callable: flowCallable, + }); + + // Call the flow and check the returned message + const result = await myFlow({ + question: "What is the capital of the France?", + }); + expect(result?.content.toLowerCase()).toContain("france"); + + // Wait for logs to be created + await new Promise((resolve) => setTimeout(resolve, 5000)); + + // Verify flow logs + const flowRetrieveResponse = + await testSetup.humanloopClient.files.retrieveByPath({ + path: flowPath, + }); + expect(flowRetrieveResponse).not.toBeNull(); + flowId = flowRetrieveResponse.id; + + const flowLogsResponse = await testSetup.humanloopClient.logs.list({ + fileId: flowRetrieveResponse.id, + page: 1, + size: 50, + }); + expect(flowLogsResponse.data.length).toBe(1); + + const flowLog = flowLogsResponse.data[0]; + expect(flowLog.outputMessage).not.toBeUndefined(); + expect(flowLog.output).toBeUndefined(); + expect(flowLog.error).toBeUndefined(); + } finally { + // Clean up resources + if (flowId) { + await testSetup!.humanloopClient.flows.delete(flowId); + } + if (testSetup) { + await cleanupTestEnvironment( + testSetup, + flowId + ? [ + { + type: "flow", + id: flowId, + }, + ] + : [], + ); + } + } + }); + + it("should run evaluations on a flow decorator", async () => { + let testSetup: TestSetup | undefined = undefined; + let flowId: string | null = null; + + try { + // Use fixtures from testSetup + testSetup = await setupTestEnvironment("eval-flow-decorator"); + if (!testSetup.evalDataset || !testSetup.outputNotNullEvaluator) { + throw new Error("Required fixtures are not initialized"); + } + + // Create test flow path + const flowPath = `${testSetup.sdkTestDir.path}/test_flow_evaluate`; + + // Define flow decorated function + const myFlow = testSetup.humanloopClient.flow({ + path: flowPath, + callable: async (inputs: { question: string }) => { + return "paris"; + }, + }); + + // Run evaluation on the flow + await testSetup.humanloopClient.evaluations.run({ + name: "Evaluate Flow Decorator", + file: { + path: flowPath, + callable: myFlow, + type: "flow", + }, + dataset: { + path: testSetup.evalDataset.path, + }, + evaluators: [ + { + path: testSetup.outputNotNullEvaluator.path, + }, + ], + }); + + // Get the flow ID for cleanup + const flowResponse = await testSetup.humanloopClient.files.retrieveByPath({ + path: flowPath, + }); + flowId = flowResponse.id; + } finally { + if (testSetup) { + await cleanupTestEnvironment( + testSetup, + flowId + ? [ + { + type: "flow", + id: flowId, + }, + ] + : [], + ); + } + } + }); + + it("should throw error when using non-existent file ID instead of path", async () => { + // Use fixtures from testSetup + let testSetup: TestSetup | undefined = undefined; + try { + testSetup = await setupTestEnvironment("eval-flow-decorator"); + if (!testSetup.evalDataset || !testSetup.outputNotNullEvaluator) { + throw new Error("Required fixtures are not initialized"); + } + // Define a simple callable + const simpleCallable = (x: any) => x; + + // Expect the evaluation to throw an error with a non-existent file ID + try { + await testSetup.humanloopClient.evaluations.run({ + name: "Evaluate Flow Decorator", + file: { + id: "non-existent-file-id", + type: "flow", + version: { + attributes: { + foo: "bar", + }, + }, + callable: simpleCallable, + }, + dataset: { + path: testSetup.evalDataset.path, + }, + evaluators: [ + { + path: testSetup.outputNotNullEvaluator.path, + }, + ], + }); + + // If we get here, the test should fail + throw new Error("Expected HumanloopRuntimeError but none was thrown"); + } catch (error) { + expect(error).toBeInstanceOf(HumanloopRuntimeError); + expect((error as HumanloopRuntimeError).message).toContain( + "File does not exist on Humanloop. Please provide a `file.path` and a version to create a new version.", + ); + } + } finally { + if (testSetup) { + await cleanupTestEnvironment(testSetup); + } + } + }); +}); diff --git a/tests/custom/integration/evals.test.ts b/tests/custom/integration/evals.test.ts new file mode 100644 index 00000000..09e3b9bf --- /dev/null +++ b/tests/custom/integration/evals.test.ts @@ -0,0 +1,577 @@ +import { FlowResponse } from "../../../src/api"; +import { HumanloopRuntimeError } from "../../../src/error"; +import { HumanloopClient } from "../../../src/humanloop.client"; +import { + cleanupTestEnvironment, + readEnvironment, + setupTestEnvironment, +} from "./fixtures"; + +// process.stdout.moveCursor is undefined in jest; mocking it since STDOUT is not relevant +if (typeof process.stdout.moveCursor !== "function") { + process.stdout.moveCursor = ( + dx: number, + dy: number, + callback?: () => void, + ): boolean => { + if (callback) callback(); + return true; + }; +} + +// Long timeout per test; evals might take a while to run +jest.setTimeout(30 * 1000); + +interface TestIdentifiers { + id: string; + path: string; +} + +interface TestSetup { + sdkTestDir: TestIdentifiers; + outputNotNullEvaluator: TestIdentifiers; + evalDataset: TestIdentifiers; + evalPrompt: TestIdentifiers; + stagingEnvironmentId: string; +} + +describe("Evals", () => { + let humanloopClient: HumanloopClient; + let openaiApiKey: string; + + beforeAll(async () => { + readEnvironment(); + if (!process.env.HUMANLOOP_API_KEY) { + throw new Error("HUMANLOOP_API_KEY is not set"); + } + if (!process.env.OPENAI_API_KEY) { + throw new Error("OPENAI_API_KEY is not set for integration tests"); + } + openaiApiKey = process.env.OPENAI_API_KEY; + humanloopClient = new HumanloopClient({ + apiKey: process.env.HUMANLOOP_API_KEY, + }); + }); + + it("should be able to import HumanloopClient", async () => { + const client = new HumanloopClient({ apiKey: process.env.HUMANLOOP_API_KEY }); + expect(client).toBeDefined(); + }); + + it("should run evaluation on online files", async () => { + // Setup test-specific environment + const setup = await setupTestEnvironment("online_files"); + + try { + await humanloopClient.evaluations.run({ + file: { + path: setup.evalPrompt.path, + type: "prompt", + }, + dataset: { + path: setup.evalDataset.path, + }, + name: "test_eval_run", + evaluators: [ + { + path: setup.outputNotNullEvaluator.path, + }, + ], + }); + + // Wait for evaluation to complete + await new Promise((resolve) => setTimeout(resolve, 5000)); + + const evalResponse = await humanloopClient.evaluations.list({ + fileId: setup.evalPrompt.id, + }); + expect(evalResponse.data.length).toBe(1); + + const evaluationId = evalResponse.data[0].id; + const runsResponse = + await humanloopClient.evaluations.listRunsForEvaluation(evaluationId); + expect(runsResponse.runs[0].status).toBe("completed"); + } finally { + // Clean up test-specific resources + await cleanupTestEnvironment(setup); + } + }); + + it("should run evaluation with version_id", async () => { + // Setup test-specific environment + const setup = await setupTestEnvironment("version_id"); + + try { + // Create a new prompt version + const newPromptVersionResponse = await humanloopClient.prompts.upsert({ + path: setup.evalPrompt.path, + provider: "openai", + model: "gpt-4o-mini", + temperature: 0, + template: [ + { + role: "system", + content: + "You are a helpful assistant. You must answer the user's question truthfully and at the level of a 5th grader.", + }, + { + role: "user", + content: "{{question}}", + }, + ], + }); + + // Run evaluation with version_id + await humanloopClient.evaluations.run({ + file: { + id: newPromptVersionResponse.id, + versionId: newPromptVersionResponse.versionId, + type: "prompt", + }, + dataset: { + path: setup.evalDataset.path, + }, + name: "test_eval_run", + evaluators: [ + { + path: setup.outputNotNullEvaluator.path, + }, + ], + }); + + // Verify evaluation + const evaluationsResponse = await humanloopClient.evaluations.list({ + fileId: newPromptVersionResponse.id, + }); + expect(evaluationsResponse.data.length).toBe(1); + + const evaluationId = evaluationsResponse.data[0].id; + const runsResponse = + await humanloopClient.evaluations.listRunsForEvaluation(evaluationId); + expect(runsResponse.runs[0].status).toBe("completed"); + if (runsResponse.runs[0].version) { + expect(runsResponse.runs[0].version.versionId).toBe( + newPromptVersionResponse.versionId, + ); + } + + // Verify version is not the default + const response = await humanloopClient.prompts.get( + newPromptVersionResponse.id, + ); + expect(response.versionId).not.toBe(newPromptVersionResponse.versionId); + } finally { + // Clean up test-specific resources + await cleanupTestEnvironment(setup); + } + }); + + it("should run evaluation with environment", async () => { + // Setup test-specific environment + const setup = await setupTestEnvironment("environment"); + + try { + // Create a new prompt version and deploy to staging + const newPromptVersionResponse = await humanloopClient.prompts.upsert({ + path: setup.evalPrompt.path, + provider: "openai", + model: "gpt-4o-mini", + temperature: 0, + template: [ + { + role: "system", + content: + "You are a helpful assistant. You must answer the user's question truthfully and at the level of a 5th grader.", + }, + { + role: "user", + content: "{{question}}", + }, + ], + }); + + await humanloopClient.prompts.setDeployment( + newPromptVersionResponse.id, + setup.stagingEnvironmentId, + { + versionId: newPromptVersionResponse.versionId, + }, + ); + + // Run evaluation with environment + await humanloopClient.evaluations.run({ + file: { + id: newPromptVersionResponse.id, + type: "prompt", + environment: "staging", + }, + dataset: { + path: setup.evalDataset.path, + }, + name: "test_eval_run", + evaluators: [ + { + path: setup.outputNotNullEvaluator.path, + }, + ], + }); + + // Verify evaluation + const evaluationsResponse = await humanloopClient.evaluations.list({ + fileId: newPromptVersionResponse.id, + }); + expect(evaluationsResponse.data.length).toBe(1); + + const evaluationId = evaluationsResponse.data[0].id; + const runsResponse = + await humanloopClient.evaluations.listRunsForEvaluation(evaluationId); + expect(runsResponse.runs[0].status).toBe("completed"); + if (runsResponse.runs[0].version) { + expect(runsResponse.runs[0].version.versionId).toBe( + newPromptVersionResponse.versionId, + ); + } + + const defaultPromptVersionResponse = await humanloopClient.prompts.get( + newPromptVersionResponse.id, + ); + expect(defaultPromptVersionResponse.versionId).not.toBe( + newPromptVersionResponse.versionId, + ); + } finally { + // Clean up test-specific resources + await cleanupTestEnvironment(setup); + } + }); + + it("should fail when using version_id with path", async () => { + // Setup test-specific environment + const setup = await setupTestEnvironment("fail_with_version_id"); + + try { + try { + await humanloopClient.evaluations.run({ + file: { + path: setup.evalPrompt.path, + type: "prompt", + versionId: "will_not_work", + }, + dataset: { + path: setup.evalDataset.path, + }, + name: "test_eval_run", + evaluators: [ + { + path: setup.outputNotNullEvaluator.path, + }, + ], + }); + // If we got here, the test failed + throw new Error("Expected runtime error but none was thrown"); + } catch (error: any) { + if (error instanceof HumanloopRuntimeError) { + expect(error.message).toContain( + "You must provide the `file.id` when addressing a file by version ID or environment", + ); + } else { + throw new Error( + `Expected test to fail for version_id but got ${error}`, + ); + } + } + } finally { + // Clean up test-specific resources + await cleanupTestEnvironment(setup); + } + }); + + it("should fail when using environment with path", async () => { + // Setup test-specific environment + const setup = await setupTestEnvironment("fail_with_environment"); + + try { + await humanloopClient.evaluations.run({ + file: { + path: setup.evalPrompt.path, + type: "prompt", + environment: "staging", + }, + dataset: { + path: setup.evalDataset.path, + }, + name: "test_eval_run", + evaluators: [ + { + path: setup.outputNotNullEvaluator.path, + }, + ], + }); + // If we got here, the test failed + throw new Error("Expected runtime error but none was thrown"); + } catch (error: any) { + if (error instanceof HumanloopRuntimeError) { + expect(error.message).toContain( + "You must provide the `file.id` when addressing a file by version ID or environment", + ); + } else { + throw new Error( + `Expected test to fail for environment but got ${error}`, + ); + } + } finally { + // Clean up test-specific resources + await cleanupTestEnvironment(setup); + } + }); + + it("should run evaluation with version upsert", async () => { + // Setup test-specific environment + const setup = await setupTestEnvironment("version_upsert"); + + try { + await humanloopClient.evaluations.run({ + file: { + path: setup.evalPrompt.path, + type: "prompt", + version: { + provider: "openai", + model: "gpt-4o-mini", + temperature: 1, + template: [ + { + role: "system", + content: + "You are a helpful assistant. You must answer the user's question truthfully and at the level of a 5th grader.", + }, + { + role: "user", + content: "{{question}}", + }, + ], + }, + }, + dataset: { + path: setup.evalDataset.path, + }, + name: "test_eval_run", + evaluators: [ + { + path: setup.outputNotNullEvaluator.path, + }, + ], + }); + + // Verify evaluation + const evaluationsResponse = await humanloopClient.evaluations.list({ + fileId: setup.evalPrompt.id, + }); + expect(evaluationsResponse.data.length).toBe(1); + + const evaluationId = evaluationsResponse.data[0].id; + const runsResponse = + await humanloopClient.evaluations.listRunsForEvaluation(evaluationId); + expect(runsResponse.runs[0].status).toBe("completed"); + + // Verify version upsert + const listPromptVersionsResponse = + await humanloopClient.prompts.listVersions(setup.evalPrompt.id); + expect(listPromptVersionsResponse.records.length).toBe(2); + } finally { + // Clean up test-specific resources + await cleanupTestEnvironment(setup); + } + }); + + it("should fail flow eval without callable", async () => { + // Setup test-specific environment + const setup = await setupTestEnvironment("flow_fail_without_callable"); + + try { + try { + await humanloopClient.evaluations.run({ + file: { + path: "Test Flow", + type: "flow", + version: { + attributes: { + foo: "bar", + }, + }, + }, + dataset: { + path: setup.evalDataset.path, + }, + name: "test_eval_run", + evaluators: [ + { + path: setup.outputNotNullEvaluator.path, + }, + ], + }); + // If we got here, the test failed + fail("Expected runtime error but none was thrown"); + } catch (error: any) { + expect(error.message).toContain( + "You must provide a `callable` for your Flow `file` to run a local eval.", + ); + } + } finally { + // Clean up test-specific resources + await cleanupTestEnvironment(setup); + } + }); + + it("should run flow eval with callable", async () => { + // Setup test-specific environment + const setup = await setupTestEnvironment("flow_with_callable"); + + try { + const flowPath = `${setup.sdkTestDir.path}/Test Flow`; + + // Create flow + const flowResponse = await humanloopClient.flows.upsert({ + path: flowPath, + attributes: { + foo: "bar", + }, + }); + + try { + const flow = await humanloopClient.flows.upsert({ + path: flowPath, + attributes: { + foo: "bar", + }, + }); + + // Run evaluation with flow + await humanloopClient.evaluations.run({ + file: { + id: flow.id, + type: "flow", + callable: ({ question }) => + "It's complicated don't worry about it", + version: { + attributes: { + foo: "bar", + }, + }, + }, + dataset: { + path: setup.evalDataset.path, + }, + name: "test_eval_run", + evaluators: [ + { + path: setup.outputNotNullEvaluator.path, + }, + ], + }); + + // Verify evaluation + const evaluationsResponse = await humanloopClient.evaluations.list({ + fileId: flow.id, + }); + expect(evaluationsResponse.data.length).toBe(1); + + const evaluationId = evaluationsResponse.data[0].id; + const runsResponse = + await humanloopClient.evaluations.listRunsForEvaluation( + evaluationId, + ); + expect(runsResponse.runs[0].status).toBe("completed"); + } finally { + await humanloopClient.flows.delete(flowResponse.id); + } + } finally { + // Clean up test-specific resources + await cleanupTestEnvironment(setup); + } + }); + + it("should not allow evaluating agent with callable", async () => { + // Setup test-specific environment + const setup = await setupTestEnvironment("agent_with_callable"); + + try { + try { + await humanloopClient.evaluations.run({ + file: { + path: "Test Agent", + type: "agent", + callable: (inputs: any) => "bar", + }, + dataset: { + path: setup.evalDataset.path, + }, + name: "test_eval_run", + evaluators: [ + { + path: setup.outputNotNullEvaluator.path, + }, + ], + }); + // If we got here, the test failed + fail("Expected ValueError but none was thrown"); + } catch (error: any) { + expect(error.message).toBe( + "Agent evaluation is only possible on the Humanloop runtime, do not provide a `callable`.", + ); + } + } finally { + // Clean up test-specific resources + await cleanupTestEnvironment(setup); + } + }); + + it("should resolve to default flow version when callable is provided without version", async () => { + // Setup test-specific environment + const setup = await setupTestEnvironment("flow_with_callable_without_version"); + let flowResponse: FlowResponse; + try { + const flowPath = `${setup.sdkTestDir.path}/Test Flow`; + + // Create flow + flowResponse = await humanloopClient.flows.upsert({ + path: flowPath, + attributes: { + foo: "bar", + }, + }); + + // Run evaluation with flow + await humanloopClient.evaluations.run({ + file: { + id: flowResponse.id, + type: "flow", + callable: ({ question }) => "It's complicated don't worry about it", + }, + dataset: { + path: setup.evalDataset.path, + }, + name: "test_eval_run", + evaluators: [ + { + path: setup.outputNotNullEvaluator.path, + }, + ], + }); + + // Verify evaluation + const evaluationsResponse = await humanloopClient.evaluations.list({ + fileId: flowResponse.id, + }); + expect(evaluationsResponse.data.length).toBe(1); + + const evaluationId = evaluationsResponse.data[0].id; + const runsResponse = + await humanloopClient.evaluations.listRunsForEvaluation(evaluationId); + expect(runsResponse.runs[0].status).toBe("completed"); + } finally { + // Clean up test-specific resources + await cleanupTestEnvironment(setup, [ + { id: flowResponse!.id, type: "flow" }, + ]); + } + }); +}); diff --git a/tests/custom/integration/fixtures.ts b/tests/custom/integration/fixtures.ts new file mode 100644 index 00000000..45e1fc41 --- /dev/null +++ b/tests/custom/integration/fixtures.ts @@ -0,0 +1,246 @@ +import dotenv from "dotenv"; +import { OpenAI } from "openai"; +import { v4 as uuidv4 } from "uuid"; + +import { FileType, PromptRequest, PromptResponse } from "../../../src/api"; +import { HumanloopClient } from "../../../src/humanloop.client"; + +export interface TestIdentifiers { + id: string; + path: string; +} + +export interface TestPrompt { + id: string; + path: string; + response: PromptResponse; +} + +export interface TestSetup { + sdkTestDir: TestIdentifiers; + testPromptConfig: PromptRequest; + openaiApiKey: string; + humanloopClient: HumanloopClient; + evalDataset: TestIdentifiers; + evalPrompt: TestIdentifiers; + stagingEnvironmentId: string; + outputNotNullEvaluator: TestIdentifiers; +} + +export interface CleanupResources { + type: FileType; + id: string; +} + +export function readEnvironment(): void { + if (![process.env.HUMANLOOP_API_KEY, process.env.OPENAI_API_KEY].every(Boolean)) { + // Testing locally not in CI, running dotenv.config() would override the secrets set for GitHub Action + dotenv.config({}); + } + if (!process.env.HUMANLOOP_API_KEY) { + throw new Error("HUMANLOOP_API_KEY is not set"); + } + if (!process.env.OPENAI_API_KEY) { + throw new Error("OPENAI_API_KEY is not set for integration tests"); + } +} + +export function getSubclient(client: HumanloopClient, type: FileType) { + switch (type) { + case "prompt": + return client.prompts; + case "tool": + return client.tools; + case "flow": + return client.flows; + case "agent": + return client.agents; + case "dataset": + return client.datasets; + case "evaluator": + return client.evaluators; + default: + throw new Error(`Unsupported file type: ${type}`); + } +} + +export async function setupTestEnvironment(testName: string): Promise { + readEnvironment(); + + const openaiApiKey = process.env.OPENAI_API_KEY!; + const humanloopClient = new HumanloopClient({ + apiKey: process.env.HUMANLOOP_API_KEY, + instrumentProviders: { + OpenAI: OpenAI, + }, + }); + + // Create a test directory + const directoryPath = `SDK_TEST_${testName}_${uuidv4()}`; + const response = await humanloopClient.directories.create({ + path: directoryPath, + }); + + const sdkTestDir = { + id: response.id, + path: response.path, + }; + + // Create test prompt config + const testPromptConfig: PromptRequest = { + provider: "openai", + model: "gpt-4o-mini", + temperature: 0.5, + template: [ + { + role: "system", + content: "You are a helpful assistant. Answer concisely.", + }, + { + role: "user", + content: "{{question}}", + }, + ], + }; + + // Create evaluator for testing + const evaluatorPath = `${sdkTestDir.path}/output_not_null_evaluator`; + const evaluatorResponse = await humanloopClient.evaluators.upsert({ + path: evaluatorPath, + spec: { + argumentsType: "target_required", + returnType: "boolean", + code: ` +def output_not_null(log: dict) -> bool: + return log["output"] is not None + `, + evaluatorType: "python", + }, + }); + const outputNotNullEvaluator = { + id: evaluatorResponse.id, + path: evaluatorPath, + }; + + // Create dataset for testing + const datasetPath = `${sdkTestDir.path}/eval_dataset`; + const datasetResponse = await humanloopClient.datasets.upsert({ + path: datasetPath, + datapoints: [ + { + inputs: { question: "What is the capital of the France?" }, + target: { output: "Paris" }, + }, + { + inputs: { question: "What is the capital of the Germany?" }, + target: { output: "Berlin" }, + }, + { + inputs: { question: "What is 2+2?" }, + target: { output: "4" }, + }, + ], + }); + const evalDataset = { + id: datasetResponse.id, + path: datasetResponse.path, + }; + + // Create prompt + const promptPath = `${sdkTestDir.path}/eval_prompt`; + const promptResponse = await humanloopClient.prompts.upsert({ + path: promptPath, + ...(testPromptConfig as PromptRequest), + }); + const evalPrompt = { + id: promptResponse.id, + path: promptResponse.path, + }; + + // Get staging environment ID + const environmentsResponse = await humanloopClient.prompts.listEnvironments( + evalPrompt.id, + ); + let stagingEnvironmentId = ""; + for (const environment of environmentsResponse) { + if (environment.name === "staging") { + stagingEnvironmentId = environment.id; + break; + } + } + if (!stagingEnvironmentId) { + throw new Error("Staging environment not found"); + } + + return { + testPromptConfig, + openaiApiKey, + humanloopClient, + sdkTestDir, + outputNotNullEvaluator, + evalDataset, + evalPrompt, + stagingEnvironmentId, + }; +} + +/** + * Cleans up all test resources + * @param setup The test setup containing the resources + * @param resources Additional resources to clean up + */ +export async function cleanupTestEnvironment( + setup: TestSetup, + resources?: CleanupResources[], +): Promise { + try { + // First clean up any additional resources + if (resources) { + for (const resource of resources) { + const subclient = getSubclient(setup.humanloopClient, resource.type); + if (resource.id) { + await subclient.delete(resource.id); + } + } + } + + // Clean up fixed test resources + if (setup.outputNotNullEvaluator?.id) { + try { + await setup.humanloopClient.evaluators.delete( + setup.outputNotNullEvaluator.id, + ); + } catch (error) { + console.warn( + `Failed to delete evaluator ${setup.outputNotNullEvaluator.id}:`, + error, + ); + } + } + + if (setup.evalDataset?.id) { + try { + await setup.humanloopClient.datasets.delete(setup.evalDataset.id); + } catch (error) { + console.warn( + `Failed to delete dataset ${setup.evalDataset.id}:`, + error, + ); + } + } + + // Finally, clean up the test directory + if (setup.sdkTestDir.id) { + try { + await setup.humanloopClient.directories.delete(setup.sdkTestDir.id); + } catch (error) { + console.warn( + `Failed to delete directory ${setup.sdkTestDir.id}:`, + error, + ); + } + } + } catch (error) { + console.error("Error during cleanup:", error); + } +} diff --git a/tests/custom/unit/LRUCache.test.ts b/tests/custom/unit/LRUCache.test.ts new file mode 100644 index 00000000..f518641f --- /dev/null +++ b/tests/custom/unit/LRUCache.test.ts @@ -0,0 +1,61 @@ +import LRUCache from "../../../../src/cache/LRUCache"; + +describe("LRUCache", () => { + let cache: LRUCache; + + beforeEach(() => { + cache = new LRUCache(3); // Test with small capacity + }); + + describe("basic operations", () => { + it("should set and get values", () => { + cache.set("key1", 1); + expect(cache.get("key1")).toBe(1); + }); + + it("should return undefined for non-existent keys", () => { + expect(cache.get("nonexistent")).toBeUndefined(); + }); + + it("should handle setting same key multiple times", () => { + cache.set("key1", 1); + cache.set("key1", 2); + expect(cache.get("key1")).toBe(2); + }); + }); + + describe("capacity and eviction", () => { + it("should evict least recently used item when capacity is reached", () => { + cache.set("key1", 1); + cache.set("key2", 2); + cache.set("key3", 3); + cache.set("key4", 4); // Should evict key1 + + expect(cache.get("key1")).toBeUndefined(); + expect(cache.get("key4")).toBe(4); + }); + + it("should update LRU order on get operations", () => { + cache.set("key1", 1); + cache.set("key2", 2); + cache.set("key3", 3); + + cache.get("key1"); // Make key1 most recently used + cache.set("key4", 4); // Should evict key2, not key1 + + expect(cache.get("key1")).toBe(1); + expect(cache.get("key2")).toBeUndefined(); + }); + }); + + describe("clear operation", () => { + it("should clear all items from cache", () => { + cache.set("key1", 1); + cache.set("key2", 2); + cache.clear(); + + expect(cache.get("key1")).toBeUndefined(); + expect(cache.get("key2")).toBeUndefined(); + }); + }); +}); diff --git a/tests/custom/unit/Logger.test.ts b/tests/custom/unit/Logger.test.ts new file mode 100644 index 00000000..40e67ab6 --- /dev/null +++ b/tests/custom/unit/Logger.test.ts @@ -0,0 +1,79 @@ +import Logger from "../../../../src/utils/Logger"; + +describe("Logger", () => { + let consoleSpy: { + log: jest.SpyInstance; + info: jest.SpyInstance; + warn: jest.SpyInstance; + debug: jest.SpyInstance; + }; + + beforeEach(() => { + // Spy on all console methods + consoleSpy = { + log: jest.spyOn(console, "log"), + info: jest.spyOn(console, "info"), + warn: jest.spyOn(console, "warn"), + debug: jest.spyOn(console, "debug"), + }; + // Reset log level before each test + Logger.setLevel("warn"); + }); + + afterEach(() => { + // Restore all spies + Object.values(consoleSpy).forEach((spy) => spy.mockRestore()); + }); + + describe("log levels", () => { + it("should respect log level settings", () => { + Logger.setLevel("warn"); + Logger.debug("debug message"); + Logger.info("info message"); + Logger.warn("warn message"); + + expect(consoleSpy.debug).not.toHaveBeenCalled(); + expect(consoleSpy.info).not.toHaveBeenCalled(); + expect(consoleSpy.warn).toHaveBeenCalledWith( + expect.stringContaining("warn"), + ); + }); + + it("should show all messages when level is set to debug", () => { + Logger.setLevel("debug"); + Logger.debug("debug message"); + Logger.info("info message"); + Logger.warn("warn message"); + + expect(consoleSpy.debug).toHaveBeenCalledWith( + expect.stringContaining("debug"), + ); + expect(consoleSpy.info).toHaveBeenCalledWith( + expect.stringContaining("info"), + ); + expect(consoleSpy.warn).toHaveBeenCalledWith( + expect.stringContaining("warn"), + ); + }); + }); + + describe("message formatting", () => { + it("should handle different input types", () => { + Logger.setLevel("info"); + + Logger.info(undefined); + Logger.info(null); + Logger.info({ key: "value" }); + + expect(consoleSpy.info).toHaveBeenCalledWith( + expect.stringContaining("undefined"), + ); + expect(consoleSpy.info).toHaveBeenCalledWith( + expect.stringContaining("null"), + ); + expect(consoleSpy.info).toHaveBeenCalledWith( + expect.stringContaining("[object Object]"), + ); + }); + }); +}); From f8e8cbeb0a75a5cdce512fa98c1349a2a9467d6b Mon Sep 17 00:00:00 2001 From: Ale Pouroullis Date: Tue, 20 May 2025 11:26:56 +0100 Subject: [PATCH 02/16] refactor(sync): rename SyncClient to FileSyncer and align with Python SDK --- src/humanloop.client.ts | 4 +- src/sync/FileSyncer.ts | 467 ++++++++++++++++++++++++++++++++++++++++ src/sync/SyncClient.ts | 311 -------------------------- src/sync/index.ts | 2 +- 4 files changed, 470 insertions(+), 314 deletions(-) create mode 100644 src/sync/FileSyncer.ts delete mode 100644 src/sync/SyncClient.ts diff --git a/src/humanloop.client.ts b/src/humanloop.client.ts index 28c3070e..1264f7f8 100644 --- a/src/humanloop.client.ts +++ b/src/humanloop.client.ts @@ -29,7 +29,7 @@ import { import { HumanloopSpanExporter } from "./otel/exporter"; import { HumanloopSpanProcessor } from "./otel/processor"; import { overloadCall, overloadLog } from "./overload"; -import { SyncClient, SyncClientOptions } from "./sync"; +import { SyncClient, FileSyncerOptions } from "./sync"; import { SDK_VERSION } from "./version"; const RED = "\x1b[91m"; @@ -252,7 +252,7 @@ export class HumanloopClient extends BaseHumanloopClient { Anthropic?: any; CohereAI?: any; }; - sync?: SyncClientOptions; + sync?: FileSyncerOptions; }, ) { super(_options); diff --git a/src/sync/FileSyncer.ts b/src/sync/FileSyncer.ts new file mode 100644 index 00000000..55d7a04a --- /dev/null +++ b/src/sync/FileSyncer.ts @@ -0,0 +1,467 @@ +import { FileType } from "api"; +import fs from "fs"; +import path from "path"; + +import { HumanloopClient as BaseHumanloopClient } from "../Client"; +import LRUCache from "../cache/LRUCache"; +import { HumanloopRuntimeError } from "../error"; +import Logger, { LogLevel } from "../utils/Logger"; + +// Set up isolated logger for file sync operations +// This logger uses the "humanloop.sdk.file_syncer" namespace, separate from the main client's logger, +// allowing CLI commands and other consumers to control sync logging verbosity independently. +const LOGGER_NAMESPACE = "humanloop.sdk.file_syncer"; +Logger.setLevel("info"); + +// Default cache size for file content caching +const DEFAULT_CACHE_SIZE = 100; + +// File types that can be serialized to/from the filesystem +export type SerializableFileType = "prompt" | "agent"; +export const SERIALIZABLE_FILE_TYPES = new Set([ + "prompt", + "agent", +]); + +export interface FileSyncerOptions { + baseDir?: string; + cacheSize?: number; + logLevel?: LogLevel; +} + +/** + * Format API error messages to be more user-friendly. + */ +function formatApiError(error: Error): string { + const errorMsg = error.toString(); + + // If the error doesn't look like an API error with status code and body + if (!errorMsg.includes("status_code") || !errorMsg.includes("body:")) { + return errorMsg; + } + + try { + // Extract the body part and parse as JSON + const bodyParts = errorMsg.split("body:"); + if (bodyParts.length < 2) return errorMsg; + + const bodyStr = bodyParts[1].trim(); + const body = JSON.parse(bodyStr); + + // Get the detail from the body + const detail = body.detail || {}; + + // Handle both string and dictionary types for detail + if (typeof detail === "string") { + return detail; + } else if (typeof detail === "object") { + return detail.description || detail.msg || errorMsg; + } + + return errorMsg; + } catch (e) { + Logger.debug(`Failed to parse error message: ${e}`); + return errorMsg; + } +} + +/** + * Client for synchronizing Prompt and Agent files between Humanloop workspace and local filesystem. + * + * This client enables a local development workflow by: + * 1. Pulling files from Humanloop workspace to local filesystem + * 2. Maintaining the same directory structure locally as in Humanloop + * 3. Storing files in human-readable, version-control friendly formats (.prompt and .agent) + * 4. Supporting local file access in the SDK when configured with use_local_files=true + * + * Files maintain their relative paths from the Humanloop workspace (with appropriate extensions added), + * allowing for seamless reference between local and remote environments using the same path identifiers. + */ +export default class FileSyncer { + private client: BaseHumanloopClient; + private baseDir: string; + private cacheSize: number; + private fileContentCache: LRUCache; + + constructor(client: BaseHumanloopClient, options: FileSyncerOptions = {}) { + this.client = client; + this.baseDir = options.baseDir || "humanloop"; + this.cacheSize = options.cacheSize || DEFAULT_CACHE_SIZE; + this.fileContentCache = new LRUCache(this.cacheSize); + + // Set the log level using the isolated logger + Logger.setLevel(options.logLevel || "warn"); + } + + /** + * Implementation of get_file_content without the cache decorator. + * + * This is the actual implementation that gets wrapped by LRU cache. + * + * @param filePath The API path to the file (e.g. `path/to/file`) + * @param fileType The type of file to get the content of (SerializableFileType) + * @returns The raw file content + * @throws HumanloopRuntimeError If the file doesn't exist or can't be read + */ + private _getFileContentImplementation( + filePath: string, + fileType: SerializableFileType, + ): string { + // Construct path to local file + const localPath = path.join(this.baseDir, filePath); + // Add appropriate extension + const dirName = path.dirname(localPath); + const baseName = path.basename(localPath, path.extname(localPath)); + const fullPath = path.join(dirName, `${baseName}.${fileType}`); + + if (!fs.existsSync(fullPath)) { + throw new HumanloopRuntimeError(`Local file not found: ${fullPath}`); + } + + try { + // Read the raw file content + const fileContent = fs.readFileSync(fullPath, "utf8"); + Logger.debug(`Using local file content from ${fullPath}`); + return fileContent; + } catch (error) { + throw new HumanloopRuntimeError( + `Error reading local file ${fullPath}: ${error}`, + ); + } + } + + /** + * Get the raw file content of a file from cache or filesystem. + * + * This method uses an LRU cache to store file contents. When the cache is full, + * the least recently accessed files are automatically removed to make space. + * + * @param filePath The normalized path to the file (without extension) + * @param fileType The type of file (Prompt or Agent) + * @returns The raw file content + * @throws HumanloopRuntimeError If the file doesn't exist or can't be read + */ + public getFileContent(filePath: string, fileType: SerializableFileType): string { + const cacheKey = `${filePath}:${fileType}`; + + // Check if in cache + const cachedContent = this.fileContentCache.get(cacheKey); + if (cachedContent !== undefined) { + Logger.debug(`Using cached file content for ${filePath}.${fileType}`); + return cachedContent; + } + + // Not in cache, get from filesystem + const content = this._getFileContentImplementation(filePath, fileType); + + // Add to cache + this.fileContentCache.set(cacheKey, content); + + return content; + } + + /** + * Clear the LRU cache. + */ + public clearCache(): void { + this.fileContentCache.clear(); + } + + /** + * Check if the path is a file by checking for .{fileType} extension for serializable file types. + * + * Files are identified by having a supported extension (.prompt or .agent). + * This method performs case-insensitive comparison and handles whitespace. + * + * @returns True if the path ends with a supported file extension + */ + public isFile(filePath: string): boolean { + const cleanPath = filePath.trim().toLowerCase(); // Convert to lowercase for case-insensitive comparison + return Array.from(SERIALIZABLE_FILE_TYPES).some((fileType) => + cleanPath.endsWith(`.${fileType}`), + ); + } + + /** + * Save serialized file to local filesystem. + */ + private _saveSerializedFile( + serializedContent: string, + filePath: string, + fileType: SerializableFileType, + ): void { + try { + // Create full path including baseDir prefix + const fullPath = path.join(this.baseDir, filePath); + const directory = path.dirname(fullPath); + const fileName = path.basename(fullPath, path.extname(fullPath)); + + // Create directory if it doesn't exist + fs.mkdirSync(directory, { recursive: true }); + + // Add file type extension + const newPath = path.join(directory, `${fileName}.${fileType}`); + + // Write raw file content to file + fs.writeFileSync(newPath, serializedContent); + } catch (error) { + Logger.error(`Failed to write ${fileType} ${filePath} to disk: ${error}`); + throw error; + } + } + + /** + * Pull a specific file from Humanloop to local filesystem. + * + * @returns True if the file was successfully pulled, False otherwise (e.g. if the file was not found) + */ + private async _pullFile(filePath: string, environment?: string): Promise { + try { + const file = await this.client.files.retrieveByPath({ + path: filePath, + environment, + includeRawFileContent: true, + }); + + if (!SERIALIZABLE_FILE_TYPES.has(file.type as SerializableFileType)) { + Logger.error(`Unsupported file type: ${file.type}`); + return false; + } + + // Type assertion for rawFileContent since we know it exists when includeRawFileContent is true + const rawContent = (file as any).rawFileContent; + if (!rawContent) { + Logger.error(`No content found for ${file.type} ${filePath}`); + return false; + } + + this._saveSerializedFile( + rawContent, + file.path, + file.type as SerializableFileType, + ); + return true; + } catch (error) { + Logger.error(`Failed to pull file ${filePath}: ${error}`); + return false; + } + } + + /** + * Sync Prompt and Agent files from Humanloop to local filesystem. + * + * @returns Tuple of two lists: + * - First list contains paths of successfully pulled files + * - Second list contains paths of files that failed to pull. + * Failures can occur due to missing content in the response or errors during local file writing. + * @throws HumanloopRuntimeError If there's an error communicating with the API + */ + private async _pullDirectory( + dirPath?: string, + environment?: string, + ): Promise<[string[], string[]]> { + const successfulFiles: string[] = []; + const failedFiles: string[] = []; + let page = 1; + + Logger.debug( + `Fetching files from directory: ${dirPath || "(root)"} in environment: ${environment || "(default)"}`, + ); + + while (true) { + try { + Logger.debug( + `${dirPath || "(root)"}: Requesting page ${page} of files`, + ); + + const response = await this.client.files.listFiles({ + type: Array.from(SERIALIZABLE_FILE_TYPES), + page, + size: 100, + includeRawFileContent: true, + environment, + path: dirPath, + }); + + if (response.records.length === 0) { + Logger.debug( + `Finished reading files for path ${dirPath || "(root)"}`, + ); + break; + } + + Logger.debug( + `${dirPath || "(root)"}: Read page ${page} containing ${response.records.length} files`, + ); + + // Process each file + for (const file of response.records) { + // Skip if not a serializable file type + if ( + !SERIALIZABLE_FILE_TYPES.has(file.type as SerializableFileType) + ) { + Logger.warn(`Skipping unsupported file type: ${file.type}`); + continue; + } + + const fileType = file.type as SerializableFileType; + + // Type assertion for rawFileContent since we know it exists when includeRawFileContent is true + const rawContent = (file as any).rawFileContent; + if (!rawContent) { + Logger.warn(`No content found for ${file.type} ${file.path}`); + failedFiles.push(file.path); + continue; + } + + try { + Logger.debug(`Writing ${file.type} ${file.path} to disk`); + this._saveSerializedFile(rawContent, file.path, fileType); + successfulFiles.push(file.path); + } catch (error) { + failedFiles.push(file.path); + Logger.error(`Failed to save ${file.path}: ${error}`); + } + } + + page += 1; + } catch (error) { + const formattedError = formatApiError(error as Error); + throw new HumanloopRuntimeError( + `Failed to fetch page ${page}: ${formattedError}`, + ); + } + } + + if (successfulFiles.length > 0) { + Logger.info(`Successfully pulled ${successfulFiles.length} files`); + } + if (failedFiles.length > 0) { + Logger.warn(`Failed to pull ${failedFiles.length} files`); + } + + return [successfulFiles, failedFiles]; + } + + /** + * Pull files from Humanloop to local filesystem. + * + * If the path ends with `.prompt` or `.agent`, pulls that specific file. + * Otherwise, pulls all files under the specified path. + * If no path is provided, pulls all files from the root. + * + * @param filePath The path to pull from. Can be: + * - A specific file with extension (e.g. "path/to/file.prompt") + * - A directory without extension (e.g. "path/to/directory") + * - None to pull all files from root + * + * Paths should not contain leading or trailing slashes + * @param environment The environment to pull from + * @returns Tuple of two lists: + * - First list contains paths of successfully pulled files + * - Second list contains paths of files that failed to pull (e.g. failed to write to disk or missing raw content) + * @throws HumanloopRuntimeError If there's an error communicating with the API + */ + public async pull( + filePath?: string, + environment?: string, + ): Promise<[string[], string[]]> { + const startTime = Date.now(); + + let apiPath: string | undefined; + let isFilePath: boolean; + + if (filePath === undefined) { + apiPath = undefined; + isFilePath = false; + } else { + filePath = filePath.trim(); + // Check if path has leading/trailing slashes + if (filePath !== filePath.trim().replace(/^\/+|\/+$/g, "")) { + throw new HumanloopRuntimeError( + `Invalid path: ${filePath}. Path should not contain leading/trailing slashes. ` + + `Valid examples: "path/to/file.prompt" or "path/to/directory"`, + ); + } + + // Check if it's a file path (has extension) + isFilePath = this.isFile(filePath); + + // For API communication, we need path without extension + apiPath = this._normalizePath(filePath, true); + } + + Logger.info( + `Starting pull: path=${apiPath || "(root)"}, environment=${environment || "(default)"}`, + ); + + try { + let successfulFiles: string[]; + let failedFiles: string[]; + + if (apiPath === undefined) { + // Pull all from root + Logger.debug("Pulling all files from (root)"); + [successfulFiles, failedFiles] = await this._pullDirectory( + undefined, + environment, + ); + } else { + if (isFilePath) { + Logger.debug(`Pulling file: ${apiPath}`); + if (await this._pullFile(apiPath, environment)) { + successfulFiles = [apiPath]; + failedFiles = []; + } else { + successfulFiles = []; + failedFiles = [apiPath]; + } + } else { + Logger.debug(`Pulling directory: ${apiPath || "(root)"}`); + [successfulFiles, failedFiles] = await this._pullDirectory( + apiPath, + environment, + ); + } + } + + // Clear the cache at the end of each pull operation + this.clearCache(); + + const duration = Date.now() - startTime; + Logger.info( + `Pull completed in ${duration}ms: ${successfulFiles.length} files pulled`, + ); + + return [successfulFiles, failedFiles]; + } catch (error) { + throw new HumanloopRuntimeError(`Pull operation failed: ${error}`); + } + } + + /** + * Normalize the path by removing extensions, etc. + */ + private _normalizePath(filePath: string, stripExtension: boolean = false): string { + if (!filePath) return ""; + + // Remove any file extensions if requested + let normalizedPath = + stripExtension && filePath.includes(".") + ? filePath.substring(0, filePath.lastIndexOf(".")) + : filePath; + + // Convert backslashes to forward slashes + normalizedPath = normalizedPath.replace(/\\/g, "/"); + + // Remove leading/trailing whitespace and slashes + normalizedPath = normalizedPath.trim().replace(/^\/+|\/+$/g, ""); + + // Normalize multiple consecutive slashes into a single forward slash + while (normalizedPath.includes("//")) { + normalizedPath = normalizedPath.replace(/\/\//g, "/"); + } + + return normalizedPath; + } +} diff --git a/src/sync/SyncClient.ts b/src/sync/SyncClient.ts deleted file mode 100644 index df9ec0f5..00000000 --- a/src/sync/SyncClient.ts +++ /dev/null @@ -1,311 +0,0 @@ -import { FileType } from "api"; -import fs from "fs"; -import path from "path"; - -import { HumanloopClient as BaseHumanloopClient } from "../Client"; -import LRUCache from "../cache/LRUCache"; -import { HumanloopRuntimeError } from "../error"; -import Logger, { LogLevel } from "../utils/Logger"; // Import your existing Logger - -// Default cache size for file content caching -const DEFAULT_CACHE_SIZE = 100; - -export interface SyncClientOptions { - baseDir?: string; - cacheSize?: number; - logLevel?: LogLevel; -} - -/** - * Format API error messages to be more user-friendly. - */ -function formatApiError(error: Error): string { - const errorMsg = error.toString(); - - // If the error doesn't look like an API error with status code and body - if (!errorMsg.includes("status_code") || !errorMsg.includes("body:")) { - return errorMsg; - } - - try { - // Extract the body part and parse as JSON - const bodyParts = errorMsg.split("body:"); - if (bodyParts.length < 2) return errorMsg; - - const bodyStr = bodyParts[1].trim(); - const body = JSON.parse(bodyStr); - - // Get the detail from the body - const detail = body.detail || {}; - - // Prefer description, fall back to msg - return detail.description || detail.msg || errorMsg; - } catch (e) { - Logger.debug(`Failed to parse error message: ${e}`); // Use debug level - return errorMsg; - } -} - -/** - * Client for managing synchronization between local filesystem and Humanloop. - * - * This client provides file synchronization between Humanloop and the local filesystem, - * with built-in caching for improved performance. - */ -export default class SyncClient { - private client: BaseHumanloopClient; - private baseDir: string; - private cacheSize: number; - private fileContentCache: LRUCache; - - constructor( - client: BaseHumanloopClient, - options: SyncClientOptions = {} - ) { - this.client = client; - this.baseDir = options.baseDir || "humanloop"; - this.cacheSize = options.cacheSize || DEFAULT_CACHE_SIZE; - this.fileContentCache = new LRUCache(this.cacheSize); - - // Set the log level using your Logger's setLevel method - Logger.setLevel(options.logLevel || 'warn'); - } - - /** - * Get the file content from cache or filesystem. - */ - public getFileContent(filePath: string, fileType: FileType): string { - const cacheKey = `${filePath}:${fileType}`; - - // Check if in cache - const cachedContent = this.fileContentCache.get(cacheKey); - if (cachedContent !== undefined) { - // Use debug level for cache hits - Logger.debug(`Using cached file content for ${filePath}.${fileType}`); - return cachedContent; - } - - // Not in cache, get from filesystem - const localPath = path.join(this.baseDir, `${filePath}.${fileType}`); - - if (!fs.existsSync(localPath)) { - throw new HumanloopRuntimeError(`Local file not found: ${localPath}`); - } - - try { - const fileContent = fs.readFileSync(localPath, 'utf8'); - Logger.debug(`Using local file content from ${localPath}`); - - // Add to cache - this.fileContentCache.set(cacheKey, fileContent); - - return fileContent; - } catch (error) { - throw new HumanloopRuntimeError( - `Error reading local file ${localPath}: ${error}` - ); - } - } - - /** - * Clear the cache. - */ - public clearCache(): void { - this.fileContentCache.clear(); - } - - /** - * Normalize the path by removing extensions, etc. - */ - private normalizePath(filePath: string): string { - if (!filePath) return ""; - - // Remove any file extensions - let normalizedPath = filePath.includes(".") - ? filePath.substring(0, filePath.lastIndexOf(".")) - : filePath; - - // Convert backslashes to forward slashes - normalizedPath = normalizedPath.replace(/\\/g, "/"); - - // Remove leading/trailing whitespace and slashes - normalizedPath = normalizedPath.trim().replace(/^\/+|\/+$/g, ""); - - // Normalize multiple consecutive slashes into a single forward slash - while (normalizedPath.includes("//")) { - normalizedPath = normalizedPath.replace(/\/\//g, "/"); - } - - return normalizedPath; - } - - /** - * Check if the path is a file by checking for .prompt or .agent extension. - */ - private isFile(path: string): boolean { - return path.trim().endsWith(".prompt") || path.trim().endsWith(".agent"); - } - - /** - * Save serialized file to local filesystem. - */ - private saveSerializedFile( - serializedContent: string, - filePath: string, - fileType: FileType - ): void { - try { - // Create full path including baseDir prefix - const fullPath = path.join(this.baseDir, filePath); - const directory = path.dirname(fullPath); - const fileName = path.basename(fullPath); - - // Create directory if it doesn't exist - fs.mkdirSync(directory, { recursive: true }); - - // Add file type extension - const newPath = path.join(directory, `${fileName}.${fileType}`); - - // Write raw file content to file - fs.writeFileSync(newPath, serializedContent); - - // Clear the cache for this file to ensure we get fresh content next time - this.clearCache(); - } catch (error) { - Logger.error(`Failed to sync ${fileType} ${filePath}: ${error}`); - throw error; - } - } - - /** - * Pull a specific file from Humanloop to local filesystem. - */ - private async pullFile(path: string, environment?: string): Promise { - const file = await this.client.files.retrieveByPath({ - path, - environment, - includeRawFileContent: true, - }); - - if (file.type !== "prompt" && file.type !== "agent") { - throw new Error(`Unsupported file type: ${file.type}`); - } - - this.saveSerializedFile(file.rawFileContent!, file.path, file.type); - } - - /** - * Pull all files from a directory in Humanloop to local filesystem. - */ - private async pullDirectory( - path?: string, - environment?: string, - ): Promise { - const successfulFiles: string[] = []; - const failedFiles: string[] = []; - let page = 1; - - Logger.debug(`Fetching files from directory: ${path || '(root)'} in environment: ${environment || '(default)'}`); - - while (true) { - try { - Logger.debug(`Requesting page ${page} of files`); - - const response = await this.client.files.listFiles({ - type: ["prompt", "agent"], - page, - includeRawFileContent: true, - environment, - path, - }); - - if (response.records.length === 0) { - Logger.debug("No more files found"); - break; - } - - Logger.debug(`Found ${response.records.length} files from page ${page}`); - - // Process each file - for (const file of response.records) { - // Skip if not a Prompt or Agent - if (file.type !== "prompt" && file.type !== "agent") { - Logger.warn(`Skipping unsupported file type: ${file.type}`); - continue; - } - - // Skip if no raw file content - if (!file.rawFileContent) { - Logger.warn(`No content found for ${file.type} ${file.id || ""}`); - continue; - } - - try { - Logger.debug(`Saving ${file.type} ${file.path}`); - - this.saveSerializedFile( - file.rawFileContent, - file.path, - file.type, - ); - successfulFiles.push(file.path); - } catch (error) { - failedFiles.push(file.path); - Logger.error(`Task failed for ${file.path}: ${error}`); - } - } - - page += 1; - } catch (error) { - const formattedError = formatApiError(error as Error); - throw new HumanloopRuntimeError( - `Failed to pull files: ${formattedError}` - ); - } - } - - if (successfulFiles.length > 0) { - Logger.info(`Successfully pulled ${successfulFiles.length} files`); - } - if (failedFiles.length > 0) { - Logger.warn(`Failed to pull ${failedFiles.length} files`); - } - - return successfulFiles; - } - - /** - * Pull files from Humanloop to local filesystem. - */ - public async pull(path?: string, environment?: string): Promise { - const startTime = Date.now(); - const normalizedPath = path ? this.normalizePath(path) : undefined; - - Logger.info(`Starting pull operation: path=${normalizedPath || '(root)'}, environment=${environment || '(default)'}`); - - let successfulFiles: string[] = []; - - if (!path) { - // Pull all files from the root - Logger.debug("Pulling all files from root"); - successfulFiles = await this.pullDirectory(undefined, environment); - } else { - if (this.isFile(path)) { - Logger.debug(`Pulling specific file: ${normalizedPath}`); - await this.pullFile(normalizedPath!, environment); - successfulFiles = [path]; - } else { - Logger.debug(`Pulling directory: ${normalizedPath}`); - successfulFiles = await this.pullDirectory( - normalizedPath, - environment, - ); - } - } - - const duration = Date.now() - startTime; - Logger.success(`Pull completed in ${duration}ms: ${successfulFiles.length} files succeeded`); - - return successfulFiles; - } -} \ No newline at end of file diff --git a/src/sync/index.ts b/src/sync/index.ts index 252b4ea8..892194b3 100644 --- a/src/sync/index.ts +++ b/src/sync/index.ts @@ -4,4 +4,4 @@ * This module provides sync functionality between Humanloop and the local filesystem. */ -export { default as SyncClient, SyncClientOptions } from './SyncClient'; \ No newline at end of file +export { default as SyncClient, FileSyncerOptions } from './FileSyncer'; \ No newline at end of file From 41d3e45b0b990684061e2582a2cc96daf488986a Mon Sep 17 00:00:00 2001 From: Ale Pouroullis Date: Tue, 20 May 2025 11:32:44 +0100 Subject: [PATCH 03/16] refactor(sync): align FileSyncer with Python SDK and improve TypeScript idioms - Rename SyncClient to FileSyncer and update client options to match Python SDK - Add useLocalFiles, localFilesDirectory, and cacheSize options - Update pull method to return [successful, failed] arrays - Improve TypeScript terminology in docstrings (array vs tuple/list) - Fix type errors and error handling Makes the codebase more consistent with Python SDK while improving TypeScript idioms and type safety. --- src/humanloop.client.ts | 115 ++++++++++++++++++++++++++++------------ src/sync/FileSyncer.ts | 12 ++--- 2 files changed, 88 insertions(+), 39 deletions(-) diff --git a/src/humanloop.client.ts b/src/humanloop.client.ts index 1264f7f8..6140e7b5 100644 --- a/src/humanloop.client.ts +++ b/src/humanloop.client.ts @@ -29,7 +29,8 @@ import { import { HumanloopSpanExporter } from "./otel/exporter"; import { HumanloopSpanProcessor } from "./otel/processor"; import { overloadCall, overloadLog } from "./overload"; -import { SyncClient, FileSyncerOptions } from "./sync"; +import { FileSyncerOptions, SyncClient } from "./sync"; +import Logger from "./utils/Logger"; import { SDK_VERSION } from "./version"; const RED = "\x1b[91m"; @@ -200,6 +201,39 @@ class HumanloopTracerSingleton { } } +export interface HumanloopClientOptions extends BaseHumanloopClient.Options { + /** + * Whether to use local files for prompts and agents + */ + useLocalFiles?: boolean; + + /** + * Base directory where local prompt and agent files are stored (default: "humanloop"). + * This is relative to the current working directory. For example: + * - "humanloop" will look for files in "./humanloop/" + * - "data/humanloop" will look for files in "./data/humanloop/" + * When using paths in the API, they must be relative to this directory. For example, + * if localFilesDirectory="humanloop" and you have a file at "humanloop/samples/test.prompt", + * you would reference it as "samples/test" in your code. + */ + localFilesDirectory?: string; + + /** + * Maximum number of files to cache when useLocalFiles is true (default: DEFAULT_CACHE_SIZE). + * This parameter has no effect if useLocalFiles is false. + */ + cacheSize?: number; + + /** + * LLM provider modules to instrument. Allows the prompt decorator to spy on provider calls and log them to Humanloop + */ + instrumentProviders?: { + OpenAI?: any; + Anthropic?: any; + CohereAI?: any; + }; +} + export class HumanloopClient extends BaseHumanloopClient { protected readonly _evaluations: ExtendedEvaluations; protected readonly _prompts_overloaded: Prompts; @@ -212,6 +246,7 @@ export class HumanloopClient extends BaseHumanloopClient { CohereAI?: any; }; protected readonly _syncClient: SyncClient; + protected readonly useLocalFiles: boolean; protected get opentelemetryTracer(): Tracer { return HumanloopTracerSingleton.getInstance({ @@ -245,21 +280,25 @@ export class HumanloopClient extends BaseHumanloopClient { * const anthropic = new Anthropic({apiKey: process.env.ANTHROPIC_KEY}); * ``` */ - constructor( - _options: BaseHumanloopClient.Options & { - instrumentProviders?: { - OpenAI?: any; - Anthropic?: any; - CohereAI?: any; - }; - sync?: FileSyncerOptions; - }, - ) { - super(_options); - - this._syncClient = new SyncClient(this, _options.sync); - - this.instrumentProviders = _options.instrumentProviders || {}; + constructor(options: HumanloopClientOptions = {}) { + super(options); + + this.useLocalFiles = options.useLocalFiles || false; + + // Warn user if cacheSize is non-default but useLocalFiles is false + if (!this.useLocalFiles && options.cacheSize !== undefined) { + Logger.warn( + `The specified cacheSize=${options.cacheSize} will have no effect because useLocalFiles=false. ` + + `File caching is only active when local files are enabled.`, + ); + } + + this._syncClient = new SyncClient(this, { + baseDir: options.localFilesDirectory || "humanloop", + cacheSize: options.cacheSize, + }); + + this.instrumentProviders = options.instrumentProviders || {}; this._prompts_overloaded = overloadLog(super.prompts); this._prompts_overloaded = overloadCall(this._prompts_overloaded); @@ -270,7 +309,7 @@ export class HumanloopClient extends BaseHumanloopClient { this._evaluators_overloaded = overloadLog(super.evaluators); - this._evaluations = new ExtendedEvaluations(_options, this); + this._evaluations = new ExtendedEvaluations(options, this); // Initialize the tracer singleton HumanloopTracerSingleton.getInstance({ @@ -364,14 +403,14 @@ ${RESET}`, * temperature: 0.5, * }); * const openaiContent = openaiResponse.choices[0].message.content; - * + * const anthropicClient = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY }); * const anthropicResponse = await anthropicClient.messages.create({ * model: "claude-3-5-sonnet-20240620", * temperature: 0.5, * }); * const anthropicContent = anthropicResponse.content; - * + * return { openaiContent, anthropicContent }; * } * }); @@ -570,40 +609,50 @@ ${RESET}`, * * This method will: * 1. Fetch Prompt and Agent files from your Humanloop workspace - * 2. Save them to the local filesystem using the client's files_directory (set during initialization) + * 2. Save them to your local filesystem (directory specified by `localFilesDirectory`, default: "humanloop") * 3. Maintain the same directory structure as in Humanloop - * 4. Add appropriate file extensions (.prompt or .agent) + * 4. Add appropriate file extensions (`.prompt` or `.agent`) * * The path parameter can be used in two ways: * - If it points to a specific file (e.g. "path/to/file.prompt" or "path/to/file.agent"), only that file will be pulled - * - If it points to a directory (e.g. "path/to/directory"), all Prompt and Agent files in that directory will be pulled + * - If it points to a directory (e.g. "path/to/directory"), all Prompt and Agent files in that directory and its subdirectories will be pulled * - If no path is provided, all Prompt and Agent files will be pulled * * The operation will overwrite existing files with the latest version from Humanloop * but will not delete local files that don't exist in the remote workspace. * - * Currently only supports syncing prompt and agent files. Other file types will be skipped. + * Currently only supports syncing Prompt and Agent files. Other file types will be skipped. * - * The files will be saved with the following structure: + * For example, with the default `localFilesDirectory="humanloop"`, files will be saved as: * ``` - * {files_directory}/ - * ├── prompts/ - * │ ├── my_prompt.prompt - * │ └── nested/ - * │ └── another_prompt.prompt - * └── agents/ - * └── my_agent.agent + * ./humanloop/ + * ├── my_project/ + * │ ├── prompts/ + * │ │ ├── my_prompt.prompt + * │ │ └── nested/ + * │ │ └── another_prompt.prompt + * │ └── agents/ + * │ └── my_agent.agent + * └── another_project/ + * └── prompts/ + * └── other_prompt.prompt * ``` * + * If you specify `localFilesDirectory="data/humanloop"`, files will be saved in ./data/humanloop/ instead. + * * @param path - Optional path to either a specific file (e.g. "path/to/file.prompt") or a directory (e.g. "path/to/directory"). * If not provided, all Prompt and Agent files will be pulled. * @param environment - The environment to pull the files from. - * @returns List of successfully processed file paths. + * @returns An array containing two string arrays: + * - First array contains paths of successfully synced files + * - Second array contains paths of files that failed to sync (due to API errors, missing content, + * or filesystem issues) + * @throws HumanloopRuntimeError If there's an error communicating with the API */ public async pull( path?: string, environment?: string, - ): Promise { + ): Promise<[string[], string[]]> { return this._syncClient.pull(path, environment); } diff --git a/src/sync/FileSyncer.ts b/src/sync/FileSyncer.ts index 55d7a04a..bc726513 100644 --- a/src/sync/FileSyncer.ts +++ b/src/sync/FileSyncer.ts @@ -250,9 +250,9 @@ export default class FileSyncer { /** * Sync Prompt and Agent files from Humanloop to local filesystem. * - * @returns Tuple of two lists: - * - First list contains paths of successfully pulled files - * - Second list contains paths of files that failed to pull. + * @returns An array containing two string arrays: + * - First array contains paths of successfully pulled files + * - Second array contains paths of files that failed to pull. * Failures can occur due to missing content in the response or errors during local file writing. * @throws HumanloopRuntimeError If there's an error communicating with the API */ @@ -357,9 +357,9 @@ export default class FileSyncer { * * Paths should not contain leading or trailing slashes * @param environment The environment to pull from - * @returns Tuple of two lists: - * - First list contains paths of successfully pulled files - * - Second list contains paths of files that failed to pull (e.g. failed to write to disk or missing raw content) + * @returns An array containing two string arrays: + * - First array contains paths of successfully pulled files + * - Second array contains paths of files that failed to pull (e.g. failed to write to disk or missing raw content) * @throws HumanloopRuntimeError If there's an error communicating with the API */ public async pull( From b03fe62744cd1f995b89b4d5915b65fc1c110fc1 Mon Sep 17 00:00:00 2001 From: Ale Pouroullis Date: Tue, 20 May 2025 12:40:48 +0100 Subject: [PATCH 04/16] Match CLI functionality with Python SDK; Simplify logging --- .fernignore | 1 - src/cli.ts | 186 +++++++++++++++++++++++++------ src/humanloop.client.ts | 5 +- src/sync/FileSyncer.ts | 183 +++++++++++++++--------------- src/utils/Logger.ts | 106 ------------------ src/utils/index.ts | 1 - tests/custom/unit/Logger.test.ts | 79 ------------- 7 files changed, 244 insertions(+), 317 deletions(-) delete mode 100644 src/utils/Logger.ts delete mode 100644 src/utils/index.ts delete mode 100644 tests/custom/unit/Logger.test.ts diff --git a/.fernignore b/.fernignore index c24f34a4..7f3bf4cb 100644 --- a/.fernignore +++ b/.fernignore @@ -13,7 +13,6 @@ src/context.ts src/cli.ts src/cache src/sync -src/utils # Tests diff --git a/src/cli.ts b/src/cli.ts index b80d1c94..6df8d05a 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,69 +1,183 @@ #!/usr/bin/env node import * as dotenv from "dotenv"; import { Command } from "commander"; +import path from "path"; import { HumanloopClient } from "./humanloop.client"; -import Logger from "./utils/Logger"; +import FileSyncer from "./sync/FileSyncer"; +import { SDK_VERSION } from "./version"; -const { version } = require("../package.json"); - -// Load environment variables dotenv.config(); +const LogType = { + SUCCESS: "\x1b[92m", // green + ERROR: "\x1b[91m", // red + INFO: "\x1b[96m", // cyan + WARN: "\x1b[93m", // yellow + RESET: "\x1b[0m", +} as const; + +function log(message: string, type: keyof typeof LogType): void { + console.log(`${LogType[type]}${message}${LogType.RESET}`); +} + const program = new Command(); program .name("humanloop") .description("Humanloop CLI for managing sync operations") - .version(version); + .version(SDK_VERSION); + +interface CommonOptions { + apiKey?: string; + envFile?: string; + baseUrl?: string; + localFilesDirectory?: string; +} + +interface PullOptions extends CommonOptions { + path?: string; + environment?: string; + verbose?: boolean; + quiet?: boolean; +} -// Common auth options -const addAuthOptions = (command: Command) => +const addCommonOptions = (command: Command) => command .option("--api-key ", "Humanloop API key") .option("--env-file ", "Path to .env file") - .option("--base-url ", "Base URL for Humanloop API"); + .option("--base-url ", "Base URL for Humanloop API") + .option( + "--local-dir, --local-files-directory ", + "Directory where Humanloop files are stored locally (default: humanloop/)", + "humanloop", + ); + +// Instantiate a HumanloopClient for the CLI +function getClient(options: CommonOptions): HumanloopClient { + if (options.envFile) { + const result = dotenv.config({ path: options.envFile }); + if (result.error) { + log( + `Failed to load environment file: ${options.envFile} (file not found or invalid format)`, + "ERROR", + ); + process.exit(1); + } + } -// Helper to get client -function getClient(options: { - envFile?: string; - apiKey?: string; - baseUrl?: string; - baseDir?: string; -}): HumanloopClient { - if (options.envFile) dotenv.config({ path: options.envFile }); const apiKey = options.apiKey || process.env.HUMANLOOP_API_KEY; if (!apiKey) { - Logger.error( - "No API key found. Set HUMANLOOP_API_KEY in .env file or use --api-key", + log( + "No API key found. Set HUMANLOOP_API_KEY in .env file or environment, or use --api-key", + "ERROR", ); process.exit(1); } + return new HumanloopClient({ apiKey, baseUrl: options.baseUrl, - sync: { baseDir: options.baseDir }, + localFilesDirectory: options.localFilesDirectory, }); } +// Helper to handle sync errors +function handleSyncErrors(fn: (options: T) => Promise) { + return async (options: T) => { + try { + await fn(options); + } catch (error) { + log(`Error: ${error}`, "ERROR"); + process.exit(1); + } + }; +} + // Pull command -addAuthOptions( +addCommonOptions( program .command("pull") - .description("Pull files from Humanloop to local filesystem") - .option("-p, --path ", "Path to pull (file or directory)") - .option("-e, --environment ", "Environment to pull from") - .option("--base-dir ", "Base directory for synced files", "humanloop"), -).action(async (options) => { - Logger.info("Pulling files from Humanloop..."); - // try { - // Logger.info("Pulling files from Humanloop..."); - // const client = getClient(options); - // const files = await client.pull(options.path, options.environment); - // Logger.success(`Successfully synced ${files.length} files`); - // } catch (error) { - // Logger.error(`Error: ${error}`); - // process.exit(1); - // } -}); + .description( + "Pull Prompt and Agent files from Humanloop to your local filesystem.\n\n" + + "This command will:\n" + + "1. Fetch Prompt and Agent files from your Humanloop workspace\n" + + "2. Save them to your local filesystem (directory specified by --local-files-directory, default: humanloop/)\n" + + "3. Maintain the same directory structure as in Humanloop\n" + + "4. Add appropriate file extensions (.prompt or .agent)\n\n" + + "For example, with the default --local-files-directory=humanloop, files will be saved as:\n" + + "./humanloop/\n" + + "├── my_project/\n" + + "│ ├── prompts/\n" + + "│ │ ├── my_prompt.prompt\n" + + "│ │ └── nested/\n" + + "│ │ └── another_prompt.prompt\n" + + "│ └── agents/\n" + + "│ └── my_agent.agent\n" + + "└── another_project/\n" + + " └── prompts/\n" + + " └── other_prompt.prompt\n\n" + + "If you specify --local-files-directory=data/humanloop, files will be saved in ./data/humanloop/ instead.\n\n" + + "If a file exists both locally and in the Humanloop workspace, the local file will be overwritten\n" + + "with the version from Humanloop. Files that only exist locally will not be affected.\n\n" + + "Currently only supports syncing Prompt and Agent files. Other file types will be skipped.", + ) + .option( + "-p, --path ", + "Path in the Humanloop workspace to pull from (file or directory). " + + "You can pull an entire directory (e.g. 'my/directory') or a specific file (e.g. 'my/directory/my_prompt.prompt'). " + + "When pulling a directory, all files within that directory and its subdirectories will be included. " + + "Paths should not contain leading or trailing slashes. " + + "If not specified, pulls from the root of the remote workspace.", + ) + .option( + "-e, --environment ", + "Environment to pull from (e.g. 'production', 'staging')", + ) + .option("-v, --verbose", "Show detailed information about the operation") + .option("-q, --quiet", "Suppress output of successful files"), +).action( + handleSyncErrors(async (options: PullOptions) => { + const client = getClient(options); + + // Create a separate FileSyncer instance with log level based on verbose flag only + const fileSyncer = new FileSyncer(client, { + baseDir: options.localFilesDirectory, + verbose: options.verbose, + }); + + log("Pulling files from Humanloop...", "INFO"); + log(`Path: ${options.path || "(root)"}`, "INFO"); + log(`Environment: ${options.environment || "(default)"}`, "INFO"); + + const startTime = Date.now(); + const [successfulFiles, failedFiles] = await fileSyncer.pull( + options.path, + options.environment, + ); + const duration = Date.now() - startTime; + + // Always show operation result + const isSuccessful = failedFiles.length === 0; + log(`Pull completed in ${duration}ms`, isSuccessful ? "SUCCESS" : "ERROR"); + + // Only suppress successful files output if quiet flag is set + if (successfulFiles.length > 0 && !options.quiet) { + console.log(); // Empty line + log(`Successfully pulled ${successfulFiles.length} files:`, "SUCCESS"); + for (const file of successfulFiles) { + log(` ✓ ${file}`, "SUCCESS"); + } + } + + // Always show failed files + if (failedFiles.length > 0) { + console.log(); // Empty line + log(`Failed to pull ${failedFiles.length} files:`, "ERROR"); + for (const file of failedFiles) { + log(` ✗ ${file}`, "ERROR"); + } + } + }), +); program.parse(process.argv); diff --git a/src/humanloop.client.ts b/src/humanloop.client.ts index 6140e7b5..06aa3f91 100644 --- a/src/humanloop.client.ts +++ b/src/humanloop.client.ts @@ -29,8 +29,7 @@ import { import { HumanloopSpanExporter } from "./otel/exporter"; import { HumanloopSpanProcessor } from "./otel/processor"; import { overloadCall, overloadLog } from "./overload"; -import { FileSyncerOptions, SyncClient } from "./sync"; -import Logger from "./utils/Logger"; +import { SyncClient } from "./sync"; import { SDK_VERSION } from "./version"; const RED = "\x1b[91m"; @@ -287,7 +286,7 @@ export class HumanloopClient extends BaseHumanloopClient { // Warn user if cacheSize is non-default but useLocalFiles is false if (!this.useLocalFiles && options.cacheSize !== undefined) { - Logger.warn( + console.warn( `The specified cacheSize=${options.cacheSize} will have no effect because useLocalFiles=false. ` + `File caching is only active when local files are enabled.`, ); diff --git a/src/sync/FileSyncer.ts b/src/sync/FileSyncer.ts index bc726513..16890813 100644 --- a/src/sync/FileSyncer.ts +++ b/src/sync/FileSyncer.ts @@ -2,16 +2,9 @@ import { FileType } from "api"; import fs from "fs"; import path from "path"; -import { HumanloopClient as BaseHumanloopClient } from "../Client"; import LRUCache from "../cache/LRUCache"; import { HumanloopRuntimeError } from "../error"; -import Logger, { LogLevel } from "../utils/Logger"; - -// Set up isolated logger for file sync operations -// This logger uses the "humanloop.sdk.file_syncer" namespace, separate from the main client's logger, -// allowing CLI commands and other consumers to control sync logging verbosity independently. -const LOGGER_NAMESPACE = "humanloop.sdk.file_syncer"; -Logger.setLevel("info"); +import { HumanloopClient } from "../humanloop.client"; // Default cache size for file content caching const DEFAULT_CACHE_SIZE = 100; @@ -26,41 +19,43 @@ export const SERIALIZABLE_FILE_TYPES = new Set([ export interface FileSyncerOptions { baseDir?: string; cacheSize?: number; - logLevel?: LogLevel; + verbose?: boolean; +} + +// Simple logging with color and verbosity control +const LogType = { + DEBUG: "\x1b[90m", // gray + INFO: "\x1b[96m", // cyan + WARN: "\x1b[93m", // yellow + ERROR: "\x1b[91m", // red + RESET: "\x1b[0m", +} as const; + +function log( + message: string, + type: keyof typeof LogType, + verbose: boolean = false, +): void { + // Only show debug/info if verbose is true + if ((type === "DEBUG" || type === "INFO") && !verbose) return; + console.log(`${LogType[type]}${message}${LogType.RESET}`); } /** * Format API error messages to be more user-friendly. */ -function formatApiError(error: Error): string { - const errorMsg = error.toString(); - - // If the error doesn't look like an API error with status code and body - if (!errorMsg.includes("status_code") || !errorMsg.includes("body:")) { - return errorMsg; - } - +function formatApiError(error: Error, verbose: boolean = false): string { + const errorMsg = error.message || String(error); try { - // Extract the body part and parse as JSON - const bodyParts = errorMsg.split("body:"); - if (bodyParts.length < 2) return errorMsg; - - const bodyStr = bodyParts[1].trim(); - const body = JSON.parse(bodyStr); - - // Get the detail from the body - const detail = body.detail || {}; - - // Handle both string and dictionary types for detail + const detail = JSON.parse(errorMsg); if (typeof detail === "string") { return detail; } else if (typeof detail === "object") { return detail.description || detail.msg || errorMsg; } - return errorMsg; } catch (e) { - Logger.debug(`Failed to parse error message: ${e}`); + log(`Failed to parse error message: ${e}`, "DEBUG", verbose); return errorMsg; } } @@ -78,19 +73,21 @@ function formatApiError(error: Error): string { * allowing for seamless reference between local and remote environments using the same path identifiers. */ export default class FileSyncer { - private client: BaseHumanloopClient; - private baseDir: string; - private cacheSize: number; - private fileContentCache: LRUCache; + // Default page size for API pagination when listing Files + private static readonly PAGE_SIZE = 100; - constructor(client: BaseHumanloopClient, options: FileSyncerOptions = {}) { + private readonly client: HumanloopClient; + private readonly baseDir: string; + private readonly cacheSize: number; + private readonly fileContentCache: LRUCache; + private readonly verbose: boolean; + + constructor(client: HumanloopClient, options: FileSyncerOptions = {}) { this.client = client; this.baseDir = options.baseDir || "humanloop"; this.cacheSize = options.cacheSize || DEFAULT_CACHE_SIZE; this.fileContentCache = new LRUCache(this.cacheSize); - - // Set the log level using the isolated logger - Logger.setLevel(options.logLevel || "warn"); + this.verbose = options.verbose || false; } /** @@ -107,25 +104,15 @@ export default class FileSyncer { filePath: string, fileType: SerializableFileType, ): string { - // Construct path to local file - const localPath = path.join(this.baseDir, filePath); - // Add appropriate extension - const dirName = path.dirname(localPath); - const baseName = path.basename(localPath, path.extname(localPath)); - const fullPath = path.join(dirName, `${baseName}.${fileType}`); - - if (!fs.existsSync(fullPath)) { - throw new HumanloopRuntimeError(`Local file not found: ${fullPath}`); - } - + const fullPath = path.join(this.baseDir, `${filePath}.${fileType}`); try { // Read the raw file content const fileContent = fs.readFileSync(fullPath, "utf8"); - Logger.debug(`Using local file content from ${fullPath}`); + log(`Using local file content from ${fullPath}`, "DEBUG", this.verbose); return fileContent; } catch (error) { throw new HumanloopRuntimeError( - `Error reading local file ${fullPath}: ${error}`, + `Failed to read ${fileType} ${filePath} from disk: ${error}`, ); } } @@ -147,7 +134,11 @@ export default class FileSyncer { // Check if in cache const cachedContent = this.fileContentCache.get(cacheKey); if (cachedContent !== undefined) { - Logger.debug(`Using cached file content for ${filePath}.${fileType}`); + log( + `Using cached file content for ${filePath}.${fileType}`, + "DEBUG", + this.verbose, + ); return cachedContent; } @@ -204,8 +195,9 @@ export default class FileSyncer { // Write raw file content to file fs.writeFileSync(newPath, serializedContent); + log(`Writing ${fileType} ${filePath} to disk`, "DEBUG", this.verbose); } catch (error) { - Logger.error(`Failed to write ${fileType} ${filePath} to disk: ${error}`); + log(`Failed to write ${fileType} ${filePath} to disk: ${error}`, "ERROR"); throw error; } } @@ -224,14 +216,13 @@ export default class FileSyncer { }); if (!SERIALIZABLE_FILE_TYPES.has(file.type as SerializableFileType)) { - Logger.error(`Unsupported file type: ${file.type}`); + log(`Unsupported file type: ${file.type}`, "ERROR"); return false; } - // Type assertion for rawFileContent since we know it exists when includeRawFileContent is true const rawContent = (file as any).rawFileContent; if (!rawContent) { - Logger.error(`No content found for ${file.type} ${filePath}`); + log(`No content found for ${file.type} ${filePath}`, "ERROR"); return false; } @@ -242,7 +233,7 @@ export default class FileSyncer { ); return true; } catch (error) { - Logger.error(`Failed to pull file ${filePath}: ${error}`); + log(`Failed to pull file ${filePath}: ${error}`, "ERROR"); return false; } } @@ -263,81 +254,82 @@ export default class FileSyncer { const successfulFiles: string[] = []; const failedFiles: string[] = []; let page = 1; + let totalPages = 0; - Logger.debug( - `Fetching files from directory: ${dirPath || "(root)"} in environment: ${environment || "(default)"}`, + log( + `Fetching files from ${dirPath || "root"} (environment: ${environment || "default"})`, + "INFO", + this.verbose, ); while (true) { try { - Logger.debug( - `${dirPath || "(root)"}: Requesting page ${page} of files`, - ); - const response = await this.client.files.listFiles({ type: Array.from(SERIALIZABLE_FILE_TYPES), page, - size: 100, + size: FileSyncer.PAGE_SIZE, includeRawFileContent: true, environment, path: dirPath, }); + // Calculate total pages on first response + if (page === 1) { + totalPages = Math.ceil(response.total / FileSyncer.PAGE_SIZE); + } + if (response.records.length === 0) { - Logger.debug( - `Finished reading files for path ${dirPath || "(root)"}`, - ); break; } - Logger.debug( - `${dirPath || "(root)"}: Read page ${page} containing ${response.records.length} files`, + log( + `Reading page ${page}/${totalPages} (${response.records.length} Files)`, + "DEBUG", + this.verbose, ); // Process each file for (const file of response.records) { - // Skip if not a serializable file type if ( !SERIALIZABLE_FILE_TYPES.has(file.type as SerializableFileType) ) { - Logger.warn(`Skipping unsupported file type: ${file.type}`); + log(`Skipping unsupported file type: ${file.type}`, "WARN"); continue; } const fileType = file.type as SerializableFileType; - - // Type assertion for rawFileContent since we know it exists when includeRawFileContent is true const rawContent = (file as any).rawFileContent; if (!rawContent) { - Logger.warn(`No content found for ${file.type} ${file.path}`); + log(`No content found for ${file.type} ${file.path}`, "WARN"); failedFiles.push(file.path); continue; } try { - Logger.debug(`Writing ${file.type} ${file.path} to disk`); this._saveSerializedFile(rawContent, file.path, fileType); successfulFiles.push(file.path); } catch (error) { failedFiles.push(file.path); - Logger.error(`Failed to save ${file.path}: ${error}`); + log(`Failed to save ${file.path}: ${error}`, "ERROR"); } } + // Update pagination based on items received + if (response.records.length < FileSyncer.PAGE_SIZE) { + // Last page (either partial or empty) + break; + } page += 1; } catch (error) { - const formattedError = formatApiError(error as Error); + const formattedError = formatApiError(error as Error, this.verbose); throw new HumanloopRuntimeError( `Failed to fetch page ${page}: ${formattedError}`, ); } } - if (successfulFiles.length > 0) { - Logger.info(`Successfully pulled ${successfulFiles.length} files`); - } if (failedFiles.length > 0) { - Logger.warn(`Failed to pull ${failedFiles.length} files`); + log(`Failed to pull ${failedFiles.length} files`, "WARN"); } return [successfulFiles, failedFiles]; @@ -391,24 +383,17 @@ export default class FileSyncer { apiPath = this._normalizePath(filePath, true); } - Logger.info( - `Starting pull: path=${apiPath || "(root)"}, environment=${environment || "(default)"}`, - ); - try { let successfulFiles: string[]; let failedFiles: string[]; if (apiPath === undefined) { - // Pull all from root - Logger.debug("Pulling all files from (root)"); [successfulFiles, failedFiles] = await this._pullDirectory( undefined, environment, ); } else { if (isFilePath) { - Logger.debug(`Pulling file: ${apiPath}`); if (await this._pullFile(apiPath, environment)) { successfulFiles = [apiPath]; failedFiles = []; @@ -417,7 +402,6 @@ export default class FileSyncer { failedFiles = [apiPath]; } } else { - Logger.debug(`Pulling directory: ${apiPath || "(root)"}`); [successfulFiles, failedFiles] = await this._pullDirectory( apiPath, environment, @@ -429,8 +413,10 @@ export default class FileSyncer { this.clearCache(); const duration = Date.now() - startTime; - Logger.info( - `Pull completed in ${duration}ms: ${successfulFiles.length} files pulled`, + log( + `Successfully pulled ${successfulFiles.length} files in ${duration}ms`, + "INFO", + this.verbose, ); return [successfulFiles, failedFiles]; @@ -464,4 +450,19 @@ export default class FileSyncer { return normalizedPath; } + + private _parseErrorResponse(response: any): string { + try { + if (response?.error?.message) { + return response.error.message; + } + if (typeof response === "string") { + return response; + } + return JSON.stringify(response); + } catch (e) { + log(`Failed to parse error message: ${e}`, "DEBUG", this.verbose); + return String(response); + } + } } diff --git a/src/utils/Logger.ts b/src/utils/Logger.ts deleted file mode 100644 index 3c74a47b..00000000 --- a/src/utils/Logger.ts +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Logger utility for consistent colored console output across the Humanloop SDK. - */ - -// ANSI escape codes for colors -export const Colors = { - YELLOW: "\x1b[93m", - CYAN: "\x1b[96m", - GREEN: "\x1b[92m", - RED: "\x1b[91m", - RESET: "\x1b[0m", -} as const; - -export type LogLevel = 'error' | 'warn' | 'info' | 'debug'; - -/** - * Helper class for colored console output with log level filtering - */ -export default class Logger { - private static currentLevel: number = 1; // Default to 'warn' - private static readonly levels: Record = { - 'error': 0, - 'warn': 1, - 'info': 2, - 'debug': 3 - }; - - /** - * Set the log level for filtering - */ - static setLevel(level: LogLevel): void { - this.currentLevel = this.levels[level] || 1; - } - - /** - * Safely converts any value to a string, handling undefined/null - */ - private static toString(value: any): string { - if (value === undefined) return "undefined"; - if (value === null) return "null"; - return String(value); - } - - /** - * Log a warning message in yellow - */ - static warn(message: any): void { - if (this.currentLevel >= 1) { - console.warn(`${Colors.YELLOW}${Logger.toString(message)}${Colors.RESET}`); - } - } - - /** - * Log an info message in cyan - */ - static info(message: any): void { - if (this.currentLevel >= 2) { - console.info(`${Colors.CYAN}${Logger.toString(message)}${Colors.RESET}`); - } - } - - /** - * Log a success message in green - */ - static success(message: any): void { - if (this.currentLevel >= 2) { // Success is info level - console.log(`${Colors.GREEN}${Logger.toString(message)}${Colors.RESET}`); - } - } - - /** - * Log an error message in red - */ - static error(message: any): void { - if (this.currentLevel >= 0) { - console.error(`${Colors.RED}${Logger.toString(message)}${Colors.RESET}`); - } - } - - /** - * Log a plain message without any color (at info level) - */ - static log(message: any): void { - if (this.currentLevel >= 2) { - console.log(Logger.toString(message)); - } - } - - /** - * Log a debug message (for detailed information) - */ - static debug(message: any): void { - if (this.currentLevel >= 3) { - console.debug(Logger.toString(message)); - } - } - - /** - * Log a message with custom color (at info level) - */ - static withColor(message: any, color: keyof typeof Colors): void { - if (this.currentLevel >= 2) { - console.log(`${Colors[color]}${Logger.toString(message)}${Colors.RESET}`); - } - } -} \ No newline at end of file diff --git a/src/utils/index.ts b/src/utils/index.ts deleted file mode 100644 index a20b6187..00000000 --- a/src/utils/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./Logger"; \ No newline at end of file diff --git a/tests/custom/unit/Logger.test.ts b/tests/custom/unit/Logger.test.ts deleted file mode 100644 index 40e67ab6..00000000 --- a/tests/custom/unit/Logger.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -import Logger from "../../../../src/utils/Logger"; - -describe("Logger", () => { - let consoleSpy: { - log: jest.SpyInstance; - info: jest.SpyInstance; - warn: jest.SpyInstance; - debug: jest.SpyInstance; - }; - - beforeEach(() => { - // Spy on all console methods - consoleSpy = { - log: jest.spyOn(console, "log"), - info: jest.spyOn(console, "info"), - warn: jest.spyOn(console, "warn"), - debug: jest.spyOn(console, "debug"), - }; - // Reset log level before each test - Logger.setLevel("warn"); - }); - - afterEach(() => { - // Restore all spies - Object.values(consoleSpy).forEach((spy) => spy.mockRestore()); - }); - - describe("log levels", () => { - it("should respect log level settings", () => { - Logger.setLevel("warn"); - Logger.debug("debug message"); - Logger.info("info message"); - Logger.warn("warn message"); - - expect(consoleSpy.debug).not.toHaveBeenCalled(); - expect(consoleSpy.info).not.toHaveBeenCalled(); - expect(consoleSpy.warn).toHaveBeenCalledWith( - expect.stringContaining("warn"), - ); - }); - - it("should show all messages when level is set to debug", () => { - Logger.setLevel("debug"); - Logger.debug("debug message"); - Logger.info("info message"); - Logger.warn("warn message"); - - expect(consoleSpy.debug).toHaveBeenCalledWith( - expect.stringContaining("debug"), - ); - expect(consoleSpy.info).toHaveBeenCalledWith( - expect.stringContaining("info"), - ); - expect(consoleSpy.warn).toHaveBeenCalledWith( - expect.stringContaining("warn"), - ); - }); - }); - - describe("message formatting", () => { - it("should handle different input types", () => { - Logger.setLevel("info"); - - Logger.info(undefined); - Logger.info(null); - Logger.info({ key: "value" }); - - expect(consoleSpy.info).toHaveBeenCalledWith( - expect.stringContaining("undefined"), - ); - expect(consoleSpy.info).toHaveBeenCalledWith( - expect.stringContaining("null"), - ); - expect(consoleSpy.info).toHaveBeenCalledWith( - expect.stringContaining("[object Object]"), - ); - }); - }); -}); From 209b15e00bce139ff86adf34963ad46c28ba2965 Mon Sep 17 00:00:00 2001 From: Ale Pouroullis Date: Tue, 20 May 2025 14:34:36 +0100 Subject: [PATCH 05/16] refactor: implement unified client overloading system - Create new unified `overloadClient` function that handles both log and call methods - Simplify client initialization by directly overloading parent class instances - Replace SyncClient with direct FileSyncer instance for cleaner file handling - Standardize tracing context, local file handling, and evaluation context across all client types --- src/humanloop.client.ts | 46 ++---- src/overload.ts | 321 ++++++++++++++++++++++++++++------------ src/sync/index.ts | 2 +- 3 files changed, 233 insertions(+), 136 deletions(-) diff --git a/src/humanloop.client.ts b/src/humanloop.client.ts index 06aa3f91..998632a2 100644 --- a/src/humanloop.client.ts +++ b/src/humanloop.client.ts @@ -8,10 +8,6 @@ import { OpenAIInstrumentation } from "@traceloop/instrumentation-openai"; import { HumanloopClient as BaseHumanloopClient } from "./Client"; import { ChatMessage } from "./api"; import { Evaluations as BaseEvaluations } from "./api/resources/evaluations/client/Client"; -import { Evaluators } from "./api/resources/evaluators/client/Client"; -import { Flows } from "./api/resources/flows/client/Client"; -import { Prompts } from "./api/resources/prompts/client/Client"; -import { Tools } from "./api/resources/tools/client/Client"; import { ToolKernelRequest } from "./api/types/ToolKernelRequest"; import { flowUtilityFactory } from "./decorators/flow"; import { promptDecoratorFactory } from "./decorators/prompt"; @@ -28,8 +24,8 @@ import { } from "./evals/types"; import { HumanloopSpanExporter } from "./otel/exporter"; import { HumanloopSpanProcessor } from "./otel/processor"; -import { overloadCall, overloadLog } from "./overload"; -import { SyncClient } from "./sync"; +import { overloadClient } from "./overload"; +import { FileSyncer } from "./sync"; import { SDK_VERSION } from "./version"; const RED = "\x1b[91m"; @@ -235,16 +231,12 @@ export interface HumanloopClientOptions extends BaseHumanloopClient.Options { export class HumanloopClient extends BaseHumanloopClient { protected readonly _evaluations: ExtendedEvaluations; - protected readonly _prompts_overloaded: Prompts; - protected readonly _flows_overloaded: Flows; - protected readonly _tools_overloaded: Tools; - protected readonly _evaluators_overloaded: Evaluators; protected readonly instrumentProviders: { OpenAI?: any; Anthropic?: any; CohereAI?: any; }; - protected readonly _syncClient: SyncClient; + protected readonly _fileSyncer: FileSyncer; protected readonly useLocalFiles: boolean; protected get opentelemetryTracer(): Tracer { @@ -292,21 +284,17 @@ export class HumanloopClient extends BaseHumanloopClient { ); } - this._syncClient = new SyncClient(this, { + this._fileSyncer = new FileSyncer(this, { baseDir: options.localFilesDirectory || "humanloop", cacheSize: options.cacheSize, }); this.instrumentProviders = options.instrumentProviders || {}; - this._prompts_overloaded = overloadLog(super.prompts); - this._prompts_overloaded = overloadCall(this._prompts_overloaded); - - this._tools_overloaded = overloadLog(super.tools); - - this._flows_overloaded = overloadLog(super.flows); - - this._evaluators_overloaded = overloadLog(super.evaluators); + overloadClient(super.prompts, this._fileSyncer, this.useLocalFiles); + overloadClient(super.flows, this._fileSyncer, this.useLocalFiles); + overloadClient(super.tools, this._fileSyncer, this.useLocalFiles); + overloadClient(super.evaluators, this._fileSyncer, this.useLocalFiles); this._evaluations = new ExtendedEvaluations(options, this); @@ -652,26 +640,10 @@ ${RESET}`, path?: string, environment?: string, ): Promise<[string[], string[]]> { - return this._syncClient.pull(path, environment); + return this._fileSyncer.pull(path, environment); } public get evaluations(): ExtendedEvaluations { return this._evaluations; } - - public get prompts(): Prompts { - return this._prompts_overloaded; - } - - public get flows(): Flows { - return this._flows_overloaded; - } - - public get tools(): Tools { - return this._tools_overloaded; - } - - public get evaluators(): Evaluators { - return this._evaluators_overloaded; - } } diff --git a/src/overload.ts b/src/overload.ts index dd66643a..f1d59ed8 100644 --- a/src/overload.ts +++ b/src/overload.ts @@ -1,44 +1,61 @@ +import path from "path"; + import { - CreateEvaluatorLogRequest, - FlowLogRequest, - PromptCallResponse, - PromptLogRequest, - ToolLogRequest, + CreateEvaluatorLogRequest, FlowLogRequest, PromptLogRequest, + ToolLogRequest } from "./api"; +import { Agents } from "./api/resources/agents/client/Client"; import { Evaluators } from "./api/resources/evaluators/client/Client"; import { Flows } from "./api/resources/flows/client/Client"; import { Prompts } from "./api/resources/prompts/client/Client"; import { Tools } from "./api/resources/tools/client/Client"; import { getDecoratorContext, getEvaluationContext, getTraceId } from "./context"; import { HumanloopRuntimeError } from "./error"; +import FileSyncer, { + SERIALIZABLE_FILE_TYPES, + SerializableFileType, +} from "./sync/FileSyncer"; -export function overloadLog( - client: T, +type ClientType = Flows | Agents | Prompts | Tools | Evaluators; +type LogRequestType = + | FlowLogRequest + | PromptLogRequest + | ToolLogRequest + | CreateEvaluatorLogRequest; + +/** + * Get the file type based on the client type. + * Only returns types that can be loaded from local filesystem. + */ +function getFileTypeFromClient(client: ClientType): SerializableFileType | null { + if (client instanceof Prompts) { + return "prompt"; + } else if (client instanceof Agents) { + return "agent"; + } else if (client instanceof Tools) { + return null; // Tools don't support local files + } else if (client instanceof Flows) { + return null; // Flows don't support local files + } else if (client instanceof Evaluators) { + return null; // Evaluators don't support local files + } else { + throw new HumanloopRuntimeError( + // @ts-ignore Client shouldn't be of a type other than those checked above, but included as a safeguard + `Unsupported client type: ${client.constructor.name}`, + ); + } +} + +/** + * Handle tracing context for both log and call methods. + */ +function handleTracingContext( + request: T, + client: ClientType, ): T { - const originalLog = client.log.bind(client); - - const _overloadedLog = async ( - request: T extends Flows - ? FlowLogRequest - : T extends Prompts - ? PromptLogRequest - : T extends Tools - ? ToolLogRequest - : T extends Evaluators - ? CreateEvaluatorLogRequest - : never, - options?: T extends Flows - ? Flows.RequestOptions - : T extends Prompts - ? Prompts.RequestOptions - : T extends Tools - ? Tools.RequestOptions - : T extends Evaluators - ? Evaluators.RequestOptions - : never, - ) => { - const traceId = getTraceId(); - if (traceId !== undefined && client instanceof Flows) { + const traceId = getTraceId(); + if (traceId !== undefined) { + if (client instanceof Flows) { const context = getDecoratorContext(); if (context === undefined) { throw new HumanloopRuntimeError( @@ -50,81 +67,189 @@ export function overloadLog( ); } - if (traceId !== undefined) { - if ("traceParentId" in request) { - console.warn( - "Ignoring trace_parent_id argument: the Flow decorator manages tracing.", - ); - } - request = { - ...request, - traceParentId: traceId, - }; + if ("traceParentId" in request) { + console.warn( + "Ignoring trace_parent_id argument: the Flow decorator manages tracing.", + ); } + return { + ...request, + traceParentId: traceId, + }; + } + return request; +} - const evaluationContext = getEvaluationContext(); - if (evaluationContext !== undefined) { - const [kwargsEval, evalCallback] = evaluationContext.logArgsWithContext({ - logArgs: request, - forOtel: true, - path: request.path, - }); - try { - // @ts-ignore Polymorphism alarms the type checker - const response = await originalLog(kwargsEval, options); - if (evalCallback !== null) { - await evalCallback(response.id); - } - return response; - } catch (e) { - throw new HumanloopRuntimeError(String(e)); - } - } else { - try { - // @ts-ignore Polymorphism alarms the type checker - return await originalLog(request, options); - } catch (e) { - throw new HumanloopRuntimeError(String(e)); - } - } - }; +/** + * Load .prompt/.agent file content from local filesystem into API request. + */ +function handleLocalFiles( + request: T, + client: ClientType, + fileSyncer: FileSyncer, +): T { + // Validate request has either id or path, but not both + if ("id" in request && "path" in request) { + throw new HumanloopRuntimeError("Cannot specify both `id` and `path`"); + } + if (!("id" in request) && !("path" in request)) { + throw new HumanloopRuntimeError("Must specify either `id` or `path`"); + } - // @ts-ignore - client.log = _overloadedLog.bind(client); - // @ts-ignore - client._log = originalLog.bind(client); + // If using id, we can't use local files + if ("id" in request) { + return request; + } - return client; + const filePath = request.path; + if (!filePath) { + throw new HumanloopRuntimeError("Path cannot be empty"); + } + + // Check for path format issues (absolute paths or leading/trailing slashes) + const normalizedPath = filePath.trim().replace(/^\/+|\/+$/g, ""); + if (path.isAbsolute(filePath) || filePath !== normalizedPath) { + throw new HumanloopRuntimeError( + `Path '${filePath}' format is invalid. ` + + `Paths must follow the standard API format 'path/to/resource' without leading or trailing slashes. ` + + `Please use '${normalizedPath}' instead.`, + ); + } + + // Check for file extensions + if (fileSyncer.isFile(filePath)) { + const pathWithoutExtension = path.join( + path.dirname(filePath), + path.basename(filePath, path.extname(filePath)), + ); + throw new HumanloopRuntimeError( + `Path '${filePath}' includes a file extension which is not supported in API calls. ` + + `When referencing files via the \`path\` parameter, use the path without extensions: '${pathWithoutExtension}'. ` + + `Note: File extensions are only used when pulling specific files via the CLI.`, + ); + } + + // Check if version_id or environment is specified + const useRemote = "versionId" in request || "environment" in request; + if (useRemote) { + throw new HumanloopRuntimeError( + `Cannot use local file for \`${filePath}\` as version_id or environment was specified. ` + + "Please either remove version_id/environment to use local files, or set use_local_files=False to use remote files.", + ); + } + + const fileType = getFileTypeFromClient(client); + if (!fileType || !SERIALIZABLE_FILE_TYPES.has(fileType)) { + throw new HumanloopRuntimeError( + `Local files are not supported for this client type: '${filePath}'.`, + ); + } + + // If file_type is already specified in request, prioritize user-provided value + if (fileType in request && typeof request[fileType as keyof T] !== "string") { + console.warn( + `Ignoring local file for \`${filePath}\` as ${fileType} parameters were directly provided. ` + + "Using provided parameters instead.", + ); + return request; + } + + try { + const fileContent = fileSyncer.getFileContent(filePath, fileType); + return { + ...request, + [fileType]: fileContent, + } as T; + } catch (error) { + throw new HumanloopRuntimeError( + `Failed to use local file for \`${filePath}\`: ${error}`, + ); + } } -export function overloadCall(client: Prompts): Prompts { - const originalCall = client.call.bind(client); - - const _overloadedCall = async ( - request: PromptLogRequest, - options?: Prompts.RequestOptions, - ): Promise => { - const traceId = getTraceId(); - if (traceId !== undefined) { - if ("traceParentId" in request) { - console.warn( - "Ignoring trace_parent_id argument: the Flow decorator manages tracing.", - ); - } - request = { - ...request, - traceParentId: traceId, - }; - } +/** + * Overloads a client with local file handling and tracing capabilities. + * This is the preferred way to overload clients, replacing individual overloadLog and overloadCall methods. + */ +export function overloadClient( + client: T, + fileSyncer?: FileSyncer, + useLocalFiles: boolean = false, +): T { + // Handle log method if it exists + if ("log" in client) { + const originalLog = (client as any).log.bind(client); + const _overloadedLog = async (request: LogRequestType, options?: any) => { + try { + request = handleTracingContext(request, client); + if ( + useLocalFiles && + (client instanceof Prompts || client instanceof Agents) + ) { + if (!fileSyncer) { + throw new HumanloopRuntimeError( + "SDK initialization error: fileSyncer is missing but required for local file operations.", + ); + } + request = handleLocalFiles(request, client, fileSyncer); + } - try { - return await originalCall(request, options); - } catch (e) { - throw new HumanloopRuntimeError(String(e)); - } - }; + const evaluationContext = getEvaluationContext(); + if (evaluationContext !== undefined) { + const [kwargsEval, evalCallback] = + evaluationContext.logArgsWithContext({ + logArgs: request, + forOtel: true, + path: request.path, + }); + try { + const response = await originalLog(kwargsEval as any, options); + if (evalCallback !== null) { + await evalCallback(response.id); + } + return response; + } catch (error) { + throw new HumanloopRuntimeError(String(error)); + } + } + return await originalLog(request as any, options); + } catch (error) { + if (error instanceof HumanloopRuntimeError) { + throw error; + } + throw new HumanloopRuntimeError(String(error)); + } + }; + (client as any)._log = originalLog; + (client as any).log = _overloadedLog.bind(client); + } - client.call = _overloadedCall.bind(client); + // Handle call method if it exists (for Prompts and Agents). Note that we can't use `"call" in client` + // because Tools also have a call method. + if (client instanceof Prompts || client instanceof Agents) { + const originalCall = (client as any).call.bind(client); + const _overloadedCall = async (request: PromptLogRequest, options?: any) => { + try { + request = handleTracingContext(request, client); + if (useLocalFiles) { + if (!fileSyncer) { + throw new HumanloopRuntimeError( + "fileSyncer is required for clients that support call operations", + ); + } + request = handleLocalFiles(request, client, fileSyncer); + } + return await originalCall(request, options); + } catch (error) { + if (error instanceof HumanloopRuntimeError) { + throw error; + } + throw new HumanloopRuntimeError(String(error)); + } + }; + (client as any)._call = originalCall; + (client as any).call = _overloadedCall.bind(client); + } return client; } diff --git a/src/sync/index.ts b/src/sync/index.ts index 892194b3..bf310919 100644 --- a/src/sync/index.ts +++ b/src/sync/index.ts @@ -4,4 +4,4 @@ * This module provides sync functionality between Humanloop and the local filesystem. */ -export { default as SyncClient, FileSyncerOptions } from './FileSyncer'; \ No newline at end of file +export { default as FileSyncer, FileSyncerOptions } from './FileSyncer'; \ No newline at end of file From 7178ecc12362e05e4af632174d4a429be8d40e5d Mon Sep 17 00:00:00 2001 From: Ale Pouroullis Date: Tue, 20 May 2025 15:02:46 +0100 Subject: [PATCH 06/16] refactor: extract path normalization in FileSyncer to separate util --- src/pathUtils.ts | 64 ++++++++++++++++++++++++++++++++++++++++++ src/sync/FileSyncer.ts | 44 ++--------------------------- 2 files changed, 66 insertions(+), 42 deletions(-) create mode 100644 src/pathUtils.ts diff --git a/src/pathUtils.ts b/src/pathUtils.ts new file mode 100644 index 00000000..fcdf3cf4 --- /dev/null +++ b/src/pathUtils.ts @@ -0,0 +1,64 @@ +import * as path from "path"; + +/** + * Normalize a path to the standard Humanloop API format. + * + * This function is primarily used when interacting with the Humanloop API to ensure paths + * follow the standard format: 'path/to/resource' without leading/trailing slashes. + * It's used when pulling files from Humanloop to local filesystem (see FileSyncer.pull) + * + * The function: + * - Converts Windows backslashes to forward slashes + * - Normalizes consecutive slashes + * - Optionally strips file extensions (e.g. .prompt, .agent) + * - Removes leading/trailing slashes to match API conventions + * + * Leading/trailing slashes are stripped because the Humanloop API expects paths in the + * format 'path/to/resource' without them. This is consistent with how the API stores + * and references files, and ensures paths work correctly in both API calls and local + * filesystem operations. + * + * @param pathStr - The path to normalize. Can be a Windows or Unix-style path. + * @param stripExtension - If true, removes the file extension (e.g. .prompt, .agent) + * @returns Normalized path string in the format 'path/to/resource' + * + * @example + * normalizePath("path/to/file.prompt") + * // => 'path/to/file.prompt' + * + * @example + * normalizePath("path/to/file.prompt", true) + * // => 'path/to/file' + * + * @example + * normalizePath("\\windows\\style\\path.prompt") + * // => 'windows/style/path.prompt' + * + * @example + * normalizePath("/leading/slash/path/") + * // => 'leading/slash/path' + * + * @example + * normalizePath("multiple//slashes//path") + * // => 'multiple/slashes/path' + */ +export function normalizePath( + pathStr: string, + stripExtension: boolean = false, +): string { + // Convert Windows backslashes to forward slashes + const normalizedSeparators = pathStr.replace(/\\/g, "/"); + + // Use path.posix to handle path normalization (handles consecutive slashes) + // We use posix to ensure forward slashes are used consistently + let normalizedPath = path.posix.normalize(normalizedSeparators); + + // Strip extension if requested + if (stripExtension) { + const ext = path.posix.extname(normalizedPath); + normalizedPath = normalizedPath.slice(0, -ext.length); + } + + // Remove leading/trailing slashes + return normalizedPath.replace(/^\/+|\/+$/g, ""); +} diff --git a/src/sync/FileSyncer.ts b/src/sync/FileSyncer.ts index 16890813..86ae25f9 100644 --- a/src/sync/FileSyncer.ts +++ b/src/sync/FileSyncer.ts @@ -5,6 +5,7 @@ import path from "path"; import LRUCache from "../cache/LRUCache"; import { HumanloopRuntimeError } from "../error"; import { HumanloopClient } from "../humanloop.client"; +import * as pathUtils from "../pathUtils"; // Default cache size for file content caching const DEFAULT_CACHE_SIZE = 100; @@ -380,7 +381,7 @@ export default class FileSyncer { isFilePath = this.isFile(filePath); // For API communication, we need path without extension - apiPath = this._normalizePath(filePath, true); + apiPath = pathUtils.normalizePath(filePath, true); } try { @@ -424,45 +425,4 @@ export default class FileSyncer { throw new HumanloopRuntimeError(`Pull operation failed: ${error}`); } } - - /** - * Normalize the path by removing extensions, etc. - */ - private _normalizePath(filePath: string, stripExtension: boolean = false): string { - if (!filePath) return ""; - - // Remove any file extensions if requested - let normalizedPath = - stripExtension && filePath.includes(".") - ? filePath.substring(0, filePath.lastIndexOf(".")) - : filePath; - - // Convert backslashes to forward slashes - normalizedPath = normalizedPath.replace(/\\/g, "/"); - - // Remove leading/trailing whitespace and slashes - normalizedPath = normalizedPath.trim().replace(/^\/+|\/+$/g, ""); - - // Normalize multiple consecutive slashes into a single forward slash - while (normalizedPath.includes("//")) { - normalizedPath = normalizedPath.replace(/\/\//g, "/"); - } - - return normalizedPath; - } - - private _parseErrorResponse(response: any): string { - try { - if (response?.error?.message) { - return response.error.message; - } - if (typeof response === "string") { - return response; - } - return JSON.stringify(response); - } catch (e) { - log(`Failed to parse error message: ${e}`, "DEBUG", this.verbose); - return String(response); - } - } } From 5be7cf7eb32a4525b902f7499dde3082e7c5fb1e Mon Sep 17 00:00:00 2001 From: Ale Pouroullis Date: Tue, 20 May 2025 15:34:40 +0100 Subject: [PATCH 07/16] Add tests for pathUtils and fix regression found from new tests --- src/pathUtils.ts | 43 +++++----------- tests/custom/unit/LRUCache.test.ts | 2 +- tests/custom/unit/pathUtils.test.ts | 78 +++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 31 deletions(-) create mode 100644 tests/custom/unit/pathUtils.test.ts diff --git a/src/pathUtils.ts b/src/pathUtils.ts index fcdf3cf4..5a2ba125 100644 --- a/src/pathUtils.ts +++ b/src/pathUtils.ts @@ -21,44 +21,27 @@ import * as path from "path"; * @param pathStr - The path to normalize. Can be a Windows or Unix-style path. * @param stripExtension - If true, removes the file extension (e.g. .prompt, .agent) * @returns Normalized path string in the format 'path/to/resource' - * - * @example - * normalizePath("path/to/file.prompt") - * // => 'path/to/file.prompt' - * - * @example - * normalizePath("path/to/file.prompt", true) - * // => 'path/to/file' - * - * @example - * normalizePath("\\windows\\style\\path.prompt") - * // => 'windows/style/path.prompt' - * - * @example - * normalizePath("/leading/slash/path/") - * // => 'leading/slash/path' - * - * @example - * normalizePath("multiple//slashes//path") - * // => 'multiple/slashes/path' */ export function normalizePath( pathStr: string, stripExtension: boolean = false, ): string { // Convert Windows backslashes to forward slashes - const normalizedSeparators = pathStr.replace(/\\/g, "/"); + let normalizedPath = pathStr.replace(/\\/g, "/"); + + // Use path.posix to handle path normalization (handles consecutive slashes and . /..) + normalizedPath = path.posix.normalize(normalizedPath); - // Use path.posix to handle path normalization (handles consecutive slashes) - // We use posix to ensure forward slashes are used consistently - let normalizedPath = path.posix.normalize(normalizedSeparators); + // Remove leading/trailing slashes + normalizedPath = normalizedPath.replace(/^\/+|\/+$/g, ""); - // Strip extension if requested - if (stripExtension) { - const ext = path.posix.extname(normalizedPath); - normalizedPath = normalizedPath.slice(0, -ext.length); + // Strip extension if requested + if (stripExtension && normalizedPath.includes(".")) { + normalizedPath = path.posix.join( + path.posix.dirname(normalizedPath), + path.posix.basename(normalizedPath, path.posix.extname(normalizedPath)), + ); } - // Remove leading/trailing slashes - return normalizedPath.replace(/^\/+|\/+$/g, ""); + return normalizedPath; } diff --git a/tests/custom/unit/LRUCache.test.ts b/tests/custom/unit/LRUCache.test.ts index f518641f..f377165c 100644 --- a/tests/custom/unit/LRUCache.test.ts +++ b/tests/custom/unit/LRUCache.test.ts @@ -1,4 +1,4 @@ -import LRUCache from "../../../../src/cache/LRUCache"; +import LRUCache from "../../../src/cache/LRUCache"; describe("LRUCache", () => { let cache: LRUCache; diff --git a/tests/custom/unit/pathUtils.test.ts b/tests/custom/unit/pathUtils.test.ts new file mode 100644 index 00000000..db71121e --- /dev/null +++ b/tests/custom/unit/pathUtils.test.ts @@ -0,0 +1,78 @@ +import { normalizePath } from "../../../src/pathUtils"; + +describe("normalizePath", () => { + const testCases = [ + // Basic cases + { + input: "path/to/file.prompt", + expectedWithExtension: "path/to/file.prompt", + expectedWithoutExtension: "path/to/file", + }, + { + input: "path\\to\\file.agent", + expectedWithExtension: "path/to/file.agent", + expectedWithoutExtension: "path/to/file", + }, + { + input: "/leading/slashes/file.prompt", + expectedWithExtension: "leading/slashes/file.prompt", + expectedWithoutExtension: "leading/slashes/file", + }, + { + input: "trailing/slashes/file.agent/", + expectedWithExtension: "trailing/slashes/file.agent", + expectedWithoutExtension: "trailing/slashes/file", + }, + { + input: "multiple//slashes//file.prompt", + expectedWithExtension: "multiple/slashes/file.prompt", + expectedWithoutExtension: "multiple/slashes/file", + }, + // Edge cases + { + input: "path/to/file with spaces.prompt", + expectedWithExtension: "path/to/file with spaces.prompt", + expectedWithoutExtension: "path/to/file with spaces", + }, + { + input: "path/to/file\\with\\backslashes.prompt", + expectedWithExtension: "path/to/file/with/backslashes.prompt", + expectedWithoutExtension: "path/to/file/with/backslashes", + }, + { + input: "path/to/unicode/文件.prompt", + expectedWithExtension: "path/to/unicode/文件.prompt", + expectedWithoutExtension: "path/to/unicode/文件", + }, + { + input: "path/to/special/chars/!@#$%^&*().prompt", + expectedWithExtension: "path/to/special/chars/!@#$%^&*().prompt", + expectedWithoutExtension: "path/to/special/chars/!@#$%^&*()", + }, + ]; + + test.each(testCases)( + "normalizes path '$input' correctly", + ({ input, expectedWithExtension, expectedWithoutExtension }) => { + // Test without stripping extension + const resultWithExtension = normalizePath(input, false); + expect(resultWithExtension).toBe(expectedWithExtension); + + // Test with extension stripping + const resultWithoutExtension = normalizePath(input, true); + expect(resultWithoutExtension).toBe(expectedWithoutExtension); + + // Add custom failure messages if needed + if (resultWithExtension !== expectedWithExtension) { + throw new Error( + `Failed with stripExtension=false for '${input}'. Expected '${expectedWithExtension}', got '${resultWithExtension}'`, + ); + } + if (resultWithoutExtension !== expectedWithoutExtension) { + throw new Error( + `Failed with stripExtension=true for '${input}'. Expected '${expectedWithoutExtension}', got '${resultWithoutExtension}'`, + ); + } + }, + ); +}); From af8ebbe07f6b243af82f32bc404c798439c3fe24 Mon Sep 17 00:00:00 2001 From: Ale Pouroullis Date: Tue, 20 May 2025 16:07:31 +0100 Subject: [PATCH 08/16] tests: Write unit tests for FileSyncer; fix total page calculation bug --- src/sync/FileSyncer.ts | 14 +- tests/custom/FileSyncer.test.ts | 364 ++++++++++++++++++++++++++++++++ tests/custom/fixtures.ts | 21 ++ 3 files changed, 392 insertions(+), 7 deletions(-) create mode 100644 tests/custom/FileSyncer.test.ts create mode 100644 tests/custom/fixtures.ts diff --git a/src/sync/FileSyncer.ts b/src/sync/FileSyncer.ts index 86ae25f9..23aa97d1 100644 --- a/src/sync/FileSyncer.ts +++ b/src/sync/FileSyncer.ts @@ -2,10 +2,10 @@ import { FileType } from "api"; import fs from "fs"; import path from "path"; +import * as pathUtils from "../pathUtils"; import LRUCache from "../cache/LRUCache"; import { HumanloopRuntimeError } from "../error"; import { HumanloopClient } from "../humanloop.client"; -import * as pathUtils from "../pathUtils"; // Default cache size for file content caching const DEFAULT_CACHE_SIZE = 100; @@ -257,8 +257,8 @@ export default class FileSyncer { let page = 1; let totalPages = 0; - log( - `Fetching files from ${dirPath || "root"} (environment: ${environment || "default"})`, + log( + `Fetching files from ${dirPath || "root"} (environment: ${environment || "default"})`, "INFO", this.verbose, ); @@ -276,7 +276,8 @@ export default class FileSyncer { // Calculate total pages on first response if (page === 1) { - totalPages = Math.ceil(response.total / FileSyncer.PAGE_SIZE); + const actualPageSize = response.size || FileSyncer.PAGE_SIZE; + totalPages = Math.ceil(response.total / actualPageSize); } if (response.records.length === 0) { @@ -315,9 +316,8 @@ export default class FileSyncer { } } - // Update pagination based on items received - if (response.records.length < FileSyncer.PAGE_SIZE) { - // Last page (either partial or empty) + // Check if we've reached the last page + if (page >= totalPages) { break; } page += 1; diff --git a/tests/custom/FileSyncer.test.ts b/tests/custom/FileSyncer.test.ts new file mode 100644 index 00000000..ed003c95 --- /dev/null +++ b/tests/custom/FileSyncer.test.ts @@ -0,0 +1,364 @@ +import * as fs from "fs"; +import * as path from "path"; +import { v4 as uuidv4 } from "uuid"; + +import { HumanloopRuntimeError } from "../../src/error"; +import FileSyncer, { + SERIALIZABLE_FILE_TYPES, + SerializableFileType, +} from "../../src/sync/FileSyncer"; + +// Mock for HumanloopClient +class MockHumanloopClient { + files = { + retrieveByPath: jest.fn(), + listFiles: jest.fn(), + }; +} + +describe("FileSyncer", () => { + let mockClient: MockHumanloopClient; + let fileSyncer: FileSyncer; + let tempDir: string; + + beforeEach(() => { + mockClient = new MockHumanloopClient(); + tempDir = path.join(process.cwd(), "test-tmp", uuidv4()); + + // Create temporary directory + fs.mkdirSync(tempDir, { recursive: true }); + + fileSyncer = new FileSyncer(mockClient as any, { + baseDir: tempDir, + cacheSize: 10, + verbose: true, // Enable verbose logging for tests + }); + }); + + afterEach(() => { + // Clean up temporary files + if (fs.existsSync(tempDir)) { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + + // Clear all mocks + jest.clearAllMocks(); + }); + + describe("initialization", () => { + it("should initialize with correct base directory, cache size and file types", () => { + // Check that the FileSyncer is initialized with the correct properties + expect(fileSyncer["baseDir"]).toBe(tempDir); + expect(fileSyncer["cacheSize"]).toBe(10); + expect(SERIALIZABLE_FILE_TYPES).toEqual(new Set(["prompt", "agent"])); + }); + }); + + describe("isFile", () => { + it("should correctly identify prompt and agent files with case insensitivity", () => { + // Standard lowercase extensions + expect(fileSyncer.isFile("test.prompt")).toBe(true); + expect(fileSyncer.isFile("test.agent")).toBe(true); + + // Uppercase extensions (case insensitivity) + expect(fileSyncer.isFile("test.PROMPT")).toBe(true); + expect(fileSyncer.isFile("test.AGENT")).toBe(true); + expect(fileSyncer.isFile("test.Prompt")).toBe(true); + expect(fileSyncer.isFile("test.Agent")).toBe(true); + + // With whitespace + expect(fileSyncer.isFile(" test.prompt ")).toBe(true); + expect(fileSyncer.isFile(" test.agent ")).toBe(true); + }); + + it("should return false for invalid or missing extensions", () => { + // Invalid file types + expect(fileSyncer.isFile("test.txt")).toBe(false); + expect(fileSyncer.isFile("test.json")).toBe(false); + expect(fileSyncer.isFile("test.py")).toBe(false); + + // No extension + expect(fileSyncer.isFile("test")).toBe(false); + expect(fileSyncer.isFile("prompt")).toBe(false); + expect(fileSyncer.isFile("agent")).toBe(false); + + // Partial extensions + expect(fileSyncer.isFile("test.prom")).toBe(false); + expect(fileSyncer.isFile("test.age")).toBe(false); + }); + }); + + describe("file operations", () => { + it("should save and read files correctly", () => { + // Given a file content and path + const content = "test content"; + const filePath = "test/path"; + const fileType: SerializableFileType = "prompt"; + + // When saving the file + fileSyncer["_saveSerializedFile"](content, filePath, fileType); + + // Then the file should exist on disk + const savedPath = path.join(tempDir, filePath + "." + fileType); + expect(fs.existsSync(savedPath)).toBe(true); + + // When reading the file + const readContent = fileSyncer.getFileContent(filePath, fileType); + + // Then the content should match + expect(readContent).toBe(content); + }); + + it("should throw an error when reading a nonexistent file", () => { + // When trying to read a nonexistent file + // Then a HumanloopRuntimeError should be raised + expect(() => { + fileSyncer.getFileContent("nonexistent", "prompt"); + }).toThrow(HumanloopRuntimeError); + + // Check that the error message contains expected text + expect(() => { + fileSyncer.getFileContent("nonexistent", "prompt"); + }).toThrow(/Failed to read/); + }); + + it("should return false when API calls fail during pull", async () => { + // Given an API error + mockClient.files.retrieveByPath.mockRejectedValue(new Error("API Error")); + + // When trying to pull a file + const result = await fileSyncer["_pullFile"]("test.prompt"); + + // Then it should return false + expect(result).toBe(false); + + // And the API method should have been called + expect(mockClient.files.retrieveByPath).toHaveBeenCalled(); + }); + }); + + describe("cache functionality", () => { + it("should cache file content and respect cache invalidation", () => { + // Given a test file + const content = "test content"; + const filePath = "test/path"; + const fileType: SerializableFileType = "prompt"; + fileSyncer["_saveSerializedFile"](content, filePath, fileType); + + // When reading the file for the first time + const firstRead = fileSyncer.getFileContent(filePath, fileType); + expect(firstRead).toBe(content); + + // When modifying the file on disk + const savedPath = path.join(tempDir, filePath + "." + fileType); + fs.writeFileSync(savedPath, "modified content"); + + // Then subsequent reads should use cache (and return the original content) + const secondRead = fileSyncer.getFileContent(filePath, fileType); + expect(secondRead).toBe(content); // Should return cached content, not modified + + // When clearing the cache + fileSyncer.clearCache(); + + // Then new content should be read from disk + const thirdRead = fileSyncer.getFileContent(filePath, fileType); + expect(thirdRead).toBe("modified content"); + }); + + it("should respect the cache size limit", () => { + // Create a file syncer with small cache + const smallCacheFileSyncer = new FileSyncer(mockClient as any, { + baseDir: tempDir, + cacheSize: 2, // Only 2 items in cache + }); + + // Save 3 different files + for (let i = 1; i <= 3; i++) { + const content = `content ${i}`; + const filePath = `test/path${i}`; + const fileType: SerializableFileType = "prompt"; + smallCacheFileSyncer["_saveSerializedFile"]( + content, + filePath, + fileType, + ); + + // Read to put in cache + smallCacheFileSyncer.getFileContent(filePath, fileType); + } + + // Modify the first file (which should have been evicted from cache) + const firstPath = "test/path1"; + const savedPath = path.join(tempDir, firstPath + ".prompt"); + fs.writeFileSync(savedPath, "modified content"); + + // Reading the first file should get the modified content (not cached) + const newContent = smallCacheFileSyncer.getFileContent(firstPath, "prompt"); + expect(newContent).toBe("modified content"); + + // But reading the 2nd and 3rd files should still use cache + expect(smallCacheFileSyncer.getFileContent("test/path2", "prompt")).toBe( + "content 2", + ); + expect(smallCacheFileSyncer.getFileContent("test/path3", "prompt")).toBe( + "content 3", + ); + }); + }); + + describe("pull operations", () => { + it("should handle successful file pull", async () => { + // Mock successful file pull response + mockClient.files.retrieveByPath.mockResolvedValue({ + type: "prompt", + path: "test/path", + rawFileContent: "pulled content", + }); + + // When pulling a file + const result = await fileSyncer["_pullFile"]("test/path"); + + // Then it should return true + expect(result).toBe(true); + + // And the file should be saved to disk + const savedPath = path.join(tempDir, "test/path.prompt"); + expect(fs.existsSync(savedPath)).toBe(true); + expect(fs.readFileSync(savedPath, "utf8")).toBe("pulled content"); + }); + + it("should handle unsuccessful file pull due to missing content", async () => { + // Mock response with missing content + mockClient.files.retrieveByPath.mockResolvedValue({ + type: "prompt", + path: "test/path", + // missing rawFileContent + }); + + // When pulling a file + const result = await fileSyncer["_pullFile"]("test/path"); + + // Then it should return false + expect(result).toBe(false); + }); + + it("should handle unsuccessful file pull due to unsupported type", async () => { + // Mock response with unsupported type + mockClient.files.retrieveByPath.mockResolvedValue({ + type: "dataset", // Not a serializable type + path: "test/path", + rawFileContent: "content", + }); + + // When pulling a file + const result = await fileSyncer["_pullFile"]("test/path"); + + // Then it should return false + expect(result).toBe(false); + }); + + it("should pull a directory of files", async () => { + // Mock directory listing responses (paginated) + mockClient.files.listFiles.mockResolvedValueOnce({ + records: [ + { + type: "prompt", + path: "dir/file1", + rawFileContent: "content 1", + }, + { + type: "agent", + path: "dir/file2", + rawFileContent: "content 2", + }, + ], + page: 1, + size: 2, + total: 3, + }); + + mockClient.files.listFiles.mockResolvedValueOnce({ + records: [ + { + type: "prompt", + path: "dir/file3", + rawFileContent: "content 3", + }, + ], + page: 2, + size: 2, + total: 3, + }); + + // When pulling a directory + const [successful, failed] = await fileSyncer["_pullDirectory"]("dir"); + + // Then it should succeed for all files + expect(successful.length).toBe(3); + expect(failed.length).toBe(0); + + // And all files should exist on disk + expect(fs.existsSync(path.join(tempDir, "dir/file1.prompt"))).toBe(true); + expect(fs.existsSync(path.join(tempDir, "dir/file2.agent"))).toBe(true); + expect(fs.existsSync(path.join(tempDir, "dir/file3.prompt"))).toBe(true); + }); + + it("should handle the main pull method with different path types", async () => { + // Mock methods that are called by pull + jest.spyOn(fileSyncer, "isFile").mockImplementation((p) => + p.endsWith(".prompt"), + ); + jest.spyOn(fileSyncer as any, "_pullFile").mockResolvedValue(true); + jest.spyOn(fileSyncer as any, "_pullDirectory").mockResolvedValue([ + ["dir/file1"], + [], + ]); + + // Test with file path + await fileSyncer.pull("test/path.prompt"); + expect(fileSyncer["_pullFile"]).toHaveBeenCalledWith( + "test/path", + undefined, + ); + + // Reset mocks + jest.clearAllMocks(); + + // Test with directory path + await fileSyncer.pull("test/dir"); + expect(fileSyncer["_pullDirectory"]).toHaveBeenCalledWith( + "test/dir", + undefined, + ); + + // Reset mocks + jest.clearAllMocks(); + + // Test with no path (root) + await fileSyncer.pull(); + expect(fileSyncer["_pullDirectory"]).toHaveBeenCalledWith( + undefined, + undefined, + ); + + // Test with environment parameter + await fileSyncer.pull("test/path.prompt", "staging"); + expect(fileSyncer["_pullFile"]).toHaveBeenCalledWith( + "test/path", + "staging", + ); + }); + + it("should reject paths with leading or trailing slashes", async () => { + // Test with leading slash + await expect(fileSyncer.pull("/test/path")).rejects.toThrow( + HumanloopRuntimeError, + ); + + // Test with trailing slash + await expect(fileSyncer.pull("test/path/")).rejects.toThrow( + HumanloopRuntimeError, + ); + }); + }); +}); diff --git a/tests/custom/fixtures.ts b/tests/custom/fixtures.ts new file mode 100644 index 00000000..9909abc4 --- /dev/null +++ b/tests/custom/fixtures.ts @@ -0,0 +1,21 @@ +import * as fs from "fs"; +import * as path from "path"; +import { v4 as uuidv4 } from "uuid"; + +/** + * Creates a temporary directory for tests + * @param prefix Optional prefix for the directory name + * @returns Path to the created directory and a cleanup function + */ +export function createTempDir(prefix = "test") { + const tempDir = path.join(process.cwd(), "test-tmp", `${prefix}-${uuidv4()}`); + fs.mkdirSync(tempDir, { recursive: true }); + + const cleanup = () => { + if (fs.existsSync(tempDir)) { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }; + + return { tempDir, cleanup }; +} From f26578954527b56c51f65309790e79112c755829 Mon Sep 17 00:00:00 2001 From: Ale Pouroullis Date: Tue, 20 May 2025 16:36:06 +0100 Subject: [PATCH 09/16] test: Write integration tests for FileSyncer: --- tests/custom/integration/FileSyncer.test.ts | 160 ++++++++++++++++++++ tests/custom/integration/fixtures.ts | 91 ++++++++++- 2 files changed, 246 insertions(+), 5 deletions(-) create mode 100644 tests/custom/integration/FileSyncer.test.ts diff --git a/tests/custom/integration/FileSyncer.test.ts b/tests/custom/integration/FileSyncer.test.ts new file mode 100644 index 00000000..15120e18 --- /dev/null +++ b/tests/custom/integration/FileSyncer.test.ts @@ -0,0 +1,160 @@ +import * as fs from "fs"; +import * as path from "path"; +import { v4 as uuidv4 } from "uuid"; + +import { FileType } from "../../../src/api"; +import { HumanloopRuntimeError } from "../../../src/error"; +import { HumanloopClient } from "../../../src/humanloop.client"; +import { createTempDir } from "../fixtures"; +import { + SyncableFile, + TestSetup, + cleanupTestEnvironment, + createSyncableFilesFixture, + setupTestEnvironment, +} from "./fixtures"; + +describe("FileSyncer Integration Tests", () => { + let testSetup: TestSetup; + let syncableFiles: SyncableFile[] = []; + let tempDirInfo: { tempDir: string; cleanup: () => void }; + + beforeAll(async () => { + // Set up test environment + testSetup = await setupTestEnvironment("file_sync"); + tempDirInfo = createTempDir("file-sync-integration"); + + // Create test files in Humanloop for syncing + syncableFiles = await createSyncableFilesFixture(testSetup); + }); + + afterAll(async () => { + // Clean up resources only if they were created + if (tempDirInfo) { + tempDirInfo.cleanup(); + } + if (testSetup) { + await cleanupTestEnvironment( + testSetup, + syncableFiles.map((file) => ({ + type: file.type as FileType, + id: file.id as string, + })), + ); + } + }); + + test("pull_basic: should pull all files from remote to local filesystem", async () => { + // GIVEN a set of files in the remote system (from syncableFiles) + const client = new HumanloopClient({ + apiKey: process.env.HUMANLOOP_API_KEY, + localFilesDirectory: tempDirInfo.tempDir, + useLocalFiles: true, + }); + + // WHEN running the pull operation + await client.pull(); + + // THEN our local filesystem should mirror the remote filesystem in the HL Workspace + for (const file of syncableFiles) { + const extension = `.${file.type}`; + const localPath = path.join( + tempDirInfo.tempDir, + `${file.path}${extension}`, + ); + + // THEN the file and its directory should exist + expect(fs.existsSync(localPath)).toBe(true); + expect(fs.existsSync(path.dirname(localPath))).toBe(true); + + // THEN the file should not be empty + const content = fs.readFileSync(localPath, "utf8"); + expect(content).toBeTruthy(); + } + }); + + test("pull_with_invalid_path: should handle error when path doesn't exist", async () => { + // GIVEN a client + const client = new HumanloopClient({ + apiKey: process.env.HUMANLOOP_API_KEY, + localFilesDirectory: tempDirInfo.tempDir, + useLocalFiles: true, + }); + + const nonExistentPath = `${testSetup.sdkTestDir.path}/non_existent_directory`; + + // WHEN/THEN pulling with an invalid path should throw an error + await expect(client.pull(nonExistentPath)).rejects.toThrow( + HumanloopRuntimeError, + ); + // The error message might be different in TypeScript, so we don't assert on the exact message + }); + + test("pull_with_invalid_environment: should handle error when environment doesn't exist", async () => { + // GIVEN a client + const client = new HumanloopClient({ + apiKey: process.env.HUMANLOOP_API_KEY, + localFilesDirectory: tempDirInfo.tempDir, + useLocalFiles: true, + }); + + // WHEN/THEN pulling with an invalid environment should throw an error + await expect(client.pull(undefined, "invalid_environment")).rejects.toThrow( + HumanloopRuntimeError, + ); + }); + + test("pull_with_path_filter: should only pull files from specified path", async () => { + // GIVEN a client and a clean temp directory + const pathFilterTempDir = createTempDir("file-sync-path-filter"); + + const client = new HumanloopClient({ + apiKey: process.env.HUMANLOOP_API_KEY, + localFilesDirectory: pathFilterTempDir.tempDir, + useLocalFiles: true, + }); + + // WHEN pulling only files from the testSetup.sdkTestDir.path + await client.pull(testSetup.sdkTestDir.path); + + // THEN count the total number of files pulled + let pulledFileCount = 0; + + // Collect expected file paths (relative to sdkTestDir.path) + const expectedFiles = new Set( + syncableFiles.map((file) => + path.join( + pathFilterTempDir.tempDir, + file.path + (file.type === "prompt" ? ".prompt" : ".agent"), + ), + ), + ); + + const foundFiles = new Set(); + + function countFilesRecursive(dirPath: string): void { + const entries = fs.readdirSync(dirPath, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(dirPath, entry.name); + if (entry.isDirectory()) { + countFilesRecursive(fullPath); + } else if (entry.isFile()) { + if (expectedFiles.has(fullPath)) { + const content = fs.readFileSync(fullPath, "utf8"); + expect(content).toBeTruthy(); + foundFiles.add(fullPath); + } + } + } + } + + if (fs.existsSync(pathFilterTempDir.tempDir)) { + countFilesRecursive(pathFilterTempDir.tempDir); + } + + expect(foundFiles.size).toBe(expectedFiles.size); + + // Clean up + pathFilterTempDir.cleanup(); + }); +}); diff --git a/tests/custom/integration/fixtures.ts b/tests/custom/integration/fixtures.ts index 45e1fc41..e4b40fc0 100644 --- a/tests/custom/integration/fixtures.ts +++ b/tests/custom/integration/fixtures.ts @@ -5,7 +5,7 @@ import { v4 as uuidv4 } from "uuid"; import { FileType, PromptRequest, PromptResponse } from "../../../src/api"; import { HumanloopClient } from "../../../src/humanloop.client"; -export interface TestIdentifiers { +export interface ResourceIdentifiers { id: string; path: string; } @@ -16,15 +16,23 @@ export interface TestPrompt { response: PromptResponse; } +export interface SyncableFile { + path: string; + type: "prompt" | "agent"; + model: string; + id?: string; + versionId?: string; +} + export interface TestSetup { - sdkTestDir: TestIdentifiers; + sdkTestDir: ResourceIdentifiers; testPromptConfig: PromptRequest; openaiApiKey: string; humanloopClient: HumanloopClient; - evalDataset: TestIdentifiers; - evalPrompt: TestIdentifiers; + evalDataset: ResourceIdentifiers; + evalPrompt: ResourceIdentifiers; stagingEnvironmentId: string; - outputNotNullEvaluator: TestIdentifiers; + outputNotNullEvaluator: ResourceIdentifiers; } export interface CleanupResources { @@ -244,3 +252,76 @@ export async function cleanupTestEnvironment( console.error("Error during cleanup:", error); } } + +/** + * Creates a predefined structure of files in Humanloop for testing sync, + * mirroring the Python syncable_files_fixture + */ +export async function createSyncableFilesFixture( + testSetup: TestSetup, +): Promise { + const fileDefinitions: SyncableFile[] = [ + { + path: "prompts/gpt-4", + type: "prompt", + model: "gpt-4o-mini", // Using gpt-4o-mini as safer default for tests + }, + { + path: "prompts/gpt-4o", + type: "prompt", + model: "gpt-4o-mini", + }, + { + path: "prompts/nested/complex/gpt-4o", + type: "prompt", + model: "gpt-4o-mini", + }, + { + path: "agents/gpt-4", + type: "agent", + model: "gpt-4o-mini", + }, + { + path: "agents/gpt-4o", + type: "agent", + model: "gpt-4o-mini", + }, + ]; + + const createdFiles: SyncableFile[] = []; + + for (const file of fileDefinitions) { + const fullPath = `${testSetup.sdkTestDir.path}/${file.path}`; + let response; + + try { + if (file.type === "prompt") { + response = await testSetup.humanloopClient.prompts.upsert({ + path: fullPath, + ...testSetup.testPromptConfig, + model: file.model, + }); + } else if (file.type === "agent") { + // Assuming agent creation works similar to your Python implementation + response = await testSetup.humanloopClient.agents.upsert({ + path: fullPath, + model: file.model, + }); + } + + if (response) { + createdFiles.push({ + path: fullPath, + type: file.type, + model: file.model, + id: response.id, + versionId: response.versionId, + }); + } + } catch (error) { + console.warn(`Failed to create ${file.type} at ${fullPath}: ${error}`); + } + } + + return createdFiles; +} From cd016b20a8b4a09ee7c4d489c83d8df0143e10d1 Mon Sep 17 00:00:00 2001 From: Ale Pouroullis Date: Tue, 20 May 2025 16:37:53 +0100 Subject: [PATCH 10/16] Fix broken relative paths in decorators.test.ts --- jest.config.mjs | 7 ++++++- tests/custom/integration/decorators.test.ts | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/jest.config.mjs b/jest.config.mjs index c7248211..b4a8227b 100644 --- a/jest.config.mjs +++ b/jest.config.mjs @@ -3,6 +3,11 @@ export default { preset: "ts-jest", testEnvironment: "node", moduleNameMapper: { - "(.+)\.js$": "$1", + // Only map .js files in our src directory, not node_modules + "^src/(.+)\\.js$": "/src/$1", }, + // Add transformIgnorePatterns to handle ESM modules in node_modules + transformIgnorePatterns: [ + "node_modules/(?!(@traceloop|js-tiktoken|base64-js)/)", + ], }; diff --git a/tests/custom/integration/decorators.test.ts b/tests/custom/integration/decorators.test.ts index 0cddc948..ef9e22a0 100644 --- a/tests/custom/integration/decorators.test.ts +++ b/tests/custom/integration/decorators.test.ts @@ -1,7 +1,7 @@ import OpenAI from "openai"; -import { PromptRequest } from "../../src/api"; -import { HumanloopRuntimeError } from "../../src/error"; +import { PromptRequest } from "../../../src/api"; +import { HumanloopRuntimeError } from "../../../src/error"; import { CleanupResources, TestPrompt, From 58dc882fb7d298105b6e6fbfcf5268814c188848 Mon Sep 17 00:00:00 2001 From: Ale Pouroullis Date: Tue, 20 May 2025 17:16:46 +0100 Subject: [PATCH 11/16] Improve test cleanup; write tests for local file operations --- src/overload.ts | 280 ++++++++---- tests/custom/integration/fixtures.ts | 90 ++-- .../integration/localFileOperations.test.ts | 404 ++++++++++++++++++ 3 files changed, 653 insertions(+), 121 deletions(-) create mode 100644 tests/custom/integration/localFileOperations.test.ts diff --git a/src/overload.ts b/src/overload.ts index f1d59ed8..5dff103b 100644 --- a/src/overload.ts +++ b/src/overload.ts @@ -1,10 +1,14 @@ import path from "path"; import { - CreateEvaluatorLogRequest, FlowLogRequest, PromptLogRequest, - ToolLogRequest + CreateEvaluatorLogRequest, + FileType, + FlowLogRequest, + PromptLogRequest, + ToolLogRequest, } from "./api"; import { Agents } from "./api/resources/agents/client/Client"; +import { Datasets } from "./api/resources/datasets/client/Client"; import { Evaluators } from "./api/resources/evaluators/client/Client"; import { Flows } from "./api/resources/flows/client/Client"; import { Prompts } from "./api/resources/prompts/client/Client"; @@ -16,7 +20,7 @@ import FileSyncer, { SerializableFileType, } from "./sync/FileSyncer"; -type ClientType = Flows | Agents | Prompts | Tools | Evaluators; +type ClientType = Flows | Agents | Prompts | Tools | Evaluators | Datasets; type LogRequestType = | FlowLogRequest | PromptLogRequest @@ -25,7 +29,9 @@ type LogRequestType = /** * Get the file type based on the client type. - * Only returns types that can be loaded from local filesystem. + * + * @param client Client instance to check + * @returns The file type corresponding to the client, or null if not a file type that supports local files */ function getFileTypeFromClient(client: ClientType): SerializableFileType | null { if (client instanceof Prompts) { @@ -38,6 +44,8 @@ function getFileTypeFromClient(client: ClientType): SerializableFileType | null return null; // Flows don't support local files } else if (client instanceof Evaluators) { return null; // Evaluators don't support local files + } else if (client instanceof Datasets) { + return null; // Datasets don't support local files } else { throw new HumanloopRuntimeError( // @ts-ignore Client shouldn't be of a type other than those checked above, but included as a safeguard @@ -48,6 +56,10 @@ function getFileTypeFromClient(client: ClientType): SerializableFileType | null /** * Handle tracing context for both log and call methods. + * + * @param request The API request + * @param client The client making the request + * @returns The updated request with tracing context applied */ function handleTracingContext( request: T, @@ -63,7 +75,8 @@ function handleTracingContext( ); } throw new HumanloopRuntimeError( - `Using flows.log() is not allowed: Flow decorator for File ${context.path} manages the tracing and trace completion.`, + `Using \`flows.log()\` is not allowed: Flow decorator ` + + `for File ${context.path} manages the tracing and trace completion.`, ); } @@ -81,7 +94,19 @@ function handleTracingContext( } /** - * Load .prompt/.agent file content from local filesystem into API request. + * Load prompt/agent file content from local filesystem into API request. + * + * Retrieves the file content at the specified path and adds it to request + * under the appropriate field ('prompt' or 'agent'), allowing local files + * to be used in API calls instead of fetching from Humanloop API. + * + * @param request API request object + * @param client Client instance making the call + * @param fileSyncer FileSyncer handling local file operations + * @returns Updated request with file content in the appropriate field + * @throws HumanloopRuntimeError On validation or file loading failures. + * For example, an invalid path format (absolute paths, leading/trailing slashes, etc.) + * or a file not being found. */ function handleLocalFiles( request: T, @@ -90,7 +115,7 @@ function handleLocalFiles( ): T { // Validate request has either id or path, but not both if ("id" in request && "path" in request) { - throw new HumanloopRuntimeError("Cannot specify both `id` and `path`"); + throw new HumanloopRuntimeError("Can only specify one of `id` or `path`"); } if (!("id" in request) && !("path" in request)) { throw new HumanloopRuntimeError("Must specify either `id` or `path`"); @@ -101,31 +126,31 @@ function handleLocalFiles( return request; } - const filePath = request.path; + const filePath = request.path?.trim(); if (!filePath) { throw new HumanloopRuntimeError("Path cannot be empty"); } - // Check for path format issues (absolute paths or leading/trailing slashes) + // First check for path format issues (absolute paths or leading/trailing slashes) const normalizedPath = filePath.trim().replace(/^\/+|\/+$/g, ""); if (path.isAbsolute(filePath) || filePath !== normalizedPath) { throw new HumanloopRuntimeError( `Path '${filePath}' format is invalid. ` + - `Paths must follow the standard API format 'path/to/resource' without leading or trailing slashes. ` + - `Please use '${normalizedPath}' instead.`, + `Paths must follow the standard API format 'path/to/resource' without leading or trailing slashes. ` + + `Please use '${normalizedPath}' instead.`, ); } - // Check for file extensions + // Then check for file extensions if (fileSyncer.isFile(filePath)) { const pathWithoutExtension = path.join( path.dirname(filePath), path.basename(filePath, path.extname(filePath)), ); throw new HumanloopRuntimeError( - `Path '${filePath}' includes a file extension which is not supported in API calls. ` + - `When referencing files via the \`path\` parameter, use the path without extensions: '${pathWithoutExtension}'. ` + - `Note: File extensions are only used when pulling specific files via the CLI.`, + `Path '${filePath}' should not include any file extensions in API calls. ` + + `When referencing files via the \`path\` parameter, use the path without extensions: '${pathWithoutExtension}'. ` + + `Note: File extensions are only used when pulling specific files via the CLI.`, ); } @@ -134,14 +159,14 @@ function handleLocalFiles( if (useRemote) { throw new HumanloopRuntimeError( `Cannot use local file for \`${filePath}\` as version_id or environment was specified. ` + - "Please either remove version_id/environment to use local files, or set use_local_files=False to use remote files.", + `Please either remove version_id/environment to use local files, or set use_local_files=False to use remote files.`, ); } const fileType = getFileTypeFromClient(client); if (!fileType || !SERIALIZABLE_FILE_TYPES.has(fileType)) { throw new HumanloopRuntimeError( - `Local files are not supported for this client type: '${filePath}'.`, + `Local files are not supported for \`${fileType?.charAt(0).toUpperCase()}${fileType?.slice(1)}\` files: '${filePath}'.`, ); } @@ -149,7 +174,7 @@ function handleLocalFiles( if (fileType in request && typeof request[fileType as keyof T] !== "string") { console.warn( `Ignoring local file for \`${filePath}\` as ${fileType} parameters were directly provided. ` + - "Using provided parameters instead.", + `Using provided parameters instead.`, ); return request; } @@ -168,8 +193,144 @@ function handleLocalFiles( } /** - * Overloads a client with local file handling and tracing capabilities. - * This is the preferred way to overload clients, replacing individual overloadLog and overloadCall methods. + * Handle evaluation context for logging. + * + * @param request The API request + * @returns Tuple of [updated request, callback function] + */ +function handleEvaluationContext( + request: T, +): [T, ((id: string) => Promise) | null] { + const evaluationContext = getEvaluationContext(); + if (evaluationContext !== undefined) { + const [newRequest, callback] = evaluationContext.logArgsWithContext({ + logArgs: request, + forOtel: true, + path: request.path, + }); + return [newRequest as T, callback]; + } + return [request, null]; +} + +/** + * Overloaded log method implementation. + * Handles tracing context, local file loading, and evaluation context. + * + * @param self The client instance + * @param fileSyncer Optional FileSyncer for local file operations + * @param useLocalFiles Whether to use local files + * @param request The log request + * @param options Additional options + * @returns The log response + */ +async function overloadedLog( + self: T, + fileSyncer: FileSyncer | undefined, + useLocalFiles: boolean, + request: LogRequestType, + options?: any, +) { + try { + // Special handling for flows - prevent direct log usage + if (self instanceof Flows && getTraceId() !== undefined) { + const context = getDecoratorContext(); + if (context === undefined) { + throw new HumanloopRuntimeError( + "Internal error: trace_id context is set outside a decorator context.", + ); + } + throw new HumanloopRuntimeError( + `Using \`flows.log()\` is not allowed: Flow decorator ` + + `for File ${context.path} manages the tracing and trace completion.`, + ); + } + + request = handleTracingContext(request, self); + + // Handle loading files from local filesystem when using Prompt and Agent clients + if ( + useLocalFiles && + (self instanceof Prompts || self instanceof Agents) + ) { + if (!fileSyncer) { + throw new HumanloopRuntimeError( + "SDK initialization error: fileSyncer is missing but required for local file operations. " + + "This is likely a bug in the SDK initialization - please report this issue to the Humanloop team.", + ); + } + request = handleLocalFiles(request, self, fileSyncer); + } + + const [evalRequest, evalCallback] = handleEvaluationContext(request); + const response = await (self as any)._log(evalRequest, options); + + if (evalCallback !== null) { + await evalCallback(response.id); + } + return response; + } catch (error) { + if (error instanceof HumanloopRuntimeError) { + throw error; + } + throw new HumanloopRuntimeError(String(error)); + } +} + +/** + * Overloaded call method implementation. + * Handles tracing context and local file loading. + * + * @param self The client instance + * @param fileSyncer Optional FileSyncer for local file operations + * @param useLocalFiles Whether to use local files + * @param request The call request + * @param options Additional options + * @returns The call response + */ +async function overloadedCall( + self: T, + fileSyncer: FileSyncer | undefined, + useLocalFiles: boolean, + request: any, + options?: any, +) { + try { + request = handleTracingContext(request, self); + + // If `useLocalFiles` flag is True, we should use local file content for + // `call` operations on Prompt and Agent clients. + if (useLocalFiles && (self instanceof Prompts || self instanceof Agents)) { + if (!fileSyncer) { + throw new HumanloopRuntimeError( + "fileSyncer is required for clients that support call operations", + ); + } + request = handleLocalFiles(request, self, fileSyncer); + } + + return await (self as any)._call(request, options); + } catch (error) { + if (error instanceof HumanloopRuntimeError) { + throw error; + } + throw new HumanloopRuntimeError(String(error)); + } +} + +/** + * Overloads client methods to add tracing, local file handling, and evaluation context. + * + * This function enhances clients by: + * 1. Adding tracing context to requests for Flow integration + * 2. Supporting local file loading for Prompt and Agent clients + * 3. Handling evaluation context for logging + * + * @param client The client to overload + * @param fileSyncer Optional FileSyncer for local file operations + * @param useLocalFiles Whether to use local files (default: false) + * @returns The overloaded client + * @throws HumanloopRuntimeError If fileSyncer is missing but required */ export function overloadClient( client: T, @@ -179,77 +340,26 @@ export function overloadClient( // Handle log method if it exists if ("log" in client) { const originalLog = (client as any).log.bind(client); - const _overloadedLog = async (request: LogRequestType, options?: any) => { - try { - request = handleTracingContext(request, client); - if ( - useLocalFiles && - (client instanceof Prompts || client instanceof Agents) - ) { - if (!fileSyncer) { - throw new HumanloopRuntimeError( - "SDK initialization error: fileSyncer is missing but required for local file operations.", - ); - } - request = handleLocalFiles(request, client, fileSyncer); - } - - const evaluationContext = getEvaluationContext(); - if (evaluationContext !== undefined) { - const [kwargsEval, evalCallback] = - evaluationContext.logArgsWithContext({ - logArgs: request, - forOtel: true, - path: request.path, - }); - try { - const response = await originalLog(kwargsEval as any, options); - if (evalCallback !== null) { - await evalCallback(response.id); - } - return response; - } catch (error) { - throw new HumanloopRuntimeError(String(error)); - } - } - return await originalLog(request as any, options); - } catch (error) { - if (error instanceof HumanloopRuntimeError) { - throw error; - } - throw new HumanloopRuntimeError(String(error)); - } - }; (client as any)._log = originalLog; - (client as any).log = _overloadedLog.bind(client); + (client as any).log = async (request: LogRequestType, options?: any) => { + return overloadedLog(client, fileSyncer, useLocalFiles, request, options); + }; } - // Handle call method if it exists (for Prompts and Agents). Note that we can't use `"call" in client` - // because Tools also have a call method. + // Handle call method if it exists (for Prompts and Agents) if (client instanceof Prompts || client instanceof Agents) { + // Verify fileSyncer is provided if needed + if (fileSyncer === undefined && useLocalFiles) { + console.error("fileSyncer is undefined but client has call method and useLocalFiles=%s", useLocalFiles); + throw new HumanloopRuntimeError("fileSyncer is required for clients that support call operations"); + } + const originalCall = (client as any).call.bind(client); - const _overloadedCall = async (request: PromptLogRequest, options?: any) => { - try { - request = handleTracingContext(request, client); - if (useLocalFiles) { - if (!fileSyncer) { - throw new HumanloopRuntimeError( - "fileSyncer is required for clients that support call operations", - ); - } - request = handleLocalFiles(request, client, fileSyncer); - } - return await originalCall(request, options); - } catch (error) { - if (error instanceof HumanloopRuntimeError) { - throw error; - } - throw new HumanloopRuntimeError(String(error)); - } - }; (client as any)._call = originalCall; - (client as any).call = _overloadedCall.bind(client); + (client as any).call = async (request: any, options?: any) => { + return overloadedCall(client, fileSyncer, useLocalFiles, request, options); + }; } return client; -} +} \ No newline at end of file diff --git a/tests/custom/integration/fixtures.ts b/tests/custom/integration/fixtures.ts index e4b40fc0..c7ba73b9 100644 --- a/tests/custom/integration/fixtures.ts +++ b/tests/custom/integration/fixtures.ts @@ -205,57 +205,77 @@ export async function cleanupTestEnvironment( // First clean up any additional resources if (resources) { for (const resource of resources) { - const subclient = getSubclient(setup.humanloopClient, resource.type); - if (resource.id) { - await subclient.delete(resource.id); + try { + const subclient = getSubclient( + setup.humanloopClient, + resource.type, + ); + if (resource.id) { + await subclient.delete(resource.id); + } + } catch (error) { + console.warn( + `Failed to delete ${resource.type} ${resource.id}:`, + error, + ); } } } - // Clean up fixed test resources - if (setup.outputNotNullEvaluator?.id) { - try { - await setup.humanloopClient.evaluators.delete( - setup.outputNotNullEvaluator.id, - ); - } catch (error) { - console.warn( - `Failed to delete evaluator ${setup.outputNotNullEvaluator.id}:`, - error, - ); + // Sleep a bit to let API operations settle + await new Promise((resolve) => setTimeout(resolve, 3000)); + + // Recursively clean up the test directory + try { + if (setup.sdkTestDir.id) { + await cleanupDirectory(setup.humanloopClient, setup.sdkTestDir.id); } + } catch (error) { + console.warn(`Failed to clean up test directory: ${error}`); } + } catch (error) { + console.error("Error during cleanup:", error); + } +} - if (setup.evalDataset?.id) { - try { - await setup.humanloopClient.datasets.delete(setup.evalDataset.id); - } catch (error) { - console.warn( - `Failed to delete dataset ${setup.evalDataset.id}:`, - error, - ); - } +/** + * Recursively cleans up a directory and all its contents + * Mirrors the Python SDK's cleanup_directory function + * @param client The Humanloop client + * @param directoryId ID of the directory to clean + */ +async function cleanupDirectory( + client: HumanloopClient, + directoryId: string, +): Promise { + try { + // Get directory details + const directory = await client.directories.get(directoryId); + + // First, recursively clean up subdirectories + for (const subdirectory of directory.subdirectories) { + await cleanupDirectory(client, subdirectory.id); } - // Finally, clean up the test directory - if (setup.sdkTestDir.id) { + // Then delete all files in this directory + for (const file of directory.files) { try { - await setup.humanloopClient.directories.delete(setup.sdkTestDir.id); + const subclient = getSubclient(client, file.type as FileType); + await subclient.delete(file.id); } catch (error) { - console.warn( - `Failed to delete directory ${setup.sdkTestDir.id}:`, - error, - ); + console.warn(`Failed to delete ${file.type} ${file.id}: ${error}`); } } + + // Finally delete this directory + await client.directories.delete(directoryId); } catch (error) { - console.error("Error during cleanup:", error); + console.warn(`Error cleaning directory ${directoryId}: ${error}`); } } /** - * Creates a predefined structure of files in Humanloop for testing sync, - * mirroring the Python syncable_files_fixture + * Creates a predefined structure of files in Humanloop for testing sync */ export async function createSyncableFilesFixture( testSetup: TestSetup, @@ -264,7 +284,7 @@ export async function createSyncableFilesFixture( { path: "prompts/gpt-4", type: "prompt", - model: "gpt-4o-mini", // Using gpt-4o-mini as safer default for tests + model: "gpt-4o-mini", }, { path: "prompts/gpt-4o", @@ -298,11 +318,9 @@ export async function createSyncableFilesFixture( if (file.type === "prompt") { response = await testSetup.humanloopClient.prompts.upsert({ path: fullPath, - ...testSetup.testPromptConfig, model: file.model, }); } else if (file.type === "agent") { - // Assuming agent creation works similar to your Python implementation response = await testSetup.humanloopClient.agents.upsert({ path: fullPath, model: file.model, diff --git a/tests/custom/integration/localFileOperations.test.ts b/tests/custom/integration/localFileOperations.test.ts new file mode 100644 index 00000000..4f5da986 --- /dev/null +++ b/tests/custom/integration/localFileOperations.test.ts @@ -0,0 +1,404 @@ +import * as fs from "fs"; +import * as path from "path"; + +import { ChatMessage } from "../../../src/api"; +import { HumanloopRuntimeError } from "../../../src/error"; +import { HumanloopClient } from "../../../src/humanloop.client"; +import { createTempDir } from "../fixtures"; +import { + TestSetup, + cleanupTestEnvironment, + createSyncableFilesFixture, + setupTestEnvironment, +} from "./fixtures"; + +// Define SyncableFile interface to match Python version +interface SyncableFile { + path: string; + type: "prompt" | "agent"; + model: string; + id?: string; + versionId?: string; +} + +interface PathTestCase { + name: string; + pathGenerator: (file: SyncableFile) => string; + shouldPass: boolean; + expectedError?: string; // Only required when shouldPass is false +} + +describe("Local File Operations Integration Tests", () => { + let testSetup: TestSetup; + let syncableFiles: SyncableFile[] = []; + let tempDirInfo: { tempDir: string; cleanup: () => void }; + + beforeAll(async () => { + // Increase timeout for setup operations + jest.setTimeout(30000); // 30 seconds + + // Set up test environment + testSetup = await setupTestEnvironment("local_file_ops"); + tempDirInfo = createTempDir("local-file-integration"); + + // Create test files in Humanloop for syncing + syncableFiles = await createSyncableFilesFixture(testSetup); + + // Pull files for tests that need them pre-pulled + const setupClient = new HumanloopClient({ + apiKey: process.env.HUMANLOOP_API_KEY, + localFilesDirectory: tempDirInfo.tempDir, + useLocalFiles: true, + }); + + await setupClient.pull(); + }, 30000); + + afterAll(async () => { + // Clean up resources + tempDirInfo.cleanup(); + await cleanupTestEnvironment( + testSetup, + syncableFiles.map((file) => ({ + type: file.type as any, + id: file.id as string, + })), + ); + }, 30000); + + describe("Path Validation", () => { + // Path validation test cases + const pathTestCases = [ + // Basic path test cases + { + name: "With whitespace", + pathGenerator: (file: SyncableFile) => ` ${file.path} `, + shouldPass: true, + }, + { + name: "Standard extension", + pathGenerator: (file: SyncableFile) => `${file.path}.${file.type}`, + expectedError: "should not include any file extension", + }, + { + name: "Uppercase extension", + pathGenerator: (file: SyncableFile) => + `${file.path}.${file.type.toUpperCase()}`, + expectedError: "should not include any file extension", + }, + { + name: "Mixed case extension", + pathGenerator: (file: SyncableFile) => + `${file.path}.${file.type.charAt(0).toUpperCase() + file.type.slice(1)}`, + expectedError: "should not include any file extension", + }, + // Slash path test cases + { + name: "Trailing slash", + pathGenerator: (file: SyncableFile) => `${file.path}/`, + expectedError: "Path .* format is invalid", + }, + { + name: "Leading slash", + pathGenerator: (file: SyncableFile) => `/${file.path}`, + expectedError: "Path .* format is invalid", + }, + { + name: "Both leading and trailing slashes", + pathGenerator: (file: SyncableFile) => `/${file.path}/`, + expectedError: "Path .* format is invalid", + }, + { + name: "Multiple leading and trailing slashes", + pathGenerator: (file: SyncableFile) => `//${file.path}//`, + expectedError: "Path .* format is invalid", + }, + // Combined path test cases + { + name: "Extension and trailing slash", + pathGenerator: (file: SyncableFile) => `${file.path}.${file.type}/`, + expectedError: "Path .* format is invalid", + }, + { + name: "Extension and leading slash", + pathGenerator: (file: SyncableFile) => `/${file.path}.${file.type}`, + expectedError: "Path .* format is invalid", + }, + ]; + + // Test all path validation cases + test.each(pathTestCases)( + "should $shouldPass ? 'accept' : 'reject' $name path format", + async ({ pathGenerator, expectedError, shouldPass }) => { + // GIVEN a client with local files enabled and a test file + const client = new HumanloopClient({ + apiKey: process.env.HUMANLOOP_API_KEY, + localFilesDirectory: tempDirInfo.tempDir, + useLocalFiles: true, + }); + + const testFile = syncableFiles[0]; + const testPath = pathGenerator(testFile); + const testMessage: ChatMessage[] = [ + { role: "user", content: "Testing" }, + ]; + + // WHEN using the path + if (shouldPass) { + // THEN it should work (just trimming whitespace) + if (testFile.type === "prompt") { + await expect( + client.prompts.call({ + path: testPath, + messages: testMessage, + }), + ).resolves.toBeDefined(); + } else if (testFile.type === "agent") { + await expect( + client.agents.call({ + path: testPath, + messages: testMessage, + }), + ).resolves.toBeDefined(); + } + } else { + // Type guard to ensure expectedError is defined when shouldPass is false + if (!expectedError) { + throw new Error( + "expectedError must be defined when shouldPass is false", + ); + } + + // THEN appropriate error should be raised + if (testFile.type === "prompt") { + await expect( + client.prompts.call({ + path: testPath, + messages: testMessage, + }), + ).rejects.toThrow(new RegExp(expectedError)); + } else if (testFile.type === "agent") { + await expect( + client.agents.call({ + path: testPath, + messages: testMessage, + }), + ).rejects.toThrow(new RegExp(expectedError)); + } + } + }, + ); + }); + + test("local_file_call: should call API with local prompt file", async () => { + // GIVEN a local prompt file with proper system tag + const promptContent = `--- +model: gpt-4o-mini +temperature: 1.0 +max_tokens: -1 +top_p: 1.0 +presence_penalty: 0.0 +frequency_penalty: 0.0 +provider: openai +endpoint: chat +tools: [] +--- + + +You are a helpful assistant that provides concise answers. When asked about capitals of countries, +you respond with just the capital name, lowercase, with no punctuation or additional text. + +`; + + // Create local file structure in temporary directory + const testPath = `${testSetup.sdkTestDir.path}/capital_prompt`; + const filePath = path.join(tempDirInfo.tempDir, `${testPath}.prompt`); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, promptContent); + + // GIVEN a client with local files enabled + const client = new HumanloopClient({ + apiKey: process.env.HUMANLOOP_API_KEY, + localFilesDirectory: tempDirInfo.tempDir, + useLocalFiles: true, + }); + + // WHEN calling the API with the local file path (without extension) + const callMessages: ChatMessage[] = [ + { role: "user", content: "What is the capital of France?" }, + ]; + const response = await client.prompts.call({ + path: testPath, + messages: callMessages, + }); + + // THEN the response should be successful + expect(response).toBeDefined(); + expect(response.logs).toBeDefined(); + expect(response.logs?.length).toBeGreaterThan(0); + + // AND the response should contain the expected output format (lowercase city name) + const output = response.logs?.[0].output; + expect(output).toBeDefined(); + expect(output?.toLowerCase()).toContain("paris"); + + // AND the prompt used should match our expected path + expect(response.prompt).toBeDefined(); + expect(response.prompt?.path).toBe(testPath); + }); + + test("local_file_log: should log data with local prompt file", async () => { + // GIVEN a local prompt file with proper system tag + const promptContent = `--- +model: gpt-4o-mini +temperature: 1.0 +max_tokens: -1 +top_p: 1.0 +presence_penalty: 0.0 +frequency_penalty: 0.0 +provider: openai +endpoint: chat +tools: [] +--- + + +You are a helpful assistant that answers questions about geography. + +`; + + // Create local file structure in temporary directory + const testPath = `${testSetup.sdkTestDir.path}/geography_prompt`; + const filePath = path.join(tempDirInfo.tempDir, `${testPath}.prompt`); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, promptContent); + + // GIVEN a client with local files enabled + const client = new HumanloopClient({ + apiKey: process.env.HUMANLOOP_API_KEY, + localFilesDirectory: tempDirInfo.tempDir, + useLocalFiles: true, + }); + + // GIVEN message content to log + const testOutput = "Paris is the capital of France."; + + // WHEN logging the data with the local file path + const messages: ChatMessage[] = [ + { role: "user", content: "What is the capital of France?" }, + ]; + const response = await client.prompts.log({ + path: testPath, + messages: messages, + output: testOutput, + }); + + // THEN the log should be successful + expect(response).toBeDefined(); + expect(response.promptId).toBeDefined(); + expect(response.id).toBeDefined(); // log ID + + // WHEN retrieving the logged prompt details + const promptDetails = await client.prompts.get(response.promptId); + + // THEN the details should match our expected path + expect(promptDetails).toBeDefined(); + expect(promptDetails.path).toContain(testPath); + }); + + test("overload_version_environment_handling: should handle version_id and environment parameters", async () => { + // GIVEN a client with local files enabled + const client = new HumanloopClient({ + apiKey: process.env.HUMANLOOP_API_KEY, + localFilesDirectory: tempDirInfo.tempDir, + useLocalFiles: true, + }); + + const testMessage: ChatMessage[] = [{ role: "user", content: "Testing" }]; + + // GIVEN a test file that exists locally + const testFile = syncableFiles[0]; + const extension = `.${testFile.type}`; + const localPath = path.join( + tempDirInfo.tempDir, + `${testFile.path}${extension}`, + ); + + // THEN the file should exist locally + expect(fs.existsSync(localPath)).toBe(true); + expect(fs.existsSync(path.dirname(localPath))).toBe(true); + + // WHEN calling with version_id + // THEN a HumanloopRuntimeError should be raised + if (testFile.type === "prompt") { + await expect( + client.prompts.call({ + path: testFile.path, + versionId: testFile.versionId, + messages: testMessage, + }), + ).rejects.toThrow( + /Cannot use local file.*version_id or environment was specified/, + ); + } else if (testFile.type === "agent") { + await expect( + client.agents.call({ + path: testFile.path, + versionId: testFile.versionId, + messages: testMessage, + }), + ).rejects.toThrow( + /Cannot use local file.*version_id or environment was specified/, + ); + } + + // WHEN calling with environment + // THEN a HumanloopRuntimeError should be raised + if (testFile.type === "prompt") { + await expect( + client.prompts.call({ + path: testFile.path, + environment: "production", + messages: testMessage, + }), + ).rejects.toThrow( + /Cannot use local file.*version_id or environment was specified/, + ); + } else if (testFile.type === "agent") { + await expect( + client.agents.call({ + path: testFile.path, + environment: "production", + messages: testMessage, + }), + ).rejects.toThrow( + /Cannot use local file.*version_id or environment was specified/, + ); + } + + // WHEN calling with both version_id and environment + // THEN a HumanloopRuntimeError should be raised + if (testFile.type === "prompt") { + await expect( + client.prompts.call({ + path: testFile.path, + versionId: testFile.versionId, + environment: "staging", + messages: testMessage, + }), + ).rejects.toThrow( + /Cannot use local file.*version_id or environment was specified/, + ); + } else if (testFile.type === "agent") { + await expect( + client.agents.call({ + path: testFile.path, + versionId: testFile.versionId, + environment: "staging", + messages: testMessage, + }), + ).rejects.toThrow( + /Cannot use local file.*version_id or environment was specified/, + ); + } + }); +}); From 48263c871c224c7a5fb7a7ed1012a9641e9bbb32 Mon Sep 17 00:00:00 2001 From: Ale Pouroullis Date: Tue, 20 May 2025 17:51:48 +0100 Subject: [PATCH 12/16] test: Write tests for CLI --- tests/custom/integration/cli.test.ts | 295 +++++++++++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 tests/custom/integration/cli.test.ts diff --git a/tests/custom/integration/cli.test.ts b/tests/custom/integration/cli.test.ts new file mode 100644 index 00000000..be710dc4 --- /dev/null +++ b/tests/custom/integration/cli.test.ts @@ -0,0 +1,295 @@ +import * as fs from "fs"; +import * as path from "path"; +import { spawn } from "child_process"; + +import { createTempDir } from "../fixtures"; +import { + TestSetup, + cleanupTestEnvironment, + createSyncableFilesFixture, + setupTestEnvironment, +} from "./fixtures"; + +// Helper function to run CLI commands with TypeScript +async function runCli( + args: string[], +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + return new Promise((resolve) => { + const packageRoot = path.resolve(__dirname, "../../../"); + const cliPath = path.join(packageRoot, "dist/cli.js"); + + // Use spawn to avoid shell interpretation issues + const childProcess = spawn("node", [cliPath, ...args], { + stdio: ["ignore", "pipe", "pipe"], + }); + + let stdout = ""; + let stderr = ""; + + childProcess.stdout?.on("data", (data) => { + stdout += data.toString(); + }); + + childProcess.stderr?.on("data", (data) => { + stderr += data.toString(); + }); + + childProcess.on("close", (code) => { + resolve({ + stdout, + stderr, + exitCode: code !== null ? code : 0, + }); + }); + }); +} + +describe("CLI Integration Tests", () => { + let testSetup: TestSetup; + let syncableFiles: any[] = []; + + beforeAll(async () => { + // Increase timeout for setup operations + jest.setTimeout(40000); // 40 seconds + + // Set up test environment + testSetup = await setupTestEnvironment("cli_test"); + + // Create test files in Humanloop for syncing + syncableFiles = await createSyncableFilesFixture(testSetup); + }, 30000); + + afterAll(async () => { + await cleanupTestEnvironment( + testSetup, + syncableFiles.map((file) => ({ + type: file.type as any, + id: file.id as string, + })), + ); + }, 30000); + + /** + * NOTE: This test is currently skipped due to issues with CLI environment isolation. + * + * The test attempts to verify behavior when no API key is available, but faces + * challenges with how Node.js handles process execution during tests: + * + * 1. When executed via child_process.exec, the path to nonexistent env files + * causes Node to return exit code 9 (SIGKILL) instead of the expected code 1 + * 2. Shell interpretation of arguments makes it difficult to reliably test this edge case + * + * If this functionality needs testing, consider: + * - Using child_process.spawn for better argument handling + * - Unit testing the API key validation logic directly + * - Moving this test to a separate process with full environment isolation + * + * @see https://nodejs.org/api/child_process.html for more info on process execution + */ + test.skip("pull_without_api_key: should show error when no API key is available", async () => { + // GIVEN a temporary directory and no API key + const { tempDir, cleanup } = createTempDir("cli-no-api-key"); + + // Create a path to a file that definitely doesn't exist + const nonExistentEnvFile = path.join(tempDir, "__DOES_NOT_EXIST__.env"); + + // WHEN running pull command without API key + const originalApiKey = process.env.HUMANLOOP_API_KEY; + delete process.env.HUMANLOOP_API_KEY; + + const result = await runCli([ + "pull", + "--local-files-directory", + tempDir, + "--env-file", + `"${nonExistentEnvFile}"`, + ]); + + // Restore API key + process.env.HUMANLOOP_API_KEY = originalApiKey; + + // THEN it should fail with appropriate error message + expect(result.exitCode).not.toBe(0); + expect(result.stderr + result.stdout).toContain( + "Failed to load environment file", + ); + + cleanup(); + }); + + test("pull_basic: should pull all files successfully", async () => { + // Increase timeout for this test + jest.setTimeout(30000); // 30 seconds + + // GIVEN a base directory for pulled files + const { tempDir, cleanup } = createTempDir("cli-basic-pull"); + + // WHEN running pull command + const result = await runCli([ + "pull", + "--local-files-directory", + tempDir, + "--verbose", + "--api-key", + process.env.HUMANLOOP_API_KEY || "", + ]); + + // THEN it should succeed + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("Pulling files from Humanloop"); + expect(result.stdout).toContain("Pull completed"); + + // THEN the files should exist locally + for (const file of syncableFiles) { + const extension = `.${file.type}`; + const localPath = path.join(tempDir, `${file.path}${extension}`); + + expect(fs.existsSync(localPath)).toBe(true); + expect(fs.existsSync(path.dirname(localPath))).toBe(true); + + const content = fs.readFileSync(localPath, "utf8"); + expect(content).toBeTruthy(); + } + + cleanup(); + }, 30000); + + test("pull_with_specific_path: should pull files from a specific path", async () => { + // GIVEN a base directory and specific path + const { tempDir, cleanup } = createTempDir("cli-path-pull"); + + // Get the prefix of the first file's path (test directory) + const testPath = syncableFiles[0].path.split("/")[0]; + + // WHEN running pull command with path + const result = await runCli([ + "pull", + "--local-files-directory", + tempDir, + "--path", + testPath, + "--verbose", + "--api-key", + process.env.HUMANLOOP_API_KEY || "", + ]); + + // THEN it should succeed and show the path + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain(`Path: ${testPath}`); + + // THEN only files from that path should exist locally + for (const file of syncableFiles) { + const extension = `.${file.type}`; + const localPath = path.join(tempDir, `${file.path}${extension}`); + + if (file.path.startsWith(testPath)) { + expect(fs.existsSync(localPath)).toBe(true); + } else { + expect(fs.existsSync(localPath)).toBe(false); + } + } + + cleanup(); + }); + + test("pull_with_environment: should pull files from a specific environment", async () => { + // Increase timeout for this test + jest.setTimeout(30000); // 30 seconds + + // GIVEN a base directory and environment + const { tempDir, cleanup } = createTempDir("cli-env-pull"); + + // WHEN running pull command with environment + const result = await runCli([ + "pull", + "--local-files-directory", + tempDir, + "--environment", + "staging", + "--verbose", + "--api-key", + process.env.HUMANLOOP_API_KEY || "", + ]); + + // THEN it should succeed and show the environment + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("Environment: staging"); + + cleanup(); + }, 30000); + + test("pull_with_quiet_mode: should pull files with quiet mode enabled", async () => { + // GIVEN a base directory and quiet mode + const { tempDir, cleanup } = createTempDir("cli-quiet-pull"); + + // WHEN running pull command with quiet mode + const result = await runCli([ + "pull", + "--local-files-directory", + tempDir, + "--quiet", + "--api-key", + process.env.HUMANLOOP_API_KEY || "", + ]); + + // THEN it should succeed but not show file list + expect(result.exitCode).toBe(0); + expect(result.stdout).not.toContain("Successfully pulled"); + + // THEN files should still be pulled + for (const file of syncableFiles) { + const extension = `.${file.type}`; + const localPath = path.join(tempDir, `${file.path}${extension}`); + expect(fs.existsSync(localPath)).toBe(true); + } + + cleanup(); + }); + + test("pull_with_invalid_path: should handle error when pulling from an invalid path", async () => { + // GIVEN an invalid path + const { tempDir, cleanup } = createTempDir("cli-invalid-path"); + const path = "nonexistent/path"; + + // WHEN running pull command + const result = await runCli([ + "pull", + "--local-files-directory", + tempDir, + "--path", + path, + "--api-key", + process.env.HUMANLOOP_API_KEY || "", + ]); + + // THEN it should fail + expect(result.exitCode).toBe(1); + expect(result.stderr + result.stdout).toContain("Error"); + + cleanup(); + }); + + test("pull_with_invalid_environment: should handle error when pulling from an invalid environment", async () => { + // GIVEN an invalid environment + const { tempDir, cleanup } = createTempDir("cli-invalid-env"); + const environment = "nonexistent"; + + // WHEN running pull command + const result = await runCli([ + "pull", + "--local-files-directory", + tempDir, + "--environment", + environment, + "--verbose", + "--api-key", + process.env.HUMANLOOP_API_KEY || "", + ]); + + // THEN it should fail + expect(result.exitCode).toBe(1); + expect(result.stderr + result.stdout).toContain("Error"); + + cleanup(); + }); +}); From d53897f36a82d3036e44ea14e6305ca08fbefba8 Mon Sep 17 00:00:00 2001 From: Ale Pouroullis Date: Tue, 20 May 2025 18:02:51 +0100 Subject: [PATCH 13/16] test: Increase timeout of cleanup in slow integration tests --- tests/custom/integration/FileSyncer.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/custom/integration/FileSyncer.test.ts b/tests/custom/integration/FileSyncer.test.ts index 15120e18..0f3e5f40 100644 --- a/tests/custom/integration/FileSyncer.test.ts +++ b/tests/custom/integration/FileSyncer.test.ts @@ -42,7 +42,7 @@ describe("FileSyncer Integration Tests", () => { })), ); } - }); + }, 30000); test("pull_basic: should pull all files from remote to local filesystem", async () => { // GIVEN a set of files in the remote system (from syncableFiles) From 0d5fb8535eceb78161ac92ff1ad68fe4a23c564e Mon Sep 17 00:00:00 2001 From: Ale Pouroullis Date: Tue, 20 May 2025 18:20:35 +0100 Subject: [PATCH 14/16] Updated deps needed to successfully run the tests --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index ed2af786..0374d4f0 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,8 @@ "@traceloop/instrumentation-openai": ">=0.11.3", "@traceloop/ai-semantic-conventions": ">=0.11.6", "cli-progress": "^3.12.0", + "dotenv": "^16.5.0", + "commander": "^14.0.0", "lodash": "^4.17.21" }, "devDependencies": { @@ -46,7 +48,6 @@ "openai": "^4.74.0", "@anthropic-ai/sdk": "^0.32.1", "cohere-ai": "^7.15.0", - "dotenv": "^16.4.6", "jsonschema": "^1.4.1", "@types/cli-progress": "^3.11.6", "@types/lodash": "4.14.74", From 529c25c65750dd018e7fc62c847fbf638d35901c Mon Sep 17 00:00:00 2001 From: Ale Pouroullis Date: Tue, 20 May 2025 18:29:56 +0100 Subject: [PATCH 15/16] test: Increase timeout for pull_basic test as the pagination can be slow --- tests/custom/integration/FileSyncer.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/custom/integration/FileSyncer.test.ts b/tests/custom/integration/FileSyncer.test.ts index 0f3e5f40..7afa1ab2 100644 --- a/tests/custom/integration/FileSyncer.test.ts +++ b/tests/custom/integration/FileSyncer.test.ts @@ -71,7 +71,7 @@ describe("FileSyncer Integration Tests", () => { const content = fs.readFileSync(localPath, "utf8"); expect(content).toBeTruthy(); } - }); + }, 30000); test("pull_with_invalid_path: should handle error when path doesn't exist", async () => { // GIVEN a client From 5b74f94b823913142b34fdf6c369ff64ef1bb9af Mon Sep 17 00:00:00 2001 From: Ale Pouroullis Date: Wed, 21 May 2025 12:37:49 +0100 Subject: [PATCH 16/16] docs: Add Syncing Files section to README --- README.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/README.md b/README.md index e1765775..6ac9ea37 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,47 @@ try { } ``` +## Store Humanloop Files in Code + +Humanloop allows you to maintain Prompts and Agents in your local filesystem and version control, while still leveraging Humanloop's prompt management capabilities. + +### Syncing Files with the CLI + +```bash +# Basic usage +npx humanloop pull # Pull all files to 'humanloop/' directory +npx humanloop pull --path="examples/chat" # Pull specific directory +npx humanloop pull --environment="production" # Pull from specific environment +npx humanloop pull --local-files-directory="ai" # Specify local destination (default: "humanloop") + +# View available options +npx humanloop pull --help +``` + +### Using Local Files in the SDK + +To use local Files in your code: + +```typescript +// Enable local file support +const client = new HumanloopClient({ + apiKey: "YOUR_API_KEY", + useLocalFiles: true +}); + +// Call a local Prompt file +const response = await client.prompts.call({ + path: "examples/chat/basic", // Looks for humanloop/examples/chat/basic.prompt + inputs: { query: "Hello world" } +}); + +// The same path-based approach works with prompts.log(), agents.call(), and agents.log() +``` + +For detailed instructions, see our [Guide on Storing Files in Code](https://humanloop.com/docs/v5/guides/prompts/store-prompts-in-code). + +For information about file formats, see our [File Format Reference](https://humanloop.com/docs/v5/reference/serialized-files). + ## Pagination List endpoints are paginated. The SDK provides an iterator so that you can simply loop over the items: