diff --git a/lib/langfuse/api_client.rb b/lib/langfuse/api_client.rb index dbac268..2ece303 100644 --- a/lib/langfuse/api_client.rb +++ b/lib/langfuse/api_client.rb @@ -276,6 +276,26 @@ def create_dataset_run_item(dataset_item_id:, run_name:, trace_id: nil, raise ApiError, "HTTP request failed: #{e.message}" end + # Fetch projects accessible with the current API keys + # + # @return [Hash] The parsed response body containing project data + # @raise [UnauthorizedError] if authentication fails + # @raise [ApiError] for other API errors + # + # @example + # data = api_client.get_projects + # project_id = data["data"][0]["id"] + def get_projects # rubocop:disable Naming/AccessorMethodName + response = connection.get("/api/public/projects") + handle_response(response) + rescue Faraday::RetriableResponse => e + logger.error("Faraday error: Retries exhausted - #{e.response.status}") + handle_response(e.response) + rescue Faraday::Error => e + logger.error("Faraday error: #{e.message}") + raise ApiError, "HTTP request failed: #{e.message}" + end + # Shut down the API client and release resources # # Shuts down the cache if it supports shutdown (e.g., SWR thread pool). diff --git a/lib/langfuse/client.rb b/lib/langfuse/client.rb index 4503a40..d071a49 100644 --- a/lib/langfuse/client.rb +++ b/lib/langfuse/client.rb @@ -49,6 +49,11 @@ def initialize(config) cache: cache ) + @project_id = nil + # One-shot lookup: avoids repeated blocking API calls in URL helpers + # (trace_url, dataset_url, dataset_run_url) when the project endpoint is down. + @project_id_fetched = false + # Initialize score client for batching score events @score_client = ScoreClient.new(api_client: @api_client, config: config) end @@ -235,16 +240,45 @@ def update_prompt(name:, version:, labels:) build_prompt_client(prompt_data) end + # Lazily-fetched project ID for URL generation + # + # Fetches the project ID from the API on first access and caches it. + # Returns nil if the API call fails (URL generation is non-critical). + # + # @return [String, nil] The Langfuse project ID + def project_id + return @project_id if @project_id_fetched + + fetch_project_id + end + # Generate URL for viewing a trace in Langfuse UI # # @param trace_id [String] The trace ID (hex-encoded, 32 characters) - # @return [String] URL to view the trace + # @return [String, nil] URL to view the trace, or nil if project ID unavailable # # @example # url = client.trace_url("abc123...") # puts "View trace at: #{url}" def trace_url(trace_id) - "#{config.base_url}/traces/#{trace_id}" + project_url("traces/#{trace_id}") + end + + # Generate URL for viewing a dataset in Langfuse UI + # + # @param dataset_id [String] The dataset ID + # @return [String, nil] URL to view the dataset, or nil if project ID unavailable + def dataset_url(dataset_id) + project_url("datasets/#{dataset_id}") + end + + # Generate URL for viewing a dataset run in Langfuse UI + # + # @param dataset_id [String] The dataset ID + # @param dataset_run_id [String] The dataset run ID + # @return [String, nil] URL to view the dataset run, or nil if project ID unavailable + def dataset_run_url(dataset_id:, dataset_run_id:) + project_url("datasets/#{dataset_id}/runs/#{dataset_run_id}") end # Create a score event and queue it for batching @@ -562,6 +596,26 @@ def run_experiment(name:, task:, data: nil, dataset_name: nil, description: nil, attr_reader :score_client + # Build a project-scoped URL, returning nil if project ID is unavailable + def project_url(path) + pid = project_id + return nil unless pid + + "#{config.base_url}/project/#{pid}/#{path}" + end + + # Fetch project ID from the API and cache it + # + # @return [String, nil] the project ID, or nil on failure + def fetch_project_id + data = api_client.get_projects + @project_id = data.dig("data", 0, "id") + rescue StandardError + nil + ensure + @project_id_fetched = true + end + def fetch_dataset_items_page(page:, limit:, **filters) api_client.list_dataset_items(page: page, limit: limit, **filters) end diff --git a/lib/langfuse/dataset_client.rb b/lib/langfuse/dataset_client.rb index dfc358f..2596943 100644 --- a/lib/langfuse/dataset_client.rb +++ b/lib/langfuse/dataset_client.rb @@ -63,6 +63,15 @@ def items end end + # Generate URL for viewing this dataset in Langfuse UI + # + # @return [String, nil] URL to view the dataset, or nil if client unavailable + def url + return nil unless @client + + @client.dataset_url(@id) + end + # Run an experiment against all items in this dataset # # @param name [String] experiment/run name (required) diff --git a/lib/langfuse/experiment_runner.rb b/lib/langfuse/experiment_runner.rb index 2328585..c931d97 100644 --- a/lib/langfuse/experiment_runner.rb +++ b/lib/langfuse/experiment_runner.rb @@ -34,6 +34,7 @@ def initialize(client:, name:, items:, task:, evaluators: [], run_evaluators: [] @run_name = run_name || "#{name} - #{Time.now.utc.iso8601}" @logger = Langfuse.configuration.logger @dataset_run_id = nil + @dataset_id = nil end # rubocop:enable Metrics/ParameterLists @@ -52,7 +53,8 @@ def execute run_evaluations: run_evals, run_name: @run_name, description: @description, - dataset_run_id: @dataset_run_id + dataset_run_id: @dataset_run_id, + dataset_run_url: build_dataset_run_url ) end @@ -136,13 +138,17 @@ def persist_run_score(evaluation) @logger.warn("Run score persistence failed for '#{evaluation.name}': #{e.message}") end + # Invariant: all items in a single run belong to the same dataset. def link_to_dataset_run(item, trace_id, observation_id) response = @client.create_dataset_run_item( dataset_item_id: item.id, run_name: @run_name, trace_id: trace_id, observation_id: observation_id, metadata: @metadata, run_description: @description ) - @dataset_run_id ||= response&.dig("datasetRunId") + unless @dataset_run_id + @dataset_run_id = response&.dig("datasetRunId") + @dataset_id = item.dataset_id if @dataset_run_id + end response rescue StandardError => e @logger.warn("Dataset run item linking failed: #{e.message}") @@ -223,6 +229,12 @@ def item_metadata(item) item.respond_to?(:metadata) ? item.metadata : nil end + def build_dataset_run_url + return nil unless @dataset_run_id && @dataset_id + + @client.dataset_run_url(dataset_id: @dataset_id, dataset_run_id: @dataset_run_id) + end + # @api private def log_task_failure(task_error, index) message = task_error.respond_to?(:message) ? task_error.message : task_error.to_s diff --git a/spec/langfuse/api_client_spec.rb b/spec/langfuse/api_client_spec.rb index 8682366..00da694 100644 --- a/spec/langfuse/api_client_spec.rb +++ b/spec/langfuse/api_client_spec.rb @@ -2180,6 +2180,71 @@ def set(_key, value) end end + describe "#get_projects" do + let(:projects_response) do + { + "data" => [ + { "id" => "proj-abc-123", "name" => "my-project" } + ] + } + end + + context "with successful response" do + before do + stub_request(:get, "#{base_url}/api/public/projects") + .to_return( + status: 200, + body: projects_response.to_json, + headers: { "Content-Type" => "application/json" } + ) + end + + it "returns parsed response body" do + result = api_client.get_projects + expect(result["data"].first["id"]).to eq("proj-abc-123") + end + + it "makes GET request to correct endpoint" do + api_client.get_projects + expect( + a_request(:get, "#{base_url}/api/public/projects") + ).to have_been_made.once + end + end + + context "when authentication fails" do + before do + stub_request(:get, "#{base_url}/api/public/projects") + .to_return(status: 401, body: { message: "Unauthorized" }.to_json) + end + + it "raises UnauthorizedError" do + expect { api_client.get_projects }.to raise_error(Langfuse::UnauthorizedError) + end + end + + context "when network error occurs" do + before do + stub_request(:get, "#{base_url}/api/public/projects") + .to_timeout + end + + it "raises ApiError" do + expect { api_client.get_projects }.to raise_error(Langfuse::ApiError, /HTTP request failed/) + end + end + + context "when retries exhausted" do + it "handles Faraday::RetriableResponse" do + mock_response = instance_double(Faraday::Response, status: 503, body: { "message" => "Service unavailable" }) + retriable_error = Faraday::RetriableResponse.new("Retries exhausted", mock_response) + allow(api_client.connection).to receive(:get).and_raise(retriable_error) + + expect { api_client.get_projects }.to raise_error(Langfuse::ApiError, /API request failed \(503\)/) + end + end + end + describe "#create_dataset_run_item" do let(:run_item_response) do { diff --git a/spec/langfuse/base_observation_spec.rb b/spec/langfuse/base_observation_spec.rb index 9ceef05..e90a739 100644 --- a/spec/langfuse/base_observation_spec.rb +++ b/spec/langfuse/base_observation_spec.rb @@ -553,17 +553,24 @@ def version config.secret_key = "sk_test_456" config.base_url = "https://cloud.langfuse.com" end + + stub_request(:get, "https://cloud.langfuse.com/api/public/projects") + .to_return( + status: 200, + body: { "data" => [{ "id" => "proj-abc" }] }.to_json, + headers: { "Content-Type" => "application/json" } + ) end after do Langfuse.reset! end - it "generates trace URL using client" do + it "generates trace URL with project ID" do trace_id = observation.trace_id url = observation.trace_url - expect(url).to eq("https://cloud.langfuse.com/traces/#{trace_id}") + expect(url).to eq("https://cloud.langfuse.com/project/proj-abc/traces/#{trace_id}") end it "uses configured base_url" do @@ -573,10 +580,17 @@ def version config.base_url = "https://custom.langfuse.com" end + stub_request(:get, "https://custom.langfuse.com/api/public/projects") + .to_return( + status: 200, + body: { "data" => [{ "id" => "proj-xyz" }] }.to_json, + headers: { "Content-Type" => "application/json" } + ) + trace_id = observation.trace_id url = observation.trace_url - expect(url).to eq("https://custom.langfuse.com/traces/#{trace_id}") + expect(url).to eq("https://custom.langfuse.com/project/proj-xyz/traces/#{trace_id}") end it "calls client.trace_url with correct trace_id" do diff --git a/spec/langfuse/client_spec.rb b/spec/langfuse/client_spec.rb index 98d139f..c4b00d5 100644 --- a/spec/langfuse/client_spec.rb +++ b/spec/langfuse/client_spec.rb @@ -1232,14 +1232,97 @@ class << self end end + describe "#project_id" do + let(:client) { described_class.new(valid_config) } + let(:base_url) { valid_config.base_url } + + context "when API returns project data" do + before do + stub_request(:get, "#{base_url}/api/public/projects") + .to_return( + status: 200, + body: { "data" => [{ "id" => "proj-abc-123" }] }.to_json, + headers: { "Content-Type" => "application/json" } + ) + end + + it "returns the project ID" do + expect(client.project_id).to eq("proj-abc-123") + end + + it "caches the project ID" do + client.project_id + client.project_id + + expect( + a_request(:get, "#{base_url}/api/public/projects") + ).to have_been_made.once + end + end + + context "when API call fails" do + before do + stub_request(:get, "#{base_url}/api/public/projects") + .to_return(status: 500, body: { message: "Server error" }.to_json) + end + + it "returns nil" do + expect(client.project_id).to be_nil + end + + it "does not retry on subsequent trace_url calls" do + client.trace_url("abc123") + client.trace_url("def456") + + expect( + a_request(:get, "#{base_url}/api/public/projects") + ).to have_been_made.once + end + end + + context "when API returns empty data" do + before do + stub_request(:get, "#{base_url}/api/public/projects") + .to_return( + status: 200, + body: { "data" => [] }.to_json, + headers: { "Content-Type" => "application/json" } + ) + end + + it "returns nil" do + expect(client.project_id).to be_nil + end + + it "does not retry on subsequent dataset_url calls" do + client.dataset_url("ds-1") + client.dataset_url("ds-2") + + expect( + a_request(:get, "#{base_url}/api/public/projects") + ).to have_been_made.once + end + end + end + describe "#trace_url" do let(:client) { described_class.new(valid_config) } + let(:base_url) { valid_config.base_url } - it "generates trace URL with default base_url" do - trace_id = "a" * 32 # 32 hex characters + before do + stub_request(:get, "#{base_url}/api/public/projects") + .to_return( + status: 200, + body: { "data" => [{ "id" => "proj-abc" }] }.to_json, + headers: { "Content-Type" => "application/json" } + ) + end + + it "generates trace URL with project ID" do + trace_id = "a" * 32 url = client.trace_url(trace_id) - expect(url).to eq("https://cloud.langfuse.com/traces/#{trace_id}") + expect(url).to eq("https://cloud.langfuse.com/project/proj-abc/traces/#{trace_id}") end it "generates trace URL with custom base_url" do @@ -1249,18 +1332,92 @@ class << self config.base_url = "https://custom.langfuse.com" end custom_client = described_class.new(custom_config) - trace_id = "b" * 32 + stub_request(:get, "https://custom.langfuse.com/api/public/projects") + .to_return( + status: 200, + body: { "data" => [{ "id" => "proj-xyz" }] }.to_json, + headers: { "Content-Type" => "application/json" } + ) + + trace_id = "b" * 32 url = custom_client.trace_url(trace_id) - expect(url).to eq("https://custom.langfuse.com/traces/#{trace_id}") + expect(url).to eq("https://custom.langfuse.com/project/proj-xyz/traces/#{trace_id}") end - it "handles trace IDs of any length" do - trace_id = "abc123def456" - url = client.trace_url(trace_id) + context "when project ID is unavailable" do + before do + stub_request(:get, "#{base_url}/api/public/projects") + .to_return(status: 500, body: { message: "Server error" }.to_json) + end - expect(url).to eq("https://cloud.langfuse.com/traces/#{trace_id}") + it "returns nil" do + client_without_project = described_class.new(valid_config) + expect(client_without_project.trace_url("abc123")).to be_nil + end + end + end + + describe "#dataset_url" do + let(:client) { described_class.new(valid_config) } + let(:base_url) { valid_config.base_url } + + before do + stub_request(:get, "#{base_url}/api/public/projects") + .to_return( + status: 200, + body: { "data" => [{ "id" => "proj-abc" }] }.to_json, + headers: { "Content-Type" => "application/json" } + ) + end + + it "generates dataset URL with project ID" do + url = client.dataset_url("ds-123") + expect(url).to eq("https://cloud.langfuse.com/project/proj-abc/datasets/ds-123") + end + + context "when project ID is unavailable" do + before do + stub_request(:get, "#{base_url}/api/public/projects") + .to_return(status: 500, body: { message: "Server error" }.to_json) + end + + it "returns nil" do + client_without_project = described_class.new(valid_config) + expect(client_without_project.dataset_url("ds-123")).to be_nil + end + end + end + + describe "#dataset_run_url" do + let(:client) { described_class.new(valid_config) } + let(:base_url) { valid_config.base_url } + + before do + stub_request(:get, "#{base_url}/api/public/projects") + .to_return( + status: 200, + body: { "data" => [{ "id" => "proj-abc" }] }.to_json, + headers: { "Content-Type" => "application/json" } + ) + end + + it "generates dataset run URL with project ID" do + url = client.dataset_run_url(dataset_id: "ds-123", dataset_run_id: "run-456") + expect(url).to eq("https://cloud.langfuse.com/project/proj-abc/datasets/ds-123/runs/run-456") + end + + context "when project ID is unavailable" do + before do + stub_request(:get, "#{base_url}/api/public/projects") + .to_return(status: 500, body: { message: "Server error" }.to_json) + end + + it "returns nil" do + client_without_project = described_class.new(valid_config) + expect(client_without_project.dataset_run_url(dataset_id: "ds-123", dataset_run_id: "run-456")).to be_nil + end end end diff --git a/spec/langfuse/dataset_client_spec.rb b/spec/langfuse/dataset_client_spec.rb index 4c5dc74..5a6f759 100644 --- a/spec/langfuse/dataset_client_spec.rb +++ b/spec/langfuse/dataset_client_spec.rb @@ -99,6 +99,28 @@ end end + describe "#url" do + context "without client" do + it "returns nil" do + client = described_class.new(dataset_data) + expect(client.url).to be_nil + end + end + + context "with client" do + let(:mock_client) { instance_double(Langfuse::Client) } + + it "delegates to client.dataset_url with id" do + allow(mock_client).to receive(:dataset_url) + .with("dataset-123") + .and_return("https://cloud.langfuse.com/project/proj-abc/datasets/dataset-123") + + client = described_class.new(dataset_data, client: mock_client) + expect(client.url).to eq("https://cloud.langfuse.com/project/proj-abc/datasets/dataset-123") + end + end + end + describe "#items" do context "with no items and no client" do it "returns empty array" do diff --git a/spec/langfuse/experiment_runner_spec.rb b/spec/langfuse/experiment_runner_spec.rb index 702231e..33001ab 100644 --- a/spec/langfuse/experiment_runner_spec.rb +++ b/spec/langfuse/experiment_runner_spec.rb @@ -8,6 +8,7 @@ allow(mock_client).to receive(:create_dataset_run_item) allow(mock_client).to receive(:create_score) allow(mock_client).to receive(:flush_scores) + allow(mock_client).to receive(:dataset_run_url) allow(Langfuse).to receive(:force_flush) allow(Langfuse.configuration).to receive(:logger).and_return(logger) end @@ -252,6 +253,73 @@ expect(result.dataset_run_id).to eq("run-abc-123") end + + it "populates dataset_run_url on the result" do + allow(mock_client).to receive(:create_dataset_run_item) + .and_return({ "datasetRunId" => "run-abc-123" }) + allow(mock_client).to receive(:dataset_run_url) + .with(dataset_id: "ds-1", dataset_run_id: "run-abc-123") + .and_return("https://cloud.langfuse.com/project/proj-1/datasets/ds-1/runs/run-abc-123") + + runner = described_class.new( + client: mock_client, name: "ds-exp", items: dataset_items, + task: ->(_) { "a" } + ) + result = runner.execute + + expect(result.dataset_run_url).to eq( + "https://cloud.langfuse.com/project/proj-1/datasets/ds-1/runs/run-abc-123" + ) + end + + it "sets dataset_run_url to nil when no dataset_run_id" do + allow(mock_client).to receive(:create_dataset_run_item) + .and_return({}) + + runner = described_class.new( + client: mock_client, name: "ds-exp", items: dataset_items, + task: ->(_) { "a" } + ) + result = runner.execute + + expect(result.dataset_run_url).to be_nil + end + + it "uses dataset_id from the first successfully linked item" do + items = [ + Langfuse::DatasetItemClient.new( + { "id" => "item-1", "datasetId" => "ds-1", + "input" => { "q" => "x" }, "expectedOutput" => "ax" }, + client: mock_client + ), + Langfuse::DatasetItemClient.new( + { "id" => "item-2", "datasetId" => "ds-1", + "input" => { "q" => "y" }, "expectedOutput" => "ay" }, + client: mock_client + ) + ] + + call_count = 0 + allow(mock_client).to receive(:create_dataset_run_item) do + call_count += 1 + raise StandardError, "link error" if call_count == 1 + + { "datasetRunId" => "run-1" } + end + allow(mock_client).to receive(:dataset_run_url) + .with(dataset_id: "ds-1", dataset_run_id: "run-1") + .and_return("https://example.com/datasets/ds-1/runs/run-1") + + runner = described_class.new( + client: mock_client, name: "ds-exp", items: items, + task: ->(_) { "a" } + ) + result = runner.execute + + expect(result.dataset_run_url).to eq("https://example.com/datasets/ds-1/runs/run-1") + expect(mock_client).to have_received(:dataset_run_url) + .with(dataset_id: "ds-1", dataset_run_id: "run-1") + end end context "when task fails" do