From 49f3f25ea5d4858a8fe1a95e937295cfbe45c4e6 Mon Sep 17 00:00:00 2001
From: Bob <uncoolbob@gmail.com>
Date: Tue, 16 Sep 2025 23:30:37 +0100
Subject: [PATCH 01/16] abstract Summarizer and concrete OpenAISummarizer

---
 .../ai/SingleGeneAiExpressionReporter.java    |  5 +-
 .../ai/expression/OpenAISummarizer.java       | 67 ++++++++++++++
 .../report/ai/expression/Summarizer.java      | 87 +++++--------------
 3 files changed, 90 insertions(+), 69 deletions(-)
 create mode 100644 Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java

diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
index 9b61a9f54..55b8f0d49 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
@@ -13,6 +13,7 @@
 import org.apidb.apicommon.model.report.ai.expression.DailyCostMonitor;
 import org.apidb.apicommon.model.report.ai.expression.GeneRecordProcessor;
 import org.apidb.apicommon.model.report.ai.expression.GeneRecordProcessor.GeneSummaryInputs;
+import org.apidb.apicommon.model.report.ai.expression.OpenAISummarizer;
 import org.apidb.apicommon.model.report.ai.expression.Summarizer;
 import org.gusdb.wdk.model.WdkModelException;
 import org.gusdb.wdk.model.WdkServiceTemporarilyUnavailableException;
@@ -80,7 +81,7 @@ protected void write(OutputStream out) throws IOException, WdkModelException {
     AiExpressionCache cache = AiExpressionCache.getInstance(_wdkModel);
 
     // create summarizer (interacts with OpenAI)
-    Summarizer summarizer = new Summarizer(_wdkModel, _costMonitor);
+    OpenAISummarizer summarizer = new OpenAISummarizer(_wdkModel, _costMonitor);
 
     // open record and output streams
     try (RecordStream recordStream = RecordStreamFactory.getRecordStream(_baseAnswer, List.of(), tables);
@@ -93,7 +94,7 @@ protected void write(OutputStream out) throws IOException, WdkModelException {
 
         // create summary inputs
         GeneSummaryInputs summaryInputs =
-            GeneRecordProcessor.getSummaryInputsFromRecord(record, Summarizer.OPENAI_CHAT_MODEL.toString(),
+            GeneRecordProcessor.getSummaryInputsFromRecord(record, OpenAISummarizer.OPENAI_CHAT_MODEL.toString(),
                 Summarizer::getExperimentMessage, Summarizer::getFinalSummaryMessage);
 
         // fetch summary, producing if necessary and requested
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
new file mode 100644
index 000000000..329e46c3f
--- /dev/null
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
@@ -0,0 +1,67 @@
+package org.apidb.apicommon.model.report.ai.expression;
+
+import java.util.concurrent.CompletableFuture;
+
+import org.gusdb.wdk.model.WdkModel;
+import org.gusdb.wdk.model.WdkModelException;
+
+import com.openai.client.OpenAIClientAsync;
+import com.openai.client.okhttp.OpenAIOkHttpClientAsync;
+import com.openai.models.ChatCompletionCreateParams;
+import com.openai.models.ChatModel;
+import com.openai.models.ResponseFormatJsonSchema;
+import com.openai.models.ResponseFormatJsonSchema.JsonSchema;
+
+public class OpenAISummarizer extends Summarizer {
+
+  // provide exact model number for semi-reproducibility
+  public static final ChatModel OPENAI_CHAT_MODEL = ChatModel.GPT_4O_2024_11_20; // GPT_4O_2024_08_06;
+
+  private static final String OPENAI_API_KEY_PROP_NAME = "OPENAI_API_KEY";
+
+  private final OpenAIClientAsync _openAIClient;
+
+  public OpenAISummarizer(WdkModel wdkModel, DailyCostMonitor costMonitor) throws WdkModelException {
+    super(costMonitor);
+
+    String apiKey = wdkModel.getProperties().get(OPENAI_API_KEY_PROP_NAME);
+    if (apiKey == null) {
+      throw new WdkModelException("WDK property '" + OPENAI_API_KEY_PROP_NAME + "' has not been set.");
+    }
+
+    _openAIClient = OpenAIOkHttpClientAsync.builder()
+        .apiKey(apiKey)
+        .maxRetries(32)  // Handle 429 errors
+        .build();
+  }
+
+  @Override
+  protected CompletableFuture<String> callApiForJson(String prompt, com.openai.models.ResponseFormatJsonSchema.JsonSchema.Schema schema) {
+    ChatCompletionCreateParams request = ChatCompletionCreateParams.builder()
+        .model(OPENAI_CHAT_MODEL)
+        .maxCompletionTokens(MAX_RESPONSE_TOKENS)
+        .responseFormat(ResponseFormatJsonSchema.builder()
+            .jsonSchema(JsonSchema.builder()
+                .name("structured-response")
+                .schema(schema)
+                .strict(true)
+                .build())
+            .build())
+        .addSystemMessage(SYSTEM_MESSAGE)
+        .addUserMessage(prompt)
+        .build();
+
+    return _openAIClient.chat().completions().create(request).thenApply(completion -> {
+      // update cost accumulator
+      _costMonitor.updateCost(completion.usage());
+      
+      // return JSON string
+      return completion.choices().get(0).message().content().get();
+    });
+  }
+
+  @Override
+  protected void updateCostMonitor(Object apiResponse) {
+    // OpenAI response handling is done in callApiForJson
+  }
+}
\ No newline at end of file
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
index b62338699..35fdcb388 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
@@ -17,28 +17,20 @@
 import org.json.JSONException;
 import org.json.JSONObject;
 
-import com.openai.client.OpenAIClientAsync;
-import com.openai.client.okhttp.OpenAIOkHttpClientAsync;
 import com.openai.core.JsonValue;
-import com.openai.models.ChatCompletionCreateParams;
-import com.openai.models.ChatModel;
-import com.openai.models.ResponseFormatJsonSchema;
 import com.openai.models.ResponseFormatJsonSchema.JsonSchema;
 import com.openai.models.ResponseFormatJsonSchema.JsonSchema.Schema;
 
-public class Summarizer {
+public abstract class Summarizer {
 
-  // provide exact model number for semi-reproducibility
-  public static final ChatModel OPENAI_CHAT_MODEL = ChatModel.GPT_4O_2024_11_20; // GPT_4O_2024_08_06;
-
-  private static final int MAX_RESPONSE_TOKENS = 10000;
+  protected static final int MAX_RESPONSE_TOKENS = 10000;
 
   private static final int MAX_MALFORMED_RESPONSE_RETRIES = 3;
 
-  private static final String SYSTEM_MESSAGE = "You are a bioinformatician working for VEuPathDB.org. You are an expert at providing biologist-friendly summaries of transcriptomic data";
+  protected static final String SYSTEM_MESSAGE = "You are a bioinformatician working for VEuPathDB.org. You are an expert at providing biologist-friendly summaries of transcriptomic data";
 
   // Prepare JSON schemas for structured responses
-  private static final JsonSchema.Schema experimentResponseSchema = JsonSchema.Schema.builder()
+  protected static final JsonSchema.Schema experimentResponseSchema = JsonSchema.Schema.builder()
     .putAdditionalProperty("type", JsonValue.from("object"))
     .putAdditionalProperty("properties", JsonValue.from(Map.of(
           "one_sentence_summary", Map.of("type", "string"),
@@ -57,7 +49,7 @@ public class Summarizer {
     .putAdditionalProperty("additionalProperties", JsonValue.from(false))
     .build();
 
-  private static final JsonSchema.Schema finalResponseSchema = JsonSchema.Schema.builder()
+  protected static final JsonSchema.Schema finalResponseSchema = JsonSchema.Schema.builder()
     .putAdditionalProperty("type", JsonValue.from("object"))
     .putAdditionalProperty("properties", JsonValue.from(Map.of(
           "headline", Map.of("type", "string"),
@@ -81,25 +73,11 @@ public class Summarizer {
     .putAdditionalProperty("additionalProperties", JsonValue.from(false))
     .build();
 
-  private static final String OPENAI_API_KEY_PROP_NAME = "OPENAI_API_KEY";
-
-  private final OpenAIClientAsync _openAIClient;
-  private final DailyCostMonitor _costMonitor;
+  protected final DailyCostMonitor _costMonitor;
 
   private static final Logger LOG = Logger.getLogger(Summarizer.class);
 
-  public Summarizer(WdkModel wdkModel, DailyCostMonitor costMonitor) throws WdkModelException {
-
-    String apiKey = wdkModel.getProperties().get(OPENAI_API_KEY_PROP_NAME);
-    if (apiKey == null) {
-      throw new WdkModelException("WDK property '" + OPENAI_API_KEY_PROP_NAME + "' has not been set.");
-    }
-
-    _openAIClient = OpenAIOkHttpClientAsync.builder()
-        .apiKey(apiKey)
-        .maxRetries(32)  // Handle 429 errors
-        .build();
-
+  public Summarizer(DailyCostMonitor costMonitor) {
     _costMonitor = costMonitor;
   }
 
@@ -133,12 +111,9 @@ public static String getExperimentMessage(JSONObject experiment) {
 
   public CompletableFuture<JSONObject> describeExperiment(ExperimentInputs experimentInputs) {
 
-    ChatCompletionCreateParams request = buildAiRequest(
-        "experiment-summary",
-        experimentResponseSchema,
-        getExperimentMessage(experimentInputs.getExperimentData()));
+    String prompt = getExperimentMessage(experimentInputs.getExperimentData());
 
-    return getValidatedAiResponse("dataset " + experimentInputs.getDatasetId(), request, json -> {
+    return getValidatedAiResponse("dataset " + experimentInputs.getDatasetId(), prompt, experimentResponseSchema, json -> {
       // add some fields to the result to aid the final summarization
       return json
         .put("dataset_id", experimentInputs.getDatasetId())
@@ -159,12 +134,9 @@ public static String getFinalSummaryMessage(List<JSONObject> experiments) {
   
   public JSONObject summarizeExperiments(String geneId, List<JSONObject> experiments) {
 
-    ChatCompletionCreateParams request = buildAiRequest(
-        "expression-summary",
-        finalResponseSchema,
-        getFinalSummaryMessage(experiments));
+    String prompt = getFinalSummaryMessage(experiments);
 
-    return getValidatedAiResponse("summary for gene " + geneId, request, json ->
+    return getValidatedAiResponse("summary for gene " + geneId, prompt, finalResponseSchema, json ->
       // quality control (remove bad `dataset_id`s) and add 'Others' section for any experiments not listed by AI
       consolidateSummary(json, experiments)
     ).join();
@@ -240,34 +212,17 @@ private static JSONObject consolidateSummary(JSONObject summaryResponse,
   }
 
 
-  private static ChatCompletionCreateParams buildAiRequest(String name, Schema schema, String userMessage) {
-    return ChatCompletionCreateParams.builder()
-        .model(OPENAI_CHAT_MODEL)
-        .maxCompletionTokens(MAX_RESPONSE_TOKENS)
-        .responseFormat(ResponseFormatJsonSchema.builder()
-            .jsonSchema(JsonSchema.builder()
-                .name(name)
-                .schema(schema)
-                .strict(true)
-                .build())
-            .build())
-        .addSystemMessage(SYSTEM_MESSAGE)
-        .addUserMessage(userMessage)
-        .build();
-  }
+  protected abstract CompletableFuture<String> callApiForJson(String prompt, Schema schema);
+  
+  protected abstract void updateCostMonitor(Object apiResponse);
 
   private CompletableFuture<JSONObject> getValidatedAiResponse(
       String operationDescription,
-      ChatCompletionCreateParams request,
+      String prompt,
+      Schema schema,
       Function<JSONObject,JSONObject> createFinalJson
   ) {
-    return _openAIClient.chat().completions().create(request).thenApply(completion -> {
-
-      // update cost accumulator
-      _costMonitor.updateCost(completion.usage());
-
-      // expect response to be a JSON string
-      String jsonString = completion.choices().get(0).message().content().get();
+    return callApiForJson(prompt, schema).thenApply(jsonString -> {
       int attempts = 1;
       Exception mostRecentError;
 
@@ -281,12 +236,10 @@ private CompletableFuture<JSONObject> getValidatedAiResponse(
         }
         catch (JSONException e) {
           mostRecentError = e;
-          LOG.warn("Malformed JSON from OpenAI (attempt " + attempts + ") for " + operationDescription + ". Retrying...");
+          LOG.warn("Malformed JSON from AI (attempt " + attempts + ") for " + operationDescription + ". Retrying...");
 
-          // Re-request from OpenAI
-          completion = _openAIClient.chat().completions().create(request).join();
-          _costMonitor.updateCost(completion.usage());
-          jsonString = completion.choices().get(0).message().content().get();
+          // Re-request from AI
+          jsonString = callApiForJson(prompt, schema).join();
           attempts++;
         }
       }

From 7cc64d3f5fa622ef9cc3c7c09b407150a0964ce6 Mon Sep 17 00:00:00 2001
From: Bob <uncoolbob@gmail.com>
Date: Wed, 17 Sep 2025 00:09:00 +0100
Subject: [PATCH 02/16] first attempt at ClaudeSummarizer

---
 Model/pom.xml                                 |   6 +
 .../ai/expression/ClaudeSummarizer.java       | 103 ++++++++++++++++++
 2 files changed, 109 insertions(+)
 create mode 100644 Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java

diff --git a/Model/pom.xml b/Model/pom.xml
index cc13dc180..232572ef9 100644
--- a/Model/pom.xml
+++ b/Model/pom.xml
@@ -135,6 +135,12 @@
       <artifactId>openai-java</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.anthropic</groupId>
+      <artifactId>anthropic-java</artifactId>
+      <version>2.7.0</version>
+    </dependency>
+
   </dependencies>
 
 </project>
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
new file mode 100644
index 000000000..fc070d0cf
--- /dev/null
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
@@ -0,0 +1,103 @@
+package org.apidb.apicommon.model.report.ai.expression;
+
+import java.time.Duration;
+import java.util.concurrent.CompletableFuture;
+
+import org.gusdb.wdk.model.WdkModel;
+import org.gusdb.wdk.model.WdkModelException;
+
+import com.anthropic.client.AnthropicClientAsync;
+import com.anthropic.client.okhttp.AnthropicOkHttpClientAsync;
+import com.anthropic.models.messages.MessageCreateParams;
+import com.anthropic.models.messages.Model;
+import com.openai.models.ResponseFormatJsonSchema.JsonSchema.Schema;
+
+public class ClaudeSummarizer extends Summarizer {
+
+  private static final Model CLAUDE_MODEL = Model.CLAUDE_SONNET_4_20250514;
+
+  private static final String CLAUDE_API_KEY_PROP_NAME = "CLAUDE_API_KEY";
+
+  private final AnthropicClientAsync _claudeClient;
+
+  public ClaudeSummarizer(WdkModel wdkModel, DailyCostMonitor costMonitor) throws WdkModelException {
+    super(costMonitor);
+
+    String apiKey = wdkModel.getProperties().get(CLAUDE_API_KEY_PROP_NAME);
+    if (apiKey == null) {
+      throw new WdkModelException("WDK property '" + CLAUDE_API_KEY_PROP_NAME + "' has not been set.");
+    }
+
+    _claudeClient = AnthropicOkHttpClientAsync.builder()
+        .apiKey(apiKey)
+        .build();
+  }
+
+  @Override
+  protected CompletableFuture<String> callApiForJson(String prompt, Schema schema) {
+    // Convert JSON schema to natural language description for Claude
+    String jsonFormatInstructions = convertSchemaToPromptInstructions(schema);
+    
+    String enhancedPrompt = prompt + "\n\n" + jsonFormatInstructions;
+
+    MessageCreateParams request = MessageCreateParams.builder()
+        .model(CLAUDE_MODEL)
+        .maxTokens((long) MAX_RESPONSE_TOKENS)
+        .system(SYSTEM_MESSAGE)
+        .addUserMessage(enhancedPrompt)
+        .build();
+
+    return _claudeClient.messages().create(request).thenApply(response -> {
+      // Convert Claude usage to OpenAI format for cost monitoring
+      com.anthropic.models.messages.Usage claudeUsage = response.usage();
+      com.openai.models.CompletionUsage openAiUsage = com.openai.models.CompletionUsage.builder()
+          .promptTokens(claudeUsage.inputTokens())
+          .completionTokens(claudeUsage.outputTokens())
+          .totalTokens(claudeUsage.inputTokens() + claudeUsage.outputTokens())
+          .build();
+      
+      _costMonitor.updateCost(java.util.Optional.of(openAiUsage));
+      
+      // Extract text from content blocks using stream API
+      return response.content().stream()
+          .flatMap(contentBlock -> contentBlock.text().stream())
+          .map(textBlock -> textBlock.text())
+          .findFirst()
+          .orElseThrow(() -> new RuntimeException("No text content found in Claude response"));
+    });
+  }
+
+  @Override
+  protected void updateCostMonitor(Object apiResponse) {
+    // Claude response handling is done in callApiForJson
+  }
+
+  private String convertSchemaToPromptInstructions(Schema schema) {
+    // Convert OpenAI JSON schema to Claude-friendly format instructions
+    if (schema == experimentResponseSchema) {
+      return "Respond in valid JSON format matching this exact structure:\n" +
+          "{\n" +
+          "  \"one_sentence_summary\": \"string describing gene expression\",\n" +
+          "  \"biological_importance\": \"integer 0-5\",\n" +
+          "  \"confidence\": \"integer 0-5\",\n" +
+          "  \"experiment_keywords\": [\"array\", \"of\", \"strings\"],\n" +
+          "  \"notes\": \"string with additional context\"\n" +
+          "}";
+    } else if (schema == finalResponseSchema) {
+      return "Respond in valid JSON format matching this exact structure:\n" +
+          "{\n" +
+          "  \"headline\": \"string summarizing key results\",\n" +
+          "  \"one_paragraph_summary\": \"string with ~100 words\",\n" +
+          "  \"topics\": [\n" +
+          "    {\n" +
+          "      \"headline\": \"string summarizing topic\",\n" +
+          "      \"one_sentence_summary\": \"string describing topic results\",\n" +
+          "      \"dataset_ids\": [\"array\", \"of\", \"dataset\", \"strings\"]\n" +
+          "    }\n" +
+          "  ]\n" +
+          "}";
+    } else {
+      return "Respond in valid JSON format.";
+    }
+  }
+}
\ No newline at end of file

From 6b2e3790f7490d00df22cc6598c0e090c7c3afec Mon Sep 17 00:00:00 2001
From: Bob <uncoolbob@gmail.com>
Date: Wed, 17 Sep 2025 10:38:21 +0100
Subject: [PATCH 03/16] switch to Claude and tweaks

---
 .../ai/SingleGeneAiExpressionReporter.java    | 10 ++++----
 .../ai/expression/ClaudeSummarizer.java       | 25 +++++++++++++++++--
 2 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
index 55b8f0d49..d6c65fc3c 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
@@ -13,7 +13,7 @@
 import org.apidb.apicommon.model.report.ai.expression.DailyCostMonitor;
 import org.apidb.apicommon.model.report.ai.expression.GeneRecordProcessor;
 import org.apidb.apicommon.model.report.ai.expression.GeneRecordProcessor.GeneSummaryInputs;
-import org.apidb.apicommon.model.report.ai.expression.OpenAISummarizer;
+import org.apidb.apicommon.model.report.ai.expression.ClaudeSummarizer;
 import org.apidb.apicommon.model.report.ai.expression.Summarizer;
 import org.gusdb.wdk.model.WdkModelException;
 import org.gusdb.wdk.model.WdkServiceTemporarilyUnavailableException;
@@ -53,7 +53,7 @@ public Reporter configure(JSONObject config) throws ReporterConfigException, Wdk
             " should only be assigned to " + geneRecordClass.getFullName());
       }
 
-      // check result size; limit to small results due to OpenAI cost
+      // check result size; limit to small results due to AI API cost
       if (_baseAnswer.getResultSizeFactory().getResultSize() > MAX_RESULT_SIZE) {
         throw new ReporterConfigException("This reporter cannot be called with results of size greater than " + MAX_RESULT_SIZE);
       }
@@ -80,8 +80,8 @@ protected void write(OutputStream out) throws IOException, WdkModelException {
     // open summary cache (manages persistence of expression data)
     AiExpressionCache cache = AiExpressionCache.getInstance(_wdkModel);
 
-    // create summarizer (interacts with OpenAI)
-    OpenAISummarizer summarizer = new OpenAISummarizer(_wdkModel, _costMonitor);
+    // create summarizer (interacts with Claude)
+    ClaudeSummarizer summarizer = new ClaudeSummarizer(_wdkModel, _costMonitor);
 
     // open record and output streams
     try (RecordStream recordStream = RecordStreamFactory.getRecordStream(_baseAnswer, List.of(), tables);
@@ -94,7 +94,7 @@ protected void write(OutputStream out) throws IOException, WdkModelException {
 
         // create summary inputs
         GeneSummaryInputs summaryInputs =
-            GeneRecordProcessor.getSummaryInputsFromRecord(record, OpenAISummarizer.OPENAI_CHAT_MODEL.toString(),
+            GeneRecordProcessor.getSummaryInputsFromRecord(record, ClaudeSummarizer.CLAUDE_MODEL.toString(),
                 Summarizer::getExperimentMessage, Summarizer::getFinalSummaryMessage);
 
         // fetch summary, producing if necessary and requested
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
index fc070d0cf..7b741089f 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
@@ -14,7 +14,7 @@
 
 public class ClaudeSummarizer extends Summarizer {
 
-  private static final Model CLAUDE_MODEL = Model.CLAUDE_SONNET_4_20250514;
+  public static final Model CLAUDE_MODEL = Model.CLAUDE_SONNET_4_20250514;
 
   private static final String CLAUDE_API_KEY_PROP_NAME = "CLAUDE_API_KEY";
 
@@ -30,6 +30,7 @@ public ClaudeSummarizer(WdkModel wdkModel, DailyCostMonitor costMonitor) throws
 
     _claudeClient = AnthropicOkHttpClientAsync.builder()
         .apiKey(apiKey)
+        .checkJacksonVersionCompatibility(false)
         .build();
   }
 
@@ -59,11 +60,14 @@ protected CompletableFuture<String> callApiForJson(String prompt, Schema schema)
       _costMonitor.updateCost(java.util.Optional.of(openAiUsage));
       
       // Extract text from content blocks using stream API
-      return response.content().stream()
+      String rawText = response.content().stream()
           .flatMap(contentBlock -> contentBlock.text().stream())
           .map(textBlock -> textBlock.text())
           .findFirst()
           .orElseThrow(() -> new RuntimeException("No text content found in Claude response"));
+      
+      // Strip JSON markdown formatting if present
+      return stripJsonMarkdown(rawText);
     });
   }
 
@@ -72,6 +76,23 @@ protected void updateCostMonitor(Object apiResponse) {
     // Claude response handling is done in callApiForJson
   }
 
+  private String stripJsonMarkdown(String text) {
+    String trimmed = text.trim();
+    
+    // Remove ```json and ``` markdown formatting
+    if (trimmed.startsWith("```json")) {
+      trimmed = trimmed.substring(7); // Remove "```json"
+    } else if (trimmed.startsWith("```")) {
+      trimmed = trimmed.substring(3); // Remove "```"
+    }
+    
+    if (trimmed.endsWith("```")) {
+      trimmed = trimmed.substring(0, trimmed.length() - 3); // Remove trailing "```"
+    }
+    
+    return trimmed.trim();
+  }
+
   private String convertSchemaToPromptInstructions(Schema schema) {
     // Convert OpenAI JSON schema to Claude-friendly format instructions
     if (schema == experimentResponseSchema) {

From fa12327e1224ea2e1f5353964691234fa42103cc Mon Sep 17 00:00:00 2001
From: Bob <uncoolbob@gmail.com>
Date: Wed, 17 Sep 2025 11:05:04 +0100
Subject: [PATCH 04/16] deprecate OPENAI_-prefixed daily cost env vars

---
 .../ai/expression/DailyCostMonitor.java       | 35 +++++++++++++++----
 1 file changed, 28 insertions(+), 7 deletions(-)

diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/DailyCostMonitor.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/DailyCostMonitor.java
index 2185ee365..73789dbc1 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/DailyCostMonitor.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/DailyCostMonitor.java
@@ -31,10 +31,15 @@ public class DailyCostMonitor {
   private static final String DAILY_COST_ACCUMULATION_FILE_DIR = "dailyCost";
   private static final String DAILY_COST_ACCUMULATION_FILE = "daily_cost_accumulation.txt";
 
-  // model prop keys
-  private static final String MAX_DAILY_DOLLAR_COST_PROP_NAME = "OPENAI_MAX_DAILY_AI_EXPRESSION_DOLLAR_COST";
-  private static final String DOLLAR_COST_PER_1M_INPUT_TOKENS_PROP_NAME = "OPENAI_DOLLAR_COST_PER_1M_AI_INPUT_TOKENS";
-  private static final String DOLLAR_COST_PER_1M_OUTPUT_TOKENS_PROP_NAME = "OPENAI_DOLLAR_COST_PER_1M_AI_OUTPUT_TOKENS";
+  // model prop keys (new names without OPENAI_ prefix)
+  private static final String MAX_DAILY_DOLLAR_COST_PROP_NAME = "MAX_DAILY_AI_EXPRESSION_DOLLAR_COST";
+  private static final String DOLLAR_COST_PER_1M_INPUT_TOKENS_PROP_NAME = "DOLLAR_COST_PER_1M_AI_INPUT_TOKENS";
+  private static final String DOLLAR_COST_PER_1M_OUTPUT_TOKENS_PROP_NAME = "DOLLAR_COST_PER_1M_AI_OUTPUT_TOKENS";
+  
+  // deprecated model prop keys (with OPENAI_ prefix)
+  private static final String DEPRECATED_MAX_DAILY_DOLLAR_COST_PROP_NAME = "OPENAI_MAX_DAILY_AI_EXPRESSION_DOLLAR_COST";
+  private static final String DEPRECATED_DOLLAR_COST_PER_1M_INPUT_TOKENS_PROP_NAME = "OPENAI_DOLLAR_COST_PER_1M_AI_INPUT_TOKENS";
+  private static final String DEPRECATED_DOLLAR_COST_PER_1M_OUTPUT_TOKENS_PROP_NAME = "OPENAI_DOLLAR_COST_PER_1M_AI_OUTPUT_TOKENS";
 
   // lock characteristics
   private static final long DEFAULT_TIMEOUT_MILLIS = 1000;
@@ -68,9 +73,25 @@ public DailyCostMonitor(WdkModel wdkModel) throws WdkModelException {
 
     _costMonitoringFile = _costMonitoringDir.resolve(DAILY_COST_ACCUMULATION_FILE);
 
-    _maxDailyDollarCost = getNumberProp(wdkModel, MAX_DAILY_DOLLAR_COST_PROP_NAME);
-    _costPerInputToken = getNumberProp(wdkModel, DOLLAR_COST_PER_1M_INPUT_TOKENS_PROP_NAME) / 1000000;
-    _costPerOutputToken = getNumberProp(wdkModel, DOLLAR_COST_PER_1M_OUTPUT_TOKENS_PROP_NAME) / 1000000;
+    _maxDailyDollarCost = getNumberProp(wdkModel, MAX_DAILY_DOLLAR_COST_PROP_NAME, DEPRECATED_MAX_DAILY_DOLLAR_COST_PROP_NAME);
+    _costPerInputToken = getNumberProp(wdkModel, DOLLAR_COST_PER_1M_INPUT_TOKENS_PROP_NAME, DEPRECATED_DOLLAR_COST_PER_1M_INPUT_TOKENS_PROP_NAME) / 1000000;
+    _costPerOutputToken = getNumberProp(wdkModel, DOLLAR_COST_PER_1M_OUTPUT_TOKENS_PROP_NAME, DEPRECATED_DOLLAR_COST_PER_1M_OUTPUT_TOKENS_PROP_NAME) / 1000000;
+  }
+
+  private double getNumberProp(WdkModel wdkModel, String propName, String deprecatedPropName) throws WdkModelException {
+    // First try the new property name
+    if (wdkModel.getProperties().get(propName) != null) {
+      return getNumberProp(wdkModel, propName);
+    }
+    
+    // Fall back to deprecated property name with warning
+    if (wdkModel.getProperties().get(deprecatedPropName) != null) {
+      LOG.warn("WDK property '" + deprecatedPropName + "' is deprecated. Please use '" + propName + "' instead.");
+      return getNumberProp(wdkModel, deprecatedPropName);
+    }
+    
+    // Neither property is set
+    throw new WdkModelException("WDK property '" + propName + "' (or deprecated '" + deprecatedPropName + "') has not been set.");
   }
 
   private double getNumberProp(WdkModel wdkModel, String propName) throws WdkModelException {

From 7d7ba6fba3e73694a7d2cedb63fb9003b07bf03b Mon Sep 17 00:00:00 2001
From: Bob <uncoolbob@gmail.com>
Date: Thu, 18 Sep 2025 22:21:51 +0100
Subject: [PATCH 05/16] add more retries for Claude

---
 .../apicommon/model/report/ai/expression/ClaudeSummarizer.java | 3 ++-
 .../apicommon/model/report/ai/expression/OpenAISummarizer.java | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
index 7b741089f..e87bec90c 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
@@ -30,6 +30,7 @@ public ClaudeSummarizer(WdkModel wdkModel, DailyCostMonitor costMonitor) throws
 
     _claudeClient = AnthropicOkHttpClientAsync.builder()
         .apiKey(apiKey)
+        .maxRetries(32)  // Handle 429 errors
         .checkJacksonVersionCompatibility(false)
         .build();
   }
@@ -121,4 +122,4 @@ private String convertSchemaToPromptInstructions(Schema schema) {
       return "Respond in valid JSON format.";
     }
   }
-}
\ No newline at end of file
+}
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
index 329e46c3f..fcea4ac24 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
@@ -64,4 +64,4 @@ protected CompletableFuture<String> callApiForJson(String prompt, com.openai.mod
   protected void updateCostMonitor(Object apiResponse) {
     // OpenAI response handling is done in callApiForJson
   }
-}
\ No newline at end of file
+}

From 98330e50d6a94cf91403151f7909f2770518bab4 Mon Sep 17 00:00:00 2001
From: Bob <uncoolbob@gmail.com>
Date: Fri, 24 Oct 2025 12:45:35 +0100
Subject: [PATCH 06/16] Anthropic SDK upgrade to 2.9.0; use Sonnet 4.5; tweak
 prompt for sentence case

---
 Model/pom.xml                                                 | 2 +-
 .../model/report/ai/expression/ClaudeSummarizer.java          | 4 ++--
 .../apicommon/model/report/ai/expression/Summarizer.java      | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/Model/pom.xml b/Model/pom.xml
index 232572ef9..bf7d78569 100644
--- a/Model/pom.xml
+++ b/Model/pom.xml
@@ -138,7 +138,7 @@
     <dependency>
       <groupId>com.anthropic</groupId>
       <artifactId>anthropic-java</artifactId>
-      <version>2.7.0</version>
+      <version>2.9.0</version>
     </dependency>
 
   </dependencies>
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
index e87bec90c..9f4ab38e4 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
@@ -14,7 +14,7 @@
 
 public class ClaudeSummarizer extends Summarizer {
 
-  public static final Model CLAUDE_MODEL = Model.CLAUDE_SONNET_4_20250514;
+  public static final Model CLAUDE_MODEL = Model.CLAUDE_SONNET_4_5_20250929;
 
   private static final String CLAUDE_API_KEY_PROP_NAME = "CLAUDE_API_KEY";
 
@@ -114,7 +114,7 @@ private String convertSchemaToPromptInstructions(Schema schema) {
           "    {\n" +
           "      \"headline\": \"string summarizing topic\",\n" +
           "      \"one_sentence_summary\": \"string describing topic results\",\n" +
-          "      \"dataset_ids\": [\"array\", \"of\", \"dataset\", \"strings\"]\n" +
+          "      \"dataset_ids\": [\"array\", \"of\", \"dataset_id\", \"strings\"]\n" +
           "    }\n" +
           "  ]\n" +
           "}";
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
index 35fdcb388..a1b8a39aa 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
@@ -126,6 +126,7 @@ public static String getFinalSummaryMessage(List<JSONObject> experiments) {
     return "Below are AI-generated summaries of one gene's behavior in all the transcriptomics experiments available in VEuPathDB, provided in JSON format:\n\n" +
         String.format("```json\n%s\n```\n\n", new JSONArray(experiments).toString(2)) +
         "Generate a one-paragraph summary (~100 words) describing the gene's expression. Structure it using <strong>, <ul>, and <li> tags with no attributes. If relevant, briefly speculate on the gene's potential function, but only if justified by the data. Also, generate a short, specific headline for the summary. The headline must reflect this gene's expression and **must not** include generic phrases like \"comprehensive insights into\" or the word \"gene\".\n\n" +
+        "Use sentence case for all headlines: capitalize only the first word and proper nouns, not every word.\n\n" +
     "Additionally, group the per-experiment summaries (identified by `dataset_id`) with `biological_importance > 3` and `confidence > 3` into sections by topic. For each topic, provide:\n" +
     "- A headline summarizing the key experimental results within the topic\n" +
     "- A concise one-sentence summary of the topic's experimental results\n\n" +

From 9e929e0b2553ecd6141b0bcb1c5b18a10dc263bd Mon Sep 17 00:00:00 2001
From: Bob <uncoolbob@gmail.com>
Date: Fri, 24 Oct 2025 14:20:59 +0100
Subject: [PATCH 07/16] added embedding stuff; not tested; no cost monitoring

---
 .../ai/SingleGeneAiExpressionReporter.java    |  1 +
 .../ai/expression/ClaudeSummarizer.java       |  2 +-
 .../ai/expression/GeneRecordProcessor.java    |  6 +-
 .../ai/expression/OpenAISummarizer.java       |  2 +-
 .../report/ai/expression/Summarizer.java      | 87 +++++++++++++++++--
 5 files changed, 84 insertions(+), 14 deletions(-)

diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
index d6c65fc3c..96ecf7d88 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
@@ -95,6 +95,7 @@ protected void write(OutputStream out) throws IOException, WdkModelException {
         // create summary inputs
         GeneSummaryInputs summaryInputs =
             GeneRecordProcessor.getSummaryInputsFromRecord(record, ClaudeSummarizer.CLAUDE_MODEL.toString(),
+                Summarizer.EMBEDDING_MODEL.asString(),
                 Summarizer::getExperimentMessage, Summarizer::getFinalSummaryMessage);
 
         // fetch summary, producing if necessary and requested
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
index 9f4ab38e4..13e0db5f3 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
@@ -21,7 +21,7 @@ public class ClaudeSummarizer extends Summarizer {
   private final AnthropicClientAsync _claudeClient;
 
   public ClaudeSummarizer(WdkModel wdkModel, DailyCostMonitor costMonitor) throws WdkModelException {
-    super(costMonitor);
+    super(wdkModel, costMonitor);
 
     String apiKey = wdkModel.getProperties().get(CLAUDE_API_KEY_PROP_NAME);
     if (apiKey == null) {
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/GeneRecordProcessor.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/GeneRecordProcessor.java
index 26fa39bab..c1ad3095f 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/GeneRecordProcessor.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/GeneRecordProcessor.java
@@ -32,7 +32,7 @@ public class GeneRecordProcessor {
 
   // Increment this to invalidate all previous cache entries:
   // (for example if changing first level model outputs rather than inputs which are already digestified)
-  private static final String DATA_MODEL_VERSION = "v3b";
+  private static final String DATA_MODEL_VERSION = "v4";
   
   public interface ExperimentInputs {
 
@@ -62,7 +62,7 @@ private static String getGeneId(RecordInstance record) {
     return record.getPrimaryKey().getValues().get("source_id");
   }
 
-  public static GeneSummaryInputs getSummaryInputsFromRecord(RecordInstance record, String aiChatModel, Function<JSONObject, String> getExperimentPrompt, Function<List<JSONObject>, String> getFinalSummaryPrompt) throws WdkModelException {
+  public static GeneSummaryInputs getSummaryInputsFromRecord(RecordInstance record, String aiChatModel, String embeddingModel, Function<JSONObject, String> getExperimentPrompt, Function<List<JSONObject>, String> getFinalSummaryPrompt) throws WdkModelException {
 
     String geneId = getGeneId(record);
 
@@ -90,7 +90,7 @@ public String getDigest() {
         List<JSONObject> digests = experimentsWithData.stream()
             .map(exp -> new JSONObject().put("digest", exp.getDigest()))
             .collect(Collectors.toList());
-        return EncryptionUtil.md5(aiChatModel + ":" + DATA_MODEL_VERSION + ":" + getFinalSummaryPrompt.apply(digests));
+        return EncryptionUtil.md5(aiChatModel + ":" + embeddingModel + ":" + DATA_MODEL_VERSION + ":" + getFinalSummaryPrompt.apply(digests));
       }
 
     };
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
index fcea4ac24..8c9bbb98a 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
@@ -22,7 +22,7 @@ public class OpenAISummarizer extends Summarizer {
   private final OpenAIClientAsync _openAIClient;
 
   public OpenAISummarizer(WdkModel wdkModel, DailyCostMonitor costMonitor) throws WdkModelException {
-    super(costMonitor);
+    super(wdkModel, costMonitor);
 
     String apiKey = wdkModel.getProperties().get(OPENAI_API_KEY_PROP_NAME);
     if (apiKey == null) {
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
index a1b8a39aa..f2f2eb0e5 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
@@ -7,6 +7,7 @@
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import org.apache.log4j.Logger;
 import org.apidb.apicommon.model.report.ai.expression.GeneRecordProcessor.ExperimentInputs;
@@ -17,7 +18,11 @@
 import org.json.JSONException;
 import org.json.JSONObject;
 
+import com.openai.client.OpenAIClientAsync;
+import com.openai.client.okhttp.OpenAIOkHttpClientAsync;
 import com.openai.core.JsonValue;
+import com.openai.models.EmbeddingCreateParams;
+import com.openai.models.EmbeddingModel;
 import com.openai.models.ResponseFormatJsonSchema.JsonSchema;
 import com.openai.models.ResponseFormatJsonSchema.JsonSchema.Schema;
 
@@ -25,7 +30,12 @@ public abstract class Summarizer {
 
   protected static final int MAX_RESPONSE_TOKENS = 10000;
 
+  public static final EmbeddingModel EMBEDDING_MODEL = EmbeddingModel.TEXT_EMBEDDING_3_SMALL;
+  private static final int EMBEDDING_DIMENSIONS = 512;
+  private static final int EMBEDDING_DECIMAL_PLACES = 4;
+
   private static final int MAX_MALFORMED_RESPONSE_RETRIES = 3;
+  private static final String OPENAI_API_KEY_PROP_NAME = "OPENAI_API_KEY";
 
   protected static final String SYSTEM_MESSAGE = "You are a bioinformatician working for VEuPathDB.org. You are an expert at providing biologist-friendly summaries of transcriptomic data";
 
@@ -74,11 +84,44 @@ public abstract class Summarizer {
     .build();
 
   protected final DailyCostMonitor _costMonitor;
+  private final OpenAIClientAsync _embeddingClient;
 
   private static final Logger LOG = Logger.getLogger(Summarizer.class);
 
-  public Summarizer(DailyCostMonitor costMonitor) {
+  public Summarizer(WdkModel wdkModel, DailyCostMonitor costMonitor) throws WdkModelException {
     _costMonitor = costMonitor;
+
+    String apiKey = wdkModel.getProperties().get(OPENAI_API_KEY_PROP_NAME);
+    if (apiKey == null) {
+      throw new WdkModelException("WDK property '" + OPENAI_API_KEY_PROP_NAME + "' has not been set.");
+    }
+
+    _embeddingClient = OpenAIOkHttpClientAsync.builder()
+        .apiKey(apiKey)
+        .maxRetries(32)  // Handle 429 errors
+        .build();
+  }
+
+  private CompletableFuture<List<Double>> getEmbedding(String text) {
+    EmbeddingCreateParams request = EmbeddingCreateParams.builder()
+        .model(EMBEDDING_MODEL)
+        .input(text)
+        .dimensions(EMBEDDING_DIMENSIONS)
+        .build();
+
+    return _embeddingClient.embeddings().create(request).thenApply(response -> {
+      // Extract embedding vector from first result
+      List<Double> embedding = response.data().get(0).embedding();
+
+      // Round to specified decimal places
+      double scale = Math.pow(10, EMBEDDING_DECIMAL_PLACES);
+      return embedding.stream()
+          .map(val -> Math.round(val * scale) / scale)
+          .collect(Collectors.toList());
+    }).exceptionally(e -> {
+      LOG.error("Failed to generate embedding: " + e.getMessage(), e);
+      return List.of(); // Return empty list on error
+    });
   }
 
   public static String getExperimentMessage(JSONObject experiment) {
@@ -138,12 +181,14 @@ public JSONObject summarizeExperiments(String geneId, List<JSONObject> experimen
     String prompt = getFinalSummaryMessage(experiments);
 
     return getValidatedAiResponse("summary for gene " + geneId, prompt, finalResponseSchema, json ->
+      json  // Return json as-is; consolidateSummary will be called separately
+    ).thenCompose(json ->
       // quality control (remove bad `dataset_id`s) and add 'Others' section for any experiments not listed by AI
       consolidateSummary(json, experiments)
     ).join();
   }
 
-  private static JSONObject consolidateSummary(JSONObject summaryResponse,
+  private CompletableFuture<JSONObject> consolidateSummary(JSONObject summaryResponse,
       List<JSONObject> individualResults) {
     // Gather all dataset IDs from individualResults and map them to summaries.
     // Preserving the order of individualResults.
@@ -153,7 +198,8 @@ private static JSONObject consolidateSummary(JSONObject summaryResponse,
     }
 
     Set<String> seenDatasetIds = new LinkedHashSet<>();
-    JSONArray deduplicatedTopics = new JSONArray();
+    List<JSONObject> deduplicatedTopicsList = new java.util.ArrayList<>();
+    List<CompletableFuture<Void>> embeddingFutures = new java.util.ArrayList<>();
     JSONArray topics = summaryResponse.getJSONArray("topics");
 
     for (int i = 0; i < topics.length(); i++) {
@@ -183,7 +229,19 @@ private static JSONObject consolidateSummary(JSONObject summaryResponse,
       if (summaries.length() > 0) {
         topic.put("summaries", summaries);
         topic.remove("dataset_ids");
-        deduplicatedTopics.put(topic);
+        deduplicatedTopicsList.add(topic);
+
+        // Generate embedding for non-"Other" topics
+        String headline = topic.optString("headline", "");
+        if (!headline.equals("Other")) {
+          String embeddingText = headline + "\n\n" + topic.optString("one_sentence_summary", "");
+          CompletableFuture<Void> embeddingFuture = getEmbedding(embeddingText).thenAccept(embedding -> {
+            if (!embedding.isEmpty()) {
+              topic.put("embedding_vector", embedding);
+            }
+          });
+          embeddingFutures.add(embeddingFuture);
+        }
       }
     }
 
@@ -203,13 +261,24 @@ private static JSONObject consolidateSummary(JSONObject summaryResponse,
       otherTopic.put("one_sentence_summary",
           "The AI ordered these experiments by biological importance but did not group them into topics.");
       otherTopic.put("summaries", otherSummaries);
-      deduplicatedTopics.put(otherTopic);
+      deduplicatedTopicsList.add(otherTopic);
+      // Note: no embedding for "Other" topic
     }
 
-    // Create final deduplicated summary
-    JSONObject finalSummary = new JSONObject(summaryResponse.toString());
-    finalSummary.put("topics", deduplicatedTopics);
-    return finalSummary;
+    // Wait for all embeddings to complete, then create final summary
+    return CompletableFuture.allOf(embeddingFutures.toArray(new CompletableFuture[0]))
+        .thenApply(v -> {
+          // Convert deduplicated topics list back to JSONArray
+          JSONArray deduplicatedTopics = new JSONArray();
+          for (JSONObject topic : deduplicatedTopicsList) {
+            deduplicatedTopics.put(topic);
+          }
+
+          // Create final deduplicated summary
+          JSONObject finalSummary = new JSONObject(summaryResponse.toString());
+          finalSummary.put("topics", deduplicatedTopics);
+          return finalSummary;
+        });
   }
 
 

From 6c5818f4f55c50627ec48820bdbcb6ebcc47f0e7 Mon Sep 17 00:00:00 2001
From: Bob <uncoolbob@gmail.com>
Date: Fri, 24 Oct 2025 14:39:31 +0100
Subject: [PATCH 08/16] tidy up token usage

---
 .../ai/expression/ClaudeSummarizer.java       |  9 ++-
 .../ai/expression/DailyCostMonitor.java       | 22 ++++---
 .../ai/expression/OpenAISummarizer.java       | 12 +++-
 .../report/ai/expression/Summarizer.java      |  7 +++
 .../report/ai/expression/TokenUsage.java      | 59 +++++++++++++++++++
 5 files changed, 92 insertions(+), 17 deletions(-)
 create mode 100644 Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/TokenUsage.java

diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
index 13e0db5f3..3a1280507 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
@@ -50,15 +50,14 @@ protected CompletableFuture<String> callApiForJson(String prompt, Schema schema)
         .build();
 
     return _claudeClient.messages().create(request).thenApply(response -> {
-      // Convert Claude usage to OpenAI format for cost monitoring
+      // Convert Claude usage to TokenUsage for cost monitoring
       com.anthropic.models.messages.Usage claudeUsage = response.usage();
-      com.openai.models.CompletionUsage openAiUsage = com.openai.models.CompletionUsage.builder()
+      TokenUsage tokenUsage = TokenUsage.builder()
           .promptTokens(claudeUsage.inputTokens())
           .completionTokens(claudeUsage.outputTokens())
-          .totalTokens(claudeUsage.inputTokens() + claudeUsage.outputTokens())
           .build();
-      
-      _costMonitor.updateCost(java.util.Optional.of(openAiUsage));
+
+      _costMonitor.updateCost(tokenUsage);
       
       // Extract text from content blocks using stream API
       String rawText = response.content().stream()
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/DailyCostMonitor.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/DailyCostMonitor.java
index 73789dbc1..277853496 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/DailyCostMonitor.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/DailyCostMonitor.java
@@ -21,8 +21,6 @@
 import org.json.JSONException;
 import org.json.JSONObject;
 
-import com.openai.models.CompletionUsage;
-
 public class DailyCostMonitor {
 
   private static final Logger LOG = Logger.getLogger(DailyCostMonitor.class);
@@ -35,6 +33,9 @@ public class DailyCostMonitor {
   private static final String MAX_DAILY_DOLLAR_COST_PROP_NAME = "MAX_DAILY_AI_EXPRESSION_DOLLAR_COST";
   private static final String DOLLAR_COST_PER_1M_INPUT_TOKENS_PROP_NAME = "DOLLAR_COST_PER_1M_AI_INPUT_TOKENS";
   private static final String DOLLAR_COST_PER_1M_OUTPUT_TOKENS_PROP_NAME = "DOLLAR_COST_PER_1M_AI_OUTPUT_TOKENS";
+
+  // hardcoded embedding token cost
+  private static final double DOLLAR_COST_PER_1M_EMBEDDING_TOKENS = 0.02;
   
   // deprecated model prop keys (with OPENAI_ prefix)
   private static final String DEPRECATED_MAX_DAILY_DOLLAR_COST_PROP_NAME = "OPENAI_MAX_DAILY_AI_EXPRESSION_DOLLAR_COST";
@@ -49,11 +50,11 @@ public class DailyCostMonitor {
   private static final String JSON_DATE_PROP = "currentDate";
   private static final String JSON_COST_PROP = "accumulatedCost";
 
-  // completion usage object representing 0 cost
-  private static final CompletionUsage EMPTY_COST = CompletionUsage.builder()
+  // token usage object representing 0 cost
+  private static final TokenUsage EMPTY_COST = TokenUsage.builder()
       .promptTokens(0)
       .completionTokens(0)
-      .totalTokens(0)
+      .embeddingTokens(0)
       .build();
 
   private final Path _costMonitoringDir;
@@ -62,6 +63,7 @@ public class DailyCostMonitor {
   private final double _maxDailyDollarCost;
   private final double _costPerInputToken;
   private final double _costPerOutputToken;
+  private final double _costPerEmbeddingToken;
 
   public DailyCostMonitor(WdkModel wdkModel) throws WdkModelException {
     _costMonitoringDir = AiExpressionCache.getAiExpressionCacheParentDir(wdkModel).resolve(DAILY_COST_ACCUMULATION_FILE_DIR).toAbsolutePath();
@@ -76,6 +78,7 @@ public DailyCostMonitor(WdkModel wdkModel) throws WdkModelException {
     _maxDailyDollarCost = getNumberProp(wdkModel, MAX_DAILY_DOLLAR_COST_PROP_NAME, DEPRECATED_MAX_DAILY_DOLLAR_COST_PROP_NAME);
     _costPerInputToken = getNumberProp(wdkModel, DOLLAR_COST_PER_1M_INPUT_TOKENS_PROP_NAME, DEPRECATED_DOLLAR_COST_PER_1M_INPUT_TOKENS_PROP_NAME) / 1000000;
     _costPerOutputToken = getNumberProp(wdkModel, DOLLAR_COST_PER_1M_OUTPUT_TOKENS_PROP_NAME, DEPRECATED_DOLLAR_COST_PER_1M_OUTPUT_TOKENS_PROP_NAME) / 1000000;
+    _costPerEmbeddingToken = DOLLAR_COST_PER_1M_EMBEDDING_TOKENS / 1000000;
   }
 
   private double getNumberProp(WdkModel wdkModel, String propName, String deprecatedPropName) throws WdkModelException {
@@ -109,11 +112,11 @@ public boolean isCostExceeded() {
     return updateAndGetCost(EMPTY_COST) > _maxDailyDollarCost;
   }
 
-  public void updateCost(Optional<CompletionUsage> usage) {
-    updateAndGetCost(usage.orElse(EMPTY_COST));
+  public void updateCost(TokenUsage usage) {
+    updateAndGetCost(usage);
   }
 
-  public double updateAndGetCost(CompletionUsage usageCost) {
+  public double updateAndGetCost(TokenUsage usageCost) {
     try (DirectoryLock lock = new DirectoryLock(_costMonitoringDir, DEFAULT_TIMEOUT_MILLIS, DEFAULT_POLL_FREQUENCY_MILLIS)) {
 
       // read current values from file
@@ -124,7 +127,8 @@ public double updateAndGetCost(CompletionUsage usageCost) {
       // calculate cost of the current usage
       double additionalCost =
           (usageCost.promptTokens() * _costPerInputToken) +
-          (usageCost.completionTokens() * _costPerOutputToken);
+          (usageCost.completionTokens() * _costPerOutputToken) +
+          (usageCost.embeddingTokens() * _costPerEmbeddingToken);
 
       // reset cost to zero if date has rolled over to the next day
       String newDate = getCurrentDateString();
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
index 8c9bbb98a..4f9b21169 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
@@ -52,9 +52,15 @@ protected CompletableFuture<String> callApiForJson(String prompt, com.openai.mod
         .build();
 
     return _openAIClient.chat().completions().create(request).thenApply(completion -> {
-      // update cost accumulator
-      _costMonitor.updateCost(completion.usage());
-      
+      // update cost accumulator - convert to TokenUsage
+      completion.usage().ifPresent(usage -> {
+        TokenUsage tokenUsage = TokenUsage.builder()
+            .promptTokens(usage.promptTokens())
+            .completionTokens(usage.completionTokens())
+            .build();
+        _costMonitor.updateCost(tokenUsage);
+      });
+
       // return JSON string
       return completion.choices().get(0).message().content().get();
     });
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
index f2f2eb0e5..2a6b6cd71 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
@@ -110,6 +110,13 @@ private CompletableFuture<List<Double>> getEmbedding(String text) {
         .build();
 
     return _embeddingClient.embeddings().create(request).thenApply(response -> {
+      // Update cost monitor - convert embedding usage to TokenUsage
+      com.openai.models.CreateEmbeddingResponse.Usage embeddingUsage = response.usage();
+      TokenUsage tokenUsage = TokenUsage.builder()
+          .embeddingTokens(embeddingUsage.totalTokens())
+          .build();
+      _costMonitor.updateCost(tokenUsage);
+
       // Extract embedding vector from first result
       List<Double> embedding = response.data().get(0).embedding();
 
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/TokenUsage.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/TokenUsage.java
new file mode 100644
index 000000000..ea76ca3f8
--- /dev/null
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/TokenUsage.java
@@ -0,0 +1,59 @@
+package org.apidb.apicommon.model.report.ai.expression;
+
+/**
+ * Represents token usage for AI API calls, including chat completions and embeddings.
+ * Immutable value object with builder pattern.
+ */
+public class TokenUsage {
+
+  private final long promptTokens;
+  private final long completionTokens;
+  private final long embeddingTokens;
+
+  private TokenUsage(Builder builder) {
+    this.promptTokens = builder.promptTokens;
+    this.completionTokens = builder.completionTokens;
+    this.embeddingTokens = builder.embeddingTokens;
+  }
+
+  public long promptTokens() {
+    return promptTokens;
+  }
+
+  public long completionTokens() {
+    return completionTokens;
+  }
+
+  public long embeddingTokens() {
+    return embeddingTokens;
+  }
+
+  public static Builder builder() {
+    return new Builder();
+  }
+
+  public static class Builder {
+    private long promptTokens = 0;
+    private long completionTokens = 0;
+    private long embeddingTokens = 0;
+
+    public Builder promptTokens(long promptTokens) {
+      this.promptTokens = promptTokens;
+      return this;
+    }
+
+    public Builder completionTokens(long completionTokens) {
+      this.completionTokens = completionTokens;
+      return this;
+    }
+
+    public Builder embeddingTokens(long embeddingTokens) {
+      this.embeddingTokens = embeddingTokens;
+      return this;
+    }
+
+    public TokenUsage build() {
+      return new TokenUsage(this);
+    }
+  }
+}

From bf83d47f7302a08d847300a42ee055092f27cdc1 Mon Sep 17 00:00:00 2001
From: Bob MacCallum <uncoolbob@gmail.com>
Date: Mon, 27 Oct 2025 12:08:28 +0000
Subject: [PATCH 09/16] better handling of AI API 500 responses and
 configurable MAX_CONCURRENT API calls

---
 .../ai/SingleGeneAiExpressionReporter.java    | 11 ++-
 .../ai/expression/AiExpressionCache.java      | 13 ++--
 .../ai/expression/ClaudeSummarizer.java       |  8 ++-
 .../ai/expression/DailyCostMonitor.java       |  2 +-
 .../ai/expression/OpenAISummarizer.java       |  8 ++-
 .../report/ai/expression/Summarizer.java      | 68 +++++++++++++++++++
 6 files changed, 99 insertions(+), 11 deletions(-)

diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
index d6c65fc3c..185e28ace 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
@@ -33,8 +33,11 @@ public class SingleGeneAiExpressionReporter extends AbstractReporter {
   private static final int MAX_RESULT_SIZE = 1; // one gene at a time for now
 
   private static final String POPULATION_MODE_PROP_KEY = "populateIfNotPresent";
+  private static final String AI_MAX_CONCURRENT_REQUESTS_PROP_KEY = "AI_MAX_CONCURRENT_REQUESTS";
+  private static final int DEFAULT_MAX_CONCURRENT_REQUESTS = 10;
 
   private boolean _populateIfNotPresent;
+  private int _maxConcurrentRequests;
   private DailyCostMonitor _costMonitor;
 
   @Override
@@ -43,6 +46,12 @@ public Reporter configure(JSONObject config) throws ReporterConfigException, Wdk
       // assign cache mode
       _populateIfNotPresent = config.optBoolean(POPULATION_MODE_PROP_KEY, false);
 
+      // read max concurrent requests from model properties or use default
+      String maxConcurrentRequestsStr = _wdkModel.getProperties().get(AI_MAX_CONCURRENT_REQUESTS_PROP_KEY);
+      _maxConcurrentRequests = maxConcurrentRequestsStr != null
+          ? Integer.parseInt(maxConcurrentRequestsStr)
+          : DEFAULT_MAX_CONCURRENT_REQUESTS;
+
       // instantiate cost monitor
       _costMonitor = new DailyCostMonitor(_wdkModel);
 
@@ -99,7 +108,7 @@ protected void write(OutputStream out) throws IOException, WdkModelException {
 
         // fetch summary, producing if necessary and requested
         JSONObject expressionSummary = _populateIfNotPresent
-            ? cache.populateSummary(summaryInputs, summarizer::describeExperiment, summarizer::summarizeExperiments)
+            ? cache.populateSummary(summaryInputs, summarizer::describeExperiment, summarizer::summarizeExperiments, _maxConcurrentRequests)
             : cache.readSummary(summaryInputs);
 
         // join entries with commas
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/AiExpressionCache.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/AiExpressionCache.java
index 3bc5768b7..4054eb1e6 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/AiExpressionCache.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/AiExpressionCache.java
@@ -73,7 +73,6 @@ public class AiExpressionCache {
   private static Logger LOG = Logger.getLogger(AiExpressionCache.class);
 
   // parallel processing
-  private static final int MAX_CONCURRENT_EXPERIMENT_LOOKUPS_PER_REQUEST = 10;
   private static final long VISIT_ENTRY_LOCK_MAX_WAIT_MILLIS = 50;
 
   // cache location
@@ -317,18 +316,20 @@ private static Optional<JSONObject> readCachedData(Path entryDir) {
    * @param summaryInputs gene summary inputs
    * @param experimentDescriber function to describe an experiment
    * @param experimentSummarizer function to summarize experiments into an expression summary
+   * @param maxConcurrentRequests maximum number of concurrent experiment lookups
    * @return expression summary (will always be a cache hit)
    */
   public JSONObject populateSummary(GeneSummaryInputs summaryInputs,
       FunctionWithException<ExperimentInputs, CompletableFuture<JSONObject>> experimentDescriber,
-      BiFunctionWithException<String, List<JSONObject>, JSONObject> experimentSummarizer) {
+      BiFunctionWithException<String, List<JSONObject>, JSONObject> experimentSummarizer,
+      int maxConcurrentRequests) {
     try {
       return _cache.populateAndProcessContent(summaryInputs.getGeneId(),
 
           // populator
           entryDir -> {
             // first populate each dataset entry as needed and collect experiment descriptors
-            List<JSONObject> experiments = populateExperiments(summaryInputs.getExperimentsWithData(), experimentDescriber);
+            List<JSONObject> experiments = populateExperiments(summaryInputs.getExperimentsWithData(), experimentDescriber, maxConcurrentRequests);
 
             // sort them most-interesting first so that the "Other" section will be filled
             // in that order (and also to give the AI the data in a sensible order)
@@ -362,14 +363,16 @@ public JSONObject populateSummary(GeneSummaryInputs summaryInputs,
    *
    * @param experimentData experiment inputs
    * @param experimentDescriber function to describe an experiment
+   * @param maxConcurrentRequests maximum number of concurrent experiment lookups
    * @return list of cached experiment descriptions
    * @throws Exception if unable to generate descriptions or store
    */
   private List<JSONObject> populateExperiments(List<ExperimentInputs> experimentData,
-      FunctionWithException<ExperimentInputs, CompletableFuture<JSONObject>> experimentDescriber) throws Exception {
+      FunctionWithException<ExperimentInputs, CompletableFuture<JSONObject>> experimentDescriber,
+      int maxConcurrentRequests) throws Exception {
 
     // use a thread for each experiment, up to a reasonable max
-    int threadPoolSize = Math.min(MAX_CONCURRENT_EXPERIMENT_LOOKUPS_PER_REQUEST, experimentData.size());
+    int threadPoolSize = Math.min(maxConcurrentRequests, experimentData.size());
 
     ExecutorService exec = Executors.newFixedThreadPool(threadPoolSize);
     try {
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
index 9f4ab38e4..7a49f4870 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
@@ -49,10 +49,14 @@ protected CompletableFuture<String> callApiForJson(String prompt, Schema schema)
         .addUserMessage(enhancedPrompt)
         .build();
 
-    return _claudeClient.messages().create(request).thenApply(response -> {
+    return retryOnOverload(
+        () -> _claudeClient.messages().create(request),
+        e -> e instanceof com.anthropic.errors.InternalServerException,
+        "Claude API call"
+    ).thenApply(response -> {
       // Convert Claude usage to OpenAI format for cost monitoring
       com.anthropic.models.messages.Usage claudeUsage = response.usage();
-      com.openai.models.CompletionUsage openAiUsage = com.openai.models.CompletionUsage.builder()
+      com.openai.models.completions.CompletionUsage openAiUsage = com.openai.models.completions.CompletionUsage.builder()
           .promptTokens(claudeUsage.inputTokens())
           .completionTokens(claudeUsage.outputTokens())
           .totalTokens(claudeUsage.inputTokens() + claudeUsage.outputTokens())
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/DailyCostMonitor.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/DailyCostMonitor.java
index 73789dbc1..6cd7f3574 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/DailyCostMonitor.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/DailyCostMonitor.java
@@ -21,7 +21,7 @@
 import org.json.JSONException;
 import org.json.JSONObject;
 
-import com.openai.models.CompletionUsage;
+import com.openai.models.completions.CompletionUsage;
 
 public class DailyCostMonitor {
 
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
index fcea4ac24..36811b32b 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
@@ -7,7 +7,7 @@
 
 import com.openai.client.OpenAIClientAsync;
 import com.openai.client.okhttp.OpenAIOkHttpClientAsync;
-import com.openai.models.ChatCompletionCreateParams;
+import com.openai.models.chat.completions.ChatCompletionCreateParams;
 import com.openai.models.ChatModel;
 import com.openai.models.ResponseFormatJsonSchema;
 import com.openai.models.ResponseFormatJsonSchema.JsonSchema;
@@ -51,7 +51,11 @@ protected CompletableFuture<String> callApiForJson(String prompt, com.openai.mod
         .addUserMessage(prompt)
         .build();
 
-    return _openAIClient.chat().completions().create(request).thenApply(completion -> {
+    return retryOnOverload(
+        () -> _openAIClient.chat().completions().create(request),
+        e -> e instanceof com.openai.errors.InternalServerException,
+        "OpenAI API call"
+    ).thenApply(completion -> {
       // update cost accumulator
       _costMonitor.updateCost(completion.usage());
       
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
index a1b8a39aa..a997dc086 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
@@ -81,6 +81,74 @@ public Summarizer(DailyCostMonitor costMonitor) {
     _costMonitor = costMonitor;
   }
 
+  /**
+   * Retries an operation with exponential backoff if it fails with a retriable error.
+   *
+   * @param <T> the return type of the operation
+   * @param operation supplier that produces the CompletableFuture to execute
+   * @param shouldRetry predicate to determine if an exception should trigger a retry
+   * @param operationDescription description for logging purposes
+   * @return CompletableFuture with the result of the operation
+   */
+  protected <T> CompletableFuture<T> retryOnOverload(
+      java.util.function.Supplier<CompletableFuture<T>> operation,
+      java.util.function.Predicate<Throwable> shouldRetry,
+      String operationDescription) {
+
+    final int maxRetries = 3;
+    final long[] backoffDelaysMs = {1000, 2000, 4000}; // 1s, 2s, 4s
+
+    return retryWithBackoff(operation, shouldRetry, operationDescription, 0, maxRetries, backoffDelaysMs);
+  }
+
+  private <T> CompletableFuture<T> retryWithBackoff(
+      java.util.function.Supplier<CompletableFuture<T>> operation,
+      java.util.function.Predicate<Throwable> shouldRetry,
+      String operationDescription,
+      int attemptNumber,
+      int maxRetries,
+      long[] backoffDelaysMs) {
+
+    return operation.get().exceptionallyCompose(throwable -> {
+      // Unwrap CompletionException to get the actual cause
+      Throwable actualCause = throwable instanceof java.util.concurrent.CompletionException && throwable.getCause() != null
+          ? throwable.getCause()
+          : throwable;
+
+      // Check if we should retry this exception and haven't exceeded max retries
+      if (shouldRetry.test(actualCause) && attemptNumber < maxRetries) {
+        long delayMs = backoffDelaysMs[attemptNumber];
+        LOG.warn(String.format(
+            "Retrying %s after error (attempt %d/%d, waiting %dms): %s",
+            operationDescription, attemptNumber + 1, maxRetries, delayMs, actualCause.getMessage()));
+
+        // Schedule retry after delay
+        CompletableFuture<T> delayed = new CompletableFuture<>();
+        new java.util.Timer().schedule(new java.util.TimerTask() {
+          @Override
+          public void run() {
+            retryWithBackoff(operation, shouldRetry, operationDescription, attemptNumber + 1, maxRetries, backoffDelaysMs)
+                .whenComplete((result, error) -> {
+                  if (error != null) {
+                    delayed.completeExceptionally(error);
+                  } else {
+                    delayed.complete(result);
+                  }
+                });
+          }
+        }, delayMs);
+
+        return delayed;
+      } else {
+        // No more retries or non-retriable exception
+        if (attemptNumber >= maxRetries) {
+          LOG.error(String.format("Failed %s after %d retries: %s", operationDescription, maxRetries, actualCause.getMessage()));
+        }
+        return CompletableFuture.failedFuture(throwable);
+      }
+    });
+  }
+
   public static String getExperimentMessage(JSONObject experiment) {
 
     // Possible TO DO: AI EDIT DESCRIPTION

From 410732acaea8af6b51f9f09067f5a6c50bcc16cc Mon Sep 17 00:00:00 2001
From: Bob MacCallum <uncoolbob@gmail.com>
Date: Mon, 27 Oct 2025 16:00:19 +0000
Subject: [PATCH 10/16] rewrite retry logic for Java 11 (was 12+)

---
 .../report/ai/expression/Summarizer.java      | 72 ++++++++++---------
 1 file changed, 39 insertions(+), 33 deletions(-)

diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
index a997dc086..f58b03e60 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
@@ -109,44 +109,50 @@ private <T> CompletableFuture<T> retryWithBackoff(
       int maxRetries,
       long[] backoffDelaysMs) {
 
-    return operation.get().exceptionallyCompose(throwable -> {
-      // Unwrap CompletionException to get the actual cause
-      Throwable actualCause = throwable instanceof java.util.concurrent.CompletionException && throwable.getCause() != null
-          ? throwable.getCause()
-          : throwable;
-
-      // Check if we should retry this exception and haven't exceeded max retries
-      if (shouldRetry.test(actualCause) && attemptNumber < maxRetries) {
-        long delayMs = backoffDelaysMs[attemptNumber];
-        LOG.warn(String.format(
-            "Retrying %s after error (attempt %d/%d, waiting %dms): %s",
-            operationDescription, attemptNumber + 1, maxRetries, delayMs, actualCause.getMessage()));
-
-        // Schedule retry after delay
-        CompletableFuture<T> delayed = new CompletableFuture<>();
-        new java.util.Timer().schedule(new java.util.TimerTask() {
-          @Override
-          public void run() {
-            retryWithBackoff(operation, shouldRetry, operationDescription, attemptNumber + 1, maxRetries, backoffDelaysMs)
-                .whenComplete((result, error) -> {
-                  if (error != null) {
-                    delayed.completeExceptionally(error);
-                  } else {
-                    delayed.complete(result);
-                  }
-                });
-          }
-        }, delayMs);
+    CompletableFuture<T> result = new CompletableFuture<>();
 
-        return delayed;
+    operation.get().whenComplete((value, throwable) -> {
+      if (throwable == null) {
+        // Success case
+        result.complete(value);
       } else {
-        // No more retries or non-retriable exception
-        if (attemptNumber >= maxRetries) {
-          LOG.error(String.format("Failed %s after %d retries: %s", operationDescription, maxRetries, actualCause.getMessage()));
+        // Error case - unwrap CompletionException to get the actual cause
+        Throwable actualCause = throwable instanceof java.util.concurrent.CompletionException && throwable.getCause() != null
+            ? throwable.getCause()
+            : throwable;
+
+        // Check if we should retry this exception and haven't exceeded max retries
+        if (shouldRetry.test(actualCause) && attemptNumber < maxRetries) {
+          long delayMs = backoffDelaysMs[attemptNumber];
+          LOG.warn(String.format(
+              "Retrying %s after error (attempt %d/%d, waiting %dms): %s",
+              operationDescription, attemptNumber + 1, maxRetries, delayMs, actualCause.getMessage()));
+
+          // Schedule retry after delay
+          new java.util.Timer().schedule(new java.util.TimerTask() {
+            @Override
+            public void run() {
+              retryWithBackoff(operation, shouldRetry, operationDescription, attemptNumber + 1, maxRetries, backoffDelaysMs)
+                  .whenComplete((retryValue, retryError) -> {
+                    if (retryError != null) {
+                      result.completeExceptionally(retryError);
+                    } else {
+                      result.complete(retryValue);
+                    }
+                  });
+            }
+          }, delayMs);
+        } else {
+          // No more retries or non-retriable exception
+          if (attemptNumber >= maxRetries) {
+            LOG.error(String.format("Failed %s after %d retries: %s", operationDescription, maxRetries, actualCause.getMessage()));
+          }
+          result.completeExceptionally(throwable);
         }
-        return CompletableFuture.failedFuture(throwable);
       }
     });
+
+    return result;
   }
 
   public static String getExperimentMessage(JSONObject experiment) {

From 69e8dc1a93a05798c082e47430f1511845f1b81d Mon Sep 17 00:00:00 2001
From: Bob <uncoolbob@gmail.com>
Date: Sat, 8 Nov 2025 18:46:34 +0000
Subject: [PATCH 11/16] make topic embedding configurable by hard-coding,
 default is off

---
 .../ai/SingleGeneAiExpressionReporter.java    | 10 ++++---
 .../ai/expression/ClaudeSummarizer.java       |  4 +--
 .../ai/expression/GeneRecordProcessor.java    |  4 +--
 .../ai/expression/OpenAISummarizer.java       |  4 +--
 .../report/ai/expression/Summarizer.java      | 26 +++++++++++--------
 5 files changed, 28 insertions(+), 20 deletions(-)

diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
index e739a8cc2..9f679bcff 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
@@ -14,6 +14,7 @@
 import org.apidb.apicommon.model.report.ai.expression.GeneRecordProcessor;
 import org.apidb.apicommon.model.report.ai.expression.GeneRecordProcessor.GeneSummaryInputs;
 import org.apidb.apicommon.model.report.ai.expression.ClaudeSummarizer;
+//import org.apidb.apicommon.model.report.ai.expression.OpenAISummarizer;
 import org.apidb.apicommon.model.report.ai.expression.Summarizer;
 import org.gusdb.wdk.model.WdkModelException;
 import org.gusdb.wdk.model.WdkServiceTemporarilyUnavailableException;
@@ -35,6 +36,7 @@ public class SingleGeneAiExpressionReporter extends AbstractReporter {
   private static final String POPULATION_MODE_PROP_KEY = "populateIfNotPresent";
   private static final String AI_MAX_CONCURRENT_REQUESTS_PROP_KEY = "AI_MAX_CONCURRENT_REQUESTS";
   private static final int DEFAULT_MAX_CONCURRENT_REQUESTS = 10;
+  private static final boolean MAKE_TOPIC_EMBEDDINGS = false;
 
   private boolean _populateIfNotPresent;
   private int _maxConcurrentRequests;
@@ -90,8 +92,10 @@ protected void write(OutputStream out) throws IOException, WdkModelException {
     AiExpressionCache cache = AiExpressionCache.getInstance(_wdkModel);
 
     // create summarizer (interacts with Claude)
-    ClaudeSummarizer summarizer = new ClaudeSummarizer(_wdkModel, _costMonitor);
-
+    ClaudeSummarizer summarizer = new ClaudeSummarizer(_wdkModel, _costMonitor, MAKE_TOPIC_EMBEDDINGS);
+    // or alternatively use OpenAI (with the appropriate import)
+    // OpenAISummarizer summarizer = new OpenAISummarizer(_wdkModel, _costMonitor, MAKE_TOPIC_EMBEDDINGS);
+    
     // open record and output streams
     try (RecordStream recordStream = RecordStreamFactory.getRecordStream(_baseAnswer, List.of(), tables);
         BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out))) {
@@ -104,7 +108,7 @@ protected void write(OutputStream out) throws IOException, WdkModelException {
         // create summary inputs
         GeneSummaryInputs summaryInputs =
             GeneRecordProcessor.getSummaryInputsFromRecord(record, ClaudeSummarizer.CLAUDE_MODEL.toString(),
-                Summarizer.EMBEDDING_MODEL.asString(),
+                Summarizer.EMBEDDING_MODEL.asString(), MAKE_TOPIC_EMBEDDINGS,
                 Summarizer::getExperimentMessage, Summarizer::getFinalSummaryMessage);
 
         // fetch summary, producing if necessary and requested
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
index 09637f30f..f5fa42179 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
@@ -20,8 +20,8 @@ public class ClaudeSummarizer extends Summarizer {
 
   private final AnthropicClientAsync _claudeClient;
 
-  public ClaudeSummarizer(WdkModel wdkModel, DailyCostMonitor costMonitor) throws WdkModelException {
-    super(wdkModel, costMonitor);
+  public ClaudeSummarizer(WdkModel wdkModel, DailyCostMonitor costMonitor, boolean makeTopicEmbeddings) throws WdkModelException {
+    super(wdkModel, costMonitor, makeTopicEmbeddings);
 
     String apiKey = wdkModel.getProperties().get(CLAUDE_API_KEY_PROP_NAME);
     if (apiKey == null) {
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/GeneRecordProcessor.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/GeneRecordProcessor.java
index c1ad3095f..83aceddc5 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/GeneRecordProcessor.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/GeneRecordProcessor.java
@@ -62,7 +62,7 @@ private static String getGeneId(RecordInstance record) {
     return record.getPrimaryKey().getValues().get("source_id");
   }
 
-  public static GeneSummaryInputs getSummaryInputsFromRecord(RecordInstance record, String aiChatModel, String embeddingModel, Function<JSONObject, String> getExperimentPrompt, Function<List<JSONObject>, String> getFinalSummaryPrompt) throws WdkModelException {
+  public static GeneSummaryInputs getSummaryInputsFromRecord(RecordInstance record, String aiChatModel, String embeddingModel, boolean makeTopicEmbeddings, Function<JSONObject, String> getExperimentPrompt, Function<List<JSONObject>, String> getFinalSummaryPrompt) throws WdkModelException {
 
     String geneId = getGeneId(record);
 
@@ -90,7 +90,7 @@ public String getDigest() {
         List<JSONObject> digests = experimentsWithData.stream()
             .map(exp -> new JSONObject().put("digest", exp.getDigest()))
             .collect(Collectors.toList());
-        return EncryptionUtil.md5(aiChatModel + ":" + embeddingModel + ":" + DATA_MODEL_VERSION + ":" + getFinalSummaryPrompt.apply(digests));
+        return EncryptionUtil.md5(aiChatModel + ":" + embeddingModel + ":" + makeTopicEmbeddings + ":" + DATA_MODEL_VERSION + ":" + getFinalSummaryPrompt.apply(digests));
       }
 
     };
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
index 233893d0b..bcbd4cbab 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/OpenAISummarizer.java
@@ -21,8 +21,8 @@ public class OpenAISummarizer extends Summarizer {
 
   private final OpenAIClientAsync _openAIClient;
 
-  public OpenAISummarizer(WdkModel wdkModel, DailyCostMonitor costMonitor) throws WdkModelException {
-    super(wdkModel, costMonitor);
+  public OpenAISummarizer(WdkModel wdkModel, DailyCostMonitor costMonitor, boolean makeTopicEmbeddings) throws WdkModelException {
+    super(wdkModel, costMonitor, makeTopicEmbeddings);
 
     String apiKey = wdkModel.getProperties().get(OPENAI_API_KEY_PROP_NAME);
     if (apiKey == null) {
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
index 927bb8c16..f5973a318 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
@@ -88,11 +88,13 @@ public abstract class Summarizer {
 
   protected final DailyCostMonitor _costMonitor;
   private final OpenAIClientAsync _embeddingClient;
+  protected final boolean _makeTopicEmbeddings;
 
   private static final Logger LOG = Logger.getLogger(Summarizer.class);
 
-  public Summarizer(WdkModel wdkModel, DailyCostMonitor costMonitor) throws WdkModelException {
+  public Summarizer(WdkModel wdkModel, DailyCostMonitor costMonitor, boolean makeTopicEmbeddings) throws WdkModelException {
     _costMonitor = costMonitor;
+    _makeTopicEmbeddings = makeTopicEmbeddings;
 
     String apiKey = wdkModel.getProperties().get(OPENAI_API_KEY_PROP_NAME);
     if (apiKey == null) {
@@ -315,16 +317,18 @@ private CompletableFuture<JSONObject> consolidateSummary(JSONObject summaryRespo
         topic.remove("dataset_ids");
         deduplicatedTopicsList.add(topic);
 
-        // Generate embedding for non-"Other" topics
-        String headline = topic.optString("headline", "");
-        if (!headline.equals("Other")) {
-          String embeddingText = headline + "\n\n" + topic.optString("one_sentence_summary", "");
-          CompletableFuture<Void> embeddingFuture = getEmbedding(embeddingText).thenAccept(embedding -> {
-            if (!embedding.isEmpty()) {
-              topic.put("embedding_vector", embedding);
-            }
-          });
-          embeddingFutures.add(embeddingFuture);
+        // Generate embedding for non-"Other" topics (if enabled)
+        if (_makeTopicEmbeddings) {
+          String headline = topic.optString("headline", "");
+          if (!headline.equals("Other")) {
+            String embeddingText = headline + "\n\n" + topic.optString("one_sentence_summary", "");
+            CompletableFuture<Void> embeddingFuture = getEmbedding(embeddingText).thenAccept(embedding -> {
+              if (!embedding.isEmpty()) {
+                topic.put("embedding_vector", embedding);
+              }
+            });
+            embeddingFutures.add(embeddingFuture);
+          }
         }
       }
     }

From 77b6adc8a97d4523316b7dd79a414dcb3c228055 Mon Sep 17 00:00:00 2001
From: Bob <uncoolbob@gmail.com>
Date: Sat, 8 Nov 2025 18:57:52 +0000
Subject: [PATCH 12/16] topic embeddings now configured by reporter request
 payload

---
 .../ai/SingleGeneAiExpressionReporter.java    | 38 +++++++++++++++++--
 1 file changed, 34 insertions(+), 4 deletions(-)

diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
index 9f679bcff..6c3fa447a 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
@@ -29,6 +29,32 @@
 import org.json.JSONException;
 import org.json.JSONObject;
 
+/**
+ * Reporter that generates AI-powered gene expression summaries using LLM models.
+ *
+ * <p>This reporter analyzes expression data across multiple experiments for a single gene
+ * and generates natural language summaries of expression patterns and biological significance.
+ * Results are cached to minimize API costs and response times.</p>
+ *
+ * <h3>Configuration (JSON request payload)</h3>
+ * <pre>
+ * {
+ *   "populateIfNotPresent": true|false,  // If true, generate summary if not cached (default: false)
+ *   "makeTopicEmbeddings": true|false    // If true, generate embedding vectors for topics (default: false)
+ * }
+ * </pre>
+ *
+ * <h3>Cache Invalidation Warning</h3>
+ * <p><strong>IMPORTANT:</strong> Changing the {@code makeTopicEmbeddings} setting will invalidate
+ * the entire cache for all genes, as this value is included in the cache digest. To avoid costly
+ * cache regeneration, choose a setting and stick with it across requests. Only change this value
+ * when you intentionally want to regenerate all summaries with or without embeddings.</p>
+ *
+ * <h3>Model Configuration</h3>
+ * <p>The AI model and embedding model are hardcoded in the summarizer implementations
+ * ({@link ClaudeSummarizer}, {@link org.apidb.apicommon.model.report.ai.expression.OpenAISummarizer}).
+ * Changing models will also invalidate the cache.</p>
+ */
 public class SingleGeneAiExpressionReporter extends AbstractReporter {
 
   private static final int MAX_RESULT_SIZE = 1; // one gene at a time for now
@@ -36,10 +62,11 @@ public class SingleGeneAiExpressionReporter extends AbstractReporter {
   private static final String POPULATION_MODE_PROP_KEY = "populateIfNotPresent";
   private static final String AI_MAX_CONCURRENT_REQUESTS_PROP_KEY = "AI_MAX_CONCURRENT_REQUESTS";
   private static final int DEFAULT_MAX_CONCURRENT_REQUESTS = 10;
-  private static final boolean MAKE_TOPIC_EMBEDDINGS = false;
+  private static final String MAKE_TOPIC_EMBEDDINGS_PROP_KEY = "makeTopicEmbeddings";
 
   private boolean _populateIfNotPresent;
   private int _maxConcurrentRequests;
+  private boolean _makeTopicEmbeddings;
   private DailyCostMonitor _costMonitor;
 
   @Override
@@ -48,6 +75,9 @@ public Reporter configure(JSONObject config) throws ReporterConfigException, Wdk
       // assign cache mode
       _populateIfNotPresent = config.optBoolean(POPULATION_MODE_PROP_KEY, false);
 
+      // assign topic embeddings flag
+      _makeTopicEmbeddings = config.optBoolean(MAKE_TOPIC_EMBEDDINGS_PROP_KEY, false);
+
       // read max concurrent requests from model properties or use default
       String maxConcurrentRequestsStr = _wdkModel.getProperties().get(AI_MAX_CONCURRENT_REQUESTS_PROP_KEY);
       _maxConcurrentRequests = maxConcurrentRequestsStr != null
@@ -92,9 +122,9 @@ protected void write(OutputStream out) throws IOException, WdkModelException {
     AiExpressionCache cache = AiExpressionCache.getInstance(_wdkModel);
 
     // create summarizer (interacts with Claude)
-    ClaudeSummarizer summarizer = new ClaudeSummarizer(_wdkModel, _costMonitor, MAKE_TOPIC_EMBEDDINGS);
+    ClaudeSummarizer summarizer = new ClaudeSummarizer(_wdkModel, _costMonitor, _makeTopicEmbeddings);
     // or alternatively use OpenAI (with the appropriate import)
-    // OpenAISummarizer summarizer = new OpenAISummarizer(_wdkModel, _costMonitor, MAKE_TOPIC_EMBEDDINGS);
+    // OpenAISummarizer summarizer = new OpenAISummarizer(_wdkModel, _costMonitor, _makeTopicEmbeddings);
     
     // open record and output streams
     try (RecordStream recordStream = RecordStreamFactory.getRecordStream(_baseAnswer, List.of(), tables);
@@ -108,7 +138,7 @@ protected void write(OutputStream out) throws IOException, WdkModelException {
         // create summary inputs
         GeneSummaryInputs summaryInputs =
             GeneRecordProcessor.getSummaryInputsFromRecord(record, ClaudeSummarizer.CLAUDE_MODEL.toString(),
-                Summarizer.EMBEDDING_MODEL.asString(), MAKE_TOPIC_EMBEDDINGS,
+                Summarizer.EMBEDDING_MODEL.asString(), _makeTopicEmbeddings,
                 Summarizer::getExperimentMessage, Summarizer::getFinalSummaryMessage);
 
         // fetch summary, producing if necessary and requested

From f6ab3badb902b24c1cd7ef013956bb79276f590a Mon Sep 17 00:00:00 2001
From: Bob MacCallum <uncoolbob@gmail.com>
Date: Sun, 9 Nov 2025 21:56:05 +0000
Subject: [PATCH 13/16] add Claude extended thinking - off by default

---
 .../ai/SingleGeneAiExpressionReporter.java    |  2 +-
 .../ai/expression/ClaudeSummarizer.java       | 12 ++++++---
 .../ai/expression/GeneRecordProcessor.java    |  4 +--
 .../report/ai/expression/Summarizer.java      | 27 +++++++++++++------
 4 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
index 6c3fa447a..cf2f0d1ad 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/SingleGeneAiExpressionReporter.java
@@ -138,7 +138,7 @@ protected void write(OutputStream out) throws IOException, WdkModelException {
         // create summary inputs
         GeneSummaryInputs summaryInputs =
             GeneRecordProcessor.getSummaryInputsFromRecord(record, ClaudeSummarizer.CLAUDE_MODEL.toString(),
-                Summarizer.EMBEDDING_MODEL.asString(), _makeTopicEmbeddings,
+                Summarizer.EMBEDDING_MODEL.asString(), _makeTopicEmbeddings, ClaudeSummarizer.USE_EXTENDED_THINKING,
                 Summarizer::getExperimentMessage, Summarizer::getFinalSummaryMessage);
 
         // fetch summary, producing if necessary and requested
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
index f5fa42179..2e77db18e 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/ClaudeSummarizer.java
@@ -15,6 +15,7 @@
 public class ClaudeSummarizer extends Summarizer {
 
   public static final Model CLAUDE_MODEL = Model.CLAUDE_SONNET_4_5_20250929;
+  public static final boolean USE_EXTENDED_THINKING = false;
 
   private static final String CLAUDE_API_KEY_PROP_NAME = "CLAUDE_API_KEY";
 
@@ -42,12 +43,17 @@ protected CompletableFuture<String> callApiForJson(String prompt, Schema schema)
     
     String enhancedPrompt = prompt + "\n\n" + jsonFormatInstructions;
 
-    MessageCreateParams request = MessageCreateParams.builder()
+    MessageCreateParams.Builder requestBuilder = MessageCreateParams.builder()
         .model(CLAUDE_MODEL)
         .maxTokens((long) MAX_RESPONSE_TOKENS)
         .system(SYSTEM_MESSAGE)
-        .addUserMessage(enhancedPrompt)
-        .build();
+        .addUserMessage(enhancedPrompt);
+
+    if (USE_EXTENDED_THINKING) {
+      requestBuilder.enabledThinking(1024);
+    }
+
+    MessageCreateParams request = requestBuilder.build();
 
     return retryOnOverload(
         () -> _claudeClient.messages().create(request),
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/GeneRecordProcessor.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/GeneRecordProcessor.java
index 83aceddc5..dbd11f303 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/GeneRecordProcessor.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/GeneRecordProcessor.java
@@ -62,7 +62,7 @@ private static String getGeneId(RecordInstance record) {
     return record.getPrimaryKey().getValues().get("source_id");
   }
 
-  public static GeneSummaryInputs getSummaryInputsFromRecord(RecordInstance record, String aiChatModel, String embeddingModel, boolean makeTopicEmbeddings, Function<JSONObject, String> getExperimentPrompt, Function<List<JSONObject>, String> getFinalSummaryPrompt) throws WdkModelException {
+  public static GeneSummaryInputs getSummaryInputsFromRecord(RecordInstance record, String aiChatModel, String embeddingModel, boolean makeTopicEmbeddings, boolean useExtendedThinking, Function<JSONObject, String> getExperimentPrompt, Function<List<JSONObject>, String> getFinalSummaryPrompt) throws WdkModelException {
 
     String geneId = getGeneId(record);
 
@@ -90,7 +90,7 @@ public String getDigest() {
         List<JSONObject> digests = experimentsWithData.stream()
             .map(exp -> new JSONObject().put("digest", exp.getDigest()))
             .collect(Collectors.toList());
-        return EncryptionUtil.md5(aiChatModel + ":" + embeddingModel + ":" + makeTopicEmbeddings + ":" + DATA_MODEL_VERSION + ":" + getFinalSummaryPrompt.apply(digests));
+        return EncryptionUtil.md5(aiChatModel + ":" + embeddingModel + ":" + makeTopicEmbeddings + ":" + useExtendedThinking + ":" + DATA_MODEL_VERSION + ":" + getFinalSummaryPrompt.apply(digests));
       }
 
     };
diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
index f5973a318..e64d8e27c 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/Summarizer.java
@@ -96,18 +96,29 @@ public Summarizer(WdkModel wdkModel, DailyCostMonitor costMonitor, boolean makeT
     _costMonitor = costMonitor;
     _makeTopicEmbeddings = makeTopicEmbeddings;
 
-    String apiKey = wdkModel.getProperties().get(OPENAI_API_KEY_PROP_NAME);
-    if (apiKey == null) {
-      throw new WdkModelException("WDK property '" + OPENAI_API_KEY_PROP_NAME + "' has not been set.");
-    }
+    // Only create embedding client if we need to make topic embeddings
+    if (makeTopicEmbeddings) {
+      String apiKey = wdkModel.getProperties().get(OPENAI_API_KEY_PROP_NAME);
+      if (apiKey == null) {
+        throw new WdkModelException("WDK property '" + OPENAI_API_KEY_PROP_NAME + "' has not been set.");
+      }
 
-    _embeddingClient = OpenAIOkHttpClientAsync.builder()
-        .apiKey(apiKey)
-        .maxRetries(32)  // Handle 429 errors
-        .build();
+      _embeddingClient = OpenAIOkHttpClientAsync.builder()
+          .apiKey(apiKey)
+          .maxRetries(32)  // Handle 429 errors
+          .build();
+    } else {
+      _embeddingClient = null;
+    }
   }
 
   private CompletableFuture<List<Double>> getEmbedding(String text) {
+    // Safety check: ensure embedding client was initialized
+    if (_embeddingClient == null) {
+      LOG.error("Attempted to generate embedding but embedding client was not initialized (makeTopicEmbeddings=false)");
+      return CompletableFuture.completedFuture(List.of());
+    }
+
     EmbeddingCreateParams request = EmbeddingCreateParams.builder()
         .model(EMBEDDING_MODEL)
         .input(text)

From 3b27f64138bce23999de9007967e114d451d9ed0 Mon Sep 17 00:00:00 2001
From: Bob <uncoolbob@gmail.com>
Date: Sat, 14 Feb 2026 11:38:04 +0000
Subject: [PATCH 14/16] workaround for paralog_count

---
 .../ai/expression/GeneRecordProcessor.java       | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/GeneRecordProcessor.java b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/GeneRecordProcessor.java
index dbd11f303..0b6e00653 100644
--- a/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/GeneRecordProcessor.java
+++ b/Model/src/main/java/org/apidb/apicommon/model/report/ai/expression/GeneRecordProcessor.java
@@ -6,6 +6,7 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import org.apache.log4j.Logger;
 import org.gusdb.fgputil.EncryptionUtil;
 import org.gusdb.wdk.model.WdkModelException;
 import org.gusdb.wdk.model.WdkUserException;
@@ -21,9 +22,12 @@
  */
 public class GeneRecordProcessor {
 
+  private static final Logger LOG = Logger.getLogger(GeneRecordProcessor.class);
+
   private static final Set<String> KEYS_TO_KEEP = Set.of("y_axis", "description", "genus_species",
       "project_id", "summary", "dataset_id", "assay_type", "x_axis", "module", "dataset_name", "display_name",
-      "short_attribution", "paralog_number");
+      "short_attribution");
+      // TODO: restore "paralog_number" to KEYS_TO_KEEP once it is reliably present in gene records again
 
   private static final String EXPRESSION_GRAPH_TABLE = "ExpressionGraphs";
   private static final String EXPRESSION_GRAPH_DATA_TABLE = "ExpressionGraphsDataTable";
@@ -114,6 +118,16 @@ private static List<ExperimentInputs> processExpressionData(RecordInstance recor
           experimentInfo.put(key, experimentRow.getAttributeValue(key).getValue());
         }
 
+        // TODO: remove this fallback once paralog_number is reliably present in gene records again;
+        //       restore "paralog_number" to KEYS_TO_KEEP above and delete this block.
+        try {
+          experimentInfo.put("paralog_number", experimentRow.getAttributeValue("paralog_number").getValue());
+        } catch (WdkModelException e) {
+          LOG.warn("paralog_number attribute is missing from gene record; defaulting to 0. " +
+              "This is a temporary workaround - restore it to KEYS_TO_KEEP once the field is available again.");
+          experimentInfo.put("paralog_number", "0");
+        }
+
         String datasetId = experimentRow.getAttributeValue("dataset_id").getValue();
         String assayType = experimentRow.getAttributeValue("assay_type").getValue();
         String experimentName = experimentRow.getAttributeValue("display_name").getValue();

From a7499a9b98c7c4dc5f6e816a49079d1f420b3f68 Mon Sep 17 00:00:00 2001
From: Bob <uncoolbob@gmail.com>
Date: Sat, 14 Feb 2026 15:29:25 +0000
Subject: [PATCH 15/16] add correct costings for Claude and use new
 provider-agnostic prop names

---
 Model/lib/conifer/roles/conifer/vars/ApiCommon/default.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Model/lib/conifer/roles/conifer/vars/ApiCommon/default.yml b/Model/lib/conifer/roles/conifer/vars/ApiCommon/default.yml
index be7b97146..cbaf1cef6 100644
--- a/Model/lib/conifer/roles/conifer/vars/ApiCommon/default.yml
+++ b/Model/lib/conifer/roles/conifer/vars/ApiCommon/default.yml
@@ -50,9 +50,10 @@ modelprop:
   JBROWSE_SERVICE_URL: "/{{ webapp_ctx }}/service/jbrowse"
   AI_EXPRESSION_CACHE_DIR: "/var/www/Common/ai-expr-cache"
   AI_EXPRESSION_QUALTRICS_ID: SV_38C4ZX1JxLi2SEe
-  OPENAI_MAX_DAILY_AI_EXPRESSION_DOLLAR_COST: 33
-  OPENAI_DOLLAR_COST_PER_1M_AI_INPUT_TOKENS: 2.5
-  OPENAI_DOLLAR_COST_PER_1M_AI_OUTPUT_TOKENS: 10
+  MAX_DAILY_AI_EXPRESSION_DOLLAR_COST: 33
+  # Claude Sonnet 4.5 costs from here: https://platform.claude.com/docs/en/about-claude/pricing
+  DOLLAR_COST_PER_1M_AI_INPUT_TOKENS: 3
+  DOLLAR_COST_PER_1M_AI_OUTPUT_TOKENS: 15
 
 user_datasets_uploadTypes_env_map:
   w: "genelist"

From d2b88f9dfd347a8391c84c2fab95ad3a97688563 Mon Sep 17 00:00:00 2001
From: Bob <uncoolbob@gmail.com>
Date: Sat, 14 Feb 2026 16:15:55 +0000
Subject: [PATCH 16/16] add claude api key lookup from secrets in prod

---
 .../conifer/roles/conifer/vars/ApiCommon/production/default.yml  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Model/lib/conifer/roles/conifer/vars/ApiCommon/production/default.yml b/Model/lib/conifer/roles/conifer/vars/ApiCommon/production/default.yml
index 82596e15e..81c090100 100644
--- a/Model/lib/conifer/roles/conifer/vars/ApiCommon/production/default.yml
+++ b/Model/lib/conifer/roles/conifer/vars/ApiCommon/production/default.yml
@@ -46,6 +46,7 @@ modelprop:
   GOOGLE_MAPS_API_KEY: "{{ lookup('euparc', 'attr=api_key xpath=sites/google_maps default=NOKEY') }}"
   COMMUNITY_SITE: "//{{ community_env_map[prefix]|default(community_env_map['default']) }}"
   OPENAI_API_KEY: "{{ lookup('euparc', 'attr=api_key xpath=sites/openai default=NOKEY') }}"
+  CLAUDE_API_KEY: "{{ lookup('euparc', 'attr=api_key xpath=sites/claude default=NOKEY') }}"
 
 
 # the below extends the w_ q_ prefix pattern used for workspace_env_map, which