From 84eadf5be51a9eb39fd2c0195ce283f4a0f596b2 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 3 Nov 2025 23:48:49 +0000
Subject: [PATCH 01/93] codex generated changes, need to review and refactor

---
 .../0003_subscription_output_fields.py        | 72 +++++++++++++++++++
 radis/extractions/models.py                   | 28 +++++++-
 2 files changed, 97 insertions(+), 3 deletions(-)
 create mode 100644 radis/extractions/migrations/0003_subscription_output_fields.py

diff --git a/radis/extractions/migrations/0003_subscription_output_fields.py b/radis/extractions/migrations/0003_subscription_output_fields.py
new file mode 100644
index 00000000..7c6acf18
--- /dev/null
+++ b/radis/extractions/migrations/0003_subscription_output_fields.py
@@ -0,0 +1,72 @@
+# Generated by Django 5.2.7 on 2025-11-03 22:58
+
+import django.db.models.deletion
+from django.db import migrations, models
+from django.db.models import Q
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("extractions", "0002_procrastinate_on_delete"),
+        ("subscriptions", "0008_rename_filter_fields_result_subscribeditem_answers_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="outputfield",
+            name="job",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="output_fields",
+                to="extractions.extractionjob",
+            ),
+        ),
+        migrations.RemoveConstraint(
+            model_name="outputfield",
+            name="unique_output_field_name_per_job",
+        ),
+        migrations.RemoveField(
+            model_name="outputfield",
+            name="optional",
+        ),
+        migrations.AddField(
+            model_name="outputfield",
+            name="subscription",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="extraction_fields",
+                to="subscriptions.subscription",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="outputfield",
+            constraint=models.UniqueConstraint(
+                condition=Q(job__isnull=False),
+                fields=("name", "job_id"),
+                name="unique_output_field_name_per_job",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="outputfield",
+            constraint=models.UniqueConstraint(
+                condition=Q(subscription__isnull=False),
+                fields=("name", "subscription_id"),
+                name="unique_output_field_name_per_subscription",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="outputfield",
+            constraint=models.CheckConstraint(
+                check=(
+                    Q(job__isnull=False, subscription__isnull=True)
+                    | Q(job__isnull=True, subscription__isnull=False)
+                ),
+                name="output_field_exactly_one_parent",
+            ),
+        ),
+    ]
diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index dc761b2f..f778d61a 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -4,6 +4,7 @@
 from django.conf import settings
 from django.contrib.auth.models import Group
 from django.db import models
+from django.db.models import Q
 from django.urls import reverse
 from procrastinate.contrib.django import app
 from procrastinate.contrib.django.models import ProcrastinateJob
@@ -82,16 +83,37 @@ class OutputField(models.Model):
         max_length=1, choices=OutputType.choices, default=OutputType.TEXT
     )
     get_output_type_display: Callable[[], str]
-    optional = models.BooleanField(default=False)
-    job = models.ForeignKey[ExtractionJob](
-        ExtractionJob, on_delete=models.CASCADE, related_name="output_fields"
+    job = models.ForeignKey[
+        ExtractionJob
+    ](ExtractionJob, null=True, blank=True, on_delete=models.CASCADE, related_name="output_fields")
+    subscription = models.ForeignKey[
+        "subscriptions.Subscription"
+    ](
+        "subscriptions.Subscription",
+        null=True,
+        blank=True,
+        on_delete=models.CASCADE,
+        related_name="extraction_fields",
     )
 
     class Meta:
         constraints = [
             models.UniqueConstraint(
                 fields=["name", "job_id"],
+                condition=Q(job__isnull=False),
                 name="unique_output_field_name_per_job",
+            ),
+            models.UniqueConstraint(
+                fields=["name", "subscription_id"],
+                condition=Q(subscription__isnull=False),
+                name="unique_output_field_name_per_subscription",
+            ),
+            models.CheckConstraint(
+                check=(
+                    Q(job__isnull=False, subscription__isnull=True)
+                    | Q(job__isnull=True, subscription__isnull=False)
+                ),
+                name="output_field_exactly_one_parent",
             )
         ]
 

From f39195d191ebcf6a2e6690caa4a4654534e35914 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 3 Nov 2025 23:50:06 +0000
Subject: [PATCH 02/93] forgotten files to stage

---
 radis/settings/base.py                        | 24 +++--
 radis/subscriptions/forms.py                  | 89 +++++++++++++++++--
 .../0009_separate_filter_and_extraction.py    | 49 ++++++++++
 radis/subscriptions/models.py                 | 54 +++++++++--
 radis/subscriptions/processors.py             | 76 +++++++++++-----
 .../_subscribed_item_preview.html             | 28 ++++++
 .../subscriptions/subscription_detail.html    | 37 +++++++-
 radis/subscriptions/tests/__init__.py         |  0
 .../tests/test_processor_utils.py             | 83 +++++++++++++++++
 radis/subscriptions/utils/processor_utils.py  | 85 +++++++++++++++---
 radis/subscriptions/views.py                  | 75 ++++++++++++----
 11 files changed, 527 insertions(+), 73 deletions(-)
 create mode 100644 radis/subscriptions/migrations/0009_separate_filter_and_extraction.py
 create mode 100644 radis/subscriptions/tests/__init__.py
 create mode 100644 radis/subscriptions/tests/test_processor_utils.py

diff --git a/radis/settings/base.py b/radis/settings/base.py
index dea0bb53..439c84fb 100644
--- a/radis/settings/base.py
+++ b/radis/settings/base.py
@@ -356,20 +356,34 @@
 """
 
 # Subscription
-QUESTIONS_SYSTEM_PROMPT = """
+SUBSCRIPTION_FILTER_PROMPT = """
 You are an AI medical assistant with extensive knowledge in radiology and general medicine.
 You have been trained on a wide range of medical literature, including the latest research
 and guidelines in radiological practices.
-Answer the following questions from the given radiology report. The report and questions can
-be given in any language.
-Base your answers only on the information provided in the report. Don't hallucinate.
-Return the answer in JSON format. Answer with 'true' for 'yes' and 'false' for 'no'.
+Answer the following filter questions about the radiology report. The questions can be in any
+language. Base your answers strictly on the contents of the report. Return the answers in JSON
+format using the provided field identifiers. Answer with `true` for "yes" and `false` for "no".
 
 Radiology Report:
 $report
 
 Questions:
 $questions
+
+"""
+
+SUBSCRIPTION_EXTRACTION_PROMPT = """
+You are an AI medical assistant with extensive knowledge in radiology and general medicine.
+Extract the requested information from the radiology report. Only provide data that is explicitly
+mentioned in the report and respect the expected data type. If the report does not contain the
+requested information, respond with null. Return the extracted information in JSON format using
+the provided field identifiers.
+
+Radiology Report:
+$report
+
+Fields to extract:
+$fields
 """
 
 # Extraction
diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index 59e32c29..cb7af395 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -9,7 +9,9 @@
 from radis.reports.models import Language, Modality
 from radis.search.forms import AGE_STEP, MAX_AGE, MIN_AGE
 
-from .models import Question, Subscription
+from radis.extractions.models import OutputField
+
+from .models import FilterQuestion, Subscription
 from .site import subscription_retrieval_providers
 
 
@@ -95,7 +97,16 @@ def build_layout(self):
                     "provider",
                     "query",
                     "send_finished_mail",
-                    Formset("formset", legend="Questions", add_form_label="Add Question"),
+                    Formset(
+                        "filter_formset",
+                        legend="Filter Questions",
+                        add_form_label="Add Filter Question",
+                    ),
+                    Formset(
+                        "extraction_formset",
+                        legend="Extraction Fields",
+                        add_form_label="Add Extraction Field",
+                    ),
                 ),
                 Column(
                     "patient_id",
@@ -139,15 +150,21 @@ def clean(self) -> dict[str, Any] | None:
         return super().clean()
 
 
-class QuestionForm(forms.ModelForm):
+class FilterQuestionForm(forms.ModelForm):
     class Meta:
-        model = Question
-        fields = ["question"]
+        model = FilterQuestion
+        fields = ["question", "expected_answer"]
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
         self.fields["question"].required = False
+        self.fields["expected_answer"].required = False
+        self.fields["expected_answer"].choices = [
+            ("", "Select expected answer"),
+            *FilterQuestion.ExpectedAnswer.choices,
+        ]
+        self.fields["expected_answer"].label = "Accept when answer is"
 
         self.helper = FormHelper()
         self.helper.form_tag = False
@@ -157,17 +174,73 @@ def __init__(self, *args, **kwargs):
                 Field("id", type="hidden"),
                 Field("DELETE", type="hidden"),
                 "question",
+                "expected_answer",
             ),
         )
 
+    def clean(self) -> dict[str, Any]:
+        cleaned_data = super().clean()
+        question = cleaned_data.get("question")
+        expected_answer = cleaned_data.get("expected_answer")
+
+        if not question and not expected_answer:
+            return cleaned_data
+
+        if question and not expected_answer:
+            cleaned_data["expected_answer"] = FilterQuestion.ExpectedAnswer.YES
+
+        if not question:
+            raise forms.ValidationError("Question text is required when specifying a filter.")
+
+        return cleaned_data
+
+
+class ExtractionFieldForm(forms.ModelForm):
+    class Meta:
+        model = OutputField
+        fields = ["name", "description", "output_type"]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.fields["name"].required = True
+        self.fields["description"].required = False
+
+        self.helper = FormHelper()
+        self.helper.form_tag = False
+        self.helper.disable_csrf = True
+        self.helper.layout = Layout(
+            Div(
+                Field("id", type="hidden"),
+                Field("DELETE", type="hidden"),
+                Row(
+                    Column("name", css_class="col-6"),
+                    Column("output_type", css_class="col-4"),
+                ),
+                "description",
+            )
+        )
+
 
-QuestionFormSet = forms.inlineformset_factory(
+FilterQuestionFormSet = forms.inlineformset_factory(
     Subscription,
-    Question,
-    form=QuestionForm,
+    FilterQuestion,
+    form=FilterQuestionForm,
     extra=1,
     min_num=0,
     max_num=3,
     validate_max=True,
     can_delete=False,
 )
+
+ExtractionFieldFormSet = forms.inlineformset_factory(
+    Subscription,
+    OutputField,
+    form=ExtractionFieldForm,
+    fk_name="subscription",
+    extra=1,
+    min_num=0,
+    max_num=10,
+    validate_max=True,
+    can_delete=False,
+)
diff --git a/radis/subscriptions/migrations/0009_separate_filter_and_extraction.py b/radis/subscriptions/migrations/0009_separate_filter_and_extraction.py
new file mode 100644
index 00000000..7c923225
--- /dev/null
+++ b/radis/subscriptions/migrations/0009_separate_filter_and_extraction.py
@@ -0,0 +1,49 @@
+# Generated by Django 5.2.7 on 2025-11-03 21:25
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        (
+            "subscriptions",
+            "0008_rename_filter_fields_result_subscribeditem_answers_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.RenameModel(
+            old_name="Question",
+            new_name="FilterQuestion",
+        ),
+        migrations.AddField(
+            model_name="filterquestion",
+            name="expected_answer",
+            field=models.CharField(
+                choices=[("Y", "Yes"), ("N", "No")],
+                default="Y",
+                max_length=1,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="filterquestion",
+            name="subscription",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="filter_questions",
+                to="subscriptions.subscription",
+            ),
+        ),
+        migrations.RenameField(
+            model_name="subscribeditem",
+            old_name="answers",
+            new_name="filter_results",
+        ),
+        migrations.AddField(
+            model_name="subscribeditem",
+            name="extraction_results",
+            field=models.JSONField(blank=True, null=True),
+        ),
+    ]
diff --git a/radis/subscriptions/models.py b/radis/subscriptions/models.py
index 71a680c8..115583b2 100644
--- a/radis/subscriptions/models.py
+++ b/radis/subscriptions/models.py
@@ -8,6 +8,7 @@
 from procrastinate.contrib.django.models import ProcrastinateJob
 
 from radis.core.models import AnalysisJob, AnalysisTask
+from radis.extractions.models import OutputField
 from radis.reports.models import Language, Modality, Report
 
 
@@ -50,7 +51,8 @@ class Subscription(models.Model):
     created_at = models.DateTimeField(auto_now_add=True)
     last_refreshed = models.DateTimeField(auto_now_add=True)
 
-    questions: models.QuerySet["Question"]
+    filter_questions: models.QuerySet["FilterQuestion"]
+    extraction_fields: models.QuerySet[OutputField]
     items: models.QuerySet["SubscribedItem"]
 
     send_finished_mail = models.BooleanField(default=False)
@@ -67,17 +69,30 @@ def __str__(self):
         return f"Subscription {self.name} [{self.pk}]"
 
 
-class Question(models.Model):
+class FilterQuestion(models.Model):
+    class ExpectedAnswer(models.TextChoices):
+        YES = "Y", "Yes"
+        NO = "N", "No"
+
     subscription = models.ForeignKey[Subscription](
-        Subscription, on_delete=models.CASCADE, related_name="questions"
+        Subscription, on_delete=models.CASCADE, related_name="filter_questions"
     )
     question = models.CharField(max_length=300)
+    expected_answer = models.CharField(
+        max_length=1,
+        choices=ExpectedAnswer.choices,
+        default=ExpectedAnswer.YES,
+    )
 
     def __str__(self) -> str:
         max_length = 30
-        if len(self.question) > max_length:
-            return f'Question "{self.question[:max_length]}..." [{self.pk}]'
-        return f'Question "{self.question}" [{self.pk}]'
+        truncated = self.question[:max_length]
+        suffix = "..." if len(self.question) > max_length else ""
+        return f'Filter Question "{truncated}{suffix}" [{self.pk}]'
+
+    @property
+    def expected_answer_bool(self) -> bool:
+        return self.expected_answer == self.ExpectedAnswer.YES
 
 
 class SubscribedItem(models.Model):
@@ -88,12 +103,37 @@ class SubscribedItem(models.Model):
         "SubscriptionJob", null=True, on_delete=models.SET_NULL, related_name="items"
     )
     report = models.ForeignKey[Report](Report, on_delete=models.CASCADE, related_name="+")
-    answers = models.JSONField(null=True, blank=True)
+    filter_results = models.JSONField(null=True, blank=True)
+    extraction_results = models.JSONField(null=True, blank=True)
     created_at = models.DateTimeField(auto_now_add=True)
 
     def __str__(self):
         return f"SubscribedItem of {self.subscription} [{self.pk}]"
 
+    def iter_filter_results(self) -> list[tuple[FilterQuestion, bool]]:
+        if not self.filter_results:
+            return []
+
+        results: list[tuple[FilterQuestion, bool]] = []
+        subscription_questions = {str(q.pk): q for q in self.subscription.filter_questions.all()}
+        for key, value in self.filter_results.items():
+            question = subscription_questions.get(str(key))
+            if question is not None:
+                results.append((question, bool(value)))
+        return results
+
+    def iter_extraction_results(self) -> list[tuple[OutputField, object]]:
+        if not self.extraction_results:
+            return []
+
+        results: list[tuple[OutputField, object]] = []
+        subscription_fields = {str(f.pk): f for f in self.subscription.extraction_fields.all()}
+        for key, value in self.extraction_results.items():
+            field = subscription_fields.get(str(key))
+            if field is not None:
+                results.append((field, value))
+        return results
+
 
 class SubscriptionJob(AnalysisJob):
     default_priority = settings.SUBSCRIPTION_DEFAULT_PRIORITY
diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index 002371a3..0f6b0a9e 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -1,4 +1,5 @@
 import logging
+from typing import Any
 from concurrent.futures import Future, ThreadPoolExecutor
 from string import Template
 
@@ -16,8 +17,10 @@
     SubscriptionTask,
 )
 from .utils.processor_utils import (
-    generate_questions_for_prompt,
-    generate_questions_schema,
+    build_extraction_schema,
+    build_filter_schema,
+    generate_extraction_fields_prompt,
+    generate_filter_questions_prompt,
 )
 
 logger = logging.getLogger(__name__)
@@ -47,25 +50,58 @@ def process_task(self, task: SubscriptionTask) -> None:
 
     def process_report(self, report: Report, task: SubscriptionTask) -> None:
         subscription: Subscription = task.job.subscription
-        Schema = generate_questions_schema(subscription.questions)
-        prompt = Template(settings.QUESTION_SYSTEM_PROMPT).substitute(
-            {
-                "report": report.body,
-                "questions": generate_questions_for_prompt(subscription.questions),
-            }
-        )
-        result = self.client.extract_data(prompt, Schema)
+        filter_bundle = build_filter_schema(subscription.filter_questions)
 
-        is_accepted = all(
-            [getattr(result, field_name) for field_name in result.__pydantic_fields__]
-        )
-        if is_accepted:
-            SubscribedItem.objects.create(
-                subscription=task.job.subscription,
-                job=task.job,
-                report=report,
-                filter_fields_results=result.model_dump(),
+        filter_results: dict[str, bool] = {}
+        is_accepted = True
+
+        if filter_bundle.mapping:
+            filter_prompt = Template(settings.SUBSCRIPTION_FILTER_PROMPT).substitute(
+                {
+                    "report": report.body,
+                    "questions": generate_filter_questions_prompt(filter_bundle.mapping),
+                }
             )
-            logger.debug(f"Report {report.pk} was accepted by subscription {subscription.pk}")
+            filter_response = self.client.extract_data(filter_prompt, filter_bundle.schema)
+
+            for field_name, question in filter_bundle.mapping:
+                answer = bool(getattr(filter_response, field_name))
+                filter_results[str(question.pk)] = answer
+                if answer != question.expected_answer_bool:
+                    is_accepted = False
         else:
+            logger.debug(
+                "Subscription %s has no filter questions; accepting report %s by default",
+                subscription.pk,
+                report.pk,
+            )
+
+        if not is_accepted:
             logger.debug(f"Report {report.pk} was rejected by subscription {subscription.pk}")
+            return
+
+        extraction_bundle = build_extraction_schema(subscription.extraction_fields)
+        extraction_results: dict[str, Any] = {}
+
+        if extraction_bundle.mapping:
+            extraction_prompt = Template(settings.SUBSCRIPTION_EXTRACTION_PROMPT).substitute(
+                {
+                    "report": report.body,
+                    "fields": generate_extraction_fields_prompt(extraction_bundle.mapping),
+                }
+            )
+            extraction_response = self.client.extract_data(
+                extraction_prompt, extraction_bundle.schema
+            )
+
+            for field_name, field in extraction_bundle.mapping:
+                extraction_results[str(field.pk)] = getattr(extraction_response, field_name)
+
+        SubscribedItem.objects.create(
+            subscription=task.job.subscription,
+            job=task.job,
+            report=report,
+            filter_results=filter_results or None,
+            extraction_results=extraction_results or None,
+        )
+        logger.debug(f"Report {report.pk} was accepted by subscription {subscription.pk}")
diff --git a/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html b/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
index 432891f5..fbf65f99 100644
--- a/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
+++ b/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
@@ -4,6 +4,34 @@
             <div class="d-flex justify-content-between">
                 {% include "reports/_report_header.html" with report=subscribed_item.report %}
             </div>
+            {% with filter_results=subscribed_item.iter_filter_results %}
+                {% if filter_results %}
+                    <div>
+                        <div class="fw-semibold small text-uppercase text-secondary">Filter results</div>
+                        <ul class="mb-2 small">
+                            {% for question, value in filter_results %}
+                                <li>
+                                    {{ question.question }} → {{ value|yesno:"Yes,No" }}
+                                </li>
+                            {% endfor %}
+                        </ul>
+                    </div>
+                {% endif %}
+            {% endwith %}
+            {% with extraction_results=subscribed_item.iter_extraction_results %}
+                {% if extraction_results %}
+                    <div>
+                        <div class="fw-semibold small text-uppercase text-secondary">Extraction results</div>
+                        <ul class="mb-2 small">
+                            {% for field, value in extraction_results %}
+                                <li>
+                                    {{ field.name }} → {{ value|default_if_none:"—" }}
+                                </li>
+                            {% endfor %}
+                        </ul>
+                    </div>
+                {% endif %}
+            {% endwith %}
             <div class="d-flex flex-column gap-1">
                 <div class="clamp-3" :class="{'clamp-3': !full, 'pre-line': full}">{{ subscribed_item.report.body }}</div>
                 <div class="d-flex gap-2">
diff --git a/radis/subscriptions/templates/subscriptions/subscription_detail.html b/radis/subscriptions/templates/subscriptions/subscription_detail.html
index 7002a2b1..dc6cdb46 100644
--- a/radis/subscriptions/templates/subscriptions/subscription_detail.html
+++ b/radis/subscriptions/templates/subscriptions/subscription_detail.html
@@ -118,8 +118,37 @@ <h4>Filter Details</h4>
             {% endif %}
         </dd>
     </dl>
-    <h4>Questions</h4>
-    <ul>
-        {% for question in subscription.questions.all %}<li>{{ question.question }}</li>{% endfor %}
-    </ul>
+    <h4>Filter Questions</h4>
+    {% with questions=subscription.filter_questions.all %}
+        {% if questions %}
+            <ul>
+                {% for question in questions %}
+                    <li>
+                        {{ question.question }} — accepts reports when answer is
+                        {{ question.get_expected_answer_display|lower }}
+                    </li>
+                {% endfor %}
+            </ul>
+        {% else %}
+            <p class="text-muted mb-0">No filter questions defined.</p>
+        {% endif %}
+    {% endwith %}
+    <h4 class="mt-4">Extraction Fields</h4>
+    {% with fields=subscription.extraction_fields.all %}
+        {% if fields %}
+            <dl class="row">
+                {% for field in fields %}
+                    <dt class="col-sm-3">{{ field.name }}</dt>
+                    <dd class="col-sm-9">
+                        {{ field.description|default:"No description provided." }}
+                        <div class="text-muted small">
+                            Type: {{ field.get_output_type_display }}
+                        </div>
+                    </dd>
+                {% endfor %}
+            </dl>
+        {% else %}
+            <p class="text-muted mb-0">No extraction fields configured.</p>
+        {% endif %}
+    {% endwith %}
 {% endblock content %}
diff --git a/radis/subscriptions/tests/__init__.py b/radis/subscriptions/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/radis/subscriptions/tests/test_processor_utils.py b/radis/subscriptions/tests/test_processor_utils.py
new file mode 100644
index 00000000..f2be9b21
--- /dev/null
+++ b/radis/subscriptions/tests/test_processor_utils.py
@@ -0,0 +1,83 @@
+from dataclasses import dataclass
+
+from radis.extractions.models import OutputType
+from radis.subscriptions.utils.processor_utils import (
+    build_extraction_schema,
+    build_filter_schema,
+    generate_extraction_fields_prompt,
+    generate_filter_questions_prompt,
+)
+
+
+@dataclass
+class DummyFilterQuestion:
+    pk: int
+    question: str
+    expected_answer_bool: bool
+
+
+class DummyFilterQuestionSet:
+    def __init__(self, *questions: DummyFilterQuestion):
+        self._questions = list(questions)
+
+    def all(self):
+        return list(self._questions)
+
+
+@dataclass
+class DummyExtractionField:
+    pk: int
+    name: str
+    description: str
+    output_type: str
+
+    def get_output_type_display(self) -> str:
+        return {
+            OutputType.TEXT: "Text",
+            OutputType.NUMERIC: "Numeric",
+            OutputType.BOOLEAN: "Boolean",
+        }[self.output_type]
+
+
+class DummyExtractionFieldSet:
+    def __init__(self, *fields: DummyExtractionField):
+        self._fields = list(fields)
+
+    def all(self):
+        return list(self._fields)
+
+
+def test_build_subscription_schema_and_prompts_without_database():
+    questions = DummyFilterQuestionSet(
+        DummyFilterQuestion(pk=1, question="Contains pneumonia?", expected_answer_bool=True)
+    )
+    fields = DummyExtractionFieldSet(
+        DummyExtractionField(
+            pk=10,
+            name="diagnosis",
+            description="Primary diagnosis mentioned in the report",
+            output_type=OutputType.TEXT,
+        )
+    )
+
+    filter_bundle = build_filter_schema(questions)
+    extraction_bundle = build_extraction_schema(fields)
+
+    assert len(filter_bundle.mapping) == 1
+    assert len(extraction_bundle.mapping) == 1
+
+    filter_field_name, mapped_question = filter_bundle.mapping[0]
+    assert filter_field_name.startswith("filter_")
+    assert mapped_question.pk == 1
+
+    extraction_field_name, mapped_field = extraction_bundle.mapping[0]
+    assert extraction_field_name.startswith("extraction_")
+    assert mapped_field.pk == 10
+
+    filter_prompt = generate_filter_questions_prompt(filter_bundle.mapping)
+    assert "filter_" in filter_prompt
+    assert "Contains pneumonia?" in filter_prompt
+
+    extraction_prompt = generate_extraction_fields_prompt(extraction_bundle.mapping)
+    assert "extraction_" in extraction_prompt
+    assert "diagnosis" in extraction_prompt
diff --git a/radis/subscriptions/utils/processor_utils.py b/radis/subscriptions/utils/processor_utils.py
index ab24ba1c..c728e1e8 100644
--- a/radis/subscriptions/utils/processor_utils.py
+++ b/radis/subscriptions/utils/processor_utils.py
@@ -1,22 +1,87 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
 from typing import Any
 
 from django.db.models import QuerySet
 from pydantic import BaseModel, create_model
 
-from ..models import Question
+from radis.extractions.models import OutputField, OutputType
+
+from ..models import FilterQuestion
+
+type Numeric = float | int
+
+
+@dataclass(slots=True)
+class FilterSchemaBundle:
+    schema: type[BaseModel]
+    mapping: list[tuple[str, FilterQuestion]]
+
+
+@dataclass(slots=True)
+class ExtractionSchemaBundle:
+    schema: type[BaseModel]
+    mapping: list[tuple[str, OutputField]]
+
+
+def build_filter_schema(questions: QuerySet[FilterQuestion]) -> FilterSchemaBundle:
+    field_definitions: dict[str, Any] = {}
+    mapping: list[tuple[str, FilterQuestion]] = []
+
+    for index, question in enumerate(questions.all()):
+        field_name = f"filter_{index}"
+        field_definitions[field_name] = (bool, ...)
+        mapping.append((field_name, question))
+
+    model_name = "SubscriptionFilterResultsModel"
+    schema = create_model(model_name, **field_definitions) if field_definitions else create_model(model_name)
+    return FilterSchemaBundle(schema, mapping)
 
 
-def generate_questions_schema(questions: QuerySet[Question]) -> type[BaseModel]:
+def build_extraction_schema(fields: QuerySet[OutputField]) -> ExtractionSchemaBundle:
     field_definitions: dict[str, Any] = {}
-    for index, _ in enumerate(questions.all()):
-        field_definitions[f"question_{index}"] = (bool, ...)
+    mapping: list[tuple[str, OutputField]] = []
+
+    for index, field in enumerate(fields.all()):
+        field_name = f"extraction_{index}"
+        if field.output_type == OutputType.TEXT:
+            output_type = str
+        elif field.output_type == OutputType.NUMERIC:
+            output_type = Numeric
+        elif field.output_type == OutputType.BOOLEAN:
+            output_type = bool
+        else:
+            raise ValueError(f"Unknown output type: {field.output_type}")
+
+        field_definitions[field_name] = (output_type, ...)
+
+        mapping.append((field_name, field))
+
+    model_name = "SubscriptionExtractionResultsModel"
+    schema = create_model(model_name, **field_definitions) if field_definitions else create_model(model_name)
+    return ExtractionSchemaBundle(schema, mapping)
+
+
+def generate_filter_questions_prompt(mapping: list[tuple[str, FilterQuestion]]) -> str:
+    if not mapping:
+        return "None"
 
-    return create_model("QuestionsModel", field_definitions=field_definitions)
+    lines: list[str] = []
+    for field_name, question in mapping:
+        lines.append(f"{field_name}: {question.question}")
+    return "\n".join(lines)
 
 
-def generate_questions_for_prompt(fields: QuerySet[Question]) -> str:
-    prompt = ""
-    for index, question in enumerate(fields.all()):
-        prompt += f"question_{index}: {question.question}\n"
+def generate_extraction_fields_prompt(mapping: list[tuple[str, OutputField]]) -> str:
+    if not mapping:
+        return "None"
 
-    return prompt
+    lines: list[str] = []
+    for field_name, field in mapping:
+        description = field.description or "No description provided."
+        lines.append(
+            f"{field_name}: {field.name} — {description} "
+            f"[type: {field.get_output_type_display()}]"
+        )
+    return "\n".join(lines)
diff --git a/radis/subscriptions/views.py b/radis/subscriptions/views.py
index e04fb74f..066ec0e3 100644
--- a/radis/subscriptions/views.py
+++ b/radis/subscriptions/views.py
@@ -20,8 +20,12 @@
 from radis.subscriptions.filters import SubscriptionFilter
 from radis.subscriptions.tables import SubscriptionTable
 
-from .forms import QuestionForm, QuestionFormSet, SubscriptionForm
-from .models import Question, SubscribedItem, Subscription
+from .forms import (
+    ExtractionFieldFormSet,
+    FilterQuestionFormSet,
+    SubscriptionForm,
+)
+from .models import SubscribedItem, Subscription
 
 logger = getLogger(__name__)
 
@@ -48,7 +52,12 @@ class SubscriptionDetailView(LoginRequiredMixin, DetailView):
     template_name = "subscriptions/subscription_detail.html"
 
     def get_queryset(self):
-        return super().get_queryset().filter(owner=self.request.user).prefetch_related("questions")
+        return (
+            super()
+            .get_queryset()
+            .filter(owner=self.request.user)
+            .prefetch_related("filter_questions", "extraction_fields")
+        )
 
 
 class SubscriptionCreateView(LoginRequiredMixin, CreateView):  # TODO: Add PermissionRequiredMixin
@@ -60,15 +69,18 @@ class SubscriptionCreateView(LoginRequiredMixin, CreateView):  # TODO: Add Permi
     def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
         ctx = super().get_context_data(**kwargs)
         if self.request.POST:
-            ctx["formset"] = QuestionFormSet(self.request.POST)
+            ctx["filter_formset"] = FilterQuestionFormSet(self.request.POST)
+            ctx["extraction_formset"] = ExtractionFieldFormSet(self.request.POST)
         else:
-            ctx["formset"] = QuestionFormSet()
+            ctx["filter_formset"] = FilterQuestionFormSet()
+            ctx["extraction_formset"] = ExtractionFieldFormSet()
         return ctx
 
     def form_valid(self, form) -> HttpResponse:
         ctx = self.get_context_data()
-        formset: BaseInlineFormSet[Question, Subscription, QuestionForm] = ctx["formset"]
-        if formset.is_valid():
+        filter_formset: BaseInlineFormSet = ctx["filter_formset"]
+        extraction_formset: BaseInlineFormSet = ctx["extraction_formset"]
+        if filter_formset.is_valid() and extraction_formset.is_valid():
             user = self.request.user
             form.instance.owner = user
             active_group = user.active_group
@@ -82,8 +94,11 @@ def form_valid(self, form) -> HttpResponse:
                     return self.form_invalid(form)
                 raise e
 
-            formset.instance = self.object
-            formset.save()
+            filter_formset.instance = self.object
+            filter_formset.save()
+
+            extraction_formset.instance = self.object
+            extraction_formset.save()
             return HttpResponseRedirect(self.get_success_url())
         else:
             return self.form_invalid(form)
@@ -99,21 +114,34 @@ def get_success_url(self):
         return reverse("subscription_detail", kwargs={"pk": self.object.pk})
 
     def get_queryset(self) -> QuerySet[Subscription]:
-        return super().get_queryset().filter(owner=self.request.user).prefetch_related("questions")
+        return (
+            super()
+            .get_queryset()
+            .filter(owner=self.request.user)
+            .prefetch_related("filter_questions", "extraction_fields")
+        )
 
     def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
         ctx = super().get_context_data(**kwargs)
         if self.request.POST:
-            ctx["formset"] = QuestionFormSet(self.request.POST, instance=self.object)
+            ctx["filter_formset"] = FilterQuestionFormSet(
+                self.request.POST, instance=self.object
+            )
+            ctx["extraction_formset"] = ExtractionFieldFormSet(
+                self.request.POST, instance=self.object
+            )
         else:
-            ctx["formset"] = QuestionFormSet(instance=self.object)
-        ctx["formset"].extra = 0  # no additional empty form when editing
+            ctx["filter_formset"] = FilterQuestionFormSet(instance=self.object)
+            ctx["extraction_formset"] = ExtractionFieldFormSet(instance=self.object)
+        ctx["filter_formset"].extra = 0  # no additional empty form when editing
+        ctx["extraction_formset"].extra = 0
         return ctx
 
     def form_valid(self, form) -> HttpResponse:
         ctx = self.get_context_data()
-        formset: BaseInlineFormSet[Question, Subscription, QuestionForm] = ctx["formset"]
-        if formset.is_valid():
+        filter_formset = ctx["filter_formset"]
+        extraction_formset = ctx["extraction_formset"]
+        if filter_formset.is_valid() and extraction_formset.is_valid():
             try:
                 self.object = form.save()
             except IntegrityError as e:
@@ -122,8 +150,11 @@ def form_valid(self, form) -> HttpResponse:
                     return self.form_invalid(form)
                 raise e
 
-            formset.instance = self.object
-            formset.save()
+            filter_formset.instance = self.object
+            filter_formset.save()
+
+            extraction_formset.instance = self.object
+            extraction_formset.save()
 
             return super().form_valid(form)
         else:
@@ -159,8 +190,14 @@ def get_queryset(self) -> QuerySet[Subscription]:
 
     def get_related_queryset(self) -> QuerySet[SubscribedItem]:
         subscription = cast(Subscription, self.get_object())
-        return SubscribedItem.objects.filter(subscription_id=subscription.pk).prefetch_related(
-            "report"
+        return (
+            SubscribedItem.objects.filter(subscription_id=subscription.pk)
+            .select_related("subscription")
+            .prefetch_related(
+                "report",
+                "subscription__filter_questions",
+                "subscription__extraction_fields",
+            )
         )
 
     def get_filter_queryset(self) -> QuerySet[SubscribedItem]:

From 087e8b1b46eb7108d55b0618e22318d5033536ae Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Thu, 6 Nov 2025 10:51:16 +0100
Subject: [PATCH 03/93] Adding downloads view

---
 .../extractions/extraction_job_detail.html    | 17 ++++--
 .../extractions/extraction_result_list.html   | 17 ++++--
 radis/extractions/urls.py                     |  6 +++
 radis/extractions/utils/__init__.py           |  5 ++
 radis/extractions/utils/csv_export.py         | 47 +++++++++++++++++
 radis/extractions/views.py                    | 52 +++++++++++++++++++
 6 files changed, 134 insertions(+), 10 deletions(-)
 create mode 100644 radis/extractions/utils/csv_export.py

diff --git a/radis/extractions/templates/extractions/extraction_job_detail.html b/radis/extractions/templates/extractions/extraction_job_detail.html
index 942a2fc8..4d874c80 100644
--- a/radis/extractions/templates/extractions/extraction_job_detail.html
+++ b/radis/extractions/templates/extractions/extraction_job_detail.html
@@ -121,11 +121,18 @@ <h5>Output Fields</h5>
     {% if not job.is_preparing %}
         <c-table-heading title="Extraction Tasks">
         <c-slot name="left">
-        <a href="{% url 'extraction_result_list' job.id %}"
-           class="btn btn-sm btn-primary">
-            {% bootstrap_icon "eye" %}
-            View Results
-        </a>
+        <div class="d-flex gap-2">
+            <a href="{% url 'extraction_result_list' job.id %}"
+               class="btn btn-sm btn-primary">
+                {% bootstrap_icon "eye" %}
+                View Results
+            </a>
+            <a href="{% url 'extraction_result_download' job.id %}"
+               class="btn btn-sm btn-outline-secondary">
+                {% bootstrap_icon "download" %}
+                Download CSV
+            </a>
+        </div>
         </c-slot>
         <c-slot name="right">
         {% crispy filter.form %}
diff --git a/radis/extractions/templates/extractions/extraction_result_list.html b/radis/extractions/templates/extractions/extraction_result_list.html
index a0abdb4e..6ca9b510 100644
--- a/radis/extractions/templates/extractions/extraction_result_list.html
+++ b/radis/extractions/templates/extractions/extraction_result_list.html
@@ -7,11 +7,18 @@
 {% block heading %}
     <c-page-heading title="Extraction Results">
     <c-slot name="right">
-    <a href="{% url 'extraction_job_detail' job.id %}"
-       class="btn btn-secondary">
-        {% bootstrap_icon "arrow-return-left" %}
-        View Job
-    </a>
+    <div class="d-flex gap-2">
+        <a href="{% url 'extraction_result_download' job.id %}"
+           class="btn btn-success">
+            {% bootstrap_icon "download" %}
+            Download CSV
+        </a>
+        <a href="{% url 'extraction_job_detail' job.id %}"
+           class="btn btn-secondary">
+            {% bootstrap_icon "arrow-return-left" %}
+            View Job
+        </a>
+    </div>
     </c-slot>
     </c-page-heading>
 {% endblock heading %}
diff --git a/radis/extractions/urls.py b/radis/extractions/urls.py
index df459e25..d7c44ba2 100644
--- a/radis/extractions/urls.py
+++ b/radis/extractions/urls.py
@@ -13,6 +13,7 @@
     ExtractionJobVerifyView,
     ExtractionJobWizardView,
     ExtractionResultListView,
+    ExtractionResultDownloadView,
     ExtractionTaskDeleteView,
     ExtractionTaskDetailView,
     ExtractionTaskResetView,
@@ -99,6 +100,11 @@
         ExtractionResultListView.as_view(),
         name="extraction_result_list",
     ),
+    path(
+        "jobs/<int:pk>/results/download/",
+        ExtractionResultDownloadView.as_view(),
+        name="extraction_result_download",
+    ),
     path(
         "instances/<int:pk>/",
         ExtractionInstanceDetailView.as_view(),
diff --git a/radis/extractions/utils/__init__.py b/radis/extractions/utils/__init__.py
index e69de29b..c32e9b87 100644
--- a/radis/extractions/utils/__init__.py
+++ b/radis/extractions/utils/__init__.py
@@ -0,0 +1,5 @@
+"""Utility helpers for extraction workflows."""
+
+from .csv_export import iter_extraction_result_rows
+
+__all__ = ["iter_extraction_result_rows"]
diff --git a/radis/extractions/utils/csv_export.py b/radis/extractions/utils/csv_export.py
new file mode 100644
index 00000000..9b28fc20
--- /dev/null
+++ b/radis/extractions/utils/csv_export.py
@@ -0,0 +1,47 @@
+"""Helpers for exporting extraction results in CSV format."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from typing import Any
+
+from django.db.models import QuerySet
+
+from radis.extractions.models import ExtractionInstance, ExtractionJob
+
+
+def iter_extraction_result_rows(job: ExtractionJob) -> Iterable[Sequence[str]]:
+    """Yield rows for the extraction results CSV.
+
+    Args:
+        job: The extraction job whose results should be exported.
+
+    Yields:
+        Sequences of stringified cell values suitable for csv.writer.
+    """
+
+    field_names: list[str] = list(
+        job.output_fields.order_by("pk").values_list("name", flat=True)
+    )
+
+    header = ["instance_id", "report_id", "is_processed"]
+    header.extend(field_names)
+    yield header
+
+    instances: QuerySet[ExtractionInstance] = ExtractionInstance.objects.filter(
+        task__job=job
+    ).order_by("pk")
+
+    for instance in instances.iterator():
+        row: list[str] = [
+            str(instance.pk),
+            str(instance.report_id) if instance.report_id else "",
+            "yes" if instance.is_processed else "no",
+        ]
+
+        output: dict[str, Any] = instance.output or {}
+        for field_name in field_names:
+            value = output.get(field_name)
+            row.append("" if value is None else str(value))
+
+        yield row
diff --git a/radis/extractions/views.py b/radis/extractions/views.py
index 78f4e3c2..a037931a 100644
--- a/radis/extractions/views.py
+++ b/radis/extractions/views.py
@@ -1,3 +1,5 @@
+import csv
+from collections.abc import Generator
 from typing import Any, Type, cast
 
 from adit_radis_shared.common.mixins import (
@@ -14,8 +16,10 @@
 from django.db import transaction
 from django.db.models import QuerySet
 from django.forms import BaseInlineFormSet
+from django.http import StreamingHttpResponse
 from django.shortcuts import redirect
 from django.urls import reverse_lazy
+from django.utils.text import slugify
 from django.views.generic import DetailView
 from django_tables2 import SingleTableMixin, tables
 from formtools.wizard.views import SessionWizardView
@@ -50,6 +54,7 @@
     ExtractionResultsTable,
     ExtractionTaskTable,
 )
+from .utils.csv_export import iter_extraction_result_rows
 
 EXTRACTIONS_SEARCH_PROVIDER = "extractions_search_provider"
 
@@ -272,3 +277,50 @@ def get_table(self, **kwargs):
     def get_table_data(self):
         job = cast(ExtractionJob, self.get_object())
         return ExtractionInstance.objects.filter(task__job=job)
+
+
+class _Echo:
+    """Lightweight write-only buffer for csv.writer."""
+
+    def write(self, value: str) -> str:
+        return value
+
+
+class ExtractionResultDownloadView(ExtractionsLockedMixin, LoginRequiredMixin, DetailView):
+    """Stream extraction results as a CSV download."""
+
+    model = ExtractionJob
+    request: AuthenticatedHttpRequest
+
+    def get_queryset(self) -> QuerySet[ExtractionJob]:
+        """Return the accessible extraction jobs for the current user."""
+        assert self.model
+        model = cast(Type[ExtractionJob], self.model)
+        if self.request.user.is_staff:
+            return model.objects.all()
+        return model.objects.filter(owner=self.request.user)
+
+    def get(self, request: AuthenticatedHttpRequest, *args, **kwargs) -> StreamingHttpResponse:
+        """Stream the CSV file response."""
+        job = cast(ExtractionJob, self.get_object())
+        filename = self._build_filename(job)
+
+        response = StreamingHttpResponse(
+            self._stream_rows(job),
+            content_type="text/csv",
+        )
+        response["Content-Disposition"] = f'attachment; filename="{filename}"'
+        return response
+
+    def _stream_rows(self, job: ExtractionJob) -> Generator[str, None, None]:
+        """Yield serialized CSV rows for the response."""
+        pseudo_buffer = _Echo()
+        writer = csv.writer(pseudo_buffer)
+        yield "\ufeff"
+        for row in iter_extraction_result_rows(job):
+            yield writer.writerow(row)
+
+    def _build_filename(self, job: ExtractionJob) -> str:
+        """Generate a descriptive CSV filename for the extraction job."""
+        slug = slugify(job.title) or "results"
+        return f"extraction_job_{job.pk}_{slug}.csv"

From a4a4cb2b208bacbad0dcb0707b0865a13ca4f147 Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Fri, 7 Nov 2025 10:58:25 +0100
Subject: [PATCH 04/93] Dont download report text

---
 radis/extractions/utils/csv_export.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/radis/extractions/utils/csv_export.py b/radis/extractions/utils/csv_export.py
index 9b28fc20..3d83fb43 100644
--- a/radis/extractions/utils/csv_export.py
+++ b/radis/extractions/utils/csv_export.py
@@ -5,8 +5,6 @@
 from collections.abc import Iterable, Sequence
 from typing import Any
 
-from django.db.models import QuerySet
-
 from radis.extractions.models import ExtractionInstance, ExtractionJob
 
 
@@ -28,20 +26,22 @@ def iter_extraction_result_rows(job: ExtractionJob) -> Iterable[Sequence[str]]:
     header.extend(field_names)
     yield header
 
-    instances: QuerySet[ExtractionInstance] = ExtractionInstance.objects.filter(
-        task__job=job
-    ).order_by("pk")
+    instances = (
+        ExtractionInstance.objects.filter(task__job=job)
+        .order_by("pk")
+        .values_list("pk", "report_id", "is_processed", "output")
+    )
 
-    for instance in instances.iterator():
+    for instance_id, report_id, is_processed, output in instances.iterator():
         row: list[str] = [
-            str(instance.pk),
-            str(instance.report_id) if instance.report_id else "",
-            "yes" if instance.is_processed else "no",
+            str(instance_id),
+            str(report_id) if report_id else "",
+            "yes" if is_processed else "no",
         ]
 
-        output: dict[str, Any] = instance.output or {}
+        output_dict: dict[str, Any] = output or {}
         for field_name in field_names:
-            value = output.get(field_name)
+            value = output_dict.get(field_name)
             row.append("" if value is None else str(value))
 
         yield row

From ce4091c9a863ac9a71fb012ccee985fa11e69cf5 Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Fri, 7 Nov 2025 11:08:05 +0100
Subject: [PATCH 05/93] adding tests

---
 radis/extractions/tests/test_views.py | 57 ++++++++++++++++++++++++++-
 1 file changed, 56 insertions(+), 1 deletion(-)

diff --git a/radis/extractions/tests/test_views.py b/radis/extractions/tests/test_views.py
index bb297864..14530854 100644
--- a/radis/extractions/tests/test_views.py
+++ b/radis/extractions/tests/test_views.py
@@ -1,13 +1,14 @@
 import pytest
 from adit_radis_shared.accounts.factories import GroupFactory, UserFactory
 from django.contrib.auth.models import Permission
-from django.test import Client
+from django.test import Client, override_settings
 
 from radis.core.models import AnalysisTask
 from radis.extractions.factories import (
     ExtractionInstanceFactory,
     ExtractionJobFactory,
     ExtractionTaskFactory,
+    OutputFieldFactory,
 )
 from radis.extractions.models import ExtractionJob
 from radis.reports.factories import LanguageFactory, ReportFactory
@@ -29,6 +30,10 @@ def create_test_extraction_task(job=None):
     return ExtractionTaskFactory.create(job=job)
 
 
+def _hide_toolbar(_request):
+    return False
+
+
 @pytest.mark.django_db
 def test_extraction_job_list_view(client: Client):
     user = UserFactory.create(is_active=True)
@@ -202,6 +207,56 @@ def test_extraction_result_list_view(client: Client):
     assert response.status_code == 200
 
 
+@override_settings(DEBUG_TOOLBAR_CONFIG={"SHOW_TOOLBAR_CALLBACK": _hide_toolbar})
+@pytest.mark.django_db
+def test_extraction_result_download_view(client: Client):
+    user = UserFactory.create(is_active=True)
+    job = create_test_extraction_job(owner=user)
+
+    OutputFieldFactory.create(job=job, name="field_one")
+    OutputFieldFactory.create(job=job, name="field_two")
+
+    task = create_test_extraction_task(job=job)
+    language = LanguageFactory.create(code="en")
+    report = ReportFactory.create(language=language)
+    instance = ExtractionInstanceFactory.create(
+        task=task,
+        report=report,
+        is_processed=True,
+        output={"field_one": "value", "field_two": 42},
+    )
+
+    client.force_login(user)
+    response = client.get(f"/extractions/jobs/{job.pk}/results/download/")
+    assert response.status_code == 200
+    assert response["Content-Type"].startswith("text/csv")
+    assert f"extraction_job_{job.pk}" in response["Content-Disposition"]
+
+    chunks: list[bytes] = []
+    for chunk in response.streaming_content:
+        if isinstance(chunk, bytes):
+            chunks.append(chunk)
+        else:
+            chunks.append(chunk.encode("utf-8"))
+    csv_bytes = b"".join(chunks)
+    csv_text = csv_bytes.decode("utf-8-sig")
+
+    lines = [line.strip() for line in csv_text.strip().splitlines()]
+    assert lines[0] == "instance_id,report_id,is_processed,field_one,field_two"
+    assert lines[1] == f"{instance.pk},{instance.report_id},yes,value,42"
+
+
+@override_settings(DEBUG_TOOLBAR_CONFIG={"SHOW_TOOLBAR_CALLBACK": _hide_toolbar})
+@pytest.mark.django_db
+def test_extraction_result_download_view_unauthorized(client: Client):
+    owner = UserFactory.create(is_active=True)
+    other_user = UserFactory.create(is_active=True)
+    job = create_test_extraction_job(owner=owner)
+    client.force_login(other_user)
+    response = client.get(f"/extractions/jobs/{job.pk}/results/download/")
+    assert response.status_code == 404
+
+
 @pytest.mark.django_db
 def test_extraction_task_detail_view(client: Client):
     user = UserFactory.create(is_active=True)

From 395cad6540f9bc392f6e92ed3a8a885b97eac7f7 Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Fri, 7 Nov 2025 14:26:36 +0100
Subject: [PATCH 06/93] linter error

---
 radis/extractions/urls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/radis/extractions/urls.py b/radis/extractions/urls.py
index d7c44ba2..71184dc1 100644
--- a/radis/extractions/urls.py
+++ b/radis/extractions/urls.py
@@ -12,8 +12,8 @@
     ExtractionJobRetryView,
     ExtractionJobVerifyView,
     ExtractionJobWizardView,
-    ExtractionResultListView,
     ExtractionResultDownloadView,
+    ExtractionResultListView,
     ExtractionTaskDeleteView,
     ExtractionTaskDetailView,
     ExtractionTaskResetView,

From 5a85dd5f9eaa79782b945dc95fe42aececf16cbd Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Fri, 7 Nov 2025 14:35:10 +0100
Subject: [PATCH 07/93] add test cases where there are no instances or
 extraction result has no output fields

---
 radis/extractions/tests/test_views.py | 61 +++++++++++++++++++++++----
 1 file changed, 52 insertions(+), 9 deletions(-)

diff --git a/radis/extractions/tests/test_views.py b/radis/extractions/tests/test_views.py
index 14530854..f1e25bc6 100644
--- a/radis/extractions/tests/test_views.py
+++ b/radis/extractions/tests/test_views.py
@@ -34,6 +34,17 @@ def _hide_toolbar(_request):
     return False
 
 
+def _collect_csv(response) -> str:
+    chunks: list[bytes] = []
+    for chunk in response.streaming_content:
+        if isinstance(chunk, bytes):
+            chunks.append(chunk)
+        else:
+            chunks.append(chunk.encode("utf-8"))
+    csv_bytes = b"".join(chunks)
+    return csv_bytes.decode("utf-8-sig")
+
+
 @pytest.mark.django_db
 def test_extraction_job_list_view(client: Client):
     user = UserFactory.create(is_active=True)
@@ -232,15 +243,7 @@ def test_extraction_result_download_view(client: Client):
     assert response["Content-Type"].startswith("text/csv")
     assert f"extraction_job_{job.pk}" in response["Content-Disposition"]
 
-    chunks: list[bytes] = []
-    for chunk in response.streaming_content:
-        if isinstance(chunk, bytes):
-            chunks.append(chunk)
-        else:
-            chunks.append(chunk.encode("utf-8"))
-    csv_bytes = b"".join(chunks)
-    csv_text = csv_bytes.decode("utf-8-sig")
-
+    csv_text = _collect_csv(response)
     lines = [line.strip() for line in csv_text.strip().splitlines()]
     assert lines[0] == "instance_id,report_id,is_processed,field_one,field_two"
     assert lines[1] == f"{instance.pk},{instance.report_id},yes,value,42"
@@ -257,6 +260,46 @@ def test_extraction_result_download_view_unauthorized(client: Client):
     assert response.status_code == 404
 
 
+@override_settings(DEBUG_TOOLBAR_CONFIG={"SHOW_TOOLBAR_CALLBACK": _hide_toolbar})
+@pytest.mark.django_db
+def test_extraction_result_download_view_no_instances(client: Client):
+    user = UserFactory.create(is_active=True)
+    job = create_test_extraction_job(owner=user)
+    OutputFieldFactory.create(job=job, name="field_one")
+
+    client.force_login(user)
+    response = client.get(f"/extractions/jobs/{job.pk}/results/download/")
+    assert response.status_code == 200
+
+    csv_text = _collect_csv(response)
+    assert csv_text.strip() == "instance_id,report_id,is_processed,field_one"
+
+
+@override_settings(DEBUG_TOOLBAR_CONFIG={"SHOW_TOOLBAR_CALLBACK": _hide_toolbar})
+@pytest.mark.django_db
+def test_extraction_result_download_view_no_output_fields(client: Client):
+    user = UserFactory.create(is_active=True)
+    job = create_test_extraction_job(owner=user)
+    task = create_test_extraction_task(job=job)
+    language = LanguageFactory.create(code="en")
+    report = ReportFactory.create(language=language)
+    instance = ExtractionInstanceFactory.create(
+        task=task,
+        report=report,
+        is_processed=False,
+        output={},
+    )
+
+    client.force_login(user)
+    response = client.get(f"/extractions/jobs/{job.pk}/results/download/")
+    assert response.status_code == 200
+
+    csv_text = _collect_csv(response)
+    lines = csv_text.strip().splitlines()
+    assert lines[0] == "instance_id,report_id,is_processed"
+    assert lines[1] == f"{instance.pk},{instance.report_id},no"
+
+
 @pytest.mark.django_db
 def test_extraction_task_detail_view(client: Client):
     user = UserFactory.create(is_active=True)

From 96c500a39843c93c2226e21f1ca2f725435b3127 Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Fri, 7 Nov 2025 15:02:02 +0100
Subject: [PATCH 08/93] standardize boolean values to yes/no

---
 radis/extractions/tests/test_views.py |  9 +++++----
 radis/extractions/utils/csv_export.py | 11 ++++++++++-
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/radis/extractions/tests/test_views.py b/radis/extractions/tests/test_views.py
index f1e25bc6..af099215 100644
--- a/radis/extractions/tests/test_views.py
+++ b/radis/extractions/tests/test_views.py
@@ -226,6 +226,7 @@ def test_extraction_result_download_view(client: Client):
 
     OutputFieldFactory.create(job=job, name="field_one")
     OutputFieldFactory.create(job=job, name="field_two")
+    OutputFieldFactory.create(job=job, name="field_bool")
 
     task = create_test_extraction_task(job=job)
     language = LanguageFactory.create(code="en")
@@ -234,7 +235,7 @@ def test_extraction_result_download_view(client: Client):
         task=task,
         report=report,
         is_processed=True,
-        output={"field_one": "value", "field_two": 42},
+        output={"field_one": "value", "field_two": 42, "field_bool": False},
     )
 
     client.force_login(user)
@@ -245,8 +246,8 @@ def test_extraction_result_download_view(client: Client):
 
     csv_text = _collect_csv(response)
     lines = [line.strip() for line in csv_text.strip().splitlines()]
-    assert lines[0] == "instance_id,report_id,is_processed,field_one,field_two"
-    assert lines[1] == f"{instance.pk},{instance.report_id},yes,value,42"
+    assert lines[0] == "instance_id,report_id,is_processed,field_one,field_two,field_bool"
+    assert lines[1] == f"{instance.pk},{instance.report.pk},yes,value,42,no"
 
 
 @override_settings(DEBUG_TOOLBAR_CONFIG={"SHOW_TOOLBAR_CALLBACK": _hide_toolbar})
@@ -297,7 +298,7 @@ def test_extraction_result_download_view_no_output_fields(client: Client):
     csv_text = _collect_csv(response)
     lines = csv_text.strip().splitlines()
     assert lines[0] == "instance_id,report_id,is_processed"
-    assert lines[1] == f"{instance.pk},{instance.report_id},no"
+    assert lines[1] == f"{instance.pk},{instance.report.pk},no"
 
 
 @pytest.mark.django_db
diff --git a/radis/extractions/utils/csv_export.py b/radis/extractions/utils/csv_export.py
index 3d83fb43..6759c050 100644
--- a/radis/extractions/utils/csv_export.py
+++ b/radis/extractions/utils/csv_export.py
@@ -8,6 +8,15 @@
 from radis.extractions.models import ExtractionInstance, ExtractionJob
 
 
+def _format_cell(value: Any) -> str:
+    """Format a single output value for CSV export."""
+    if value is None:
+        return ""
+    if isinstance(value, bool):
+        return "yes" if value else "no"
+    return str(value)
+
+
 def iter_extraction_result_rows(job: ExtractionJob) -> Iterable[Sequence[str]]:
     """Yield rows for the extraction results CSV.
 
@@ -42,6 +51,6 @@ def iter_extraction_result_rows(job: ExtractionJob) -> Iterable[Sequence[str]]:
         output_dict: dict[str, Any] = output or {}
         for field_name in field_names:
             value = output_dict.get(field_name)
-            row.append("" if value is None else str(value))
+            row.append(_format_cell(value))
 
         yield row

From 03e184157ff3036c0f3696d4ad7d1c77239813f9 Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Fri, 7 Nov 2025 16:11:58 +0100
Subject: [PATCH 09/93] button colrs

---
 .../templates/extractions/extraction_job_detail.html          | 2 +-
 .../templates/extractions/extraction_result_list.html         | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/radis/extractions/templates/extractions/extraction_job_detail.html b/radis/extractions/templates/extractions/extraction_job_detail.html
index 4d874c80..e580c214 100644
--- a/radis/extractions/templates/extractions/extraction_job_detail.html
+++ b/radis/extractions/templates/extractions/extraction_job_detail.html
@@ -128,7 +128,7 @@ <h5>Output Fields</h5>
                 View Results
             </a>
             <a href="{% url 'extraction_result_download' job.id %}"
-               class="btn btn-sm btn-outline-secondary">
+               class="btn btn-sm btn-success">
                 {% bootstrap_icon "download" %}
                 Download CSV
             </a>
diff --git a/radis/extractions/templates/extractions/extraction_result_list.html b/radis/extractions/templates/extractions/extraction_result_list.html
index 6ca9b510..b44fd9c5 100644
--- a/radis/extractions/templates/extractions/extraction_result_list.html
+++ b/radis/extractions/templates/extractions/extraction_result_list.html
@@ -9,12 +9,12 @@
     <c-slot name="right">
     <div class="d-flex gap-2">
         <a href="{% url 'extraction_result_download' job.id %}"
-           class="btn btn-success">
+           class="btn btn-sm btn-success">
             {% bootstrap_icon "download" %}
             Download CSV
         </a>
         <a href="{% url 'extraction_job_detail' job.id %}"
-           class="btn btn-secondary">
+           class="btn btn-sm btn-primary">
             {% bootstrap_icon "arrow-return-left" %}
             View Job
         </a>

From 9b2b368f6ea27aa820ee18b5805f9f62c433f568 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 10 Nov 2025 10:10:45 +0000
Subject: [PATCH 10/93] change models

---
 ...03_alter_extractionjob_options_and_more.py | 63 ++++++++++++++++++-
 radis/extractions/models.py                   |  4 +-
 ...9_alter_subscribeditem_options_and_more.py | 47 +++++++++++++-
 3 files changed, 107 insertions(+), 7 deletions(-)

diff --git a/radis/extractions/migrations/0003_alter_extractionjob_options_and_more.py b/radis/extractions/migrations/0003_alter_extractionjob_options_and_more.py
index 660c2081..c04ee4f9 100644
--- a/radis/extractions/migrations/0003_alter_extractionjob_options_and_more.py
+++ b/radis/extractions/migrations/0003_alter_extractionjob_options_and_more.py
@@ -1,12 +1,14 @@
-# Generated by Django 5.2.7 on 2025-11-04 14:20
+# Generated by Django 5.2.7 on 2025-11-10 09:59
 
-from django.db import migrations
+import django.db.models.deletion
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
 
     dependencies = [
         ("extractions", "0002_procrastinate_on_delete"),
+        ("subscriptions", "0009_alter_subscribeditem_options_and_more"),
     ]
 
     operations = [
@@ -14,8 +16,65 @@ class Migration(migrations.Migration):
             name="extractionjob",
             options={"ordering": ["-created_at", "title"]},
         ),
+        migrations.RemoveConstraint(
+            model_name="outputfield",
+            name="unique_output_field_name_per_job",
+        ),
         migrations.RemoveIndex(
             model_name="extractionjob",
             name="extractions_owner_i_49b05e_idx",
         ),
+        migrations.RemoveField(
+            model_name="outputfield",
+            name="optional",
+        ),
+        migrations.AddField(
+            model_name="outputfield",
+            name="subscription",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="extraction_fields",
+                to="subscriptions.subscription",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="outputfield",
+            name="job",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="output_fields",
+                to="extractions.extractionjob",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="outputfield",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("job__isnull", False)),
+                fields=("name", "job_id"),
+                name="unique_output_field_name_per_job",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="outputfield",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("subscription__isnull", False)),
+                fields=("name", "subscription_id"),
+                name="unique_output_field_name_per_subscription",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="outputfield",
+            constraint=models.CheckConstraint(
+                condition=models.Q(
+                    models.Q(("job__isnull", False), ("subscription__isnull", True)),
+                    models.Q(("job__isnull", True), ("subscription__isnull", False)),
+                    _connector="OR",
+                ),
+                name="output_field_exactly_one_parent",
+            ),
+        ),
     ]
diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index b80bb3a5..c93aeca5 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -89,9 +89,7 @@ class OutputField(models.Model):
     job = models.ForeignKey[
         ExtractionJob
     ](ExtractionJob, null=True, blank=True, on_delete=models.CASCADE, related_name="output_fields")
-    subscription = models.ForeignKey[
-        "subscriptions.Subscription"
-    ](
+    subscription = models.ForeignKey(
         "subscriptions.Subscription",
         null=True,
         blank=True,
diff --git a/radis/subscriptions/migrations/0009_alter_subscribeditem_options_and_more.py b/radis/subscriptions/migrations/0009_alter_subscribeditem_options_and_more.py
index de7e0aec..4268507b 100644
--- a/radis/subscriptions/migrations/0009_alter_subscribeditem_options_and_more.py
+++ b/radis/subscriptions/migrations/0009_alter_subscribeditem_options_and_more.py
@@ -1,6 +1,7 @@
-# Generated by Django 5.2.7 on 2025-11-04 14:20
+# Generated by Django 5.2.7 on 2025-11-10 09:59
 
-from django.db import migrations
+import django.db.models.deletion
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
@@ -21,4 +22,46 @@ class Migration(migrations.Migration):
             name="subscription",
             options={"ordering": ["-created_at"]},
         ),
+        migrations.RenameField(
+            model_name="subscribeditem",
+            old_name="answers",
+            new_name="extraction_results",
+        ),
+        migrations.AddField(
+            model_name="subscribeditem",
+            name="filter_results",
+            field=models.JSONField(blank=True, null=True),
+        ),
+        migrations.CreateModel(
+            name="FilterQuestion",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("question", models.CharField(max_length=300)),
+                (
+                    "expected_answer",
+                    models.CharField(
+                        choices=[("Y", "Yes"), ("N", "No")], default="Y", max_length=1
+                    ),
+                ),
+                (
+                    "subscription",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="filter_questions",
+                        to="subscriptions.subscription",
+                    ),
+                ),
+            ],
+        ),
+        migrations.DeleteModel(
+            name="Question",
+        ),
     ]

From 5d0cc4e820f13175343a95675431f149f6571c80 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 10 Nov 2025 10:11:21 +0000
Subject: [PATCH 11/93] remove previous migration files

---
 .../0003_subscription_output_fields.py        | 72 -------------------
 .../0009_separate_filter_and_extraction.py    | 49 -------------
 2 files changed, 121 deletions(-)
 delete mode 100644 radis/extractions/migrations/0003_subscription_output_fields.py
 delete mode 100644 radis/subscriptions/migrations/0009_separate_filter_and_extraction.py

diff --git a/radis/extractions/migrations/0003_subscription_output_fields.py b/radis/extractions/migrations/0003_subscription_output_fields.py
deleted file mode 100644
index 7c6acf18..00000000
--- a/radis/extractions/migrations/0003_subscription_output_fields.py
+++ /dev/null
@@ -1,72 +0,0 @@
-# Generated by Django 5.2.7 on 2025-11-03 22:58
-
-import django.db.models.deletion
-from django.db import migrations, models
-from django.db.models import Q
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ("extractions", "0002_procrastinate_on_delete"),
-        ("subscriptions", "0008_rename_filter_fields_result_subscribeditem_answers_and_more"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="outputfield",
-            name="job",
-            field=models.ForeignKey(
-                blank=True,
-                null=True,
-                on_delete=django.db.models.deletion.CASCADE,
-                related_name="output_fields",
-                to="extractions.extractionjob",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="outputfield",
-            name="unique_output_field_name_per_job",
-        ),
-        migrations.RemoveField(
-            model_name="outputfield",
-            name="optional",
-        ),
-        migrations.AddField(
-            model_name="outputfield",
-            name="subscription",
-            field=models.ForeignKey(
-                blank=True,
-                null=True,
-                on_delete=django.db.models.deletion.CASCADE,
-                related_name="extraction_fields",
-                to="subscriptions.subscription",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="outputfield",
-            constraint=models.UniqueConstraint(
-                condition=Q(job__isnull=False),
-                fields=("name", "job_id"),
-                name="unique_output_field_name_per_job",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="outputfield",
-            constraint=models.UniqueConstraint(
-                condition=Q(subscription__isnull=False),
-                fields=("name", "subscription_id"),
-                name="unique_output_field_name_per_subscription",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="outputfield",
-            constraint=models.CheckConstraint(
-                check=(
-                    Q(job__isnull=False, subscription__isnull=True)
-                    | Q(job__isnull=True, subscription__isnull=False)
-                ),
-                name="output_field_exactly_one_parent",
-            ),
-        ),
-    ]
diff --git a/radis/subscriptions/migrations/0009_separate_filter_and_extraction.py b/radis/subscriptions/migrations/0009_separate_filter_and_extraction.py
deleted file mode 100644
index 7c923225..00000000
--- a/radis/subscriptions/migrations/0009_separate_filter_and_extraction.py
+++ /dev/null
@@ -1,49 +0,0 @@
-# Generated by Django 5.2.7 on 2025-11-03 21:25
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        (
-            "subscriptions",
-            "0008_rename_filter_fields_result_subscribeditem_answers_and_more",
-        ),
-    ]
-
-    operations = [
-        migrations.RenameModel(
-            old_name="Question",
-            new_name="FilterQuestion",
-        ),
-        migrations.AddField(
-            model_name="filterquestion",
-            name="expected_answer",
-            field=models.CharField(
-                choices=[("Y", "Yes"), ("N", "No")],
-                default="Y",
-                max_length=1,
-            ),
-        ),
-        migrations.AlterField(
-            model_name="filterquestion",
-            name="subscription",
-            field=models.ForeignKey(
-                on_delete=django.db.models.deletion.CASCADE,
-                related_name="filter_questions",
-                to="subscriptions.subscription",
-            ),
-        ),
-        migrations.RenameField(
-            model_name="subscribeditem",
-            old_name="answers",
-            new_name="filter_results",
-        ),
-        migrations.AddField(
-            model_name="subscribeditem",
-            name="extraction_results",
-            field=models.JSONField(blank=True, null=True),
-        ),
-    ]

From 78b6c3bb0508479164389297463c4439b4cd4044 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 10 Nov 2025 10:58:41 +0000
Subject: [PATCH 12/93] fix lint errors

---
 radis/extractions/models.py                   |  8 +-
 radis/subscriptions/factories.py              |  6 +-
 radis/subscriptions/forms.py                  | 21 ++---
 radis/subscriptions/processors.py             |  2 +-
 .../_subscribed_item_preview.html             | 12 +--
 .../subscriptions/subscription_detail.html    |  4 +-
 .../tests/test_processor_utils.py             | 83 -------------------
 radis/subscriptions/tests/test_views.py       |  4 +-
 radis/subscriptions/utils/processor_utils.py  | 15 +++-
 radis/subscriptions/views.py                  |  4 +-
 10 files changed, 34 insertions(+), 125 deletions(-)
 delete mode 100644 radis/subscriptions/tests/test_processor_utils.py

diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index c93aeca5..4b25fc6e 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -86,9 +86,9 @@ class OutputField(models.Model):
         max_length=1, choices=OutputType.choices, default=OutputType.TEXT
     )
     get_output_type_display: Callable[[], str]
-    job = models.ForeignKey[
-        ExtractionJob
-    ](ExtractionJob, null=True, blank=True, on_delete=models.CASCADE, related_name="output_fields")
+    job = models.ForeignKey[ExtractionJob](
+        ExtractionJob, null=True, blank=True, on_delete=models.CASCADE, related_name="output_fields"
+    )
     subscription = models.ForeignKey(
         "subscriptions.Subscription",
         null=True,
@@ -115,7 +115,7 @@ class Meta:
                     | Q(job__isnull=True, subscription__isnull=False)
                 ),
                 name="output_field_exactly_one_parent",
-            )
+            ),
         ]
 
     def __str__(self) -> str:
diff --git a/radis/subscriptions/factories.py b/radis/subscriptions/factories.py
index da103952..73e61050 100644
--- a/radis/subscriptions/factories.py
+++ b/radis/subscriptions/factories.py
@@ -4,7 +4,7 @@
 
 from radis.reports.factories import LanguageFactory, ReportFactory
 
-from .models import Question, SubscribedItem, Subscription, SubscriptionJob, SubscriptionTask
+from .models import FilterQuestion, SubscribedItem, Subscription, SubscriptionJob, SubscriptionTask
 
 
 class SubscriptionFactory(BaseDjangoModelFactory[Subscription]):
@@ -25,9 +25,9 @@ class Meta:
     send_finished_mail = factory.Faker("boolean")
 
 
-class QuestionFactory(BaseDjangoModelFactory[Question]):
+class FilterQuestionFactory(BaseDjangoModelFactory[FilterQuestion]):
     class Meta:
-        model = Question
+        model = FilterQuestion
 
     subscription = factory.SubFactory(SubscriptionFactory)
     question = factory.Faker("sentence", nb_words=6, variable_nb_words=True)
diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index cb7af395..26cc531f 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -6,11 +6,10 @@
 
 from radis.core.constants import LANGUAGE_LABELS
 from radis.core.layouts import Formset, RangeSlider
+from radis.extractions.models import OutputField
 from radis.reports.models import Language, Modality
 from radis.search.forms import AGE_STEP, MAX_AGE, MIN_AGE
 
-from radis.extractions.models import OutputField
-
 from .models import FilterQuestion, Subscription
 from .site import subscription_retrieval_providers
 
@@ -160,7 +159,7 @@ def __init__(self, *args, **kwargs):
 
         self.fields["question"].required = False
         self.fields["expected_answer"].required = False
-        self.fields["expected_answer"].choices = [
+        self.fields["expected_answer"].choices = [  # type: ignore[attr-defined]
             ("", "Select expected answer"),
             *FilterQuestion.ExpectedAnswer.choices,
         ]
@@ -180,17 +179,15 @@ def __init__(self, *args, **kwargs):
 
     def clean(self) -> dict[str, Any]:
         cleaned_data = super().clean()
-        question = cleaned_data.get("question")
-        expected_answer = cleaned_data.get("expected_answer")
-
-        if not question and not expected_answer:
-            return cleaned_data
+        assert cleaned_data
 
-        if question and not expected_answer:
-            cleaned_data["expected_answer"] = FilterQuestion.ExpectedAnswer.YES
+        question = cleaned_data["question"]
+        expected_answer = cleaned_data["expected_answer"]
 
-        if not question:
-            raise forms.ValidationError("Question text is required when specifying a filter.")
+        if not question and not expected_answer:
+            raise forms.ValidationError(
+                "Question and Expected Answer text is required when specifying a filter."
+            )
 
         return cleaned_data
 
diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index 0f6b0a9e..2ee361f9 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -1,7 +1,7 @@
 import logging
-from typing import Any
 from concurrent.futures import Future, ThreadPoolExecutor
 from string import Template
+from typing import Any
 
 from adit_radis_shared.common.types import User
 from django import db
diff --git a/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html b/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
index fbf65f99..e8317076 100644
--- a/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
+++ b/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
@@ -9,11 +9,7 @@
                     <div>
                         <div class="fw-semibold small text-uppercase text-secondary">Filter results</div>
                         <ul class="mb-2 small">
-                            {% for question, value in filter_results %}
-                                <li>
-                                    {{ question.question }} → {{ value|yesno:"Yes,No" }}
-                                </li>
-                            {% endfor %}
+                            {% for question, value in filter_results %}<li>{{ question.question }} → {{ value|yesno:"Yes,No" }}</li>{% endfor %}
                         </ul>
                     </div>
                 {% endif %}
@@ -23,11 +19,7 @@
                     <div>
                         <div class="fw-semibold small text-uppercase text-secondary">Extraction results</div>
                         <ul class="mb-2 small">
-                            {% for field, value in extraction_results %}
-                                <li>
-                                    {{ field.name }} → {{ value|default_if_none:"—" }}
-                                </li>
-                            {% endfor %}
+                            {% for field, value in extraction_results %}<li>{{ field.name }} → {{ value|default_if_none:"—" }}</li>{% endfor %}
                         </ul>
                     </div>
                 {% endif %}
diff --git a/radis/subscriptions/templates/subscriptions/subscription_detail.html b/radis/subscriptions/templates/subscriptions/subscription_detail.html
index dc6cdb46..8ba95b04 100644
--- a/radis/subscriptions/templates/subscriptions/subscription_detail.html
+++ b/radis/subscriptions/templates/subscriptions/subscription_detail.html
@@ -141,9 +141,7 @@ <h4 class="mt-4">Extraction Fields</h4>
                     <dt class="col-sm-3">{{ field.name }}</dt>
                     <dd class="col-sm-9">
                         {{ field.description|default:"No description provided." }}
-                        <div class="text-muted small">
-                            Type: {{ field.get_output_type_display }}
-                        </div>
+                        <div class="text-muted small">Type: {{ field.get_output_type_display }}</div>
                     </dd>
                 {% endfor %}
             </dl>
diff --git a/radis/subscriptions/tests/test_processor_utils.py b/radis/subscriptions/tests/test_processor_utils.py
deleted file mode 100644
index f2be9b21..00000000
--- a/radis/subscriptions/tests/test_processor_utils.py
+++ /dev/null
@@ -1,83 +0,0 @@
-from dataclasses import dataclass
-
-from radis.extractions.models import OutputType
-from radis.subscriptions.utils.processor_utils import (
-    build_extraction_schema,
-    build_filter_schema,
-    generate_extraction_fields_prompt,
-    generate_filter_questions_prompt,
-)
-
-
-@dataclass
-class DummyFilterQuestion:
-    pk: int
-    question: str
-    expected_answer_bool: bool
-
-
-class DummyFilterQuestionSet:
-    def __init__(self, *questions: DummyFilterQuestion):
-        self._questions = list(questions)
-
-    def all(self):
-        return list(self._questions)
-
-
-@dataclass
-class DummyExtractionField:
-    pk: int
-    name: str
-    description: str
-    output_type: str
-
-    def get_output_type_display(self) -> str:
-        return {
-            OutputType.TEXT: "Text",
-            OutputType.NUMERIC: "Numeric",
-            OutputType.BOOLEAN: "Boolean",
-        }[self.output_type]
-
-
-class DummyExtractionFieldSet:
-    def __init__(self, *fields: DummyExtractionField):
-        self._fields = list(fields)
-
-    def all(self):
-        return list(self._fields)
-
-
-def test_build_subscription_schema_and_prompts_without_database():
-    questions = DummyFilterQuestionSet(
-        DummyFilterQuestion(pk=1, question="Contains pneumonia?", expected_answer_bool=True)
-    )
-    fields = DummyExtractionFieldSet(
-        DummyExtractionField(
-            pk=10,
-            name="diagnosis",
-            description="Primary diagnosis mentioned in the report",
-            output_type=OutputType.TEXT,
-        )
-    )
-
-    filter_bundle = build_filter_schema(questions)
-    extraction_bundle = build_extraction_schema(fields)
-
-    assert len(filter_bundle.mapping) == 1
-    assert len(extraction_bundle.mapping) == 1
-
-    filter_field_name, mapped_question = filter_bundle.mapping[0]
-    assert filter_field_name.startswith("filter_")
-    assert mapped_question.pk == 1
-
-    extraction_field_name, mapped_field = extraction_bundle.mapping[0]
-    assert extraction_field_name.startswith("extraction_")
-    assert mapped_field.pk == 10
-
-    filter_prompt = generate_filter_questions_prompt(filter_bundle.mapping)
-    assert "filter_" in filter_prompt
-    assert "Contains pneumonia?" in filter_prompt
-
-    extraction_prompt = generate_extraction_fields_prompt(extraction_bundle.mapping)
-    assert "extraction_" in extraction_prompt
-    assert "diagnosis" in extraction_prompt
diff --git a/radis/subscriptions/tests/test_views.py b/radis/subscriptions/tests/test_views.py
index eff79f4b..837269a6 100644
--- a/radis/subscriptions/tests/test_views.py
+++ b/radis/subscriptions/tests/test_views.py
@@ -5,7 +5,7 @@
 from radis.reports.factories import LanguageFactory
 from radis.reports.models import Modality
 from radis.subscriptions.factories import (
-    QuestionFactory,
+    FilterQuestionFactory,
     SubscriptionFactory,
 )
 from radis.subscriptions.models import Subscription
@@ -190,7 +190,7 @@ def test_subscription_update_view_unauthorized(client: Client):
 def test_subscription_update_view_post_valid(client: Client):
     user = UserFactory.create(is_active=True)
     subscription = create_test_subscription(owner=user, name="Original Name")
-    question = QuestionFactory.create(subscription=subscription)
+    question = FilterQuestionFactory.create(subscription=subscription)
 
     client.force_login(user)
 
diff --git a/radis/subscriptions/utils/processor_utils.py b/radis/subscriptions/utils/processor_utils.py
index c728e1e8..299e7eb1 100644
--- a/radis/subscriptions/utils/processor_utils.py
+++ b/radis/subscriptions/utils/processor_utils.py
@@ -35,7 +35,11 @@ def build_filter_schema(questions: QuerySet[FilterQuestion]) -> FilterSchemaBund
         mapping.append((field_name, question))
 
     model_name = "SubscriptionFilterResultsModel"
-    schema = create_model(model_name, **field_definitions) if field_definitions else create_model(model_name)
+    schema = (
+        create_model(model_name, **field_definitions)
+        if field_definitions
+        else create_model(model_name)
+    )
     return FilterSchemaBundle(schema, mapping)
 
 
@@ -59,7 +63,11 @@ def build_extraction_schema(fields: QuerySet[OutputField]) -> ExtractionSchemaBu
         mapping.append((field_name, field))
 
     model_name = "SubscriptionExtractionResultsModel"
-    schema = create_model(model_name, **field_definitions) if field_definitions else create_model(model_name)
+    schema = (
+        create_model(model_name, **field_definitions)
+        if field_definitions
+        else create_model(model_name)
+    )
     return ExtractionSchemaBundle(schema, mapping)
 
 
@@ -81,7 +89,6 @@ def generate_extraction_fields_prompt(mapping: list[tuple[str, OutputField]]) ->
     for field_name, field in mapping:
         description = field.description or "No description provided."
         lines.append(
-            f"{field_name}: {field.name} — {description} "
-            f"[type: {field.get_output_type_display()}]"
+            f"{field_name}: {field.name} — {description} [type: {field.get_output_type_display()}]"
         )
     return "\n".join(lines)
diff --git a/radis/subscriptions/views.py b/radis/subscriptions/views.py
index d54edd12..8d4fefd4 100644
--- a/radis/subscriptions/views.py
+++ b/radis/subscriptions/views.py
@@ -126,9 +126,7 @@ def get_queryset(self) -> QuerySet[Subscription]:
     def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
         ctx = super().get_context_data(**kwargs)
         if self.request.POST:
-            ctx["filter_formset"] = FilterQuestionFormSet(
-                self.request.POST, instance=self.object
-            )
+            ctx["filter_formset"] = FilterQuestionFormSet(self.request.POST, instance=self.object)
             ctx["extraction_formset"] = ExtractionFieldFormSet(
                 self.request.POST, instance=self.object
             )

From 204ff4c9eb72fdd015928d55b9772dc95f76cb6b Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 10 Nov 2025 13:53:19 +0000
Subject: [PATCH 13/93] form fixes

---
 radis/extractions/models.py  |  2 +-
 radis/subscriptions/forms.py | 38 ++++++++++++++++++------------------
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index 4b25fc6e..622e4fe2 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -110,7 +110,7 @@ class Meta:
                 name="unique_output_field_name_per_subscription",
             ),
             models.CheckConstraint(
-                check=(
+                condition=(
                     Q(job__isnull=False, subscription__isnull=True)
                     | Q(job__isnull=True, subscription__isnull=False)
                 ),
diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index 26cc531f..961d87a4 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -168,14 +168,11 @@ def __init__(self, *args, **kwargs):
         self.helper = FormHelper()
         self.helper.form_tag = False
         self.helper.disable_csrf = True
-        self.helper.layout = Layout(
-            Div(
-                Field("id", type="hidden"),
-                Field("DELETE", type="hidden"),
-                "question",
-                "expected_answer",
-            ),
-        )
+        fields = [Field("id", type="hidden"), "question", "expected_answer"]
+        if "DELETE" in self.fields:
+            fields.insert(1, Field("DELETE", type="hidden"))
+        self.helper.layout = Layout(Div(*fields))
+
 
     def clean(self) -> dict[str, Any]:
         cleaned_data = super().clean()
@@ -206,17 +203,20 @@ def __init__(self, *args, **kwargs):
         self.helper = FormHelper()
         self.helper.form_tag = False
         self.helper.disable_csrf = True
-        self.helper.layout = Layout(
-            Div(
-                Field("id", type="hidden"),
-                Field("DELETE", type="hidden"),
-                Row(
-                    Column("name", css_class="col-6"),
-                    Column("output_type", css_class="col-4"),
-                ),
-                "description",
-            )
-        )
+        
+        fields = [
+            Field("id", type="hidden"),
+            Row(
+                Column("name", css_class="col-6"),
+                Column("output_type", css_class="col-4"),
+            ),
+            "description",
+        ]
+
+        if "DELETE" in self.fields:
+            fields.insert(1, Field("DELETE", type="hidden"))
+
+        self.helper.layout = Layout(Div(*fields))
 
 
 FilterQuestionFormSet = forms.inlineformset_factory(

From b5add3971f71c04b6133af960282006ecb044116 Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Tue, 11 Nov 2025 13:36:47 +0000
Subject: [PATCH 14/93] Adding Selection output type to the existing types for
 extracting output

---
 radis/core/static/core/core.js                | 82 +++++++++++++++++
 radis/core/templates/cotton/formset.html      | 12 +--
 radis/extractions/forms.py                    | 92 ++++++++++++++++++-
 .../0004_outputfield_selection_options.py     | 29 ++++++
 radis/extractions/models.py                   | 30 ++++++
 .../extractions/_selection_options_field.html | 48 ++++++++++
 .../extractions/extraction_job_detail.html    | 10 ++
 .../extraction_job_wizard_summary.html        | 10 ++
 8 files changed, 305 insertions(+), 8 deletions(-)
 create mode 100644 radis/extractions/migrations/0004_outputfield_selection_options.py
 create mode 100644 radis/extractions/templates/extractions/_selection_options_field.html

diff --git a/radis/core/static/core/core.js b/radis/core/static/core/core.js
index cfff69a0..1b6c4d8f 100644
--- a/radis/core/static/core/core.js
+++ b/radis/core/static/core/core.js
@@ -89,3 +89,85 @@ function FormSet(rootEl) {
     },
   };
 }
+
+/**
+ * Manages the dynamic selection options input for extraction output fields.
+ *
+ * @param {HTMLElement} rootEl
+ * @returns {Object}
+ */
+function SelectionOptions(rootEl) {
+  const hiddenInput = rootEl.querySelector("[data-selection-input]");
+  const formContainer =
+    rootEl.closest(".formset-form") ?? rootEl.closest("form") ?? rootEl;
+  const outputTypeField =
+    formContainer.querySelector('select[name$="-output_type"]') ??
+    formContainer.querySelector('select[name="output_type"]');
+
+  return {
+    options: [],
+    maxOptions: 7,
+    supportsSelection: false,
+    init() {
+      this.options = this.parseOptions(hiddenInput?.value);
+      this.updateSupports();
+      if (outputTypeField) {
+        outputTypeField.addEventListener("change", () => {
+          const wasSelection = this.supportsSelection;
+          this.updateSupports();
+          if (!this.supportsSelection) {
+            this.options = [];
+          } else if (!wasSelection && this.options.length === 0) {
+            this.options = this.parseOptions(hiddenInput?.value);
+          }
+        });
+      }
+    },
+    parseOptions(value) {
+      if (!value) {
+        return [];
+      }
+      try {
+        const parsed = JSON.parse(value);
+        if (Array.isArray(parsed)) {
+          return parsed
+            .map((opt) => (typeof opt === "string" ? opt : ""))
+            .filter((opt) => opt !== "");
+        }
+      } catch (err) {
+        console.warn("Invalid selection options payload", err);
+      }
+      return [];
+    },
+    updateSupports() {
+      this.supportsSelection = outputTypeField
+        ? outputTypeField.value === "S"
+        : false;
+    },
+    syncOptions() {
+      if (!hiddenInput) {
+        return;
+      }
+      const sanitized = this.options
+        .map((opt) => (typeof opt === "string" ? opt.trim() : ""))
+        .filter((opt) => opt !== "");
+      hiddenInput.value = JSON.stringify(sanitized);
+    },
+    addOption() {
+      if (!this.supportsSelection || this.options.length >= this.maxOptions) {
+        return;
+      }
+      this.options.push("");
+      this.$nextTick(() => {
+        const inputs = rootEl.querySelectorAll("[data-selection-option-input]");
+        const lastInput = inputs[inputs.length - 1];
+        if (lastInput instanceof HTMLInputElement) {
+          lastInput.focus();
+        }
+      });
+    },
+    removeOption(index) {
+      this.options.splice(index, 1);
+    },
+  };
+}
diff --git a/radis/core/templates/cotton/formset.html b/radis/core/templates/cotton/formset.html
index 20c6af7b..904473a1 100644
--- a/radis/core/templates/cotton/formset.html
+++ b/radis/core/templates/cotton/formset.html
@@ -3,12 +3,12 @@
 <fieldset x-data="FormSet($el)" class="mb-3">
     {% if legend %}<legend>{{ legend }}</legend>{% endif %}
     {{ formset.management_form }}
-    <template>
-        <c-formset-form>{% crispy formset.empty_form %}</c-formset-form>
-    </template>
-    <div class="formset-forms">
-        {% for form in formset %}<c-formset-form>{{ form|crispy }}</c-formset-form>{% endfor %}
-    </div>
+<template>
+    <c-formset-form>{% crispy formset.empty_form %}</c-formset-form>
+</template>
+<div class="formset-forms">
+    {% for form in formset %}<c-formset-form>{% crispy form %}</c-formset-form>{% endfor %}
+</div>
     {% if add_form_label %}
         <div class="d-flex flex-row-reverse">
             <button type="button"
diff --git a/radis/extractions/forms.py b/radis/extractions/forms.py
index 9e6a4bca..dc223eb7 100644
--- a/radis/extractions/forms.py
+++ b/radis/extractions/forms.py
@@ -1,8 +1,9 @@
+import json
 from typing import Any, cast
 
 from adit_radis_shared.accounts.models import User
 from crispy_forms.helper import FormHelper
-from crispy_forms.layout import Column, Layout, Row, Submit
+from crispy_forms.layout import Column, Div, HTML, Layout, Row, Submit
 from django import forms
 from django.conf import settings
 from django.db.models import QuerySet
@@ -14,7 +15,7 @@
 from radis.search.site import Search, SearchFilters
 from radis.search.utils.query_parser import QueryParser
 
-from .models import ExtractionJob, OutputField
+from .models import ExtractionJob, OutputField, OutputType
 from .site import extraction_retrieval_providers
 
 
@@ -185,14 +186,101 @@ def clean(self) -> dict[str, Any] | None:
 
 
 class OutputFieldForm(forms.ModelForm):
+    selection_options = forms.CharField(
+        required=False,
+        widget=forms.HiddenInput(),
+    )
+
     class Meta:
         model = OutputField
         fields = [
             "name",
             "description",
             "output_type",
+            "selection_options",
         ]
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.fields["description"].widget = forms.Textarea(attrs={"rows": 3})
+        self.fields["selection_options"].widget.attrs.update(
+            {
+                "data-selection-input": "true",
+            }
+        )
+
+        initial_options = self.instance.selection_options if self.instance.pk else []
+        self.initial["selection_options"] = json.dumps(initial_options)
+
+        self.helper = FormHelper()
+        self.helper.form_tag = False
+        self.helper.disable_csrf = True
+        self.helper.layout = Layout(
+            Row(
+                Column("name", css_class="col-8"),
+                Column("output_type", css_class="col-4"),
+                css_class="g-3",
+            ),
+            "description",
+            Div(
+                HTML('{% include "extractions/_selection_options_field.html" %}'),
+                css_class="selection-options-wrapper",
+            ),
+        )
+
+    def clean_selection_options(self) -> list[str]:
+        raw_value = self.cleaned_data.get("selection_options") or ""
+        raw_value = raw_value.strip()
+        if raw_value == "":
+            return []
+
+        try:
+            parsed = json.loads(raw_value)
+        except json.JSONDecodeError as exc:
+            raise forms.ValidationError("Invalid selection data.") from exc
+
+        if not isinstance(parsed, list):
+            raise forms.ValidationError("Selection data must be a list.")
+
+        cleaned: list[str] = []
+        for item in parsed:
+            if not isinstance(item, str):
+                raise forms.ValidationError("Selection options must be text.")
+            value = item.strip()
+            if not value:
+                raise forms.ValidationError("Selection options cannot be empty.")
+            cleaned.append(value)
+
+        if len(cleaned) > 7:
+            raise forms.ValidationError("Provide at most 7 selection options.")
+
+        return cleaned
+
+    def clean(self):
+        cleaned_data = super().clean()
+        if not cleaned_data:
+            return cleaned_data
+
+        output_type = cleaned_data.get("output_type")
+        selection_options: list[str] = cleaned_data.get("selection_options") or []
+
+        if output_type == OutputType.SELECTION:
+            if not selection_options:
+                self.add_error(
+                    "selection_options",
+                    "Add at least one selection to use the Selection type.",
+                )
+        else:
+            if selection_options:
+                self.add_error(
+                    "selection_options",
+                    "Selections are only allowed when Output Type is Selection.",
+                )
+                cleaned_data["selection_options"] = []
+
+        return cleaned_data
+
 
 OutputFieldFormSet = forms.inlineformset_factory(
     ExtractionJob,
diff --git a/radis/extractions/migrations/0004_outputfield_selection_options.py b/radis/extractions/migrations/0004_outputfield_selection_options.py
new file mode 100644
index 00000000..f01877a8
--- /dev/null
+++ b/radis/extractions/migrations/0004_outputfield_selection_options.py
@@ -0,0 +1,29 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("extractions", "0003_alter_extractionjob_options_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="outputfield",
+            name="selection_options",
+            field=models.JSONField(blank=True, default=list),
+        ),
+        migrations.AlterField(
+            model_name="outputfield",
+            name="output_type",
+            field=models.CharField(
+                choices=[
+                    ("T", "Text"),
+                    ("N", "Numeric"),
+                    ("B", "Boolean"),
+                    ("S", "Selection"),
+                ],
+                default="T",
+                max_length=1,
+            ),
+        ),
+    ]
diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index da9756da..2ae477b9 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -76,6 +76,7 @@ class OutputType(models.TextChoices):
     TEXT = "T", "Text"
     NUMERIC = "N", "Numeric"
     BOOLEAN = "B", "Boolean"
+    SELECTION = "S", "Selection"
 
 
 class OutputField(models.Model):
@@ -86,6 +87,7 @@ class OutputField(models.Model):
     )
     get_output_type_display: Callable[[], str]
     optional = models.BooleanField(default=False)
+    selection_options = models.JSONField(default=list, blank=True)
     job = models.ForeignKey[ExtractionJob](
         ExtractionJob, on_delete=models.CASCADE, related_name="output_fields"
     )
@@ -101,6 +103,34 @@ class Meta:
     def __str__(self) -> str:
         return f'Output Field "{self.name}" [{self.pk}]'
 
+    def clean(self) -> None:
+        from django.core.exceptions import ValidationError
+
+        super().clean()
+
+        if self.output_type == OutputType.SELECTION:
+            if not self.selection_options:
+                raise ValidationError({"selection_options": "Add at least one selection option."})
+            if len(self.selection_options) > 7:
+                raise ValidationError({"selection_options": "Provide at most 7 selection options."})
+            cleaned_options = []
+            for option in self.selection_options:
+                if not isinstance(option, str):
+                    raise ValidationError({"selection_options": "All selection options must be text."})
+                stripped = option.strip()
+                if not stripped:
+                    raise ValidationError(
+                        {"selection_options": "Selection options cannot be empty strings."}
+                    )
+                cleaned_options.append(stripped)
+            self.selection_options = cleaned_options
+        else:
+            if self.selection_options:
+                raise ValidationError(
+                    {"selection_options": "Selections are only allowed for the Selection type."}
+                )
+            self.selection_options = []
+
 
 class ExtractionTask(AnalysisTask):
     job = models.ForeignKey[ExtractionJob](
diff --git a/radis/extractions/templates/extractions/_selection_options_field.html b/radis/extractions/templates/extractions/_selection_options_field.html
new file mode 100644
index 00000000..91437e67
--- /dev/null
+++ b/radis/extractions/templates/extractions/_selection_options_field.html
@@ -0,0 +1,48 @@
+{% load bootstrap_icon from common_extras %}
+<div class="mt-3"
+     x-data="SelectionOptions($el)"
+     x-effect="syncOptions()"
+     data-selection-wrapper="true">
+    <div class="d-flex align-items-center gap-2">
+        {{ form.selection_options }}
+        <button type="button"
+                class="btn btn-outline-secondary btn-sm"
+                @click.prevent="addOption()"
+                :disabled="!supportsSelection || options.length >= maxOptions">
+            {% bootstrap_icon "plus-lg" %}
+            Enter a selection
+        </button>
+        <small class="text-muted" x-show="supportsSelection"
+               x-text="`${options.length} / ${maxOptions}`"></small>
+    </div>
+    <div class="mt-2" x-show="supportsSelection">
+        <template x-for="(option, index) in options" :key="index">
+            <div class="input-group mb-2">
+                <input type="text"
+                       class="form-control"
+                       :placeholder="`Selection ${index + 1}`"
+                       data-selection-option-input
+                       x-model="options[index]">
+                <button type="button"
+                        class="btn btn-outline-danger"
+                        @click.prevent="removeOption(index)"
+                        aria-label="Remove selection option">
+                    {% bootstrap_icon "trash" %}
+                </button>
+            </div>
+        </template>
+        <p class="text-muted fst-italic" x-show="options.length === 0">
+            No selections defined yet.
+        </p>
+    </div>
+    <p class="text-muted fst-italic" x-show="!supportsSelection">
+        Choose the “Selection” output type to define enumerated values.
+    </p>
+    {% if form.selection_options.errors %}
+        <div class="invalid-feedback d-block">
+            {% for error in form.selection_options.errors %}
+                {{ error }}
+            {% endfor %}
+        </div>
+    {% endif %}
+</div>
diff --git a/radis/extractions/templates/extractions/extraction_job_detail.html b/radis/extractions/templates/extractions/extraction_job_detail.html
index 942a2fc8..dde3e67f 100644
--- a/radis/extractions/templates/extractions/extraction_job_detail.html
+++ b/radis/extractions/templates/extractions/extraction_job_detail.html
@@ -114,6 +114,16 @@ <h5>Output Fields</h5>
                     <dd class="col-sm-9">
                         {{ field.output_type|human_readable_output_type }}
                     </dd>
+                    {% if field.selection_options %}
+                        <dt class="col-sm-3">Selections</dt>
+                        <dd class="col-sm-9">
+                            <ul class="mb-0 ps-3">
+                                {% for option in field.selection_options %}
+                                    <li>{{ option }}</li>
+                                {% endfor %}
+                            </ul>
+                        </dd>
+                    {% endif %}
                 </dl>
             </li>
         {% endfor %}
diff --git a/radis/extractions/templates/extractions/extraction_job_wizard_summary.html b/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
index fe7bc429..84906dbe 100644
--- a/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
+++ b/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
@@ -91,6 +91,16 @@
                                 <dd class="col-sm-9">
                                     {{ field.output_type|human_readable_output_type }}
                                 </dd>
+                                {% if field.selection_options %}
+                                    <dt class="col-sm-3">Selections</dt>
+                                    <dd class="col-sm-9">
+                                        <ul class="mb-0 ps-3">
+                                            {% for option in field.selection_options %}
+                                                <li>{{ option }}</li>
+                                            {% endfor %}
+                                        </ul>
+                                    </dd>
+                                {% endif %}
                             </dl>
                         </li>
                     {% endif %}

From fdf5cded6cbe41e84a095a90eaef6d9faf408857 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 11 Nov 2025 21:07:49 +0000
Subject: [PATCH 15/93] fixed subscription tests with new models

---
 radis/subscriptions/forms.py            |  7 ++-
 radis/subscriptions/tests/test_views.py | 62 +++++++++++++++++--------
 2 files changed, 46 insertions(+), 23 deletions(-)

diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index 961d87a4..384600f6 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -173,7 +173,6 @@ def __init__(self, *args, **kwargs):
             fields.insert(1, Field("DELETE", type="hidden"))
         self.helper.layout = Layout(Div(*fields))
 
-
     def clean(self) -> dict[str, Any]:
         cleaned_data = super().clean()
         assert cleaned_data
@@ -181,9 +180,9 @@ def clean(self) -> dict[str, Any]:
         question = cleaned_data["question"]
         expected_answer = cleaned_data["expected_answer"]
 
-        if not question and not expected_answer:
+        if bool(question) != bool(expected_answer):
             raise forms.ValidationError(
-                "Question and Expected Answer text is required when specifying a filter."
+                "Both question and expected answer are required when specifying a filter."
             )
 
         return cleaned_data
@@ -203,7 +202,7 @@ def __init__(self, *args, **kwargs):
         self.helper = FormHelper()
         self.helper.form_tag = False
         self.helper.disable_csrf = True
-        
+
         fields = [
             Field("id", type="hidden"),
             Row(
diff --git a/radis/subscriptions/tests/test_views.py b/radis/subscriptions/tests/test_views.py
index 837269a6..fb026cbe 100644
--- a/radis/subscriptions/tests/test_views.py
+++ b/radis/subscriptions/tests/test_views.py
@@ -2,12 +2,10 @@
 from adit_radis_shared.accounts.factories import GroupFactory, UserFactory
 from django.test import Client
 
+from radis.extractions.factories import OutputFieldFactory
 from radis.reports.factories import LanguageFactory
 from radis.reports.models import Modality
-from radis.subscriptions.factories import (
-    FilterQuestionFactory,
-    SubscriptionFactory,
-)
+from radis.subscriptions.factories import FilterQuestionFactory, SubscriptionFactory
 from radis.subscriptions.models import Subscription
 
 
@@ -100,11 +98,19 @@ def test_subscription_create_view_post_valid(client: Client):
         "age_till": 60,
         "patient_id": "12345",
         "send_finished_mail": True,
-        "questions-TOTAL_FORMS": "1",
-        "questions-INITIAL_FORMS": "0",
-        "questions-MIN_NUM_FORMS": "0",
-        "questions-MAX_NUM_FORMS": "3",
-        "questions-0-question": "What is the diagnosis?",
+        "filter_questions-TOTAL_FORMS": "1",
+        "filter_questions-INITIAL_FORMS": "0",
+        "filter_questions-MIN_NUM_FORMS": "0",
+        "filter_questions-MAX_NUM_FORMS": "3",
+        "filter_questions-0-question": "Does the report contain pneumothorax?",
+        "filter_questions-0-expected_answer": "Y",
+        "extraction_fields-TOTAL_FORMS": "1",
+        "extraction_fields-INITIAL_FORMS": "0",
+        "extraction_fields-MIN_NUM_FORMS": "0",
+        "extraction_fields-MAX_NUM_FORMS": "10",
+        "extraction_fields-0-name": "Pneumothorax status",
+        "extraction_fields-0-description": "Extract pneumothorax related findings",
+        "extraction_fields-0-output_type": "T",
     }
 
     response = client.post("/subscriptions/create/", data)
@@ -125,13 +131,21 @@ def test_subscription_create_view_post_duplicate_name(client: Client):
 
     client.force_login(user)
 
+    language = LanguageFactory.create(code="en")
+
     data = {
         "name": "Duplicate Name",
         "provider": "test_provider",
-        "questions-TOTAL_FORMS": "0",
-        "questions-INITIAL_FORMS": "0",
-        "questions-MIN_NUM_FORMS": "0",
-        "questions-MAX_NUM_FORMS": "3",
+        "language": language.pk,
+        "query": "",
+        "filter_questions-TOTAL_FORMS": "0",
+        "filter_questions-INITIAL_FORMS": "0",
+        "filter_questions-MIN_NUM_FORMS": "0",
+        "filter_questions-MAX_NUM_FORMS": "3",
+        "extraction_fields-TOTAL_FORMS": "0",
+        "extraction_fields-INITIAL_FORMS": "0",
+        "extraction_fields-MIN_NUM_FORMS": "0",
+        "extraction_fields-MAX_NUM_FORMS": "10",
     }
 
     response = client.post("/subscriptions/create/", data)
@@ -191,6 +205,7 @@ def test_subscription_update_view_post_valid(client: Client):
     user = UserFactory.create(is_active=True)
     subscription = create_test_subscription(owner=user, name="Original Name")
     question = FilterQuestionFactory.create(subscription=subscription)
+    output_field = OutputFieldFactory.create(subscription=subscription, job=None)
 
     client.force_login(user)
 
@@ -201,12 +216,21 @@ def test_subscription_update_view_post_valid(client: Client):
         "study_description": "Updated study",
         "patient_sex": "F",
         "send_finished_mail": False,
-        "questions-TOTAL_FORMS": "1",
-        "questions-INITIAL_FORMS": "1",
-        "questions-MIN_NUM_FORMS": "0",
-        "questions-MAX_NUM_FORMS": "3",
-        "questions-0-id": question.pk,
-        "questions-0-question": "Updated question?",
+        "filter_questions-TOTAL_FORMS": "1",
+        "filter_questions-INITIAL_FORMS": "1",
+        "filter_questions-MIN_NUM_FORMS": "0",
+        "filter_questions-MAX_NUM_FORMS": "3",
+        "filter_questions-0-id": question.pk,
+        "filter_questions-0-question": "Updated question?",
+        "filter_questions-0-expected_answer": "N",
+        "extraction_fields-TOTAL_FORMS": "1",
+        "extraction_fields-INITIAL_FORMS": "1",
+        "extraction_fields-MIN_NUM_FORMS": "0",
+        "extraction_fields-MAX_NUM_FORMS": "10",
+        "extraction_fields-0-id": output_field.pk,
+        "extraction_fields-0-name": "Volume",
+        "extraction_fields-0-description": "Volume description",
+        "extraction_fields-0-output_type": "N",
     }
 
     response = client.post(f"/subscriptions/{subscription.pk}/update/", data)

From 3b28ce4566310c29fa6dd69d75edab2cc7f004fe Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Fri, 14 Nov 2025 14:04:29 +0000
Subject: [PATCH 16/93] Add tests for Selection form added via Alpine

---
 radis/extractions/tests/test_forms.py      | 92 ++++++++++++++++++++++
 radis/extractions/utils/processor_utils.py | 21 +++--
 2 files changed, 108 insertions(+), 5 deletions(-)
 create mode 100644 radis/extractions/tests/test_forms.py

diff --git a/radis/extractions/tests/test_forms.py b/radis/extractions/tests/test_forms.py
new file mode 100644
index 00000000..2089bb42
--- /dev/null
+++ b/radis/extractions/tests/test_forms.py
@@ -0,0 +1,92 @@
+import json
+
+import pytest
+from django.core.exceptions import ValidationError
+
+from radis.extractions.factories import ExtractionJobFactory
+from radis.extractions.forms import OutputFieldForm
+from radis.extractions.models import OutputField, OutputType
+
+
+@pytest.mark.django_db
+def test_output_field_form_accepts_selection_options():
+    job = ExtractionJobFactory.create()
+    form = OutputFieldForm(
+        data={
+            "name": "tumor_grade",
+            "description": "Classified tumor grade.",
+            "output_type": OutputType.SELECTION,
+            "selection_options": json.dumps(["Grade 1", "Grade 2"]),
+        },
+        instance=OutputField(job=job),
+    )
+
+    assert form.is_valid()
+    instance = form.save(commit=False)
+
+    assert instance.selection_options == ["Grade 1", "Grade 2"]
+
+
+@pytest.mark.django_db
+def test_output_field_form_requires_options_for_selection():
+    job = ExtractionJobFactory.create()
+    form = OutputFieldForm(
+        data={
+            "name": "tumor_grade",
+            "description": "Classified tumor grade.",
+            "output_type": OutputType.SELECTION,
+            "selection_options": json.dumps([]),
+        },
+        instance=OutputField(job=job),
+    )
+
+    assert not form.is_valid()
+    assert "selection_options" in form.errors
+
+
+@pytest.mark.django_db
+def test_output_field_form_rejects_options_for_non_selection():
+    job = ExtractionJobFactory.create()
+    form = OutputFieldForm(
+        data={
+            "name": "tumor_grade",
+            "description": "Classified tumor grade.",
+            "output_type": OutputType.TEXT,
+            "selection_options": json.dumps(["Grade 1"]),
+        },
+        instance=OutputField(job=job),
+    )
+
+    assert not form.is_valid()
+    assert "selection_options" in form.errors
+
+
+@pytest.mark.django_db
+def test_output_field_clean_trims_selection_options():
+    job = ExtractionJobFactory.create()
+    field = OutputField(
+        job=job,
+        name="tumor_grade",
+        description="Classified tumor grade.",
+        output_type=OutputType.SELECTION,
+        selection_options=["  Grade 1 ", "Grade 2  "],
+    )
+
+    field.full_clean()
+
+    assert field.selection_options == ["Grade 1", "Grade 2"]
+
+
+@pytest.mark.django_db
+def test_output_field_clean_rejects_selection_options_for_other_types():
+    job = ExtractionJobFactory.create()
+    field = OutputField(
+        job=job,
+        name="tumor_grade",
+        description="Classified tumor grade.",
+        output_type=OutputType.TEXT,
+        selection_options=["Grade 1"],
+    )
+
+    with pytest.raises(ValidationError):
+        field.full_clean()
diff --git a/radis/extractions/utils/processor_utils.py b/radis/extractions/utils/processor_utils.py
index e7f12bca..8c99cfeb 100644
--- a/radis/extractions/utils/processor_utils.py
+++ b/radis/extractions/utils/processor_utils.py
@@ -1,4 +1,4 @@
-from typing import Any
+from typing import Any, Literal
 
 from django.db.models import QuerySet
 from pydantic import BaseModel, create_model
@@ -11,12 +11,18 @@
 def generate_output_fields_schema(fields: QuerySet[OutputField]) -> type[BaseModel]:
     field_definitions: dict[str, Any] = {}
     for field in fields.all():
-        if field.output_type == OutputType.TEXT:
+        field_type = OutputType(field.output_type)
+        if field_type == OutputType.TEXT:
             output_type = str
-        elif field.output_type == OutputType.NUMERIC:
+        elif field_type == OutputType.NUMERIC:
             output_type = Numeric
-        elif field.output_type == OutputType.BOOLEAN:
+        elif field_type == OutputType.BOOLEAN:
             output_type = bool
+        elif field_type == OutputType.SELECTION:
+            options = tuple(field.selection_options)
+            if not options:
+                raise ValueError("Selection output requires at least one option.")
+            output_type = Literal.__getitem__(options)
         else:
             raise ValueError(f"Unknown data type: {field.output_type}")
 
@@ -28,6 +34,11 @@ def generate_output_fields_schema(fields: QuerySet[OutputField]) -> type[BaseMod
 def generate_output_fields_prompt(fields: QuerySet[OutputField]) -> str:
     prompt = ""
     for field in fields.all():
-        prompt += f"{field.name}: {field.description}\n"
+        description = field.description
+        if OutputType(field.output_type) == OutputType.SELECTION and field.selection_options:
+            description = (
+                f"{description} (Allowed selections: {', '.join(field.selection_options)})"
+            )
+        prompt += f"{field.name}: {description}\n"
 
     return prompt

From 6292d1daceb98fbe845e9a26e67a4f7833fe30af Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 17 Nov 2025 12:29:21 +0000
Subject: [PATCH 17/93] added unit test for subscriptions that use the LLM to
 filter/extract

---
 .../tests/unit/test_processors.py             | 43 +++++++++++++++++++
 radis/subscriptions/utils/testing_helpers.py  | 43 +++++++++++++++++++
 2 files changed, 86 insertions(+)
 create mode 100644 radis/subscriptions/tests/unit/test_processors.py
 create mode 100644 radis/subscriptions/utils/testing_helpers.py

diff --git a/radis/subscriptions/tests/unit/test_processors.py b/radis/subscriptions/tests/unit/test_processors.py
new file mode 100644
index 00000000..926eccea
--- /dev/null
+++ b/radis/subscriptions/tests/unit/test_processors.py
@@ -0,0 +1,43 @@
+from unittest.mock import MagicMock, patch
+
+import pytest
+from pydantic import BaseModel
+
+from radis.chats.utils.testing_helpers import create_openai_client_mock
+from radis.subscriptions.models import SubscribedItem
+from radis.subscriptions.processors import SubscriptionTaskProcessor
+from radis.subscriptions.utils.testing_helpers import create_subscription_task
+
+
+class FilterOutput(BaseModel):
+    filter_0: bool
+
+
+class ExtractionOutput(BaseModel):
+    extraction_0: str
+
+
+@pytest.mark.django_db(transaction=True)
+def test_subscription_task_processor_filters_and_extracts():
+    task, filter_question, extraction_field, report = create_subscription_task()
+
+    filter_output = FilterOutput(filter_0=True)
+    extraction_output = ExtractionOutput(extraction_0="Pneumothorax status confirmed")
+
+    filter_response = MagicMock(choices=[MagicMock(message=MagicMock(parsed=filter_output))])
+    extraction_response = MagicMock(
+        choices=[MagicMock(message=MagicMock(parsed=extraction_output))]
+    )
+
+    openai_mock = create_openai_client_mock(extraction_output)
+    openai_mock.beta.chat.completions.parse = MagicMock(
+        side_effect=[filter_response, extraction_response]
+    )
+    with patch("openai.OpenAI", return_value=openai_mock):
+        SubscriptionTaskProcessor(task).start()
+
+    subscribed_item = SubscribedItem.objects.get(subscription=task.job.subscription, report=report)
+    assert subscribed_item.filter_results == {str(filter_question.pk): True}
+    assert subscribed_item.extraction_results == {
+        str(extraction_field.pk): "Pneumothorax status confirmed"
+    }
diff --git a/radis/subscriptions/utils/testing_helpers.py b/radis/subscriptions/utils/testing_helpers.py
new file mode 100644
index 00000000..bc1f5a0d
--- /dev/null
+++ b/radis/subscriptions/utils/testing_helpers.py
@@ -0,0 +1,43 @@
+from adit_radis_shared.accounts.factories import GroupFactory, UserFactory
+from adit_radis_shared.common.utils.testing_helpers import add_user_to_group
+
+from radis.extractions.factories import OutputFieldFactory
+from radis.extractions.models import OutputType
+from radis.reports.factories import LanguageFactory, ReportFactory
+from radis.subscriptions.factories import FilterQuestionFactory, SubscriptionFactory
+from radis.subscriptions.models import FilterQuestion, SubscriptionJob, SubscriptionTask
+
+
+def create_subscription_task():
+    language = LanguageFactory.create(code="en")
+
+    user = UserFactory(is_active=True)
+    group = GroupFactory()
+    add_user_to_group(user, group)
+    user.active_group = group
+    user.save()
+
+    subscription = SubscriptionFactory.create(owner=user, group=group, language=language)
+
+    filter_question = FilterQuestionFactory.create(
+        subscription=subscription, expected_answer=FilterQuestion.ExpectedAnswer.YES
+    )
+    extraction_field = OutputFieldFactory.create(
+        subscription=subscription,
+        job=None,
+        output_type=OutputType.TEXT,
+    )
+
+    job = SubscriptionJob.objects.create(
+        subscription=subscription,
+        owner=user,
+        owner_id=user.id,
+        status=SubscriptionJob.Status.PENDING,
+    )
+    task = SubscriptionTask.objects.create(job=job, status=SubscriptionTask.Status.PENDING)
+
+    report = ReportFactory.create(language=language, body="Pneumothorax observed.")
+    report.groups.add(group)
+    task.reports.add(report)
+
+    return task, filter_question, extraction_field, report

From 197faca4cfe1cd188f1224e45e4b18522d7b6bb8 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 17 Nov 2025 14:31:42 +0000
Subject: [PATCH 18/93] match font size of legend to other elements

---
 radis/subscriptions/forms.py                  | 19 -------------------
 .../static/subscriptions/subscriptions.css    |  6 ++++++
 2 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index 384600f6..e7fd92a1 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -11,7 +11,6 @@
 from radis.search.forms import AGE_STEP, MAX_AGE, MIN_AGE
 
 from .models import FilterQuestion, Subscription
-from .site import subscription_retrieval_providers
 
 
 class SubscriptionForm(forms.ModelForm):
@@ -19,7 +18,6 @@ class Meta:
         model = Subscription
         fields = [
             "name",
-            "provider",
             "query",
             "language",
             "modalities",
@@ -33,21 +31,12 @@ class Meta:
         labels = {"patient_id": "Patient ID"}
         help_texts = {
             "name": "Name of the Subscription",
-            "provider": "The search provider to use for the database query",
             "query": "A query to filter reports",
         }
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        self.fields["provider"].widget = forms.Select(
-            choices=sorted(
-                [
-                    (provider.name, provider.name)
-                    for provider in subscription_retrieval_providers.values()
-                ]
-            )
-        )
         self.fields["language"].choices = [  # type: ignore
             (language.pk, LANGUAGE_LABELS[language.code])
             for language in Language.objects.order_by("code")
@@ -93,7 +82,6 @@ def build_layout(self):
             Row(
                 Column(
                     "name",
-                    "provider",
                     "query",
                     "send_finished_mail",
                     Formset(
@@ -139,13 +127,6 @@ def clean(self) -> dict[str, Any] | None:
         if age_from is not None and age_till is not None and age_from >= age_till:
             raise forms.ValidationError("Age from must be less than age till")
 
-        provider = self.cleaned_data["provider"]
-        query = self.cleaned_data["query"]
-        if query != "" and not provider:
-            raise forms.ValidationError(
-                "Setup of RADIS is incomplete. No retrieval providers are registered."
-            )
-
         return super().clean()
 
 
diff --git a/radis/subscriptions/static/subscriptions/subscriptions.css b/radis/subscriptions/static/subscriptions/subscriptions.css
index e69de29b..016f8965 100644
--- a/radis/subscriptions/static/subscriptions/subscriptions.css
+++ b/radis/subscriptions/static/subscriptions/subscriptions.css
@@ -0,0 +1,6 @@
+fieldset > legend {
+  font-size: 1rem !important;
+  font-weight: 500 !important;
+  line-height: 1.5;
+  margin-bottom: 0.5rem;
+}

From 838cf79924541127625a07cf61f068d7575fc95e Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 17 Nov 2025 15:32:23 +0000
Subject: [PATCH 19/93] fix filter question form validation

---
 radis/subscriptions/forms.py            | 24 ++++++++++++-----
 radis/subscriptions/tests/test_views.py | 36 +++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 7 deletions(-)

diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index e7fd92a1..d0ef835d 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -141,7 +141,7 @@ def __init__(self, *args, **kwargs):
         self.fields["question"].required = False
         self.fields["expected_answer"].required = False
         self.fields["expected_answer"].choices = [  # type: ignore[attr-defined]
-            ("", "Select expected answer"),
+            ("", "Select the expected answer"),
             *FilterQuestion.ExpectedAnswer.choices,
         ]
         self.fields["expected_answer"].label = "Accept when answer is"
@@ -154,17 +154,27 @@ def __init__(self, *args, **kwargs):
             fields.insert(1, Field("DELETE", type="hidden"))
         self.helper.layout = Layout(Div(*fields))
 
+    def has_changed(self) -> bool:
+        if not self.is_bound:
+            return super().has_changed()
+
+        question = (self.data.get(self.add_prefix("question")) or "").strip()
+        expected_answer = self.data.get(self.add_prefix("expected_answer")) or ""
+
+        if not question and not expected_answer:
+            return False
+
+        return super().has_changed()
+
     def clean(self) -> dict[str, Any]:
         cleaned_data = super().clean()
         assert cleaned_data
 
-        question = cleaned_data["question"]
-        expected_answer = cleaned_data["expected_answer"]
+        question = cleaned_data.get("question")
+        expected_answer = cleaned_data.get("expected_answer")
 
-        if bool(question) != bool(expected_answer):
-            raise forms.ValidationError(
-                "Both question and expected answer are required when specifying a filter."
-            )
+        if bool(question) ^ bool(expected_answer):
+            raise forms.ValidationError("You must provide both a question and an expected answer.")
 
         return cleaned_data
 
diff --git a/radis/subscriptions/tests/test_views.py b/radis/subscriptions/tests/test_views.py
index fb026cbe..8a473545 100644
--- a/radis/subscriptions/tests/test_views.py
+++ b/radis/subscriptions/tests/test_views.py
@@ -119,6 +119,42 @@ def test_subscription_create_view_post_valid(client: Client):
     assert Subscription.objects.filter(name="Test Subscription").exists()
 
 
+@pytest.mark.django_db
+def test_subscription_create_view_ignores_empty_filter_question(client: Client):
+    user = UserFactory.create(is_active=True)
+    group = GroupFactory.create()
+    user.groups.add(group)
+    user.active_group = group
+    user.save()
+
+    language = LanguageFactory.create(code="en")
+
+    client.force_login(user)
+
+    data = {
+        "name": "Subscription Without Filter",
+        "provider": "test_provider",
+        "language": language.pk,
+        "query": "",
+        "filter_questions-TOTAL_FORMS": "1",
+        "filter_questions-INITIAL_FORMS": "0",
+        "filter_questions-MIN_NUM_FORMS": "0",
+        "filter_questions-MAX_NUM_FORMS": "3",
+        "filter_questions-0-question": "",
+        "filter_questions-0-expected_answer": "",
+        "extraction_fields-TOTAL_FORMS": "0",
+        "extraction_fields-INITIAL_FORMS": "0",
+        "extraction_fields-MIN_NUM_FORMS": "0",
+        "extraction_fields-MAX_NUM_FORMS": "10",
+    }
+
+    response = client.post("/subscriptions/create/", data)
+    assert response.status_code == 302
+
+    subscription = Subscription.objects.get(name="Subscription Without Filter")
+    assert subscription.filter_questions.count() == 0
+
+
 @pytest.mark.django_db
 def test_subscription_create_view_post_duplicate_name(client: Client):
     user = UserFactory.create(is_active=True)

From b602c5fe782d1d3d9e1bb421de7cd11f99cbf055 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 17 Nov 2025 23:42:51 +0000
Subject: [PATCH 20/93] fix dependency cycle

---
 ...03_alter_extractionjob_options_and_more.py | 63 +------------------
 .../0004_remove_outputfield_unique.py         | 33 ++++++++++
 .../0005_outputfield_subscription_and_more.py | 53 ++++++++++++++++
 radis/extractions/models.py                   |  2 +-
 ...9_alter_subscribeditem_options_and_more.py | 47 +-------------
 ..._subscribeditem_filter_results_and_more.py | 56 +++++++++++++++++
 6 files changed, 147 insertions(+), 107 deletions(-)
 create mode 100644 radis/extractions/migrations/0004_remove_outputfield_unique.py
 create mode 100644 radis/extractions/migrations/0005_outputfield_subscription_and_more.py
 create mode 100644 radis/subscriptions/migrations/0010_rename_answers_subscribeditem_filter_results_and_more.py

diff --git a/radis/extractions/migrations/0003_alter_extractionjob_options_and_more.py b/radis/extractions/migrations/0003_alter_extractionjob_options_and_more.py
index c04ee4f9..660c2081 100644
--- a/radis/extractions/migrations/0003_alter_extractionjob_options_and_more.py
+++ b/radis/extractions/migrations/0003_alter_extractionjob_options_and_more.py
@@ -1,14 +1,12 @@
-# Generated by Django 5.2.7 on 2025-11-10 09:59
+# Generated by Django 5.2.7 on 2025-11-04 14:20
 
-import django.db.models.deletion
-from django.db import migrations, models
+from django.db import migrations
 
 
 class Migration(migrations.Migration):
 
     dependencies = [
         ("extractions", "0002_procrastinate_on_delete"),
-        ("subscriptions", "0009_alter_subscribeditem_options_and_more"),
     ]
 
     operations = [
@@ -16,65 +14,8 @@ class Migration(migrations.Migration):
             name="extractionjob",
             options={"ordering": ["-created_at", "title"]},
         ),
-        migrations.RemoveConstraint(
-            model_name="outputfield",
-            name="unique_output_field_name_per_job",
-        ),
         migrations.RemoveIndex(
             model_name="extractionjob",
             name="extractions_owner_i_49b05e_idx",
         ),
-        migrations.RemoveField(
-            model_name="outputfield",
-            name="optional",
-        ),
-        migrations.AddField(
-            model_name="outputfield",
-            name="subscription",
-            field=models.ForeignKey(
-                blank=True,
-                null=True,
-                on_delete=django.db.models.deletion.CASCADE,
-                related_name="extraction_fields",
-                to="subscriptions.subscription",
-            ),
-        ),
-        migrations.AlterField(
-            model_name="outputfield",
-            name="job",
-            field=models.ForeignKey(
-                blank=True,
-                null=True,
-                on_delete=django.db.models.deletion.CASCADE,
-                related_name="output_fields",
-                to="extractions.extractionjob",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="outputfield",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(("job__isnull", False)),
-                fields=("name", "job_id"),
-                name="unique_output_field_name_per_job",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="outputfield",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(("subscription__isnull", False)),
-                fields=("name", "subscription_id"),
-                name="unique_output_field_name_per_subscription",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="outputfield",
-            constraint=models.CheckConstraint(
-                condition=models.Q(
-                    models.Q(("job__isnull", False), ("subscription__isnull", True)),
-                    models.Q(("job__isnull", True), ("subscription__isnull", False)),
-                    _connector="OR",
-                ),
-                name="output_field_exactly_one_parent",
-            ),
-        ),
     ]
diff --git a/radis/extractions/migrations/0004_remove_outputfield_unique.py b/radis/extractions/migrations/0004_remove_outputfield_unique.py
new file mode 100644
index 00000000..de73e8a5
--- /dev/null
+++ b/radis/extractions/migrations/0004_remove_outputfield_unique.py
@@ -0,0 +1,33 @@
+# Generated by Django 5.2.8 on 2025-11-17 23:19
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("extractions", "0003_alter_extractionjob_options_and_more"),
+    ]
+
+    operations = [
+        migrations.RemoveConstraint(
+            model_name="outputfield",
+            name="unique_output_field_name_per_job",
+        ),
+        migrations.RemoveField(
+            model_name="outputfield",
+            name="optional",
+        ),
+        migrations.AlterField(
+            model_name="outputfield",
+            name="job",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="output_fields",
+                to="extractions.extractionjob",
+            ),
+        ),
+    ]
diff --git a/radis/extractions/migrations/0005_outputfield_subscription_and_more.py b/radis/extractions/migrations/0005_outputfield_subscription_and_more.py
new file mode 100644
index 00000000..72f0c484
--- /dev/null
+++ b/radis/extractions/migrations/0005_outputfield_subscription_and_more.py
@@ -0,0 +1,53 @@
+# Generated by Django 5.2.8 on 2025-11-17 23:39
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("extractions", "0004_remove_outputfield_unique"),
+        ("subscriptions", "0010_rename_answers_subscribeditem_filter_results_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="outputfield",
+            name="subscription",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="output_fields",
+                to="subscriptions.subscription",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="outputfield",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("job__isnull", False)),
+                fields=("name", "job_id"),
+                name="unique_output_field_name_per_job",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="outputfield",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("subscription__isnull", False)),
+                fields=("name", "subscription_id"),
+                name="unique_output_field_name_per_subscription",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="outputfield",
+            constraint=models.CheckConstraint(
+                condition=models.Q(
+                    models.Q(("job__isnull", False), ("subscription__isnull", True)),
+                    models.Q(("job__isnull", True), ("subscription__isnull", False)),
+                    _connector="OR",
+                ),
+                name="output_field_exactly_one_parent",
+            ),
+        ),
+    ]
diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index 622e4fe2..8e360b67 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -94,7 +94,7 @@ class OutputField(models.Model):
         null=True,
         blank=True,
         on_delete=models.CASCADE,
-        related_name="extraction_fields",
+        related_name="output_fields",
     )
 
     class Meta:
diff --git a/radis/subscriptions/migrations/0009_alter_subscribeditem_options_and_more.py b/radis/subscriptions/migrations/0009_alter_subscribeditem_options_and_more.py
index 4268507b..de7e0aec 100644
--- a/radis/subscriptions/migrations/0009_alter_subscribeditem_options_and_more.py
+++ b/radis/subscriptions/migrations/0009_alter_subscribeditem_options_and_more.py
@@ -1,7 +1,6 @@
-# Generated by Django 5.2.7 on 2025-11-10 09:59
+# Generated by Django 5.2.7 on 2025-11-04 14:20
 
-import django.db.models.deletion
-from django.db import migrations, models
+from django.db import migrations
 
 
 class Migration(migrations.Migration):
@@ -22,46 +21,4 @@ class Migration(migrations.Migration):
             name="subscription",
             options={"ordering": ["-created_at"]},
         ),
-        migrations.RenameField(
-            model_name="subscribeditem",
-            old_name="answers",
-            new_name="extraction_results",
-        ),
-        migrations.AddField(
-            model_name="subscribeditem",
-            name="filter_results",
-            field=models.JSONField(blank=True, null=True),
-        ),
-        migrations.CreateModel(
-            name="FilterQuestion",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True,
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="ID",
-                    ),
-                ),
-                ("question", models.CharField(max_length=300)),
-                (
-                    "expected_answer",
-                    models.CharField(
-                        choices=[("Y", "Yes"), ("N", "No")], default="Y", max_length=1
-                    ),
-                ),
-                (
-                    "subscription",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        related_name="filter_questions",
-                        to="subscriptions.subscription",
-                    ),
-                ),
-            ],
-        ),
-        migrations.DeleteModel(
-            name="Question",
-        ),
     ]
diff --git a/radis/subscriptions/migrations/0010_rename_answers_subscribeditem_filter_results_and_more.py b/radis/subscriptions/migrations/0010_rename_answers_subscribeditem_filter_results_and_more.py
new file mode 100644
index 00000000..23a13958
--- /dev/null
+++ b/radis/subscriptions/migrations/0010_rename_answers_subscribeditem_filter_results_and_more.py
@@ -0,0 +1,56 @@
+# Generated by Django 5.2.8 on 2025-11-17 23:36
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("subscriptions", "0009_alter_subscribeditem_options_and_more"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="subscribeditem",
+            old_name="answers",
+            new_name="filter_results",
+        ),
+        migrations.AddField(
+            model_name="subscribeditem",
+            name="extraction_results",
+            field=models.JSONField(blank=True, null=True),
+        ),
+        migrations.CreateModel(
+            name="FilterQuestion",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("question", models.CharField(max_length=300)),
+                (
+                    "expected_answer",
+                    models.CharField(
+                        choices=[("Y", "Yes"), ("N", "No")], default="Y", max_length=1
+                    ),
+                ),
+                (
+                    "subscription",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="filter_questions",
+                        to="subscriptions.subscription",
+                    ),
+                ),
+            ],
+        ),
+        migrations.DeleteModel(
+            name="Question",
+        ),
+    ]

From 7410f9d4df994c8249be8d2ed0a2a4f2cb31188e Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 17 Nov 2025 23:55:54 +0000
Subject: [PATCH 21/93] fix files that use the new renamed output_fields

---
 radis/subscriptions/models.py                |  4 +-
 radis/subscriptions/processors.py            |  6 +--
 radis/subscriptions/tests/test_views.py      | 46 ++++++++++----------
 radis/subscriptions/utils/processor_utils.py |  2 +-
 4 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/radis/subscriptions/models.py b/radis/subscriptions/models.py
index 46eda99f..f049f065 100644
--- a/radis/subscriptions/models.py
+++ b/radis/subscriptions/models.py
@@ -52,7 +52,7 @@ class Subscription(models.Model):
     last_refreshed = models.DateTimeField(auto_now_add=True)
 
     filter_questions: models.QuerySet["FilterQuestion"]
-    extraction_fields: models.QuerySet[OutputField]
+    output_fields: models.QuerySet[OutputField]
     items: models.QuerySet["SubscribedItem"]
 
     send_finished_mail = models.BooleanField(default=False)
@@ -131,7 +131,7 @@ def iter_extraction_results(self) -> list[tuple[OutputField, object]]:
             return []
 
         results: list[tuple[OutputField, object]] = []
-        subscription_fields = {str(f.pk): f for f in self.subscription.extraction_fields.all()}
+        subscription_fields = {str(f.pk): f for f in self.subscription.output_fields.all()}
         for key, value in self.extraction_results.items():
             field = subscription_fields.get(str(key))
             if field is not None:
diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index 2ee361f9..a94f3555 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -19,8 +19,8 @@
 from .utils.processor_utils import (
     build_extraction_schema,
     build_filter_schema,
-    generate_extraction_fields_prompt,
     generate_filter_questions_prompt,
+    generate_output_fields_prompt,
 )
 
 logger = logging.getLogger(__name__)
@@ -80,14 +80,14 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
             logger.debug(f"Report {report.pk} was rejected by subscription {subscription.pk}")
             return
 
-        extraction_bundle = build_extraction_schema(subscription.extraction_fields)
+        extraction_bundle = build_extraction_schema(subscription.output_fields)
         extraction_results: dict[str, Any] = {}
 
         if extraction_bundle.mapping:
             extraction_prompt = Template(settings.SUBSCRIPTION_EXTRACTION_PROMPT).substitute(
                 {
                     "report": report.body,
-                    "fields": generate_extraction_fields_prompt(extraction_bundle.mapping),
+                    "fields": generate_output_fields_prompt(extraction_bundle.mapping),
                 }
             )
             extraction_response = self.client.extract_data(
diff --git a/radis/subscriptions/tests/test_views.py b/radis/subscriptions/tests/test_views.py
index 8a473545..f5ce9f88 100644
--- a/radis/subscriptions/tests/test_views.py
+++ b/radis/subscriptions/tests/test_views.py
@@ -104,13 +104,13 @@ def test_subscription_create_view_post_valid(client: Client):
         "filter_questions-MAX_NUM_FORMS": "3",
         "filter_questions-0-question": "Does the report contain pneumothorax?",
         "filter_questions-0-expected_answer": "Y",
-        "extraction_fields-TOTAL_FORMS": "1",
-        "extraction_fields-INITIAL_FORMS": "0",
-        "extraction_fields-MIN_NUM_FORMS": "0",
-        "extraction_fields-MAX_NUM_FORMS": "10",
-        "extraction_fields-0-name": "Pneumothorax status",
-        "extraction_fields-0-description": "Extract pneumothorax related findings",
-        "extraction_fields-0-output_type": "T",
+        "output_fields-TOTAL_FORMS": "1",
+        "output_fields-INITIAL_FORMS": "0",
+        "output_fields-MIN_NUM_FORMS": "0",
+        "output_fields-MAX_NUM_FORMS": "10",
+        "output_fields-0-name": "Pneumothorax status",
+        "output_fields-0-description": "Extract pneumothorax related findings",
+        "output_fields-0-output_type": "T",
     }
 
     response = client.post("/subscriptions/create/", data)
@@ -142,10 +142,10 @@ def test_subscription_create_view_ignores_empty_filter_question(client: Client):
         "filter_questions-MAX_NUM_FORMS": "3",
         "filter_questions-0-question": "",
         "filter_questions-0-expected_answer": "",
-        "extraction_fields-TOTAL_FORMS": "0",
-        "extraction_fields-INITIAL_FORMS": "0",
-        "extraction_fields-MIN_NUM_FORMS": "0",
-        "extraction_fields-MAX_NUM_FORMS": "10",
+        "output_fields-TOTAL_FORMS": "0",
+        "output_fields-INITIAL_FORMS": "0",
+        "output_fields-MIN_NUM_FORMS": "0",
+        "output_fields-MAX_NUM_FORMS": "10",
     }
 
     response = client.post("/subscriptions/create/", data)
@@ -178,10 +178,10 @@ def test_subscription_create_view_post_duplicate_name(client: Client):
         "filter_questions-INITIAL_FORMS": "0",
         "filter_questions-MIN_NUM_FORMS": "0",
         "filter_questions-MAX_NUM_FORMS": "3",
-        "extraction_fields-TOTAL_FORMS": "0",
-        "extraction_fields-INITIAL_FORMS": "0",
-        "extraction_fields-MIN_NUM_FORMS": "0",
-        "extraction_fields-MAX_NUM_FORMS": "10",
+        "output_fields-TOTAL_FORMS": "0",
+        "output_fields-INITIAL_FORMS": "0",
+        "output_fields-MIN_NUM_FORMS": "0",
+        "output_fields-MAX_NUM_FORMS": "10",
     }
 
     response = client.post("/subscriptions/create/", data)
@@ -259,14 +259,14 @@ def test_subscription_update_view_post_valid(client: Client):
         "filter_questions-0-id": question.pk,
         "filter_questions-0-question": "Updated question?",
         "filter_questions-0-expected_answer": "N",
-        "extraction_fields-TOTAL_FORMS": "1",
-        "extraction_fields-INITIAL_FORMS": "1",
-        "extraction_fields-MIN_NUM_FORMS": "0",
-        "extraction_fields-MAX_NUM_FORMS": "10",
-        "extraction_fields-0-id": output_field.pk,
-        "extraction_fields-0-name": "Volume",
-        "extraction_fields-0-description": "Volume description",
-        "extraction_fields-0-output_type": "N",
+        "output_fields-TOTAL_FORMS": "1",
+        "output_fields-INITIAL_FORMS": "1",
+        "output_fields-MIN_NUM_FORMS": "0",
+        "output_fields-MAX_NUM_FORMS": "10",
+        "output_fields-0-id": output_field.pk,
+        "output_fields-0-name": "Volume",
+        "output_fields-0-description": "Volume description",
+        "output_fields-0-output_type": "N",
     }
 
     response = client.post(f"/subscriptions/{subscription.pk}/update/", data)
diff --git a/radis/subscriptions/utils/processor_utils.py b/radis/subscriptions/utils/processor_utils.py
index 299e7eb1..ac78b9f3 100644
--- a/radis/subscriptions/utils/processor_utils.py
+++ b/radis/subscriptions/utils/processor_utils.py
@@ -81,7 +81,7 @@ def generate_filter_questions_prompt(mapping: list[tuple[str, FilterQuestion]])
     return "\n".join(lines)
 
 
-def generate_extraction_fields_prompt(mapping: list[tuple[str, OutputField]]) -> str:
+def generate_output_fields_prompt(mapping: list[tuple[str, OutputField]]) -> str:
     if not mapping:
         return "None"
 

From df06a769bbc677fb39b6b345f4a7feb0ddbac343 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 18 Nov 2025 00:22:37 +0000
Subject: [PATCH 22/93] complete renaming of output field

---
 radis/subscriptions/forms.py                  |  8 ++---
 .../tests/unit/test_processors.py             |  4 +--
 radis/subscriptions/utils/testing_helpers.py  |  4 +--
 radis/subscriptions/views.py                  | 36 +++++++++----------
 4 files changed, 25 insertions(+), 27 deletions(-)

diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index d0ef835d..05274a62 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -90,7 +90,7 @@ def build_layout(self):
                         add_form_label="Add Filter Question",
                     ),
                     Formset(
-                        "extraction_formset",
+                        "output_formset",
                         legend="Extraction Fields",
                         add_form_label="Add Extraction Field",
                     ),
@@ -179,7 +179,7 @@ def clean(self) -> dict[str, Any]:
         return cleaned_data
 
 
-class ExtractionFieldForm(forms.ModelForm):
+class OutputFieldForm(forms.ModelForm):
     class Meta:
         model = OutputField
         fields = ["name", "description", "output_type"]
@@ -220,10 +220,10 @@ def __init__(self, *args, **kwargs):
     can_delete=False,
 )
 
-ExtractionFieldFormSet = forms.inlineformset_factory(
+OutputFieldFormSet = forms.inlineformset_factory(
     Subscription,
     OutputField,
-    form=ExtractionFieldForm,
+    form=OutputFieldForm,
     fk_name="subscription",
     extra=1,
     min_num=0,
diff --git a/radis/subscriptions/tests/unit/test_processors.py b/radis/subscriptions/tests/unit/test_processors.py
index 926eccea..457c1891 100644
--- a/radis/subscriptions/tests/unit/test_processors.py
+++ b/radis/subscriptions/tests/unit/test_processors.py
@@ -19,7 +19,7 @@ class ExtractionOutput(BaseModel):
 
 @pytest.mark.django_db(transaction=True)
 def test_subscription_task_processor_filters_and_extracts():
-    task, filter_question, extraction_field, report = create_subscription_task()
+    task, filter_question, output_field, report = create_subscription_task()
 
     filter_output = FilterOutput(filter_0=True)
     extraction_output = ExtractionOutput(extraction_0="Pneumothorax status confirmed")
@@ -39,5 +39,5 @@ def test_subscription_task_processor_filters_and_extracts():
     subscribed_item = SubscribedItem.objects.get(subscription=task.job.subscription, report=report)
     assert subscribed_item.filter_results == {str(filter_question.pk): True}
     assert subscribed_item.extraction_results == {
-        str(extraction_field.pk): "Pneumothorax status confirmed"
+        str(output_field.pk): "Pneumothorax status confirmed"
     }
diff --git a/radis/subscriptions/utils/testing_helpers.py b/radis/subscriptions/utils/testing_helpers.py
index bc1f5a0d..9ec03e91 100644
--- a/radis/subscriptions/utils/testing_helpers.py
+++ b/radis/subscriptions/utils/testing_helpers.py
@@ -22,7 +22,7 @@ def create_subscription_task():
     filter_question = FilterQuestionFactory.create(
         subscription=subscription, expected_answer=FilterQuestion.ExpectedAnswer.YES
     )
-    extraction_field = OutputFieldFactory.create(
+    output_field = OutputFieldFactory.create(
         subscription=subscription,
         job=None,
         output_type=OutputType.TEXT,
@@ -40,4 +40,4 @@ def create_subscription_task():
     report.groups.add(group)
     task.reports.add(report)
 
-    return task, filter_question, extraction_field, report
+    return task, filter_question, output_field, report
diff --git a/radis/subscriptions/views.py b/radis/subscriptions/views.py
index 8d4fefd4..a73cc49c 100644
--- a/radis/subscriptions/views.py
+++ b/radis/subscriptions/views.py
@@ -21,8 +21,8 @@
 from radis.subscriptions.tables import SubscriptionTable
 
 from .forms import (
-    ExtractionFieldFormSet,
     FilterQuestionFormSet,
+    OutputFieldFormSet,
     SubscriptionForm,
 )
 from .models import SubscribedItem, Subscription
@@ -58,7 +58,7 @@ def get_queryset(self):
             super()
             .get_queryset()
             .filter(owner=self.request.user)
-            .prefetch_related("filter_questions", "extraction_fields")
+            .prefetch_related("filter_questions", "output_fields")
         )
 
 
@@ -72,17 +72,17 @@ def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
         ctx = super().get_context_data(**kwargs)
         if self.request.POST:
             ctx["filter_formset"] = FilterQuestionFormSet(self.request.POST)
-            ctx["extraction_formset"] = ExtractionFieldFormSet(self.request.POST)
+            ctx["output_formset"] = OutputFieldFormSet(self.request.POST)
         else:
             ctx["filter_formset"] = FilterQuestionFormSet()
-            ctx["extraction_formset"] = ExtractionFieldFormSet()
+            ctx["output_formset"] = OutputFieldFormSet()
         return ctx
 
     def form_valid(self, form) -> HttpResponse:
         ctx = self.get_context_data()
         filter_formset: BaseInlineFormSet = ctx["filter_formset"]
-        extraction_formset: BaseInlineFormSet = ctx["extraction_formset"]
-        if filter_formset.is_valid() and extraction_formset.is_valid():
+        output_formset: BaseInlineFormSet = ctx["output_formset"]
+        if filter_formset.is_valid() and output_formset.is_valid():
             user = self.request.user
             form.instance.owner = user
             active_group = user.active_group
@@ -99,8 +99,8 @@ def form_valid(self, form) -> HttpResponse:
             filter_formset.instance = self.object
             filter_formset.save()
 
-            extraction_formset.instance = self.object
-            extraction_formset.save()
+            output_formset.instance = self.object
+            output_formset.save()
             return HttpResponseRedirect(self.get_success_url())
         else:
             return self.form_invalid(form)
@@ -120,28 +120,26 @@ def get_queryset(self) -> QuerySet[Subscription]:
             super()
             .get_queryset()
             .filter(owner=self.request.user)
-            .prefetch_related("filter_questions", "extraction_fields")
+            .prefetch_related("filter_questions", "output_fields")
         )
 
     def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
         ctx = super().get_context_data(**kwargs)
         if self.request.POST:
             ctx["filter_formset"] = FilterQuestionFormSet(self.request.POST, instance=self.object)
-            ctx["extraction_formset"] = ExtractionFieldFormSet(
-                self.request.POST, instance=self.object
-            )
+            ctx["output_formset"] = OutputFieldFormSet(self.request.POST, instance=self.object)
         else:
             ctx["filter_formset"] = FilterQuestionFormSet(instance=self.object)
-            ctx["extraction_formset"] = ExtractionFieldFormSet(instance=self.object)
+            ctx["output_formset"] = OutputFieldFormSet(instance=self.object)
         ctx["filter_formset"].extra = 0  # no additional empty form when editing
-        ctx["extraction_formset"].extra = 0
+        ctx["output_formset"].extra = 0
         return ctx
 
     def form_valid(self, form) -> HttpResponse:
         ctx = self.get_context_data()
         filter_formset = ctx["filter_formset"]
-        extraction_formset = ctx["extraction_formset"]
-        if filter_formset.is_valid() and extraction_formset.is_valid():
+        output_formset = ctx["output_formset"]
+        if filter_formset.is_valid() and output_formset.is_valid():
             try:
                 self.object = form.save()
             except IntegrityError as e:
@@ -153,8 +151,8 @@ def form_valid(self, form) -> HttpResponse:
             filter_formset.instance = self.object
             filter_formset.save()
 
-            extraction_formset.instance = self.object
-            extraction_formset.save()
+            output_formset.instance = self.object
+            output_formset.save()
 
             return super().form_valid(form)
         else:
@@ -196,7 +194,7 @@ def get_related_queryset(self) -> QuerySet[SubscribedItem]:
             .prefetch_related(
                 "report",
                 "subscription__filter_questions",
-                "subscription__extraction_fields",
+                "subscription__output_fields",
             )
         )
 

From 80b8293990d14c7eaffba01910e3b84ba4551760 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 18 Nov 2025 00:31:32 +0000
Subject: [PATCH 23/93] missed one renaming, maybe use generic fk for

---
 .../templates/subscriptions/subscription_detail.html            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/radis/subscriptions/templates/subscriptions/subscription_detail.html b/radis/subscriptions/templates/subscriptions/subscription_detail.html
index 8ba95b04..61170e02 100644
--- a/radis/subscriptions/templates/subscriptions/subscription_detail.html
+++ b/radis/subscriptions/templates/subscriptions/subscription_detail.html
@@ -134,7 +134,7 @@ <h4>Filter Questions</h4>
         {% endif %}
     {% endwith %}
     <h4 class="mt-4">Extraction Fields</h4>
-    {% with fields=subscription.extraction_fields.all %}
+    {% with fields=subscription.output_fields.all %}
         {% if fields %}
             <dl class="row">
                 {% for field in fields %}

From def36f1d85dbdfc0323f4bf505b05ef69e8ba356 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 18 Nov 2025 01:01:10 +0000
Subject: [PATCH 24/93] remove unnecessary iter

---
 radis/subscriptions/models.py                 | 24 -------------------
 .../_subscribed_item_preview.html             | 20 ----------------
 2 files changed, 44 deletions(-)

diff --git a/radis/subscriptions/models.py b/radis/subscriptions/models.py
index f049f065..470c1fa2 100644
--- a/radis/subscriptions/models.py
+++ b/radis/subscriptions/models.py
@@ -114,30 +114,6 @@ class Meta:
     def __str__(self):
         return f"SubscribedItem of {self.subscription} [{self.pk}]"
 
-    def iter_filter_results(self) -> list[tuple[FilterQuestion, bool]]:
-        if not self.filter_results:
-            return []
-
-        results: list[tuple[FilterQuestion, bool]] = []
-        subscription_questions = {str(q.pk): q for q in self.subscription.filter_questions.all()}
-        for key, value in self.filter_results.items():
-            question = subscription_questions.get(str(key))
-            if question is not None:
-                results.append((question, bool(value)))
-        return results
-
-    def iter_extraction_results(self) -> list[tuple[OutputField, object]]:
-        if not self.extraction_results:
-            return []
-
-        results: list[tuple[OutputField, object]] = []
-        subscription_fields = {str(f.pk): f for f in self.subscription.output_fields.all()}
-        for key, value in self.extraction_results.items():
-            field = subscription_fields.get(str(key))
-            if field is not None:
-                results.append((field, value))
-        return results
-
 
 class SubscriptionJob(AnalysisJob):
     default_priority = settings.SUBSCRIPTION_DEFAULT_PRIORITY
diff --git a/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html b/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
index e8317076..432891f5 100644
--- a/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
+++ b/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
@@ -4,26 +4,6 @@
             <div class="d-flex justify-content-between">
                 {% include "reports/_report_header.html" with report=subscribed_item.report %}
             </div>
-            {% with filter_results=subscribed_item.iter_filter_results %}
-                {% if filter_results %}
-                    <div>
-                        <div class="fw-semibold small text-uppercase text-secondary">Filter results</div>
-                        <ul class="mb-2 small">
-                            {% for question, value in filter_results %}<li>{{ question.question }} → {{ value|yesno:"Yes,No" }}</li>{% endfor %}
-                        </ul>
-                    </div>
-                {% endif %}
-            {% endwith %}
-            {% with extraction_results=subscribed_item.iter_extraction_results %}
-                {% if extraction_results %}
-                    <div>
-                        <div class="fw-semibold small text-uppercase text-secondary">Extraction results</div>
-                        <ul class="mb-2 small">
-                            {% for field, value in extraction_results %}<li>{{ field.name }} → {{ value|default_if_none:"—" }}</li>{% endfor %}
-                        </ul>
-                    </div>
-                {% endif %}
-            {% endwith %}
             <div class="d-flex flex-column gap-1">
                 <div class="clamp-3" :class="{'clamp-3': !full, 'pre-line': full}">{{ subscribed_item.report.body }}</div>
                 <div class="d-flex gap-2">

From da048fb6821f57948c3ce1801bff5f461b709d69 Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Wed, 19 Nov 2025 13:25:50 +0000
Subject: [PATCH 25/93] Allow middleware to detect locale from users browser
 (via Accept-Language header) and use it downstream to deal with format
 parsing and translations

---
 radis/settings/base.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/radis/settings/base.py b/radis/settings/base.py
index dea0bb53..feeba638 100644
--- a/radis/settings/base.py
+++ b/radis/settings/base.py
@@ -92,6 +92,7 @@
     "django.middleware.security.SecurityMiddleware",
     "whitenoise.middleware.WhiteNoiseMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
+    "django.middleware.locale.LocaleMiddleware",
     "django.middleware.common.CommonMiddleware",
     "django.middleware.csrf.CsrfViewMiddleware",
     "django.contrib.auth.middleware.AuthenticationMiddleware",

From eb0829919b7df9ffc474a04dd708d3f1d6d5d72c Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Wed, 19 Nov 2025 19:31:52 +0000
Subject: [PATCH 26/93] readd provider form field and use
 procrastinate_on_delete_sql

---
 radis/subscriptions/forms.py                  | 12 ++++++++++
 .../0011_procrastinate_on_delete.py           | 23 +++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 radis/subscriptions/migrations/0011_procrastinate_on_delete.py

diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index 05274a62..157363fa 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -11,6 +11,7 @@
 from radis.search.forms import AGE_STEP, MAX_AGE, MIN_AGE
 
 from .models import FilterQuestion, Subscription
+from .site import subscription_retrieval_providers
 
 
 class SubscriptionForm(forms.ModelForm):
@@ -18,6 +19,7 @@ class Meta:
         model = Subscription
         fields = [
             "name",
+            "provider",
             "query",
             "language",
             "modalities",
@@ -31,12 +33,21 @@ class Meta:
         labels = {"patient_id": "Patient ID"}
         help_texts = {
             "name": "Name of the Subscription",
+            "provider": "The search provider to use for the database query",
             "query": "A query to filter reports",
         }
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
+        self.fields["provider"].widget = forms.Select(
+            choices=sorted(
+                [
+                    (provider.name, provider.name)
+                    for provider in subscription_retrieval_providers.values()
+                ]
+            )
+        )
         self.fields["language"].choices = [  # type: ignore
             (language.pk, LANGUAGE_LABELS[language.code])
             for language in Language.objects.order_by("code")
@@ -82,6 +93,7 @@ def build_layout(self):
             Row(
                 Column(
                     "name",
+                    "provider",
                     "query",
                     "send_finished_mail",
                     Formset(
diff --git a/radis/subscriptions/migrations/0011_procrastinate_on_delete.py b/radis/subscriptions/migrations/0011_procrastinate_on_delete.py
new file mode 100644
index 00000000..bd5cd156
--- /dev/null
+++ b/radis/subscriptions/migrations/0011_procrastinate_on_delete.py
@@ -0,0 +1,23 @@
+# Generated by Django 5.1.6 on 2025-02-26 08:51
+
+from django.db import migrations
+
+from adit_radis_shared.common.utils.migration_utils import procrastinate_on_delete_sql
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("subscriptions", "0010_rename_answers_subscribeditem_filter_results_and_more"),
+        ("procrastinate", "0028_add_cancel_states"),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            sql=procrastinate_on_delete_sql("subscriptions", "subscriptionjob"),
+            reverse_sql=procrastinate_on_delete_sql("subscriptions", "subscriptionjob", reverse=True),
+        ),
+        migrations.RunSQL(
+            sql=procrastinate_on_delete_sql("subscriptions", "subscriptiontask"),
+            reverse_sql=procrastinate_on_delete_sql("subscriptions", "subscriptiontask", reverse=True),
+        ),
+    ]

From 018d8aede29a602742cc461c1a220fe907639e07 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Wed, 19 Nov 2025 20:04:42 +0000
Subject: [PATCH 27/93] small fixes

---
 radis/subscriptions/forms.py                 | 2 +-
 radis/subscriptions/processors.py            | 7 ++++---
 radis/subscriptions/utils/processor_utils.py | 4 ++--
 radis/subscriptions/views.py                 | 4 ++--
 4 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index 157363fa..e6cfe226 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -200,7 +200,7 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
         self.fields["name"].required = True
-        self.fields["description"].required = False
+        self.fields["description"].required = True
 
         self.helper = FormHelper()
         self.helper.form_tag = False
diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index a94f3555..4ed4558e 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -65,9 +65,10 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
             filter_response = self.client.extract_data(filter_prompt, filter_bundle.schema)
 
             for field_name, question in filter_bundle.mapping:
-                answer = bool(getattr(filter_response, field_name))
-                filter_results[str(question.pk)] = answer
-                if answer != question.expected_answer_bool:
+                answer = getattr(filter_response, field_name, None)
+                answer_bool = False if answer is None else bool(answer)
+                filter_results[str(question.pk)] = answer_bool
+                if answer_bool != question.expected_answer_bool:
                     is_accepted = False
         else:
             logger.debug(
diff --git a/radis/subscriptions/utils/processor_utils.py b/radis/subscriptions/utils/processor_utils.py
index ac78b9f3..d0df2e91 100644
--- a/radis/subscriptions/utils/processor_utils.py
+++ b/radis/subscriptions/utils/processor_utils.py
@@ -29,7 +29,7 @@ def build_filter_schema(questions: QuerySet[FilterQuestion]) -> FilterSchemaBund
     field_definitions: dict[str, Any] = {}
     mapping: list[tuple[str, FilterQuestion]] = []
 
-    for index, question in enumerate(questions.all()):
+    for index, question in enumerate(questions.order_by("pk")):
         field_name = f"filter_{index}"
         field_definitions[field_name] = (bool, ...)
         mapping.append((field_name, question))
@@ -47,7 +47,7 @@ def build_extraction_schema(fields: QuerySet[OutputField]) -> ExtractionSchemaBu
     field_definitions: dict[str, Any] = {}
     mapping: list[tuple[str, OutputField]] = []
 
-    for index, field in enumerate(fields.all()):
+    for index, field in enumerate(fields.order_by("pk")):
         field_name = f"extraction_{index}"
         if field.output_type == OutputType.TEXT:
             output_type = str
diff --git a/radis/subscriptions/views.py b/radis/subscriptions/views.py
index a73cc49c..4a84b332 100644
--- a/radis/subscriptions/views.py
+++ b/radis/subscriptions/views.py
@@ -92,7 +92,7 @@ def form_valid(self, form) -> HttpResponse:
                 self.object: Subscription = form.save()
             except IntegrityError as e:
                 if "unique_subscription_name_per_user" in str(e):
-                    form.add_error("name", "An subscription with this name already exists.")
+                    form.add_error("name", "A subscription with this name already exists.")
                     return self.form_invalid(form)
                 raise e
 
@@ -144,7 +144,7 @@ def form_valid(self, form) -> HttpResponse:
                 self.object = form.save()
             except IntegrityError as e:
                 if "unique_subscription_name_per_user" in str(e):
-                    form.add_error("name", "An subscription with this name already exists.")
+                    form.add_error("name", "A subscription with this name already exists.")
                     return self.form_invalid(form)
                 raise e
 

From e7e6c7c4395b0ad516c748825696de8c2f7ad83e Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Wed, 19 Nov 2025 20:14:42 +0000
Subject: [PATCH 28/93] stronger form validation

---
 radis/subscriptions/forms.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index e6cfe226..f91aa322 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -178,6 +178,12 @@ def has_changed(self) -> bool:
 
         return super().has_changed()
 
+    def clean_question(self):
+        question = self.cleaned_data["question"]
+        if len(question) > 300:  # already enforced by model
+            raise forms.ValidationError("Question too long")
+        return question
+
     def clean(self) -> dict[str, Any]:
         cleaned_data = super().clean()
         assert cleaned_data
@@ -185,7 +191,7 @@ def clean(self) -> dict[str, Any]:
         question = cleaned_data.get("question")
         expected_answer = cleaned_data.get("expected_answer")
 
-        if bool(question) ^ bool(expected_answer):
+        if (question and not expected_answer) or (expected_answer and not question):
             raise forms.ValidationError("You must provide both a question and an expected answer.")
 
         return cleaned_data

From a67fe075876cc4e3aa0c262419dab23e07ac4256 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Wed, 19 Nov 2025 20:31:03 +0000
Subject: [PATCH 29/93] fix filter logic

---
 radis/subscriptions/forms.py      |  6 ------
 radis/subscriptions/processors.py | 11 +++++++----
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index f91aa322..e5eaf7fb 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -178,12 +178,6 @@ def has_changed(self) -> bool:
 
         return super().has_changed()
 
-    def clean_question(self):
-        question = self.cleaned_data["question"]
-        if len(question) > 300:  # already enforced by model
-            raise forms.ValidationError("Question too long")
-        return question
-
     def clean(self) -> dict[str, Any]:
         cleaned_data = super().clean()
         assert cleaned_data
diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index 4ed4558e..6f129aeb 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -66,10 +66,13 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
 
             for field_name, question in filter_bundle.mapping:
                 answer = getattr(filter_response, field_name, None)
-                answer_bool = False if answer is None else bool(answer)
-                filter_results[str(question.pk)] = answer_bool
-                if answer_bool != question.expected_answer_bool:
+                if answer is None:
                     is_accepted = False
+                else:
+                    answer_bool = bool(answer)
+                    filter_results[str(question.pk)] = answer_bool
+                    if answer_bool != question.expected_answer_bool:
+                        is_accepted = False
         else:
             logger.debug(
                 "Subscription %s has no filter questions; accepting report %s by default",
@@ -96,7 +99,7 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
             )
 
             for field_name, field in extraction_bundle.mapping:
-                extraction_results[str(field.pk)] = getattr(extraction_response, field_name)
+                extraction_results[str(field.pk)] = getattr(extraction_response, field_name, None)
 
         SubscribedItem.objects.create(
             subscription=task.job.subscription,

From ed9949515a01c009c7923037722dad4acfcf63b8 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Wed, 19 Nov 2025 21:47:14 +0000
Subject: [PATCH 30/93] rework pydantic model generation

---
 radis/subscriptions/processors.py             | 40 +++++----
 .../tests/unit/test_processors.py             | 29 +++---
 radis/subscriptions/utils/processor_utils.py  | 88 ++++---------------
 3 files changed, 59 insertions(+), 98 deletions(-)

diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index 6f129aeb..a9a31804 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -9,6 +9,10 @@
 
 from radis.chats.utils.chat_client import ChatClient
 from radis.core.processors import AnalysisTaskProcessor
+from radis.extractions.utils.processor_utils import (
+    generate_output_fields_prompt,
+    generate_output_fields_schema,
+)
 from radis.reports.models import Report
 
 from .models import (
@@ -17,10 +21,10 @@
     SubscriptionTask,
 )
 from .utils.processor_utils import (
-    build_extraction_schema,
-    build_filter_schema,
     generate_filter_questions_prompt,
-    generate_output_fields_prompt,
+    generate_filter_questions_schema,
+    get_filter_question_field_name,
+    get_output_field_name,
 )
 
 logger = logging.getLogger(__name__)
@@ -50,21 +54,24 @@ def process_task(self, task: SubscriptionTask) -> None:
 
     def process_report(self, report: Report, task: SubscriptionTask) -> None:
         subscription: Subscription = task.job.subscription
-        filter_bundle = build_filter_schema(subscription.filter_questions)
 
         filter_results: dict[str, bool] = {}
         is_accepted = True
 
-        if filter_bundle.mapping:
+        filter_questions = subscription.filter_questions.order_by("pk")
+
+        if filter_questions.exists():
             filter_prompt = Template(settings.SUBSCRIPTION_FILTER_PROMPT).substitute(
                 {
                     "report": report.body,
-                    "questions": generate_filter_questions_prompt(filter_bundle.mapping),
+                    "questions": generate_filter_questions_prompt(filter_questions),
                 }
             )
-            filter_response = self.client.extract_data(filter_prompt, filter_bundle.schema)
+            filter_schema = generate_filter_questions_schema(filter_questions)
+            filter_response = self.client.extract_data(filter_prompt, filter_schema)
 
-            for field_name, question in filter_bundle.mapping:
+            for question in filter_questions.all():
+                field_name = get_filter_question_field_name(question)
                 answer = getattr(filter_response, field_name, None)
                 if answer is None:
                     is_accepted = False
@@ -84,22 +91,23 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
             logger.debug(f"Report {report.pk} was rejected by subscription {subscription.pk}")
             return
 
-        extraction_bundle = build_extraction_schema(subscription.output_fields)
         extraction_results: dict[str, Any] = {}
+        output_fields = subscription.output_fields.order_by("pk")
 
-        if extraction_bundle.mapping:
+        if output_fields.exists():
             extraction_prompt = Template(settings.SUBSCRIPTION_EXTRACTION_PROMPT).substitute(
                 {
                     "report": report.body,
-                    "fields": generate_output_fields_prompt(extraction_bundle.mapping),
+                    "fields": generate_output_fields_prompt(output_fields),
                 }
             )
-            extraction_response = self.client.extract_data(
-                extraction_prompt, extraction_bundle.schema
-            )
+            extraction_schema = generate_output_fields_schema(output_fields)
+            extraction_response = self.client.extract_data(extraction_prompt, extraction_schema)
 
-            for field_name, field in extraction_bundle.mapping:
-                extraction_results[str(field.pk)] = getattr(extraction_response, field_name, None)
+            for field in output_fields.all():
+                extraction_results[str(field.pk)] = getattr(
+                    extraction_response, get_output_field_name(field), None
+                )
 
         SubscribedItem.objects.create(
             subscription=task.job.subscription,
diff --git a/radis/subscriptions/tests/unit/test_processors.py b/radis/subscriptions/tests/unit/test_processors.py
index 457c1891..ca323c2e 100644
--- a/radis/subscriptions/tests/unit/test_processors.py
+++ b/radis/subscriptions/tests/unit/test_processors.py
@@ -1,28 +1,35 @@
 from unittest.mock import MagicMock, patch
 
 import pytest
-from pydantic import BaseModel
+from pydantic import create_model
 
 from radis.chats.utils.testing_helpers import create_openai_client_mock
 from radis.subscriptions.models import SubscribedItem
 from radis.subscriptions.processors import SubscriptionTaskProcessor
+from radis.subscriptions.utils.processor_utils import (
+    get_filter_question_field_name,
+    get_output_field_name,
+)
 from radis.subscriptions.utils.testing_helpers import create_subscription_task
 
 
-class FilterOutput(BaseModel):
-    filter_0: bool
-
-
-class ExtractionOutput(BaseModel):
-    extraction_0: str
-
-
 @pytest.mark.django_db(transaction=True)
 def test_subscription_task_processor_filters_and_extracts():
     task, filter_question, output_field, report = create_subscription_task()
 
-    filter_output = FilterOutput(filter_0=True)
-    extraction_output = ExtractionOutput(extraction_0="Pneumothorax status confirmed")
+    filter_field_name = get_filter_question_field_name(filter_question)
+    extraction_field_name = get_output_field_name(output_field)
+    filter_field_definitions = {}
+    filter_field_definitions[filter_field_name] = (bool, ...)
+
+    extraction_field_definitions = {}
+    extraction_field_definitions[extraction_field_name] = (str, ...)
+
+    FilterOutput = create_model("FilterOutput", **filter_field_definitions)
+    ExtractionOutput = create_model("ExtractionOutput", **extraction_field_definitions)
+
+    filter_output = FilterOutput(**{filter_field_name: True})
+    extraction_output = ExtractionOutput(**{extraction_field_name: "Pneumothorax status confirmed"})
 
     filter_response = MagicMock(choices=[MagicMock(message=MagicMock(parsed=filter_output))])
     extraction_response = MagicMock(
diff --git a/radis/subscriptions/utils/processor_utils.py b/radis/subscriptions/utils/processor_utils.py
index d0df2e91..1c25ef2b 100644
--- a/radis/subscriptions/utils/processor_utils.py
+++ b/radis/subscriptions/utils/processor_utils.py
@@ -1,94 +1,40 @@
 from __future__ import annotations
 
-from dataclasses import dataclass
 from typing import Any
 
 from django.db.models import QuerySet
 from pydantic import BaseModel, create_model
 
-from radis.extractions.models import OutputField, OutputType
+from radis.extractions.models import OutputField
 
 from ..models import FilterQuestion
 
-type Numeric = float | int
 
+def _filter_question_field_name(question: FilterQuestion) -> str:
+    return f"question_{question.pk}"
 
-@dataclass(slots=True)
-class FilterSchemaBundle:
-    schema: type[BaseModel]
-    mapping: list[tuple[str, FilterQuestion]]
 
+def get_filter_question_field_name(question: FilterQuestion) -> str:
+    return _filter_question_field_name(question)
 
-@dataclass(slots=True)
-class ExtractionSchemaBundle:
-    schema: type[BaseModel]
-    mapping: list[tuple[str, OutputField]]
 
+def get_output_field_name(field: OutputField) -> str:
+    return field.name
 
-def build_filter_schema(questions: QuerySet[FilterQuestion]) -> FilterSchemaBundle:
+
+def generate_filter_questions_schema(questions: QuerySet[FilterQuestion]) -> type[BaseModel]:
     field_definitions: dict[str, Any] = {}
-    mapping: list[tuple[str, FilterQuestion]] = []
 
-    for index, question in enumerate(questions.order_by("pk")):
-        field_name = f"filter_{index}"
+    for question in questions.order_by("pk").all():
+        field_name = _filter_question_field_name(question)
         field_definitions[field_name] = (bool, ...)
-        mapping.append((field_name, question))
 
     model_name = "SubscriptionFilterResultsModel"
-    schema = (
-        create_model(model_name, **field_definitions)
-        if field_definitions
-        else create_model(model_name)
-    )
-    return FilterSchemaBundle(schema, mapping)
+    return create_model(model_name, **field_definitions)
 
 
-def build_extraction_schema(fields: QuerySet[OutputField]) -> ExtractionSchemaBundle:
-    field_definitions: dict[str, Any] = {}
-    mapping: list[tuple[str, OutputField]] = []
-
-    for index, field in enumerate(fields.order_by("pk")):
-        field_name = f"extraction_{index}"
-        if field.output_type == OutputType.TEXT:
-            output_type = str
-        elif field.output_type == OutputType.NUMERIC:
-            output_type = Numeric
-        elif field.output_type == OutputType.BOOLEAN:
-            output_type = bool
-        else:
-            raise ValueError(f"Unknown output type: {field.output_type}")
-
-        field_definitions[field_name] = (output_type, ...)
-
-        mapping.append((field_name, field))
-
-    model_name = "SubscriptionExtractionResultsModel"
-    schema = (
-        create_model(model_name, **field_definitions)
-        if field_definitions
-        else create_model(model_name)
-    )
-    return ExtractionSchemaBundle(schema, mapping)
-
-
-def generate_filter_questions_prompt(mapping: list[tuple[str, FilterQuestion]]) -> str:
-    if not mapping:
-        return "None"
-
-    lines: list[str] = []
-    for field_name, question in mapping:
-        lines.append(f"{field_name}: {question.question}")
-    return "\n".join(lines)
-
-
-def generate_output_fields_prompt(mapping: list[tuple[str, OutputField]]) -> str:
-    if not mapping:
-        return "None"
-
-    lines: list[str] = []
-    for field_name, field in mapping:
-        description = field.description or "No description provided."
-        lines.append(
-            f"{field_name}: {field.name} — {description} [type: {field.get_output_type_display()}]"
-        )
-    return "\n".join(lines)
+def generate_filter_questions_prompt(questions: QuerySet[FilterQuestion]) -> str:
+    prompt = ""
+    for question in questions.order_by("pk").all():
+        prompt += f"{_filter_question_field_name(question)}: {question.question}\n"
+    return prompt

From 362bfea8b514501b0871e34081dbeffe202297fe Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Wed, 19 Nov 2025 22:00:43 +0000
Subject: [PATCH 31/93] add more tests for subscription processor

---
 .../tests/unit/test_processors.py             | 75 +++++++++++++++++++
 1 file changed, 75 insertions(+)

diff --git a/radis/subscriptions/tests/unit/test_processors.py b/radis/subscriptions/tests/unit/test_processors.py
index ca323c2e..bb7416bb 100644
--- a/radis/subscriptions/tests/unit/test_processors.py
+++ b/radis/subscriptions/tests/unit/test_processors.py
@@ -48,3 +48,78 @@ def test_subscription_task_processor_filters_and_extracts():
     assert subscribed_item.extraction_results == {
         str(output_field.pk): "Pneumothorax status confirmed"
     }
+
+
+@pytest.mark.django_db(transaction=True)
+def test_subscription_task_processor_handles_llm_null_response():
+    task, _, _, report = create_subscription_task()
+
+    processor = SubscriptionTaskProcessor(task)
+    processor.client.extract_data = MagicMock(return_value=None)
+
+    processor.start()
+
+    assert not SubscribedItem.objects.filter(
+        subscription=task.job.subscription, report=report
+    ).exists()
+    processor.client.extract_data.assert_called_once()
+
+
+@pytest.mark.django_db(transaction=True)
+def test_subscription_task_processor_with_no_expected_answer():
+    task, filter_question, _, report = create_subscription_task()
+
+    filter_field_name = get_filter_question_field_name(filter_question)
+    filter_response = MagicMock()
+    setattr(filter_response, filter_field_name, None)
+
+    processor = SubscriptionTaskProcessor(task)
+    processor.client.extract_data = MagicMock(return_value=filter_response)
+
+    processor.start()
+
+    assert not SubscribedItem.objects.filter(
+        subscription=task.job.subscription, report=report
+    ).exists()
+    processor.client.extract_data.assert_called_once()
+
+
+@pytest.mark.django_db(transaction=True)
+def test_subscription_task_processor_extraction_only():
+    task, _, output_field, report = create_subscription_task()
+    task.job.subscription.filter_questions.all().delete()
+
+    extraction_field_name = get_output_field_name(output_field)
+    extraction_field_definitions = {}
+    extraction_field_definitions[extraction_field_name] = (str, ...)
+
+    ExtractionOutput = create_model("ExtractionOnlyOutput", **extraction_field_definitions)
+    extraction_output = ExtractionOutput(**{extraction_field_name: "Only extraction response"})
+
+    openai_mock = create_openai_client_mock(extraction_output)
+    with patch("openai.OpenAI", return_value=openai_mock):
+        SubscriptionTaskProcessor(task).start()
+
+    subscribed_item = SubscribedItem.objects.get(subscription=task.job.subscription, report=report)
+    assert subscribed_item.filter_results is None
+    assert subscribed_item.extraction_results == {str(output_field.pk): "Only extraction response"}
+
+
+@pytest.mark.django_db(transaction=True)
+def test_subscription_task_processor_filter_only():
+    task, filter_question, output_field, report = create_subscription_task()
+    output_field.delete()
+
+    filter_field_name = get_filter_question_field_name(filter_question)
+    filter_field_definitions = {}
+    filter_field_definitions[filter_field_name] = (bool, ...)
+    FilterOutput = create_model("FilterOnlyOutput", **filter_field_definitions)
+    filter_output = FilterOutput(**{filter_field_name: True})
+
+    openai_mock = create_openai_client_mock(filter_output)
+    with patch("openai.OpenAI", return_value=openai_mock):
+        SubscriptionTaskProcessor(task).start()
+
+    subscribed_item = SubscribedItem.objects.get(subscription=task.job.subscription, report=report)
+    assert subscribed_item.filter_results == {str(filter_question.pk): True}
+    assert subscribed_item.extraction_results is None

From d839e58f39b1505018374bb05bac27831fbbb230 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Wed, 19 Nov 2025 22:11:15 +0000
Subject: [PATCH 32/93] remove redundant db query

---
 radis/subscriptions/processors.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index a9a31804..a512b10c 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -70,7 +70,7 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
             filter_schema = generate_filter_questions_schema(filter_questions)
             filter_response = self.client.extract_data(filter_prompt, filter_schema)
 
-            for question in filter_questions.all():
+            for question in filter_questions:
                 field_name = get_filter_question_field_name(question)
                 answer = getattr(filter_response, field_name, None)
                 if answer is None:
@@ -104,7 +104,7 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
             extraction_schema = generate_output_fields_schema(output_fields)
             extraction_response = self.client.extract_data(extraction_prompt, extraction_schema)
 
-            for field in output_fields.all():
+            for field in output_fields:
                 extraction_results[str(field.pk)] = getattr(
                     extraction_response, get_output_field_name(field), None
                 )

From 2fc97d78516d2af2181b4873d68fc6d10b91de54 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Wed, 19 Nov 2025 22:50:49 +0000
Subject: [PATCH 33/93] readd provider validation

---
 radis/subscriptions/forms.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index e5eaf7fb..32c0cd59 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -139,6 +139,13 @@ def clean(self) -> dict[str, Any] | None:
         if age_from is not None and age_till is not None and age_from >= age_till:
             raise forms.ValidationError("Age from must be less than age till")
 
+        provider = self.cleaned_data["provider"]
+        query = self.cleaned_data["query"]
+        if query != "" and not provider:
+            raise forms.ValidationError(
+                "Setup of RADIS is incomplete. No retrieval providers are registered."
+            )
+
         return super().clean()
 
 

From d726ea9e6c2e37e5f2b625ba0bf908204ab7b35c Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Thu, 20 Nov 2025 11:11:17 +0000
Subject: [PATCH 34/93] display extraction results in subscription inbox

---
 .../_subscribed_item_preview.html             | 16 ++++++++++++++++
 radis/subscriptions/templatetags/__init__.py  |  1 +
 .../templatetags/subscriptions_extras.py      | 19 +++++++++++++++++++
 radis/subscriptions/views.py                  |  1 -
 4 files changed, 36 insertions(+), 1 deletion(-)
 create mode 100644 radis/subscriptions/templatetags/__init__.py
 create mode 100644 radis/subscriptions/templatetags/subscriptions_extras.py

diff --git a/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html b/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
index 432891f5..40dd041d 100644
--- a/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
+++ b/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
@@ -1,3 +1,4 @@
+{% load subscriptions_extras %}
 <div class="card mb-3">
     <div class="card-body" x-data="{full: false, questions: false}">
         <div class="d-flex flex-column gap-1">
@@ -6,6 +7,21 @@
             </div>
             <div class="d-flex flex-column gap-1">
                 <div class="clamp-3" :class="{'clamp-3': !full, 'pre-line': full}">{{ subscribed_item.report.body }}</div>
+                {% if subscribed_item.extraction_results %}
+                    <div class="border-top pt-2 mt-1">
+                        <div class="fw-semibold small text-muted mb-1">Extracted Fields</div>
+                        <dl class="row mb-0 small">
+                            {% for field in subscribed_item.subscription.output_fields.all %}
+                                {% with field_key=field.pk|stringformat:"s" %}
+                                    <dt class="col-sm-4">{{ field.name }}</dt>
+                                    <dd class="col-sm-8">
+                                        {{ subscribed_item.extraction_results|get_item:field_key|default_if_none:"—" }}
+                                    </dd>
+                                {% endwith %}
+                            {% endfor %}
+                        </dl>
+                    </div>
+                {% endif %}
                 <div class="d-flex gap-2">
                     <button type="button"
                             class="btn btn-sm btn-link p-0 border-0"
diff --git a/radis/subscriptions/templatetags/__init__.py b/radis/subscriptions/templatetags/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/radis/subscriptions/templatetags/__init__.py
@@ -0,0 +1 @@
+
diff --git a/radis/subscriptions/templatetags/subscriptions_extras.py b/radis/subscriptions/templatetags/subscriptions_extras.py
new file mode 100644
index 00000000..ee0f3d7a
--- /dev/null
+++ b/radis/subscriptions/templatetags/subscriptions_extras.py
@@ -0,0 +1,19 @@
+from collections.abc import Mapping
+from typing import Any
+
+from django import template
+
+register = template.Library()
+
+
+@register.filter
+def get_item(mapping: Mapping[str, Any] | None, key: Any) -> Any:
+    """
+    Safely retrieve a value from a mapping for template usage.
+    """
+    if mapping is None:
+        return None
+    try:
+        return mapping.get(str(key))
+    except AttributeError:
+        return None
diff --git a/radis/subscriptions/views.py b/radis/subscriptions/views.py
index 4a84b332..7038bd23 100644
--- a/radis/subscriptions/views.py
+++ b/radis/subscriptions/views.py
@@ -193,7 +193,6 @@ def get_related_queryset(self) -> QuerySet[SubscribedItem]:
             .select_related("subscription")
             .prefetch_related(
                 "report",
-                "subscription__filter_questions",
                 "subscription__output_fields",
             )
         )

From 1cbe27a5f8b0db6d1d5c1e4d39a861fb1f2de12e Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Thu, 20 Nov 2025 11:40:01 +0000
Subject: [PATCH 35/93] small refactors: remove has_changed, add logger, fix
 date for migration

---
 radis/subscriptions/forms.py                         | 12 ------------
 .../migrations/0011_procrastinate_on_delete.py       |  2 +-
 radis/subscriptions/processors.py                    |  3 +++
 3 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index 32c0cd59..be95d7f3 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -173,18 +173,6 @@ def __init__(self, *args, **kwargs):
             fields.insert(1, Field("DELETE", type="hidden"))
         self.helper.layout = Layout(Div(*fields))
 
-    def has_changed(self) -> bool:
-        if not self.is_bound:
-            return super().has_changed()
-
-        question = (self.data.get(self.add_prefix("question")) or "").strip()
-        expected_answer = self.data.get(self.add_prefix("expected_answer")) or ""
-
-        if not question and not expected_answer:
-            return False
-
-        return super().has_changed()
-
     def clean(self) -> dict[str, Any]:
         cleaned_data = super().clean()
         assert cleaned_data
diff --git a/radis/subscriptions/migrations/0011_procrastinate_on_delete.py b/radis/subscriptions/migrations/0011_procrastinate_on_delete.py
index bd5cd156..7e88a8e4 100644
--- a/radis/subscriptions/migrations/0011_procrastinate_on_delete.py
+++ b/radis/subscriptions/migrations/0011_procrastinate_on_delete.py
@@ -1,4 +1,4 @@
-# Generated by Django 5.1.6 on 2025-02-26 08:51
+# Generated by Django 5.1.6 on 2025-11-19 08:51
 
 from django.db import migrations
 
diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index a512b10c..2caa9e17 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -74,6 +74,9 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
                 field_name = get_filter_question_field_name(question)
                 answer = getattr(filter_response, field_name, None)
                 if answer is None:
+                    logger.debug(
+                        f"LLM returned None for question {question.pk} on report {report.pk}"
+                    )
                     is_accepted = False
                 else:
                     answer_bool = bool(answer)

From 8ebae989324184d8120e91cf98017600c86a9ae2 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Thu, 20 Nov 2025 12:55:12 +0000
Subject: [PATCH 36/93] resolve n+1 query during processing, readd empty form
 validator

---
 radis/subscriptions/forms.py                         | 12 ++++++++++++
 .../migrations/0011_procrastinate_on_delete.py       |  2 +-
 radis/subscriptions/processors.py                    |  4 ++--
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index be95d7f3..32c0cd59 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -173,6 +173,18 @@ def __init__(self, *args, **kwargs):
             fields.insert(1, Field("DELETE", type="hidden"))
         self.helper.layout = Layout(Div(*fields))
 
+    def has_changed(self) -> bool:
+        if not self.is_bound:
+            return super().has_changed()
+
+        question = (self.data.get(self.add_prefix("question")) or "").strip()
+        expected_answer = self.data.get(self.add_prefix("expected_answer")) or ""
+
+        if not question and not expected_answer:
+            return False
+
+        return super().has_changed()
+
     def clean(self) -> dict[str, Any]:
         cleaned_data = super().clean()
         assert cleaned_data
diff --git a/radis/subscriptions/migrations/0011_procrastinate_on_delete.py b/radis/subscriptions/migrations/0011_procrastinate_on_delete.py
index 7e88a8e4..1892eb73 100644
--- a/radis/subscriptions/migrations/0011_procrastinate_on_delete.py
+++ b/radis/subscriptions/migrations/0011_procrastinate_on_delete.py
@@ -1,4 +1,4 @@
-# Generated by Django 5.1.6 on 2025-11-19 08:51
+# Generated by Django 5.1.6 on 2025-11-18 08:51
 
 from django.db import migrations
 
diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index 2caa9e17..46bdba5f 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -60,7 +60,7 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
 
         filter_questions = subscription.filter_questions.order_by("pk")
 
-        if filter_questions.exists():
+        if filter_questions.first():
             filter_prompt = Template(settings.SUBSCRIPTION_FILTER_PROMPT).substitute(
                 {
                     "report": report.body,
@@ -97,7 +97,7 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
         extraction_results: dict[str, Any] = {}
         output_fields = subscription.output_fields.order_by("pk")
 
-        if output_fields.exists():
+        if output_fields.first():
             extraction_prompt = Template(settings.SUBSCRIPTION_EXTRACTION_PROMPT).substitute(
                 {
                     "report": report.body,

From e72da19faf6fb0528249cfd485b9c99f9fd92d01 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Thu, 20 Nov 2025 13:39:42 +0000
Subject: [PATCH 37/93] prefetch filterquestions

---
 radis/subscriptions/processors.py            |  4 ++--
 radis/subscriptions/utils/processor_utils.py | 11 +++++------
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index 46bdba5f..2328dbaf 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -58,9 +58,9 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
         filter_results: dict[str, bool] = {}
         is_accepted = True
 
-        filter_questions = subscription.filter_questions.order_by("pk")
+        filter_questions = list(subscription.filter_questions.order_by("pk"))
 
-        if filter_questions.first():
+        if filter_questions:
             filter_prompt = Template(settings.SUBSCRIPTION_FILTER_PROMPT).substitute(
                 {
                     "report": report.body,
diff --git a/radis/subscriptions/utils/processor_utils.py b/radis/subscriptions/utils/processor_utils.py
index 1c25ef2b..f0786c78 100644
--- a/radis/subscriptions/utils/processor_utils.py
+++ b/radis/subscriptions/utils/processor_utils.py
@@ -1,8 +1,7 @@
 from __future__ import annotations
 
-from typing import Any
+from typing import Any, Iterable
 
-from django.db.models import QuerySet
 from pydantic import BaseModel, create_model
 
 from radis.extractions.models import OutputField
@@ -22,10 +21,10 @@ def get_output_field_name(field: OutputField) -> str:
     return field.name
 
 
-def generate_filter_questions_schema(questions: QuerySet[FilterQuestion]) -> type[BaseModel]:
+def generate_filter_questions_schema(questions: Iterable[FilterQuestion]) -> type[BaseModel]:
     field_definitions: dict[str, Any] = {}
 
-    for question in questions.order_by("pk").all():
+    for question in questions:
         field_name = _filter_question_field_name(question)
         field_definitions[field_name] = (bool, ...)
 
@@ -33,8 +32,8 @@ def generate_filter_questions_schema(questions: QuerySet[FilterQuestion]) -> typ
     return create_model(model_name, **field_definitions)
 
 
-def generate_filter_questions_prompt(questions: QuerySet[FilterQuestion]) -> str:
+def generate_filter_questions_prompt(questions: Iterable[FilterQuestion]) -> str:
     prompt = ""
-    for question in questions.order_by("pk").all():
+    for question in questions:
         prompt += f"{_filter_question_field_name(question)}: {question.question}\n"
     return prompt

From 6ef403ace06cd12161e386a500dbf8cb4964527e Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Thu, 20 Nov 2025 13:49:48 +0000
Subject: [PATCH 38/93] early exit if expected_answer does not match

---
 radis/subscriptions/processors.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index 2328dbaf..d35287a5 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -78,11 +78,13 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
                         f"LLM returned None for question {question.pk} on report {report.pk}"
                     )
                     is_accepted = False
+                    break
                 else:
                     answer_bool = bool(answer)
                     filter_results[str(question.pk)] = answer_bool
                     if answer_bool != question.expected_answer_bool:
                         is_accepted = False
+                        break
         else:
             logger.debug(
                 "Subscription %s has no filter questions; accepting report %s by default",

From d44453dd8e0a71ab336e4e0810b2ebc2b6094222 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Thu, 20 Nov 2025 13:56:15 +0000
Subject: [PATCH 39/93] remove duplicate getter

---
 radis/subscriptions/utils/processor_utils.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/radis/subscriptions/utils/processor_utils.py b/radis/subscriptions/utils/processor_utils.py
index f0786c78..3d9ee654 100644
--- a/radis/subscriptions/utils/processor_utils.py
+++ b/radis/subscriptions/utils/processor_utils.py
@@ -9,12 +9,8 @@
 from ..models import FilterQuestion
 
 
-def _filter_question_field_name(question: FilterQuestion) -> str:
-    return f"question_{question.pk}"
-
-
 def get_filter_question_field_name(question: FilterQuestion) -> str:
-    return _filter_question_field_name(question)
+    return f"question_{question.pk}"
 
 
 def get_output_field_name(field: OutputField) -> str:
@@ -25,7 +21,7 @@ def generate_filter_questions_schema(questions: Iterable[FilterQuestion]) -> typ
     field_definitions: dict[str, Any] = {}
 
     for question in questions:
-        field_name = _filter_question_field_name(question)
+        field_name = get_filter_question_field_name(question)
         field_definitions[field_name] = (bool, ...)
 
     model_name = "SubscriptionFilterResultsModel"
@@ -35,5 +31,5 @@ def generate_filter_questions_schema(questions: Iterable[FilterQuestion]) -> typ
 def generate_filter_questions_prompt(questions: Iterable[FilterQuestion]) -> str:
     prompt = ""
     for question in questions:
-        prompt += f"{_filter_question_field_name(question)}: {question.question}\n"
+        prompt += f"{get_filter_question_field_name(question)}: {question.question}\n"
     return prompt

From 69712f067270cc4e7d5101bde29fa2a0220e8fc8 Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Tue, 25 Nov 2025 15:27:33 +0000
Subject: [PATCH 40/93] Adding option to select "array type" for the Output
 Field.

---
 radis/core/static/core/core.css               | 31 ++++++++++++++
 radis/core/static/core/core.js                | 40 +++++++++++++++++++
 radis/extractions/forms.py                    | 33 +++++++++++++--
 .../migrations/0005_outputfield_is_array.py   | 15 +++++++
 radis/extractions/models.py                   |  1 +
 .../extractions/_selection_options_field.html | 27 +++++++------
 .../extractions/extraction_job_detail.html    |  6 +++
 .../extraction_job_wizard_summary.html        |  6 +++
 radis/extractions/tests/test_forms.py         | 22 ++++++++++
 radis/extractions/utils/processor_utils.py    |  8 ++++
 10 files changed, 174 insertions(+), 15 deletions(-)
 create mode 100644 radis/extractions/migrations/0005_outputfield_is_array.py

diff --git a/radis/core/static/core/core.css b/radis/core/static/core/core.css
index 0dba50c7..c86469b7 100644
--- a/radis/core/static/core/core.css
+++ b/radis/core/static/core/core.css
@@ -39,3 +39,34 @@
   -webkit-line-clamp: 3;
   -webkit-box-orient: vertical;
 }
+
+.selection-options-controls,
+.selection-options-actions {
+  gap: 0.5rem;
+}
+
+.array-toggle-field {
+  display: flex;
+  align-items: center;
+  justify-content: flex-end;
+  padding-top: 14px;
+}
+
+.array-toggle-btn {
+  width: calc(2.5rem - 6px);
+  height: calc(2.5rem - 6px);
+  border-radius: 50%;
+  font-weight: 600;
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  padding: 0;
+  line-height: 1;
+  font-size: 1rem;
+}
+
+.array-toggle-btn.active {
+  background-color: var(--bs-primary);
+  border-color: var(--bs-primary);
+  color: #fff;
+}
diff --git a/radis/core/static/core/core.js b/radis/core/static/core/core.js
index 1b6c4d8f..7b870203 100644
--- a/radis/core/static/core/core.js
+++ b/radis/core/static/core/core.js
@@ -98,19 +98,38 @@ function FormSet(rootEl) {
  */
 function SelectionOptions(rootEl) {
   const hiddenInput = rootEl.querySelector("[data-selection-input]");
+  const arrayInput = rootEl.querySelector("[data-array-input]");
   const formContainer =
     rootEl.closest(".formset-form") ?? rootEl.closest("form") ?? rootEl;
   const outputTypeField =
     formContainer.querySelector('select[name$="-output_type"]') ??
     formContainer.querySelector('select[name="output_type"]');
+  const arrayToggleButton =
+    formContainer.querySelector("[data-array-toggle]") ?? null;
+  const parseArrayValue = (value) => {
+    if (!value) {
+      return false;
+    }
+    const normalized = value.trim().toLowerCase();
+    return normalized === "true" || normalized === "1" || normalized === "on";
+  };
 
   return {
     options: [],
     maxOptions: 7,
     supportsSelection: false,
+    isArray: parseArrayValue(arrayInput?.value),
     init() {
       this.options = this.parseOptions(hiddenInput?.value);
+      this.isArray = parseArrayValue(arrayInput?.value);
       this.updateSupports();
+      if (arrayToggleButton) {
+        arrayToggleButton.addEventListener("click", (event) => {
+          event.preventDefault();
+          this.toggleArray();
+        });
+        this.updateArrayButton();
+      }
       if (outputTypeField) {
         outputTypeField.addEventListener("change", () => {
           const wasSelection = this.supportsSelection;
@@ -153,6 +172,17 @@ function SelectionOptions(rootEl) {
         .filter((opt) => opt !== "");
       hiddenInput.value = JSON.stringify(sanitized);
     },
+    syncArray() {
+      if (!arrayInput) {
+        return;
+      }
+      arrayInput.value = this.isArray ? "true" : "false";
+    },
+    syncState() {
+      this.syncOptions();
+      this.syncArray();
+      this.updateArrayButton();
+    },
     addOption() {
       if (!this.supportsSelection || this.options.length >= this.maxOptions) {
         return;
@@ -169,5 +199,15 @@ function SelectionOptions(rootEl) {
     removeOption(index) {
       this.options.splice(index, 1);
     },
+    toggleArray() {
+      this.isArray = !this.isArray;
+    },
+    updateArrayButton() {
+      if (!arrayToggleButton) {
+        return;
+      }
+      arrayToggleButton.classList.toggle("active", this.isArray);
+      arrayToggleButton.setAttribute("aria-pressed", this.isArray ? "true" : "false");
+    },
   };
 }
diff --git a/radis/extractions/forms.py b/radis/extractions/forms.py
index dc223eb7..2545339c 100644
--- a/radis/extractions/forms.py
+++ b/radis/extractions/forms.py
@@ -190,6 +190,10 @@ class OutputFieldForm(forms.ModelForm):
         required=False,
         widget=forms.HiddenInput(),
     )
+    is_array = forms.CharField(
+        required=False,
+        widget=forms.HiddenInput(),
+    )
 
     class Meta:
         model = OutputField
@@ -198,6 +202,7 @@ class Meta:
             "description",
             "output_type",
             "selection_options",
+            "is_array",
         ]
 
     def __init__(self, *args, **kwargs):
@@ -209,18 +214,34 @@ def __init__(self, *args, **kwargs):
                 "data-selection-input": "true",
             }
         )
+        self.fields["is_array"].widget.attrs.update(
+            {
+                "data-array-input": "true",
+            }
+        )
 
         initial_options = self.instance.selection_options if self.instance.pk else []
         self.initial["selection_options"] = json.dumps(initial_options)
+        self.initial["is_array"] = "true" if self.instance.is_array else "false"
 
         self.helper = FormHelper()
         self.helper.form_tag = False
         self.helper.disable_csrf = True
         self.helper.layout = Layout(
             Row(
-                Column("name", css_class="col-8"),
-                Column("output_type", css_class="col-4"),
-                css_class="g-3",
+                Column("name", css_class="col-md-7 col-12"),
+                Column("output_type", css_class="col-md-4 col-10"),
+                Column(
+                    HTML(
+                        '<button type="button" '
+                        'class="btn btn-outline-secondary btn-sm array-toggle-btn form-array-toggle" '
+                        'data-array-toggle="true" '
+                        'aria-pressed="false" '
+                        'title="Toggle array output">[ ]</button>'
+                    ),
+                    css_class="col-md-1 col-2 d-flex align-items-center justify-content-end array-toggle-field",
+                ),
+                css_class="g-3 align-items-center",
             ),
             "description",
             Div(
@@ -257,6 +278,12 @@ def clean_selection_options(self) -> list[str]:
 
         return cleaned
 
+    def clean_is_array(self) -> bool:
+        raw_value = (self.cleaned_data.get("is_array") or "").strip().lower()
+        if raw_value in {"1", "true", "on"}:
+            return True
+        return False
+
     def clean(self):
         cleaned_data = super().clean()
         if not cleaned_data:
diff --git a/radis/extractions/migrations/0005_outputfield_is_array.py b/radis/extractions/migrations/0005_outputfield_is_array.py
new file mode 100644
index 00000000..568a02c3
--- /dev/null
+++ b/radis/extractions/migrations/0005_outputfield_is_array.py
@@ -0,0 +1,15 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("extractions", "0004_outputfield_selection_options"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="outputfield",
+            name="is_array",
+            field=models.BooleanField(default=False),
+        ),
+    ]
diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index 2ae477b9..371f3b9e 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -88,6 +88,7 @@ class OutputField(models.Model):
     get_output_type_display: Callable[[], str]
     optional = models.BooleanField(default=False)
     selection_options = models.JSONField(default=list, blank=True)
+    is_array = models.BooleanField(default=False)
     job = models.ForeignKey[ExtractionJob](
         ExtractionJob, on_delete=models.CASCADE, related_name="output_fields"
     )
diff --git a/radis/extractions/templates/extractions/_selection_options_field.html b/radis/extractions/templates/extractions/_selection_options_field.html
index 91437e67..ed50f0e9 100644
--- a/radis/extractions/templates/extractions/_selection_options_field.html
+++ b/radis/extractions/templates/extractions/_selection_options_field.html
@@ -1,19 +1,22 @@
 {% load bootstrap_icon from common_extras %}
 <div class="mt-3"
      x-data="SelectionOptions($el)"
-     x-effect="syncOptions()"
+     x-effect="syncState()"
      data-selection-wrapper="true">
-    <div class="d-flex align-items-center gap-2">
-        {{ form.selection_options }}
-        <button type="button"
-                class="btn btn-outline-secondary btn-sm"
-                @click.prevent="addOption()"
-                :disabled="!supportsSelection || options.length >= maxOptions">
-            {% bootstrap_icon "plus-lg" %}
-            Enter a selection
-        </button>
-        <small class="text-muted" x-show="supportsSelection"
-               x-text="`${options.length} / ${maxOptions}`"></small>
+    {{ form.selection_options }}
+    {{ form.is_array }}
+    <div class="d-flex align-items-center flex-wrap selection-options-controls">
+        <div class="d-flex align-items-center gap-2 flex-wrap selection-options-actions">
+            <button type="button"
+                    class="btn btn-outline-secondary btn-sm"
+                    @click.prevent="addOption()"
+                    :disabled="!supportsSelection || options.length >= maxOptions">
+                {% bootstrap_icon "plus-lg" %}
+                Enter a selection
+            </button>
+            <small class="text-muted" x-show="supportsSelection"
+                   x-text="`${options.length} / ${maxOptions}`"></small>
+        </div>
     </div>
     <div class="mt-2" x-show="supportsSelection">
         <template x-for="(option, index) in options" :key="index">
diff --git a/radis/extractions/templates/extractions/extraction_job_detail.html b/radis/extractions/templates/extractions/extraction_job_detail.html
index dde3e67f..0b76330a 100644
--- a/radis/extractions/templates/extractions/extraction_job_detail.html
+++ b/radis/extractions/templates/extractions/extraction_job_detail.html
@@ -114,6 +114,12 @@ <h5>Output Fields</h5>
                     <dd class="col-sm-9">
                         {{ field.output_type|human_readable_output_type }}
                     </dd>
+                    {% if field.is_array %}
+                        <dt class="col-sm-3">Array Output</dt>
+                        <dd class="col-sm-9">
+                            Yes — return multiple {{ field.output_type|human_readable_output_type|lower }} values
+                        </dd>
+                    {% endif %}
                     {% if field.selection_options %}
                         <dt class="col-sm-3">Selections</dt>
                         <dd class="col-sm-9">
diff --git a/radis/extractions/templates/extractions/extraction_job_wizard_summary.html b/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
index 84906dbe..9ead6f85 100644
--- a/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
+++ b/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
@@ -91,6 +91,12 @@
                                 <dd class="col-sm-9">
                                     {{ field.output_type|human_readable_output_type }}
                                 </dd>
+                                {% if field.is_array %}
+                                    <dt class="col-sm-3">Array Output</dt>
+                                    <dd class="col-sm-9">
+                                        Yes — return multiple {{ field.output_type|human_readable_output_type|lower }} values
+                                    </dd>
+                                {% endif %}
                                 {% if field.selection_options %}
                                     <dt class="col-sm-3">Selections</dt>
                                     <dd class="col-sm-9">
diff --git a/radis/extractions/tests/test_forms.py b/radis/extractions/tests/test_forms.py
index 2089bb42..0264306b 100644
--- a/radis/extractions/tests/test_forms.py
+++ b/radis/extractions/tests/test_forms.py
@@ -17,6 +17,7 @@ def test_output_field_form_accepts_selection_options():
             "description": "Classified tumor grade.",
             "output_type": OutputType.SELECTION,
             "selection_options": json.dumps(["Grade 1", "Grade 2"]),
+            "is_array": "false",
         },
         instance=OutputField(job=job),
     )
@@ -36,6 +37,7 @@ def test_output_field_form_requires_options_for_selection():
             "description": "Classified tumor grade.",
             "output_type": OutputType.SELECTION,
             "selection_options": json.dumps([]),
+            "is_array": "false",
         },
         instance=OutputField(job=job),
     )
@@ -53,6 +55,7 @@ def test_output_field_form_rejects_options_for_non_selection():
             "description": "Classified tumor grade.",
             "output_type": OutputType.TEXT,
             "selection_options": json.dumps(["Grade 1"]),
+            "is_array": "false",
         },
         instance=OutputField(job=job),
     )
@@ -90,3 +93,22 @@ def test_output_field_clean_rejects_selection_options_for_other_types():
 
     with pytest.raises(ValidationError):
         field.full_clean()
+
+
+@pytest.mark.django_db
+def test_output_field_form_handles_array_toggle():
+    job = ExtractionJobFactory.create()
+    form = OutputFieldForm(
+        data={
+            "name": "measurements",
+            "description": "Multiple numeric values.",
+            "output_type": OutputType.NUMERIC,
+            "selection_options": json.dumps([]),
+            "is_array": "true",
+        },
+        instance=OutputField(job=job),
+    )
+
+    assert form.is_valid()
+    instance = form.save(commit=False)
+    assert instance.is_array is True
diff --git a/radis/extractions/utils/processor_utils.py b/radis/extractions/utils/processor_utils.py
index 8c99cfeb..26cfb991 100644
--- a/radis/extractions/utils/processor_utils.py
+++ b/radis/extractions/utils/processor_utils.py
@@ -26,6 +26,9 @@ def generate_output_fields_schema(fields: QuerySet[OutputField]) -> type[BaseMod
         else:
             raise ValueError(f"Unknown data type: {field.output_type}")
 
+        if field.is_array:
+            output_type = list[output_type]
+
         field_definitions[field.name] = (output_type, ...)
 
     return create_model("OutputFieldsModel", **field_definitions)
@@ -39,6 +42,11 @@ def generate_output_fields_prompt(fields: QuerySet[OutputField]) -> str:
             description = (
                 f"{description} (Allowed selections: {', '.join(field.selection_options)})"
             )
+        if field.is_array:
+            description = (
+                f"{description} (Return an array of "
+                f"{field.get_output_type_display().lower()} values.)"
+            )
         prompt += f"{field.name}: {description}\n"
 
     return prompt

From 7b58df9d805eca9515caf494232d003bd4365df0 Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Tue, 25 Nov 2025 15:32:33 +0000
Subject: [PATCH 41/93] prevent accidental Enter from submitting the form

---
 radis/core/static/core/core.js                  | 17 +++++++++++++++++
 .../extraction_job_wizard_summary.html          |  5 ++++-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/radis/core/static/core/core.js b/radis/core/static/core/core.js
index 7b870203..963778c1 100644
--- a/radis/core/static/core/core.js
+++ b/radis/core/static/core/core.js
@@ -46,6 +46,23 @@ document.addEventListener("alpine:init", () => {
   });
 });
 
+document.addEventListener("DOMContentLoaded", () => {
+  const preventAttr = "[data-prevent-enter-submit]";
+  document.querySelectorAll(preventAttr).forEach((formEl) => {
+    formEl.addEventListener("keydown", (event) => {
+      if (event.key !== "Enter") {
+        return;
+      }
+      const target = event.target;
+      const isTextInput =
+        target instanceof HTMLInputElement &&
+        !["submit", "button"].includes(target.type);
+      if (isTextInput) {
+        event.preventDefault();
+      }
+    });
+  });
+});
 /**
  * An Alpine component that controls a Django FormSet
  *
diff --git a/radis/extractions/templates/extractions/extraction_job_wizard_summary.html b/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
index 9ead6f85..a9580b82 100644
--- a/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
+++ b/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
@@ -114,7 +114,10 @@
             </ul>
         </dd>
     </dl>
-    <form action="" method="post" class="pt-3">
+    <form action=""
+          method="post"
+          class="pt-3"
+          data-prevent-enter-submit="true">
         {% csrf_token %}
         {{ wizard.management_form }}
         {% crispy form %}

From db1aa5ac7e3e1a80d1e335cf51ba1a7b78f1c645 Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Tue, 25 Nov 2025 15:38:14 +0000
Subject: [PATCH 42/93] fixing ruff errors

---
 radis/extractions/forms.py  | 20 +++++++++++++-------
 radis/extractions/models.py |  4 +++-
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/radis/extractions/forms.py b/radis/extractions/forms.py
index 2545339c..a11fd678 100644
--- a/radis/extractions/forms.py
+++ b/radis/extractions/forms.py
@@ -3,7 +3,7 @@
 
 from adit_radis_shared.accounts.models import User
 from crispy_forms.helper import FormHelper
-from crispy_forms.layout import Column, Div, HTML, Layout, Row, Submit
+from crispy_forms.layout import HTML, Column, Div, Layout, Row, Submit
 from django import forms
 from django.conf import settings
 from django.db.models import QuerySet
@@ -233,13 +233,19 @@ def __init__(self, *args, **kwargs):
                 Column("output_type", css_class="col-md-4 col-10"),
                 Column(
                     HTML(
-                        '<button type="button" '
-                        'class="btn btn-outline-secondary btn-sm array-toggle-btn form-array-toggle" '
-                        'data-array-toggle="true" '
-                        'aria-pressed="false" '
-                        'title="Toggle array output">[ ]</button>'
+                        (
+                            '<button type="button" '
+                            'class="btn btn-outline-secondary btn-sm array-toggle-btn '
+                            'form-array-toggle" '
+                            'data-array-toggle="true" '
+                            'aria-pressed="false" '
+                            'title="Toggle array output">[ ]</button>'
+                        )
+                    ),
+                    css_class=(
+                        "col-md-1 col-2 d-flex align-items-center "
+                        "justify-content-end array-toggle-field"
                     ),
-                    css_class="col-md-1 col-2 d-flex align-items-center justify-content-end array-toggle-field",
                 ),
                 css_class="g-3 align-items-center",
             ),
diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index 371f3b9e..5d503aff 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -117,7 +117,9 @@ def clean(self) -> None:
             cleaned_options = []
             for option in self.selection_options:
                 if not isinstance(option, str):
-                    raise ValidationError({"selection_options": "All selection options must be text."})
+                    raise ValidationError(
+                        {"selection_options": "All selection options must be text."}
+                    )
                 stripped = option.strip()
                 if not stripped:
                     raise ValidationError(

From 1f3b92944128357dc5451318df1b12557707c95e Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Wed, 26 Nov 2025 10:13:56 +0000
Subject: [PATCH 43/93] move expand button above extracted fields

---
 .../_subscribed_item_preview.html             | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html b/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
index 40dd041d..47c47eb7 100644
--- a/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
+++ b/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
@@ -7,6 +7,17 @@
             </div>
             <div class="d-flex flex-column gap-1">
                 <div class="clamp-3" :class="{'clamp-3': !full, 'pre-line': full}">{{ subscribed_item.report.body }}</div>
+                <div class="d-flex gap-2">
+                    <button type="button"
+                            class="btn btn-sm btn-link p-0 border-0"
+                            @click.prevent="full=true"
+                            x-show="!full">[Show full report]</button>
+                    <button type="button"
+                            class="btn btn-sm btn-link p-0 border-0"
+                            @click.prevent="full=false"
+                            x-cloak
+                            x-show="full">[Show summary]</button>
+                </div>
                 {% if subscribed_item.extraction_results %}
                     <div class="border-top pt-2 mt-1">
                         <div class="fw-semibold small text-muted mb-1">Extracted Fields</div>
@@ -22,17 +33,6 @@
                         </dl>
                     </div>
                 {% endif %}
-                <div class="d-flex gap-2">
-                    <button type="button"
-                            class="btn btn-sm btn-link p-0 border-0"
-                            @click.prevent="full=true"
-                            x-show="!full">[Show full report]</button>
-                    <button type="button"
-                            class="btn btn-sm btn-link p-0 border-0"
-                            @click.prevent="full=false"
-                            x-cloak
-                            x-show="full">[Show summary]</button>
-                </div>
             </div>
             {% include "reports/_report_buttons_panel.html" with report=subscribed_item.report %}
         </div>

From 6b426a2a33f9eae459d4e4c3113e35e8a9b5b783 Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Wed, 26 Nov 2025 15:07:47 +0000
Subject: [PATCH 44/93] Disallow duplicate Selection Options, add test cases
 for the same and update the Alpine stuff to save user input while switching
 between output types

---
 radis/core/static/core/core.js                | 10 ++-
 radis/extractions/forms.py                    |  2 +
 radis/extractions/models.py                   |  4 ++
 radis/extractions/tests/test_forms.py         | 72 +++++++++++++++++++
 .../tests/unit/test_processor_utils.py        | 48 +++++++++++++
 5 files changed, 135 insertions(+), 1 deletion(-)
 create mode 100644 radis/extractions/tests/unit/test_processor_utils.py

diff --git a/radis/core/static/core/core.js b/radis/core/static/core/core.js
index 963778c1..0e1765cc 100644
--- a/radis/core/static/core/core.js
+++ b/radis/core/static/core/core.js
@@ -136,9 +136,11 @@ function SelectionOptions(rootEl) {
     maxOptions: 7,
     supportsSelection: false,
     isArray: parseArrayValue(arrayInput?.value),
+    lastSelectionOptions: [],
     init() {
       this.options = this.parseOptions(hiddenInput?.value);
       this.isArray = parseArrayValue(arrayInput?.value);
+      this.lastSelectionOptions = [...this.options];
       this.updateSupports();
       if (arrayToggleButton) {
         arrayToggleButton.addEventListener("click", (event) => {
@@ -152,9 +154,14 @@ function SelectionOptions(rootEl) {
           const wasSelection = this.supportsSelection;
           this.updateSupports();
           if (!this.supportsSelection) {
+            this.lastSelectionOptions = [...this.options];
             this.options = [];
           } else if (!wasSelection && this.options.length === 0) {
-            this.options = this.parseOptions(hiddenInput?.value);
+            if (this.lastSelectionOptions.length > 0) {
+              this.options = [...this.lastSelectionOptions];
+            } else {
+              this.options = this.parseOptions(hiddenInput?.value);
+            }
           }
         });
       }
@@ -188,6 +195,7 @@ function SelectionOptions(rootEl) {
         .map((opt) => (typeof opt === "string" ? opt.trim() : ""))
         .filter((opt) => opt !== "");
       hiddenInput.value = JSON.stringify(sanitized);
+      this.lastSelectionOptions = [...sanitized];
     },
     syncArray() {
       if (!arrayInput) {
diff --git a/radis/extractions/forms.py b/radis/extractions/forms.py
index a11fd678..e39831da 100644
--- a/radis/extractions/forms.py
+++ b/radis/extractions/forms.py
@@ -281,6 +281,8 @@ def clean_selection_options(self) -> list[str]:
 
         if len(cleaned) > 7:
             raise forms.ValidationError("Provide at most 7 selection options.")
+        if len(set(cleaned)) != len(cleaned):
+            raise forms.ValidationError("Selection options must be unique.")
 
         return cleaned
 
diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index 5d503aff..e4660e61 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -126,6 +126,10 @@ def clean(self) -> None:
                         {"selection_options": "Selection options cannot be empty strings."}
                     )
                 cleaned_options.append(stripped)
+            if len(set(cleaned_options)) != len(cleaned_options):
+                raise ValidationError(
+                    {"selection_options": "Selection options must be unique."}
+                )
             self.selection_options = cleaned_options
         else:
             if self.selection_options:
diff --git a/radis/extractions/tests/test_forms.py b/radis/extractions/tests/test_forms.py
index 0264306b..d91fc909 100644
--- a/radis/extractions/tests/test_forms.py
+++ b/radis/extractions/tests/test_forms.py
@@ -112,3 +112,75 @@ def test_output_field_form_handles_array_toggle():
     assert form.is_valid()
     instance = form.save(commit=False)
     assert instance.is_array is True
+
+
+@pytest.mark.django_db
+def test_output_field_form_rejects_duplicate_selection_options():
+    job = ExtractionJobFactory.create()
+    form = OutputFieldForm(
+        data={
+            "name": "tumor_grade",
+            "description": "Classified tumor grade.",
+            "output_type": OutputType.SELECTION,
+            "selection_options": json.dumps(["Grade 1", "Grade 1"]),
+            "is_array": "false",
+        },
+        instance=OutputField(job=job),
+    )
+
+    assert not form.is_valid()
+    assert "selection_options" in form.errors
+
+
+@pytest.mark.django_db
+def test_output_field_clean_rejects_duplicate_selection_options():
+    job = ExtractionJobFactory.create()
+    field = OutputField(
+        job=job,
+        name="tumor_grade",
+        description="Classified tumor grade.",
+        output_type=OutputType.SELECTION,
+        selection_options=["Grade 1", "Grade 1"],
+    )
+
+    with pytest.raises(ValidationError):
+        field.full_clean()
+
+
+@pytest.mark.django_db
+def test_output_field_form_rejects_whitespace_only_selection_option():
+    job = ExtractionJobFactory.create()
+    form = OutputFieldForm(
+        data={
+            "name": "tumor_grade",
+            "description": "Classified tumor grade.",
+            "output_type": OutputType.SELECTION,
+            "selection_options": json.dumps(["Grade 1", "   "]),
+            "is_array": "false",
+        },
+        instance=OutputField(job=job),
+    )
+
+    assert not form.is_valid()
+    assert "selection_options" in form.errors
+
+
+@pytest.mark.django_db
+def test_output_field_form_accepts_unicode_and_long_selection_options():
+    job = ExtractionJobFactory.create()
+    unicode_option = "Grädé αβγ測試"
+    long_option = "Grade " + ("X" * 150)
+    form = OutputFieldForm(
+        data={
+            "name": "tumor_grade",
+            "description": "Classified tumor grade.",
+            "output_type": OutputType.SELECTION,
+            "selection_options": json.dumps([unicode_option, long_option]),
+            "is_array": "false",
+        },
+        instance=OutputField(job=job),
+    )
+
+    assert form.is_valid()
+    instance = form.save(commit=False)
+    assert instance.selection_options == [unicode_option.strip(), long_option.strip()]
diff --git a/radis/extractions/tests/unit/test_processor_utils.py b/radis/extractions/tests/unit/test_processor_utils.py
new file mode 100644
index 00000000..4f71b127
--- /dev/null
+++ b/radis/extractions/tests/unit/test_processor_utils.py
@@ -0,0 +1,48 @@
+from typing import Literal, get_args, get_origin
+
+import pytest
+
+from radis.extractions.factories import ExtractionJobFactory, OutputFieldFactory
+from radis.extractions.models import OutputType
+from radis.extractions.utils.processor_utils import generate_output_fields_schema
+
+
+@pytest.mark.django_db
+def test_generate_output_fields_schema_uses_literal_for_selection_fields():
+    job = ExtractionJobFactory.create()
+    field = OutputFieldFactory(
+        job=job,
+        name="grade",
+        output_type=OutputType.SELECTION,
+    )
+    field.selection_options = ["Grade 1", "Grade 2"]
+    field.save()
+
+    schema = generate_output_fields_schema(job.output_fields)
+
+    grade_field = schema.model_fields["grade"]
+    annotation = grade_field.annotation
+    assert get_origin(annotation) is Literal
+    assert set(get_args(annotation)) == {"Grade 1", "Grade 2"}
+
+
+@pytest.mark.django_db
+def test_generate_output_fields_schema_wraps_literal_in_list_for_array_selections():
+    job = ExtractionJobFactory.create()
+    field = OutputFieldFactory(
+        job=job,
+        name="grade_multi",
+        output_type=OutputType.SELECTION,
+    )
+    field.selection_options = ["High", "Low"]
+    field.is_array = True
+    field.save()
+
+    schema = generate_output_fields_schema(job.output_fields)
+
+    grade_field = schema.model_fields["grade_multi"]
+    annotation = grade_field.annotation
+    assert get_origin(annotation) is list
+    (inner_annotation,) = get_args(annotation)
+    assert get_origin(inner_annotation) is Literal
+    assert set(get_args(inner_annotation)) == {"High", "Low"}

From bed99fa6f3e0a4278efb84c217e01a09395106f0 Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Wed, 26 Nov 2025 15:11:01 +0000
Subject: [PATCH 45/93] Revert "Allow middleware to detect locale from users
 browser (via Accept-Language header) and use it downstream to deal with
 format parsing and translations"

This reverts commit da048fb6821f57948c3ce1801bff5f461b709d69.
---
 radis/settings/base.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/radis/settings/base.py b/radis/settings/base.py
index feeba638..dea0bb53 100644
--- a/radis/settings/base.py
+++ b/radis/settings/base.py
@@ -92,7 +92,6 @@
     "django.middleware.security.SecurityMiddleware",
     "whitenoise.middleware.WhiteNoiseMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
-    "django.middleware.locale.LocaleMiddleware",
     "django.middleware.common.CommonMiddleware",
     "django.middleware.csrf.CsrfViewMiddleware",
     "django.contrib.auth.middleware.AuthenticationMiddleware",

From 1c13a0a0e153bb87c610ec1e15346fa06d3368ed Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Wed, 26 Nov 2025 15:13:38 +0000
Subject: [PATCH 46/93] pagination, sorting and filtering added to inbox view

---
 radis/subscriptions/factories.py              |   1 -
 radis/subscriptions/filters.py                |  62 ++++-
 .../subscriptions/subscription_inbox.html     |  68 ++++-
 radis/subscriptions/tests/test_views.py       | 258 +++++++++++++++++-
 radis/subscriptions/views.py                  |  54 +++-
 5 files changed, 435 insertions(+), 8 deletions(-)

diff --git a/radis/subscriptions/factories.py b/radis/subscriptions/factories.py
index 73e61050..75fb1cec 100644
--- a/radis/subscriptions/factories.py
+++ b/radis/subscriptions/factories.py
@@ -57,4 +57,3 @@ class Meta:
         SubscriptionJobFactory, subscription=factory.SelfAttribute("..subscription")
     )
     report = factory.SubFactory(ReportFactory)
-    answers = factory.LazyFunction(lambda: {"question_1": "answer_1", "question_2": "answer_2"})
diff --git a/radis/subscriptions/filters.py b/radis/subscriptions/filters.py
index 0ef57e17..6e3cddb4 100644
--- a/radis/subscriptions/filters.py
+++ b/radis/subscriptions/filters.py
@@ -1,9 +1,14 @@
 import django_filters
 from adit_radis_shared.common.forms import SingleFilterFieldFormHelper
 from adit_radis_shared.common.types import with_form_helper
+from crispy_forms.helper import FormHelper
+from crispy_forms.layout import HTML, Div, Field, Layout, Submit
+from django import forms
 from django.http import HttpRequest
 
-from .models import Subscription
+from radis.reports.models import Modality
+
+from .models import SubscribedItem, Subscription
 
 
 class SubscriptionFilter(django_filters.FilterSet):
@@ -18,3 +23,58 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
         with_form_helper(self.form).helper = SingleFilterFieldFormHelper(self.request.GET, "name")
+
+
+class SubscribedItemFilter(django_filters.FilterSet):
+    patient_id = django_filters.CharFilter(
+        label="Patient ID",
+        field_name="report__patient_id",
+        lookup_expr="icontains",
+    )
+    study_description = django_filters.CharFilter(
+        label="Study Description",
+        field_name="report__study_description",
+        lookup_expr="icontains",
+    )
+    study_date_from = django_filters.DateFilter(
+        label="Study Date From",
+        field_name="report__study_datetime",
+        lookup_expr="date__gte",
+        widget=forms.DateInput(attrs={"type": "date"}),
+    )
+    study_date_till = django_filters.DateFilter(
+        label="Study Date Till",
+        field_name="report__study_datetime",
+        lookup_expr="date__lte",
+        widget=forms.DateInput(attrs={"type": "date"}),
+    )
+    modalities = django_filters.ModelMultipleChoiceFilter(
+        queryset=Modality.objects.order_by("code"),
+        field_name="report__modalities__code",
+        to_field_name="code",
+    )
+    request: HttpRequest
+
+    class Meta:
+        model = SubscribedItem
+        fields = ()
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        form_helper = FormHelper()
+        form_helper.form_tag = False
+        form_helper.disable_csrf = True
+        form_helper.layout = Layout(
+            Field("patient_id", css_class="form-control-sm"),
+            Field("study_description", css_class="form-control-sm"),
+            Field("study_date_from", css_class="form-control-sm"),
+            Field("study_date_till", css_class="form-control-sm"),
+            Field("modalities", css_class="form-select-sm"),
+            Div(
+                Submit("submit", "Apply Filters", css_class="btn btn-sm btn-primary"),
+                HTML("<a class='btn btn-sm btn-secondary' href='{{ request.path }}'>Reset</a>"),
+                css_class="d-flex justify-content-center gap-2",
+            ),
+        )
+        with_form_helper(self.form).helper = form_helper
diff --git a/radis/subscriptions/templates/subscriptions/subscription_inbox.html b/radis/subscriptions/templates/subscriptions/subscription_inbox.html
index 0e1eec5c..f2f2f27b 100644
--- a/radis/subscriptions/templates/subscriptions/subscription_inbox.html
+++ b/radis/subscriptions/templates/subscriptions/subscription_inbox.html
@@ -1,4 +1,6 @@
 {% extends 'subscriptions/subscription_layout.html' %}
+{% load crispy from crispy_forms_tags %}
+
 {% block title %}
     Subscription Inbox
 {% endblock title %}
@@ -6,7 +8,67 @@
     <c-page-heading title="Subscription Inbox" />
 {% endblock heading %}
 {% block content %}
-    {% for subscribed_item in object_list %}
-        {% include "subscriptions/_subscribed_item_preview.html" %}
-    {% endfor %}
+    <div class="row">
+        <div class="col">
+            {# Sorting Controls #}
+            <div class="mb-3">
+                <div class="btn-group btn-group-sm" role="group" aria-label="Sort options">
+                    <a href="?sort_by=created_at&order=desc{% if request.GET.per_page %}&per_page={{ request.GET.per_page }}{% endif %}{% for key, value in request.GET.items %}{% if key != 'sort_by' and key != 'order' and key != 'page' and key != 'per_page' %}&{{ key }}={{ value }}{% endif %}{% endfor %}"
+                       class="btn btn-outline-secondary {% if current_sort_by == 'created_at' and current_order == 'desc' %}active{% endif %}">
+                        Newest First
+                    </a>
+                    <a href="?sort_by=created_at&order=asc{% if request.GET.per_page %}&per_page={{ request.GET.per_page }}{% endif %}{% for key, value in request.GET.items %}{% if key != 'sort_by' and key != 'order' and key != 'page' and key != 'per_page' %}&{{ key }}={{ value }}{% endif %}{% endfor %}"
+                       class="btn btn-outline-secondary {% if current_sort_by == 'created_at' and current_order == 'asc' %}active{% endif %}">
+                        Oldest First
+                    </a>
+                    <a href="?sort_by=study_date&order=desc{% if request.GET.per_page %}&per_page={{ request.GET.per_page }}{% endif %}{% for key, value in request.GET.items %}{% if key != 'sort_by' and key != 'order' and key != 'page' and key != 'per_page' %}&{{ key }}={{ value }}{% endif %}{% endfor %}"
+                       class="btn btn-outline-secondary {% if current_sort_by == 'study_date' and current_order == 'desc' %}active{% endif %}">
+                        Study Date (Newest)
+                    </a>
+                    <a href="?sort_by=study_date&order=asc{% if request.GET.per_page %}&per_page={{ request.GET.per_page }}{% endif %}{% for key, value in request.GET.items %}{% if key != 'sort_by' and key != 'order' and key != 'page' and key != 'per_page' %}&{{ key }}={{ value }}{% endif %}{% endfor %}"
+                       class="btn btn-outline-secondary {% if current_sort_by == 'study_date' and current_order == 'asc' %}active{% endif %}">
+                        Study Date (Oldest)
+                    </a>
+                </div>
+            </div>
+            {# Card List #}
+            {% for subscribed_item in object_list %}
+                {% include "subscriptions/_subscribed_item_preview.html" %}
+            {% empty %}
+                <div class="alert alert-light" role="alert">
+                    No items found
+                </div>
+            {% endfor %}
+            {# Pagination Controls #}
+            <div class="d-flex justify-content-around align-items-center mt-3">
+                <div>
+                    <c-pagination :page-obj="page_obj" :page-sizes="page_sizes" />
+                </div>
+            </div>
+        </div>
+        {# Filter Sidebar #}
+        <div class="col col-auto">
+            <form action="{% url 'subscription_inbox' object.pk %}"
+                  method="get"
+                  x-data
+                  x-ignore-empty-inputs>
+                <div id="filters" class="card">
+                    <div class="card-body">
+                        <h5 class="card-title">Filters</h5>
+                        {# Hidden inputs to preserve sort and pagination state #}
+                        <input type="hidden"
+                               name="per_page"
+                               value="{{ request.GET.per_page|default:'' }}">
+                        <input type="hidden"
+                               name="sort_by"
+                               value="{{ request.GET.sort_by|default:'' }}">
+                        <input type="hidden"
+                               name="order"
+                               value="{{ request.GET.order|default:'' }}">
+                        {% crispy filter.form %}
+                    </div>
+                </div>
+            </form>
+        </div>
+    </div>
 {% endblock content %}
diff --git a/radis/subscriptions/tests/test_views.py b/radis/subscriptions/tests/test_views.py
index f5ce9f88..93a63a4c 100644
--- a/radis/subscriptions/tests/test_views.py
+++ b/radis/subscriptions/tests/test_views.py
@@ -1,11 +1,17 @@
+from datetime import datetime, timedelta, timezone
+
 import pytest
 from adit_radis_shared.accounts.factories import GroupFactory, UserFactory
 from django.test import Client
 
 from radis.extractions.factories import OutputFieldFactory
-from radis.reports.factories import LanguageFactory
+from radis.reports.factories import LanguageFactory, ReportFactory
 from radis.reports.models import Modality
-from radis.subscriptions.factories import FilterQuestionFactory, SubscriptionFactory
+from radis.subscriptions.factories import (
+    FilterQuestionFactory,
+    SubscribedItemFactory,
+    SubscriptionFactory,
+)
 from radis.subscriptions.models import Subscription
 
 
@@ -363,3 +369,251 @@ def test_unauthenticated_access_redirects_to_login(client: Client):
         response = client.get(endpoint)
         assert response.status_code == 302
         assert "/accounts/login/" in response["Location"]
+
+
+# Subscription Inbox Pagination and Sorting Tests
+
+
+@pytest.mark.django_db
+def test_subscription_inbox_pagination(client: Client):
+    """Test that pagination works correctly in subscription inbox."""
+
+    user = UserFactory.create(is_active=True)
+    subscription = create_test_subscription(owner=user)
+    language = LanguageFactory.create(code="en")
+
+    # Create 25 subscribed items
+    base_time = datetime.now(timezone.utc)
+    for i in range(25):
+        item = SubscribedItemFactory.create(
+            subscription=subscription, report=ReportFactory.create(language=language)
+        )
+        # Set created_at to ensure consistent ordering
+        item.created_at = base_time - timedelta(hours=i)
+        item.save()
+
+    client.force_login(user)
+
+    # Test first page with default page size (10)
+    response = client.get(f"/subscriptions/{subscription.pk}/inbox/")
+    assert response.status_code == 200
+    assert len(response.context["object_list"]) == 10
+    assert response.context["page_obj"].number == 1
+    assert response.context["page_obj"].paginator.num_pages == 3
+
+    # Test second page
+    response = client.get(f"/subscriptions/{subscription.pk}/inbox/?page=2")
+    assert response.status_code == 200
+    assert response.context["page_obj"].number == 2
+
+    # Test custom page size
+    response = client.get(f"/subscriptions/{subscription.pk}/inbox/?per_page=25")
+    assert len(response.context["object_list"]) == 25
+    assert response.context["page_obj"].paginator.num_pages == 1
+
+
+@pytest.mark.django_db
+def test_subscription_inbox_sorting_by_created_date(client: Client):
+    """Test sorting by created_at date."""
+
+    user = UserFactory.create(is_active=True)
+    subscription = create_test_subscription(owner=user)
+    language = LanguageFactory.create(code="en")
+
+    # Create items with different created_at times
+    item1 = SubscribedItemFactory.create(
+        subscription=subscription, report=ReportFactory.create(language=language)
+    )
+    item2 = SubscribedItemFactory.create(
+        subscription=subscription, report=ReportFactory.create(language=language)
+    )
+    item3 = SubscribedItemFactory.create(
+        subscription=subscription, report=ReportFactory.create(language=language)
+    )
+
+    # Manually set created_at to ensure order
+    base_time = datetime.now(timezone.utc)
+    item1.created_at = base_time - timedelta(days=2)
+    item1.save()
+    item2.created_at = base_time - timedelta(days=1)
+    item2.save()
+    item3.created_at = base_time
+    item3.save()
+
+    client.force_login(user)
+
+    # Test descending (newest first - default)
+    response = client.get(f"/subscriptions/{subscription.pk}/inbox/?sort_by=created_at&order=desc")
+    assert response.status_code == 200
+    items = list(response.context["object_list"])
+    assert items[0].pk == item3.pk
+    assert items[1].pk == item2.pk
+    assert items[2].pk == item1.pk
+
+    # Test ascending (oldest first)
+    response = client.get(f"/subscriptions/{subscription.pk}/inbox/?sort_by=created_at&order=asc")
+    assert response.status_code == 200
+    items = list(response.context["object_list"])
+    assert items[0].pk == item1.pk
+    assert items[1].pk == item2.pk
+    assert items[2].pk == item3.pk
+
+
+@pytest.mark.django_db
+def test_subscription_inbox_sorting_by_study_date(client: Client):
+    """Test sorting by study date."""
+
+    user = UserFactory.create(is_active=True)
+    subscription = create_test_subscription(owner=user)
+
+    # Create reports with different study dates
+    base_time = datetime.now(timezone.utc)
+    language = LanguageFactory.create(code="en")
+    report1 = ReportFactory.create(language=language, study_datetime=base_time - timedelta(days=10))
+    report2 = ReportFactory.create(language=language, study_datetime=base_time - timedelta(days=5))
+    report3 = ReportFactory.create(language=language, study_datetime=base_time - timedelta(days=1))
+
+    item1 = SubscribedItemFactory.create(subscription=subscription, report=report1)
+    item2 = SubscribedItemFactory.create(subscription=subscription, report=report2)
+    item3 = SubscribedItemFactory.create(subscription=subscription, report=report3)
+
+    client.force_login(user)
+
+    # Test descending (newest study date first)
+    response = client.get(f"/subscriptions/{subscription.pk}/inbox/?sort_by=study_date&order=desc")
+    assert response.status_code == 200
+    items = list(response.context["object_list"])
+    assert items[0].pk == item3.pk
+    assert items[1].pk == item2.pk
+    assert items[2].pk == item1.pk
+
+    # Test ascending (oldest study date first)
+    response = client.get(f"/subscriptions/{subscription.pk}/inbox/?sort_by=study_date&order=asc")
+    assert response.status_code == 200
+    items = list(response.context["object_list"])
+    assert items[0].pk == item1.pk
+    assert items[1].pk == item2.pk
+    assert items[2].pk == item3.pk
+
+
+@pytest.mark.django_db
+def test_subscription_inbox_invalid_sort_parameters(client: Client):
+    """Test that invalid sort parameters fall back to defaults."""
+
+    user = UserFactory.create(is_active=True)
+    subscription = create_test_subscription(owner=user)
+    language = LanguageFactory.create(code="en")
+
+    SubscribedItemFactory.create(
+        subscription=subscription, report=ReportFactory.create(language=language)
+    )
+
+    client.force_login(user)
+
+    # Invalid sort_by should default to created_at
+    response = client.get(
+        f"/subscriptions/{subscription.pk}/inbox/?sort_by=invalid_field&order=desc"
+    )
+    assert response.status_code == 200
+    assert response.context["current_sort_by"] == "created_at"
+
+    # Invalid order should default to desc
+    response = client.get(
+        f"/subscriptions/{subscription.pk}/inbox/?sort_by=created_at&order=invalid"
+    )
+    assert response.status_code == 200
+    assert response.context["current_order"] == "desc"
+
+
+@pytest.mark.django_db
+def test_subscription_inbox_filtering_by_patient_id(client: Client):
+    """Test filtering by patient ID."""
+
+    user = UserFactory.create(is_active=True)
+    subscription = create_test_subscription(owner=user)
+    language = LanguageFactory.create(code="en")
+    report1 = ReportFactory.create(language=language, patient_id="12345")
+    report2 = ReportFactory.create(language=language, patient_id="67890")
+    report3 = ReportFactory.create(language=language, patient_id="54321")
+
+    item1 = SubscribedItemFactory.create(subscription=subscription, report=report1)
+    item2 = SubscribedItemFactory.create(subscription=subscription, report=report2)
+    item3 = SubscribedItemFactory.create(subscription=subscription, report=report3)
+
+    client.force_login(user)
+
+    # Filter by patient_id (partial match)
+    response = client.get(f"/subscriptions/{subscription.pk}/inbox/?patient_id=123")
+    assert response.status_code == 200
+    items = list(response.context["object_list"])
+    assert len(items) == 1  # Should match "12345"
+    assert item1 in items
+    assert item2 and item3 not in items
+
+
+@pytest.mark.django_db
+def test_subscription_inbox_filtering_by_date_range(client: Client):
+    """Test filtering by study date range."""
+
+    user = UserFactory.create(is_active=True)
+    subscription = create_test_subscription(owner=user)
+
+    base_date = datetime.now(timezone.utc)
+    language = LanguageFactory.create(code="en")
+    report1 = ReportFactory.create(language=language, study_datetime=base_date - timedelta(days=10))
+    report2 = ReportFactory.create(language=language, study_datetime=base_date - timedelta(days=5))
+    report3 = ReportFactory.create(language=language, study_datetime=base_date - timedelta(days=1))
+
+    item1 = SubscribedItemFactory.create(subscription=subscription, report=report1)
+    item2 = SubscribedItemFactory.create(subscription=subscription, report=report2)
+    item3 = SubscribedItemFactory.create(subscription=subscription, report=report3)
+
+    client.force_login(user)
+
+    # Filter by date range
+    date_from = (base_date - timedelta(days=7)).strftime("%Y-%m-%d")
+    date_till = (base_date - timedelta(days=2)).strftime("%Y-%m-%d")
+
+    response = client.get(
+        f"/subscriptions/{subscription.pk}/inbox/?study_date_from={date_from}&study_date_till={date_till}"
+    )
+    assert response.status_code == 200
+    items = list(response.context["object_list"])
+    assert len(items) == 1
+    assert item2 in items
+
+
+@pytest.mark.django_db
+def test_subscription_inbox_combined_filter_and_sort(client: Client):
+    """Test that filtering and sorting work together."""
+
+    user = UserFactory.create(is_active=True)
+    subscription = create_test_subscription(owner=user)
+
+    # Create items with same patient but different study dates
+    base_date = datetime.now(timezone.utc)
+    language = LanguageFactory.create(code="en")
+    report1 = ReportFactory.create(
+        language=language, patient_id="12345", study_datetime=base_date - timedelta(days=5)
+    )
+    report2 = ReportFactory.create(
+        language=language, patient_id="12345", study_datetime=base_date - timedelta(days=1)
+    )
+    report3 = ReportFactory.create(language=language, patient_id="67890", study_datetime=base_date)
+
+    item1 = SubscribedItemFactory.create(subscription=subscription, report=report1)
+    item2 = SubscribedItemFactory.create(subscription=subscription, report=report2)
+    item3 = SubscribedItemFactory.create(subscription=subscription, report=report3)
+
+    client.force_login(user)
+
+    # Filter by patient_id and sort by study_date ascending
+    response = client.get(
+        f"/subscriptions/{subscription.pk}/inbox/?patient_id=12345&sort_by=study_date&order=asc"
+    )
+    assert response.status_code == 200
+    items = list(response.context["object_list"])
+    assert len(items) == 2
+    assert items[0].pk == item1.pk  # Older study date
+    assert items[1].pk == item2.pk  # Newer study date
+    assert item3 not in items  # Different patient
diff --git a/radis/subscriptions/views.py b/radis/subscriptions/views.py
index 7038bd23..b7108f50 100644
--- a/radis/subscriptions/views.py
+++ b/radis/subscriptions/views.py
@@ -17,7 +17,7 @@
 from django.views.generic import CreateView, DeleteView, DetailView, UpdateView
 from django_tables2 import SingleTableView
 
-from radis.subscriptions.filters import SubscriptionFilter
+from radis.subscriptions.filters import SubscribedItemFilter, SubscriptionFilter
 from radis.subscriptions.tables import SubscriptionTable
 
 from .forms import (
@@ -177,6 +177,9 @@ class SubscriptionInboxView(
 ):
     model = Subscription
     template_name = "subscriptions/subscription_inbox.html"
+    filterset_class = SubscribedItemFilter
+    paginate_by = 10
+    page_sizes = [10, 25, 50]
     request: AuthenticatedHttpRequest
 
     def get_queryset(self) -> QuerySet[Subscription]:
@@ -186,8 +189,31 @@ def get_queryset(self) -> QuerySet[Subscription]:
             return model.objects.all()
         return model.objects.filter(owner=self.request.user)
 
+    def get_ordering(self) -> str:
+        """Get the ordering from query parameters, defaulting to -created_at."""
+        sort_by = self.request.GET.get("sort_by", "created_at")
+        order = self.request.GET.get("order", "desc")
+
+        # Define allowed sort fields to prevent injection
+        allowed_fields = {
+            "created_at": "created_at",
+            "study_date": "report__study_datetime",
+        }
+
+        # Validate sort_by parameter
+        if sort_by not in allowed_fields:
+            sort_by = "created_at"
+
+        # Validate order parameter
+        if order not in ["asc", "desc"]:
+            order = "desc"
+
+        field = allowed_fields[sort_by]
+        return field if order == "asc" else f"-{field}"
+
     def get_related_queryset(self) -> QuerySet[SubscribedItem]:
         subscription = cast(Subscription, self.get_object())
+        ordering = self.get_ordering()
         return (
             SubscribedItem.objects.filter(subscription_id=subscription.pk)
             .select_related("subscription")
@@ -195,7 +221,33 @@ def get_related_queryset(self) -> QuerySet[SubscribedItem]:
                 "report",
                 "subscription__output_fields",
             )
+            .order_by(ordering)
         )
 
     def get_filter_queryset(self) -> QuerySet[SubscribedItem]:
         return self.get_related_queryset()
+
+    def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
+        context = super().get_context_data(**kwargs)
+
+        # Get validated sort parameters
+        sort_by = self.request.GET.get("sort_by", "created_at")
+        order = self.request.GET.get("order", "desc")
+
+        # Define allowed sort fields (same as in get_ordering)
+        allowed_fields = {
+            "created_at": "created_at",
+            "study_date": "report__study_datetime",
+        }
+
+        # Validate and default if invalid
+        if sort_by not in allowed_fields:
+            sort_by = "created_at"
+        if order not in ["asc", "desc"]:
+            order = "desc"
+
+        # Add validated sort parameters to context for template rendering
+        context["current_sort_by"] = sort_by
+        context["current_order"] = order
+
+        return context

From 9ac799b3c99fef0d92d4874bf610ff3db1dde2fc Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Wed, 26 Nov 2025 15:29:08 +0000
Subject: [PATCH 47/93] fix unused variables in test

---
 radis/subscriptions/tests/test_views.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/radis/subscriptions/tests/test_views.py b/radis/subscriptions/tests/test_views.py
index 93a63a4c..4c4806b3 100644
--- a/radis/subscriptions/tests/test_views.py
+++ b/radis/subscriptions/tests/test_views.py
@@ -581,6 +581,7 @@ def test_subscription_inbox_filtering_by_date_range(client: Client):
     items = list(response.context["object_list"])
     assert len(items) == 1
     assert item2 in items
+    assert item1 and item3 not in items
 
 
 @pytest.mark.django_db

From 323e35aa55468642dcab4c6b8053dd9990c7c6a9 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Thu, 27 Nov 2025 13:11:04 +0000
Subject: [PATCH 48/93] resolve invariant type checker error

---
 pyproject.toml               | 2 +-
 radis/subscriptions/views.py | 5 +++--
 uv.lock                      | 4 ++--
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index e6f1e98a..b6ecf3fa 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,7 +7,7 @@ version = "0.0.0"
 readme = "README.md"
 requires-python = ">=3.12,<4.0"
 dependencies = [
-    "adit-radis-shared @ git+https://github.com/openradx/adit-radis-shared.git@0.18.1",
+    "adit-radis-shared @ git+https://github.com/openradx/adit-radis-shared.git@5b78ee293d5ee186a1443eb9f49dc29e5c53ce09",
     "adrf>=0.1.9",
     "aiofiles>=24.1.0",
     "asyncinotify>=4.2.0",
diff --git a/radis/subscriptions/views.py b/radis/subscriptions/views.py
index b7108f50..75da9fce 100644
--- a/radis/subscriptions/views.py
+++ b/radis/subscriptions/views.py
@@ -1,6 +1,7 @@
 from logging import getLogger
 from typing import Any, Type, cast
 
+from adit_radis_shared.accounts.models import User
 from adit_radis_shared.common.mixins import (
     PageSizeSelectMixin,
     RelatedFilterMixin,
@@ -180,12 +181,12 @@ class SubscriptionInboxView(
     filterset_class = SubscribedItemFilter
     paginate_by = 10
     page_sizes = [10, 25, 50]
-    request: AuthenticatedHttpRequest
 
     def get_queryset(self) -> QuerySet[Subscription]:
         assert self.model
         model = cast(Type[Subscription], self.model)
-        if self.request.user.is_staff:
+        user = cast(User, self.request.user)
+        if user.is_staff:
             return model.objects.all()
         return model.objects.filter(owner=self.request.user)
 
diff --git a/uv.lock b/uv.lock
index c24e8d84..2b8b6d02 100644
--- a/uv.lock
+++ b/uv.lock
@@ -5,7 +5,7 @@ requires-python = ">=3.12, <4.0"
 [[package]]
 name = "adit-radis-shared"
 version = "0.0.0"
-source = { git = "https://github.com/openradx/adit-radis-shared.git?rev=0.18.1#47c1303cb9865b3c24109c4027dd405d1d5edbc5" }
+source = { git = "https://github.com/openradx/adit-radis-shared.git?rev=5b78ee293d5ee186a1443eb9f49dc29e5c53ce09#5b78ee293d5ee186a1443eb9f49dc29e5c53ce09" }
 dependencies = [
     { name = "channels" },
     { name = "crispy-bootstrap5" },
@@ -2336,7 +2336,7 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "adit-radis-shared", git = "https://github.com/openradx/adit-radis-shared.git?rev=0.18.1" },
+    { name = "adit-radis-shared", git = "https://github.com/openradx/adit-radis-shared.git?rev=5b78ee293d5ee186a1443eb9f49dc29e5c53ce09" },
     { name = "adrf", specifier = ">=0.1.9" },
     { name = "aiofiles", specifier = ">=24.1.0" },
     { name = "asyncinotify", specifier = ">=4.2.0" },

From 2187090fc0a67b59d739ccc875d6eda65d1f8c9f Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Thu, 27 Nov 2025 15:28:16 +0000
Subject: [PATCH 49/93] linter error

---
 radis/extractions/utils/processor_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/radis/extractions/utils/processor_utils.py b/radis/extractions/utils/processor_utils.py
index 26cfb991..0cc3ac05 100644
--- a/radis/extractions/utils/processor_utils.py
+++ b/radis/extractions/utils/processor_utils.py
@@ -22,7 +22,7 @@ def generate_output_fields_schema(fields: QuerySet[OutputField]) -> type[BaseMod
             options = tuple(field.selection_options)
             if not options:
                 raise ValueError("Selection output requires at least one option.")
-            output_type = Literal.__getitem__(options)
+            output_type = Literal[*options]
         else:
             raise ValueError(f"Unknown data type: {field.output_type}")
 

From 7f5243870cca4c92549fa423d5a741ac48d390f0 Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Thu, 27 Nov 2025 15:41:46 +0000
Subject: [PATCH 50/93] remove the magic number 7

---
 radis/core/static/core/core.js | 11 ++++++++++-
 radis/extractions/constants.py |  1 +
 radis/extractions/forms.py     |  8 ++++++--
 radis/extractions/models.py    | 12 ++++++++++--
 4 files changed, 27 insertions(+), 5 deletions(-)
 create mode 100644 radis/extractions/constants.py

diff --git a/radis/core/static/core/core.js b/radis/core/static/core/core.js
index 0e1765cc..060567d4 100644
--- a/radis/core/static/core/core.js
+++ b/radis/core/static/core/core.js
@@ -130,10 +130,19 @@ function SelectionOptions(rootEl) {
     const normalized = value.trim().toLowerCase();
     return normalized === "true" || normalized === "1" || normalized === "on";
   };
+  const parseMaxOptions = () => {
+    const datasetValue =
+      hiddenInput?.dataset.maxSelectionOptions ??
+      rootEl.dataset.maxSelectionOptions ??
+      "";
+    const parsed = Number.parseInt(datasetValue, 10);
+    return Number.isNaN(parsed) ? 0 : parsed;
+  };
+  const initialMaxOptions = parseMaxOptions();
 
   return {
     options: [],
-    maxOptions: 7,
+    maxOptions: initialMaxOptions,
     supportsSelection: false,
     isArray: parseArrayValue(arrayInput?.value),
     lastSelectionOptions: [],
diff --git a/radis/extractions/constants.py b/radis/extractions/constants.py
new file mode 100644
index 00000000..bacfd27c
--- /dev/null
+++ b/radis/extractions/constants.py
@@ -0,0 +1 @@
+MAX_SELECTION_OPTIONS = 7
diff --git a/radis/extractions/forms.py b/radis/extractions/forms.py
index e39831da..b1a9924c 100644
--- a/radis/extractions/forms.py
+++ b/radis/extractions/forms.py
@@ -15,6 +15,7 @@
 from radis.search.site import Search, SearchFilters
 from radis.search.utils.query_parser import QueryParser
 
+from .constants import MAX_SELECTION_OPTIONS
 from .models import ExtractionJob, OutputField, OutputType
 from .site import extraction_retrieval_providers
 
@@ -212,6 +213,7 @@ def __init__(self, *args, **kwargs):
         self.fields["selection_options"].widget.attrs.update(
             {
                 "data-selection-input": "true",
+                "data-max-selection-options": str(MAX_SELECTION_OPTIONS),
             }
         )
         self.fields["is_array"].widget.attrs.update(
@@ -279,8 +281,10 @@ def clean_selection_options(self) -> list[str]:
                 raise forms.ValidationError("Selection options cannot be empty.")
             cleaned.append(value)
 
-        if len(cleaned) > 7:
-            raise forms.ValidationError("Provide at most 7 selection options.")
+        if len(cleaned) > MAX_SELECTION_OPTIONS:
+            raise forms.ValidationError(
+                f"Provide at most {MAX_SELECTION_OPTIONS} selection options."
+            )
         if len(set(cleaned)) != len(cleaned):
             raise forms.ValidationError("Selection options must be unique.")
 
diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index e4660e61..7c3a9eb0 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -11,6 +11,8 @@
 from radis.core.models import AnalysisJob, AnalysisTask
 from radis.reports.models import Language, Modality, Report
 
+from .constants import MAX_SELECTION_OPTIONS
+
 
 class ExtractionsAppSettings(AppSettings):
     class Meta:
@@ -112,8 +114,14 @@ def clean(self) -> None:
         if self.output_type == OutputType.SELECTION:
             if not self.selection_options:
                 raise ValidationError({"selection_options": "Add at least one selection option."})
-            if len(self.selection_options) > 7:
-                raise ValidationError({"selection_options": "Provide at most 7 selection options."})
+            if len(self.selection_options) > MAX_SELECTION_OPTIONS:
+                raise ValidationError(
+                    {
+                        "selection_options": (
+                            f"Provide at most {MAX_SELECTION_OPTIONS} selection options."
+                        )
+                    }
+                )
             cleaned_options = []
             for option in self.selection_options:
                 if not isinstance(option, str):

From 1267257f6308b77d05616126a88b11c5bdfcd833 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Fri, 28 Nov 2025 15:07:18 +0000
Subject: [PATCH 51/93] add notification for new subscribed reports

---
 .../0012_subscription_last_viewed_at.py        | 18 ++++++++++++++++++
 radis/subscriptions/models.py                  |  1 +
 radis/subscriptions/tables.py                  | 16 ++++++++++++++++
 radis/subscriptions/views.py                   | 15 ++++++++++++++-
 4 files changed, 49 insertions(+), 1 deletion(-)
 create mode 100644 radis/subscriptions/migrations/0012_subscription_last_viewed_at.py

diff --git a/radis/subscriptions/migrations/0012_subscription_last_viewed_at.py b/radis/subscriptions/migrations/0012_subscription_last_viewed_at.py
new file mode 100644
index 00000000..803e371a
--- /dev/null
+++ b/radis/subscriptions/migrations/0012_subscription_last_viewed_at.py
@@ -0,0 +1,18 @@
+# Generated by Django 5.2.8 on 2025-11-28 10:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("subscriptions", "0011_procrastinate_on_delete"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="subscription",
+            name="last_viewed_at",
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]
diff --git a/radis/subscriptions/models.py b/radis/subscriptions/models.py
index 470c1fa2..345b2241 100644
--- a/radis/subscriptions/models.py
+++ b/radis/subscriptions/models.py
@@ -50,6 +50,7 @@ class Subscription(models.Model):
 
     created_at = models.DateTimeField(auto_now_add=True)
     last_refreshed = models.DateTimeField(auto_now_add=True)
+    last_viewed_at = models.DateTimeField(null=True, blank=True)
 
     filter_questions: models.QuerySet["FilterQuestion"]
     output_fields: models.QuerySet[OutputField]
diff --git a/radis/subscriptions/tables.py b/radis/subscriptions/tables.py
index 81ebacd3..9c2de3e4 100644
--- a/radis/subscriptions/tables.py
+++ b/radis/subscriptions/tables.py
@@ -1,4 +1,6 @@
 import django_tables2 as tables
+from django.urls import reverse
+from django.utils.html import format_html
 from django_tables2.utils import A
 
 from .models import Subscription
@@ -18,6 +20,20 @@ class SubscriptionTable(tables.Table):
         },
     )
 
+    def render_num_reports(self, value, record):
+        """Render the num_reports column with a notification badge for new reports."""
+        num_new = getattr(record, "num_new_reports", 0)
+        url = reverse("subscription_inbox", args=[record.pk])
+
+        if num_new > 0:
+            return format_html(
+                '<a href="{}">{} <span class="badge bg-primary">{} new</span></a>',
+                url,
+                value,
+                num_new,
+            )
+        return format_html('<a href="{}">{}</a>', url, value)
+
     class Meta:
         model = Subscription
         fields = ("name", "num_reports")
diff --git a/radis/subscriptions/views.py b/radis/subscriptions/views.py
index 75da9fce..15638021 100644
--- a/radis/subscriptions/views.py
+++ b/radis/subscriptions/views.py
@@ -11,10 +11,11 @@
 from django.contrib.auth.mixins import LoginRequiredMixin
 from django.contrib.messages.views import SuccessMessageMixin
 from django.db import IntegrityError
-from django.db.models import Count, QuerySet
+from django.db.models import Count, F, Q, QuerySet
 from django.forms.models import BaseInlineFormSet
 from django.http import HttpResponse, HttpResponseRedirect
 from django.urls import reverse, reverse_lazy
+from django.utils import timezone
 from django.views.generic import CreateView, DeleteView, DetailView, UpdateView
 from django_tables2 import SingleTableView
 
@@ -46,6 +47,13 @@ def get_queryset(self) -> QuerySet[Subscription]:
         return (
             Subscription.objects.filter(owner=self.request.user)
             .annotate(num_reports=Count("items"))
+            .annotate(
+                num_new_reports=Count(
+                    "items",
+                    filter=Q(items__created_at__gt=F("last_viewed_at"))
+                    | Q(last_viewed_at__isnull=True),
+                )
+            )
             .order_by("-created_at")
         )
 
@@ -251,4 +259,9 @@ def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
         context["current_sort_by"] = sort_by
         context["current_order"] = order
 
+        # Update last_viewed_at to mark all current reports as seen
+        subscription = cast(Subscription, self.object)
+        subscription.last_viewed_at = timezone.now()
+        subscription.save(update_fields=["last_viewed_at"])
+
         return context

From 57b1cd976e68f570013a76abc257a92d4b1d4115 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Thu, 4 Dec 2025 11:04:18 +0000
Subject: [PATCH 52/93] reorder migrations from merge

---
 ..._outputfield_unique.py => 0005_remove_outputfield_unique.py} | 2 +-
 ...on_and_more.py => 0006_outputfield_subscription_and_more.py} | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename radis/extractions/migrations/{0004_remove_outputfield_unique.py => 0005_remove_outputfield_unique.py} (92%)
 rename radis/extractions/migrations/{0005_outputfield_subscription_and_more.py => 0006_outputfield_subscription_and_more.py} (96%)

diff --git a/radis/extractions/migrations/0004_remove_outputfield_unique.py b/radis/extractions/migrations/0005_remove_outputfield_unique.py
similarity index 92%
rename from radis/extractions/migrations/0004_remove_outputfield_unique.py
rename to radis/extractions/migrations/0005_remove_outputfield_unique.py
index de73e8a5..1a5cd86e 100644
--- a/radis/extractions/migrations/0004_remove_outputfield_unique.py
+++ b/radis/extractions/migrations/0005_remove_outputfield_unique.py
@@ -7,7 +7,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ("extractions", "0003_alter_extractionjob_options_and_more"),
+        ("extractions", "0004_remove_extractionjob_provider"),
     ]
 
     operations = [
diff --git a/radis/extractions/migrations/0005_outputfield_subscription_and_more.py b/radis/extractions/migrations/0006_outputfield_subscription_and_more.py
similarity index 96%
rename from radis/extractions/migrations/0005_outputfield_subscription_and_more.py
rename to radis/extractions/migrations/0006_outputfield_subscription_and_more.py
index 72f0c484..2b2440e9 100644
--- a/radis/extractions/migrations/0005_outputfield_subscription_and_more.py
+++ b/radis/extractions/migrations/0006_outputfield_subscription_and_more.py
@@ -7,7 +7,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ("extractions", "0004_remove_outputfield_unique"),
+        ("extractions", "0005_remove_outputfield_unique"),
         ("subscriptions", "0010_rename_answers_subscribeditem_filter_results_and_more"),
     ]
 

From c4146a1ebd88f51bd650702983ae5b7f127785cc Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Thu, 4 Dec 2025 11:19:24 +0000
Subject: [PATCH 53/93] reorder subscription migrations

---
 ...11_rename_answers_subscribeditem_filter_results_and_more.py} | 2 +-
 ...ocrastinate_on_delete.py => 0012_procrastinate_on_delete.py} | 2 +-
 ...on_last_viewed_at.py => 0013_subscription_last_viewed_at.py} | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)
 rename radis/subscriptions/migrations/{0010_rename_answers_subscribeditem_filter_results_and_more.py => 0011_rename_answers_subscribeditem_filter_results_and_more.py} (95%)
 rename radis/subscriptions/migrations/{0011_procrastinate_on_delete.py => 0012_procrastinate_on_delete.py} (92%)
 rename radis/subscriptions/migrations/{0012_subscription_last_viewed_at.py => 0013_subscription_last_viewed_at.py} (85%)

diff --git a/radis/subscriptions/migrations/0010_rename_answers_subscribeditem_filter_results_and_more.py b/radis/subscriptions/migrations/0011_rename_answers_subscribeditem_filter_results_and_more.py
similarity index 95%
rename from radis/subscriptions/migrations/0010_rename_answers_subscribeditem_filter_results_and_more.py
rename to radis/subscriptions/migrations/0011_rename_answers_subscribeditem_filter_results_and_more.py
index 23a13958..f7b3e518 100644
--- a/radis/subscriptions/migrations/0010_rename_answers_subscribeditem_filter_results_and_more.py
+++ b/radis/subscriptions/migrations/0011_rename_answers_subscribeditem_filter_results_and_more.py
@@ -7,7 +7,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ("subscriptions", "0009_alter_subscribeditem_options_and_more"),
+        ("subscriptions", "0010_remove_subscription_provider"),
     ]
 
     operations = [
diff --git a/radis/subscriptions/migrations/0011_procrastinate_on_delete.py b/radis/subscriptions/migrations/0012_procrastinate_on_delete.py
similarity index 92%
rename from radis/subscriptions/migrations/0011_procrastinate_on_delete.py
rename to radis/subscriptions/migrations/0012_procrastinate_on_delete.py
index 1892eb73..198142f2 100644
--- a/radis/subscriptions/migrations/0011_procrastinate_on_delete.py
+++ b/radis/subscriptions/migrations/0012_procrastinate_on_delete.py
@@ -7,7 +7,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ("subscriptions", "0010_rename_answers_subscribeditem_filter_results_and_more"),
+        ("subscriptions", "0011_rename_answers_subscribeditem_filter_results_and_more"),
         ("procrastinate", "0028_add_cancel_states"),
     ]
 
diff --git a/radis/subscriptions/migrations/0012_subscription_last_viewed_at.py b/radis/subscriptions/migrations/0013_subscription_last_viewed_at.py
similarity index 85%
rename from radis/subscriptions/migrations/0012_subscription_last_viewed_at.py
rename to radis/subscriptions/migrations/0013_subscription_last_viewed_at.py
index 803e371a..cda71caa 100644
--- a/radis/subscriptions/migrations/0012_subscription_last_viewed_at.py
+++ b/radis/subscriptions/migrations/0013_subscription_last_viewed_at.py
@@ -6,7 +6,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ("subscriptions", "0011_procrastinate_on_delete"),
+        ("subscriptions", "0012_procrastinate_on_delete"),
     ]
 
     operations = [

From 14656e4c6338d193ff1eb992030b4fb44882c3aa Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Thu, 4 Dec 2025 11:24:09 +0000
Subject: [PATCH 54/93] migration fix

---
 .../migrations/0006_outputfield_subscription_and_more.py        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/radis/extractions/migrations/0006_outputfield_subscription_and_more.py b/radis/extractions/migrations/0006_outputfield_subscription_and_more.py
index 2b2440e9..2d2c9d3f 100644
--- a/radis/extractions/migrations/0006_outputfield_subscription_and_more.py
+++ b/radis/extractions/migrations/0006_outputfield_subscription_and_more.py
@@ -8,7 +8,7 @@ class Migration(migrations.Migration):
 
     dependencies = [
         ("extractions", "0005_remove_outputfield_unique"),
-        ("subscriptions", "0010_rename_answers_subscribeditem_filter_results_and_more"),
+        ("subscriptions", "0011_rename_answers_subscribeditem_filter_results_and_more"),
     ]
 
     operations = [

From 80a3edeac1095e875a317f7d5da51bfd68f5504a Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Thu, 4 Dec 2025 12:40:07 +0000
Subject: [PATCH 55/93] testing with original adit-radit-shared

---
 pyproject.toml | 2 +-
 uv.lock        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index b6ecf3fa..e6f1e98a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,7 +7,7 @@ version = "0.0.0"
 readme = "README.md"
 requires-python = ">=3.12,<4.0"
 dependencies = [
-    "adit-radis-shared @ git+https://github.com/openradx/adit-radis-shared.git@5b78ee293d5ee186a1443eb9f49dc29e5c53ce09",
+    "adit-radis-shared @ git+https://github.com/openradx/adit-radis-shared.git@0.18.1",
     "adrf>=0.1.9",
     "aiofiles>=24.1.0",
     "asyncinotify>=4.2.0",
diff --git a/uv.lock b/uv.lock
index 75de9df1..a3c0ba5b 100644
--- a/uv.lock
+++ b/uv.lock
@@ -5,7 +5,7 @@ requires-python = ">=3.12, <4.0"
 [[package]]
 name = "adit-radis-shared"
 version = "0.0.0"
-source = { git = "https://github.com/openradx/adit-radis-shared.git?rev=5b78ee293d5ee186a1443eb9f49dc29e5c53ce09#5b78ee293d5ee186a1443eb9f49dc29e5c53ce09" }
+source = { git = "https://github.com/openradx/adit-radis-shared.git?rev=0.18.1#47c1303cb9865b3c24109c4027dd405d1d5edbc5" }
 dependencies = [
     { name = "channels" },
     { name = "crispy-bootstrap5" },
@@ -2423,7 +2423,7 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "adit-radis-shared", git = "https://github.com/openradx/adit-radis-shared.git?rev=5b78ee293d5ee186a1443eb9f49dc29e5c53ce09" },
+    { name = "adit-radis-shared", git = "https://github.com/openradx/adit-radis-shared.git?rev=0.18.1" },
     { name = "adrf", specifier = ">=0.1.9" },
     { name = "aiofiles", specifier = ">=24.1.0" },
     { name = "asyncinotify", specifier = ">=4.2.0" },

From 9a9500ba6ef07aee3efbee9442ee77498d573ad7 Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Fri, 5 Dec 2025 10:25:38 +0000
Subject: [PATCH 56/93] Add selection output type with array support

---
 radis/core/static/core/core.css               |  31 +++
 radis/core/static/core/core.js                | 240 ++++++++++++++++++
 radis/core/templates/cotton/formset.html      |  12 +-
 radis/extractions/constants.py                |   2 +
 radis/extractions/forms.py                    | 145 ++++++++++-
 .../0004_outputfield_selection_options.py     |  33 +++
 .../migrations/0005_outputfield_is_array.py   |  16 ++
 radis/extractions/models.py                   |  51 ++++
 .../extractions/_selection_options_field.html |  51 ++++
 .../extractions/extraction_job_detail.html    |  16 ++
 .../extraction_job_wizard_summary.html        |  21 +-
 radis/extractions/tests/test_forms.py         | 186 ++++++++++++++
 .../tests/unit/test_processor_utils.py        |  48 ++++
 radis/extractions/utils/processor_utils.py    |  38 ++-
 14 files changed, 876 insertions(+), 14 deletions(-)
 create mode 100644 radis/extractions/constants.py
 create mode 100644 radis/extractions/migrations/0004_outputfield_selection_options.py
 create mode 100644 radis/extractions/migrations/0005_outputfield_is_array.py
 create mode 100644 radis/extractions/templates/extractions/_selection_options_field.html
 create mode 100644 radis/extractions/tests/test_forms.py
 create mode 100644 radis/extractions/tests/unit/test_processor_utils.py

diff --git a/radis/core/static/core/core.css b/radis/core/static/core/core.css
index 0dba50c7..c86469b7 100644
--- a/radis/core/static/core/core.css
+++ b/radis/core/static/core/core.css
@@ -39,3 +39,34 @@
   -webkit-line-clamp: 3;
   -webkit-box-orient: vertical;
 }
+
+.selection-options-controls,
+.selection-options-actions {
+  gap: 0.5rem;
+}
+
+.array-toggle-field {
+  display: flex;
+  align-items: center;
+  justify-content: flex-end;
+  padding-top: 14px;
+}
+
+.array-toggle-btn {
+  width: calc(2.5rem - 6px);
+  height: calc(2.5rem - 6px);
+  border-radius: 50%;
+  font-weight: 600;
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  padding: 0;
+  line-height: 1;
+  font-size: 1rem;
+}
+
+.array-toggle-btn.active {
+  background-color: var(--bs-primary);
+  border-color: var(--bs-primary);
+  color: #fff;
+}
diff --git a/radis/core/static/core/core.js b/radis/core/static/core/core.js
index cfff69a0..fd800060 100644
--- a/radis/core/static/core/core.js
+++ b/radis/core/static/core/core.js
@@ -46,6 +46,27 @@ document.addEventListener("alpine:init", () => {
   });
 });
 
+// Prevent form submission on Enter keypress for forms with data-prevent-enter-submit.
+// This was added to the Extractions Output Fields Page to prevent form submission
+// on pressing Enter while adding Output Fields.
+document.addEventListener("DOMContentLoaded", () => {
+  const preventAttr = "[data-prevent-enter-submit]";
+  document.querySelectorAll(preventAttr).forEach((formEl) => {
+    formEl.addEventListener("keydown", (event) => {
+      if (event.key !== "Enter") {
+        return;
+      }
+      const target = event.target;
+      const isTextInput =
+        target instanceof HTMLInputElement &&
+        !["submit", "button"].includes(target.type);
+      if (isTextInput) {
+        event.preventDefault();
+      }
+    });
+  });
+});
+
 /**
  * An Alpine component that controls a Django FormSet
  *
@@ -89,3 +110,222 @@ function FormSet(rootEl) {
     },
   };
 }
+
+/**
+ * Manages the dynamic selection options input for extraction output fields.
+ *
+ *  An Alpine component that controls a Django FormSet.
+ * The core idea is to create a small state machine that manages --
+ * 1. A list of "Selection Option" strings (like an enum)
+ * 2. Whether the current output field supports selections
+ * 3. Whether the field should return multiple values (isArray).
+ * 4. A snapshot of the last valid selections so we can restore them if the user
+ * toggles output types (lastSelectionOptions).
+ * 5. Keeping the hidden inputs (JSON representations of users dynamic input)
+ * synced so Django can read the state when the form submits.
+ *
+ * @param {HTMLElement} rootEl
+ * @returns {Object}
+ */
+function SelectionOptions(rootEl) {
+  /*data-selection-input and data-array-input are data attributes that are used to 
+  locate the the hidden fields that store the JSON serialized state.*/
+  const hiddenInput = rootEl.querySelector("[data-selection-input]");
+  const arrayInput = rootEl.querySelector("[data-array-input]");
+
+  /*finds the closest wrapper that contains the output-type dropdown and the toggle button */
+  const formContainer =
+    rootEl.closest(".formset-form") ?? rootEl.closest("form") ?? rootEl;
+
+  /*We are searching for the <select> element that controls the output type 
+    (e.g., Text, Numeric, Boolean, Selection). The name attribute of that 
+    <select> differs based on whether we are in a formset or a single form,
+    so we try both patterns here - this is important in case we want to use this component 
+    outside of a formset in the future. */
+  const outputTypeField =
+    formContainer.querySelector('select[name$="-output_type"]') ??
+    formContainer.querySelector('select[name="output_type"]');
+  const arrayToggleButton =
+    formContainer.querySelector("[data-array-toggle]") ?? null;
+
+  /*Takes the string from the hidden is_array field and turns it into a boolean.*/
+  const parseArrayValue = (value) => {
+    if (!value) {
+      return false;
+    }
+    const normalized = value.trim().toLowerCase();
+    return normalized === "true" || normalized === "1" || normalized === "on";
+  };
+  const maxSelectionOptions = outputTypeField?.dataset.maxSelectionOptions;
+
+  /*Reads the maximum number of selections allowed. 
+  Each form field embeds this limit via data-max-selection-options
+  which is set in the initialization of OutputFieldForm */
+  const parseMaxOptions = () => {
+    const datasetValue =
+      hiddenInput?.dataset.maxSelectionOptions ??
+      rootEl.dataset.maxSelectionOptions ??
+      "";
+    const parsed = Number.parseInt(datasetValue, 10);
+    return Number.isNaN(parsed) ? 0 : parsed;
+  };
+  const initialMaxOptions = parseMaxOptions();
+
+  //This represents the "reactive state" (summary of users dynamic input) of the component.
+  return {
+    options: [], //array of current selection strings displayed in the widget.
+    maxOptions: initialMaxOptions,
+    supportsSelection: false, //whether the current output type supports selection options.
+    isArray: parseArrayValue(arrayInput?.value), //current "array toggle" state,
+    // parsed from the hidden field.
+    lastSelectionOptions: [], // used to remember the user’s typed options if they
+    // switch away from Selection so we can restore them later.
+
+    init() {
+      //Populate options by reading the hidden JSON string (if any).
+      this.options = this.parseOptions(hiddenInput?.value);
+      //isArray based on the hidden is_array value.
+      this.isArray = parseArrayValue(arrayInput?.value);
+      //Remember the initial options in lastSelectionOptions.
+      this.lastSelectionOptions = [...this.options];
+      this.updateSupports(); //Determine if current output type supports selection.
+
+      //Clicking the button flips isArray
+      if (arrayToggleButton) {
+        arrayToggleButton.addEventListener("click", (event) => {
+          event.preventDefault();
+          this.toggleArray();
+        });
+        this.updateArrayButton();
+      }
+      if (outputTypeField) {
+        outputTypeField.addEventListener("change", () => {
+          /**
+           * When the user switches from Selection to another type, we save the current
+           * options in lastSelectionOptions and clear options and clear the UI
+           * (since we shouldn’t show them for Text/Numeric/Boolean).
+           *
+           * If they switch back to Selection and there are no options yet,
+           *  we restore the previous list (or reload from the hidden JSON).
+           *
+           * This prevents data loss when toggling between types.
+           */
+          const wasSelection = this.supportsSelection;
+          this.updateSupports();
+          if (!this.supportsSelection) {
+            this.lastSelectionOptions = [...this.options];
+            this.options = [];
+          } else if (!wasSelection && this.options.length === 0) {
+            if (this.lastSelectionOptions.length > 0) {
+              this.options = [...this.lastSelectionOptions];
+            } else {
+              this.options = this.parseOptions(hiddenInput?.value);
+            }
+          }
+        });
+      }
+    },
+
+    /* Converts the hidden field’s JSON string into a clean array of strings. 
+    Non-string entries become empty strings and get filtered out.*/
+    parseOptions(value) {
+      if (!value) {
+        return [];
+      }
+      try {
+        const parsed = JSON.parse(value);
+        if (Array.isArray(parsed)) {
+          return parsed
+            .map((opt) => (typeof opt === "string" ? opt : ""))
+            .filter((opt) => opt !== "");
+        }
+      } catch (err) {
+        console.warn("Invalid selection options payload", err);
+      }
+      return [];
+    },
+
+    //Checks the dropdown’s current value. If the field is set to S (Selection),
+    // we show the options list. Otherwise, hide it..
+    updateSupports() {
+      this.supportsSelection = outputTypeField
+        ? outputTypeField.value === "S"
+        : false;
+    },
+
+    /*
+    Whenever the options array changes, we trim entries, remove empty strings, serialize to JSON, 
+    and store it back in the hidden input. 
+    We also update lastSelectionOptions so we remember the sanitized state.
+    */
+    syncOptions() {
+      if (!hiddenInput) {
+        return;
+      }
+      const sanitized = this.options
+        .map((opt) => (typeof opt === "string" ? opt.trim() : ""))
+        .filter((opt) => opt !== "");
+      hiddenInput.value = JSON.stringify(sanitized);
+      this.lastSelectionOptions = [...sanitized];
+    },
+
+    //Writes "true" or "false" into the hidden is_array input.
+    syncArray() {
+      if (!arrayInput) {
+        return;
+      }
+      arrayInput.value = this.isArray ? "true" : "false";
+    },
+
+    /*
+      One method to sync everything. 
+      We call this via x-effect="syncState()" in the template, 
+      so Alpine runs it after every reactive update
+    */
+    syncState() {
+      this.syncOptions();
+      this.syncArray();
+      this.updateArrayButton();
+    },
+
+    //Adds a new empty option if the field is in Selection mode and hasn’t hit the max count.
+    addOption() {
+      if (!this.supportsSelection || this.options.length >= this.maxOptions) {
+        return;
+      }
+      this.options.push("");
+      this.$nextTick(() => {
+        // After adding, it waits for the DOM to update ($nextTick) and automatically shifts cursor focus
+        // to the newly added input field.
+        const inputs = rootEl.querySelectorAll("[data-selection-option-input]");
+        const lastInput = inputs[inputs.length - 1];
+        if (lastInput instanceof HTMLInputElement) {
+          lastInput.focus();
+        }
+      });
+    },
+
+    //Deletes the option at the selected index.
+    removeOption(index) {
+      this.options.splice(index, 1);
+    },
+
+    //Flips the boolean; syncState() will update the hidden input and button appearance.
+    toggleArray() {
+      this.isArray = !this.isArray;
+    },
+
+    //Updates the toggle button’s appearance based on isArray state -
+    // we want to show a blue highlight around the button when it is pressed (shows active state).
+    updateArrayButton() {
+      if (!arrayToggleButton) {
+        return;
+      }
+      arrayToggleButton.classList.toggle("active", this.isArray);
+      arrayToggleButton.setAttribute(
+        "aria-pressed",
+        this.isArray ? "true" : "false"
+      );
+    },
+  };
+}
diff --git a/radis/core/templates/cotton/formset.html b/radis/core/templates/cotton/formset.html
index 20c6af7b..904473a1 100644
--- a/radis/core/templates/cotton/formset.html
+++ b/radis/core/templates/cotton/formset.html
@@ -3,12 +3,12 @@
 <fieldset x-data="FormSet($el)" class="mb-3">
     {% if legend %}<legend>{{ legend }}</legend>{% endif %}
     {{ formset.management_form }}
-    <template>
-        <c-formset-form>{% crispy formset.empty_form %}</c-formset-form>
-    </template>
-    <div class="formset-forms">
-        {% for form in formset %}<c-formset-form>{{ form|crispy }}</c-formset-form>{% endfor %}
-    </div>
+<template>
+    <c-formset-form>{% crispy formset.empty_form %}</c-formset-form>
+</template>
+<div class="formset-forms">
+    {% for form in formset %}<c-formset-form>{% crispy form %}</c-formset-form>{% endfor %}
+</div>
     {% if add_form_label %}
         <div class="d-flex flex-row-reverse">
             <button type="button"
diff --git a/radis/extractions/constants.py b/radis/extractions/constants.py
new file mode 100644
index 00000000..2a291cf6
--- /dev/null
+++ b/radis/extractions/constants.py
@@ -0,0 +1,2 @@
+MAX_SELECTION_OPTIONS = 7.0  # Maximum number of Enum values (selection output types) a
+# user can add per Output Field the Extraction Output Fields Page.
diff --git a/radis/extractions/forms.py b/radis/extractions/forms.py
index 9e6a4bca..32efb5e5 100644
--- a/radis/extractions/forms.py
+++ b/radis/extractions/forms.py
@@ -1,8 +1,9 @@
+import json
 from typing import Any, cast
 
 from adit_radis_shared.accounts.models import User
 from crispy_forms.helper import FormHelper
-from crispy_forms.layout import Column, Layout, Row, Submit
+from crispy_forms.layout import HTML, Column, Div, Layout, Row, Submit
 from django import forms
 from django.conf import settings
 from django.db.models import QuerySet
@@ -14,7 +15,8 @@
 from radis.search.site import Search, SearchFilters
 from radis.search.utils.query_parser import QueryParser
 
-from .models import ExtractionJob, OutputField
+from .constants import MAX_SELECTION_OPTIONS
+from .models import ExtractionJob, OutputField, OutputType
 from .site import extraction_retrieval_providers
 
 
@@ -185,14 +187,153 @@ def clean(self) -> dict[str, Any] | None:
 
 
 class OutputFieldForm(forms.ModelForm):
+    """Hidden field to store selection options and array flag as JSON string.
+    This is done because the selection options are dynamic and the array toggle
+    is an alpine component that needs to be re-rendered on every change."""
+
+    selection_options = forms.CharField(
+        required=False,
+        widget=forms.HiddenInput(),
+    )
+    is_array = forms.CharField(
+        required=False,
+        widget=forms.HiddenInput(),
+    )
+
     class Meta:
         model = OutputField
         fields = [
             "name",
             "description",
             "output_type",
+            "selection_options",
+            "is_array",
         ]
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.fields["description"].widget = forms.Textarea(attrs={"rows": 3})
+
+        # Stamp data attributes so the selection-options widget can locate these fields.
+        self.fields["selection_options"].widget.attrs.update(
+            {
+                "data-selection-input": "true",
+                "data-max-selection-options": str(MAX_SELECTION_OPTIONS),
+            }
+        )
+        self.fields["is_array"].widget.attrs.update(
+            {
+                "data-array-input": "true",
+            }
+        )
+
+        initial_options = self.instance.selection_options if self.instance.pk else []
+
+        # Prepopulate the hidden fields so existing options/toggle state show up in the widget.
+        self.initial["selection_options"] = json.dumps(initial_options)
+        self.initial["is_array"] = "true" if self.instance.is_array else "false"
+
+        self.helper = FormHelper()
+        self.helper.form_tag = False  # because we want to define only the inner layout
+        self.helper.disable_csrf = True
+
+        # Build the layout for selection options and array toggle button using crispy.
+        self.helper.layout = Layout(
+            Row(
+                Column("name", css_class="col-md-7 col-12"),
+                Column("output_type", css_class="col-md-4 col-10"),
+                Column(
+                    HTML(
+                        (
+                            '<button type="button" '
+                            'class="btn btn-outline-secondary btn-sm array-toggle-btn '
+                            'form-array-toggle" '
+                            'data-array-toggle="true" '
+                            'aria-pressed="false" '
+                            'title="Toggle array output">[ ]</button>'
+                        )
+                    ),
+                    css_class=(
+                        "col-md-1 col-2 d-flex align-items-center "
+                        "justify-content-end array-toggle-field"
+                    ),
+                ),
+                css_class="g-3 align-items-center",
+            ),
+            "description",
+            # Include the selection options widget partial template here.
+            Div(
+                HTML('{% include "extractions/_selection_options_field.html" %}'),
+                css_class="selection-options-wrapper",
+            ),
+        )
+
+    # Parse and sanitize the JSON payload (trim strings, enforce uniqueness and max count).
+    # Called automatically by Django form validation.
+    def clean_selection_options(self) -> list[str]:
+        raw_value = self.cleaned_data.get("selection_options") or ""
+        raw_value = raw_value.strip()
+        if raw_value == "":
+            return []
+
+        try:
+            parsed = json.loads(raw_value)
+        except json.JSONDecodeError as exc:
+            raise forms.ValidationError("Invalid selection data.") from exc
+
+        if not isinstance(parsed, list):
+            raise forms.ValidationError("Selection data must be a list.")
+
+        cleaned: list[str] = []
+        for item in parsed:
+            if not isinstance(item, str):
+                raise forms.ValidationError("Selection options must be text.")
+            value = item.strip()
+            if not value:
+                raise forms.ValidationError("Selection options cannot be empty.")
+            cleaned.append(value)
+
+        if len(cleaned) > MAX_SELECTION_OPTIONS:
+            raise forms.ValidationError(
+                f"Provide at most {MAX_SELECTION_OPTIONS} selection options."
+            )
+        if len(set(cleaned)) != len(cleaned):
+            raise forms.ValidationError("Selection options must be unique.")
+
+        return cleaned
+
+    # not exactly needed because we set the value ourselves, but more as a double check
+    def clean_is_array(self) -> bool:
+        raw_value = (self.cleaned_data.get("is_array") or "").strip().lower()
+        if raw_value in {"1", "true", "on"}:
+            return True
+        return False
+
+    def clean(self):
+        cleaned_data = super().clean()
+        if not cleaned_data:
+            return cleaned_data
+
+        output_type = cleaned_data.get("output_type")
+        selection_options: list[str] = cleaned_data.get("selection_options") or []
+
+        if output_type == OutputType.SELECTION:
+            if not selection_options:
+                self.add_error(
+                    "selection_options",
+                    "Add at least one selection to use the Selection type.",
+                )
+        else:
+            if selection_options:
+                self.add_error(
+                    "selection_options",
+                    "Selections are only allowed when Output Type is Selection.",
+                )
+                cleaned_data["selection_options"] = []
+
+        return cleaned_data
+
 
 OutputFieldFormSet = forms.inlineformset_factory(
     ExtractionJob,
diff --git a/radis/extractions/migrations/0004_outputfield_selection_options.py b/radis/extractions/migrations/0004_outputfield_selection_options.py
new file mode 100644
index 00000000..fcf5887e
--- /dev/null
+++ b/radis/extractions/migrations/0004_outputfield_selection_options.py
@@ -0,0 +1,33 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("extractions", "0003_alter_extractionjob_options_and_more"),
+    ]
+
+    operations = [
+
+        # Introduce a JSON column to store the Selection output fields.
+        migrations.AddField(
+            model_name="outputfield",
+            name="selection_options",
+            field=models.JSONField(blank=True, default=list),
+        ),
+
+        # Extend the output_type choices so the database accepts the new Selection code ("S").
+        migrations.AlterField(
+            model_name="outputfield",
+            name="output_type",
+            field=models.CharField(
+                choices=[
+                    ("T", "Text"),
+                    ("N", "Numeric"),
+                    ("B", "Boolean"),
+                    ("S", "Selection"),
+                ],
+                default="T",
+                max_length=1,
+            ),
+        ),
+    ]
diff --git a/radis/extractions/migrations/0005_outputfield_is_array.py b/radis/extractions/migrations/0005_outputfield_is_array.py
new file mode 100644
index 00000000..97d6023e
--- /dev/null
+++ b/radis/extractions/migrations/0005_outputfield_is_array.py
@@ -0,0 +1,16 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("extractions", "0004_outputfield_selection_options"),
+    ]
+
+    operations = [
+        # Track whether an output field should return a list of values rather than a single value.
+        migrations.AddField(
+            model_name="outputfield",
+            name="is_array",
+            field=models.BooleanField(default=False),
+        ),
+    ]
diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index da9756da..fd29a99a 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -11,6 +11,8 @@
 from radis.core.models import AnalysisJob, AnalysisTask
 from radis.reports.models import Language, Modality, Report
 
+from .constants import MAX_SELECTION_OPTIONS
+
 
 class ExtractionsAppSettings(AppSettings):
     class Meta:
@@ -76,6 +78,9 @@ class OutputType(models.TextChoices):
     TEXT = "T", "Text"
     NUMERIC = "N", "Numeric"
     BOOLEAN = "B", "Boolean"
+    SELECTION = "S", "Selection"  # New output type which works like a enum
+    # Allows user to define a set of options which the LLM can
+    # choose from to answer questions.
 
 
 class OutputField(models.Model):
@@ -86,6 +91,11 @@ class OutputField(models.Model):
     )
     get_output_type_display: Callable[[], str]
     optional = models.BooleanField(default=False)
+    # For Selection output type, stores the list of allowed options.
+    # Only used if output_type == OutputType.SELECTION.
+    selection_options = models.JSONField(default=list, blank=True)
+    # Whether the field should return an array of values or just values.
+    is_array = models.BooleanField(default=False)
     job = models.ForeignKey[ExtractionJob](
         ExtractionJob, on_delete=models.CASCADE, related_name="output_fields"
     )
@@ -101,6 +111,47 @@ class Meta:
     def __str__(self) -> str:
         return f'Output Field "{self.name}" [{self.pk}]'
 
+    def clean(self) -> None:
+        """Custom validation logic for Selection output type
+        Checks for non empty, uniqueness, and max number of options."""
+
+        from django.core.exceptions import ValidationError
+
+        super().clean()  # Call Django's field validators and other checks first
+
+        if self.output_type == OutputType.SELECTION:
+            if not self.selection_options:
+                raise ValidationError({"selection_options": "Add at least one selection option."})
+            if len(self.selection_options) > MAX_SELECTION_OPTIONS:
+                raise ValidationError(
+                    {
+                        "selection_options": (
+                            f"Provide at most {MAX_SELECTION_OPTIONS} selection options."
+                        )
+                    }
+                )
+            cleaned_options = []
+            for option in self.selection_options:
+                if not isinstance(option, str):
+                    raise ValidationError(
+                        {"selection_options": "All selection options must be text."}
+                    )
+                stripped = option.strip()
+                if not stripped:
+                    raise ValidationError(
+                        {"selection_options": "Selection options cannot be empty strings."}
+                    )
+                cleaned_options.append(stripped)
+            if len(set(cleaned_options)) != len(cleaned_options):
+                raise ValidationError({"selection_options": "Selection options must be unique."})
+            self.selection_options = cleaned_options
+        else:
+            if self.selection_options:
+                raise ValidationError(
+                    {"selection_options": "Selections are only allowed for the Selection type."}
+                )
+            self.selection_options = []
+
 
 class ExtractionTask(AnalysisTask):
     job = models.ForeignKey[ExtractionJob](
diff --git a/radis/extractions/templates/extractions/_selection_options_field.html b/radis/extractions/templates/extractions/_selection_options_field.html
new file mode 100644
index 00000000..5aff83c2
--- /dev/null
+++ b/radis/extractions/templates/extractions/_selection_options_field.html
@@ -0,0 +1,51 @@
+<!-- Provide a dynamic UI for entering selection options (add/remove inputs) and toggling array output. 
+There’s no built-in HTML control for “editable list,” so we build one with Alpine  -->
+{% load bootstrap_icon from common_extras %}
+<div class="mt-3"
+     x-data="SelectionOptions($el)"
+     x-effect="syncState()"
+     data-selection-wrapper="true">
+    <!-- Hidden fields must appear first so the Alpine component can read/write its state -->
+    {{ form.selection_options }}
+    {{ form.is_array }}
+    <div class="d-flex align-items-center flex-wrap selection-options-controls">
+        <div class="d-flex align-items-center gap-2 flex-wrap selection-options-actions">
+            <button type="button"
+                    class="btn btn-outline-secondary btn-sm"
+                    @click.prevent="addOption()"
+                    :disabled="!supportsSelection || options.length >= maxOptions">
+                {% bootstrap_icon "plus-lg" %}
+                Enter a selection
+            </button>
+            <small class="text-muted"
+                   x-show="supportsSelection"
+                   x-text="`${options.length} / ${maxOptions}`"></small>
+        </div>
+    </div>
+    <div class="mt-2" x-show="supportsSelection">
+        <!-- Each entry binds to options[index] so that we can list all the added selection options -->
+        <template x-for="(option, index) in options" :key="index">
+            <div class="input-group mb-2">
+                <input type="text"
+                       class="form-control"
+                       :placeholder="`Selection ${index + 1}`"
+                       data-selection-option-input
+                       x-model="options[index]">
+                <button type="button"
+                        class="btn btn-outline-danger"
+                        @click.prevent="removeOption(index)"
+                        aria-label="Remove selection option">{% bootstrap_icon "trash" %}</button>
+            </div>
+        </template>
+        <p class="text-muted fst-italic" x-show="options.length === 0">No selections defined yet.</p>
+    </div>
+    <p class="text-muted fst-italic" x-show="!supportsSelection">
+        Choose the “Selection” output type to define enumerated values.
+    </p>
+    <!-- Server-side validation errors from the Django form appear here -->
+    {% if form.selection_options.errors %}
+        <div class="invalid-feedback d-block">
+            {% for error in form.selection_options.errors %}{{ error }}{% endfor %}
+        </div>
+    {% endif %}
+</div>
diff --git a/radis/extractions/templates/extractions/extraction_job_detail.html b/radis/extractions/templates/extractions/extraction_job_detail.html
index 942a2fc8..0b76330a 100644
--- a/radis/extractions/templates/extractions/extraction_job_detail.html
+++ b/radis/extractions/templates/extractions/extraction_job_detail.html
@@ -114,6 +114,22 @@ <h5>Output Fields</h5>
                     <dd class="col-sm-9">
                         {{ field.output_type|human_readable_output_type }}
                     </dd>
+                    {% if field.is_array %}
+                        <dt class="col-sm-3">Array Output</dt>
+                        <dd class="col-sm-9">
+                            Yes — return multiple {{ field.output_type|human_readable_output_type|lower }} values
+                        </dd>
+                    {% endif %}
+                    {% if field.selection_options %}
+                        <dt class="col-sm-3">Selections</dt>
+                        <dd class="col-sm-9">
+                            <ul class="mb-0 ps-3">
+                                {% for option in field.selection_options %}
+                                    <li>{{ option }}</li>
+                                {% endfor %}
+                            </ul>
+                        </dd>
+                    {% endif %}
                 </dl>
             </li>
         {% endfor %}
diff --git a/radis/extractions/templates/extractions/extraction_job_wizard_summary.html b/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
index fe7bc429..a9580b82 100644
--- a/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
+++ b/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
@@ -91,6 +91,22 @@
                                 <dd class="col-sm-9">
                                     {{ field.output_type|human_readable_output_type }}
                                 </dd>
+                                {% if field.is_array %}
+                                    <dt class="col-sm-3">Array Output</dt>
+                                    <dd class="col-sm-9">
+                                        Yes — return multiple {{ field.output_type|human_readable_output_type|lower }} values
+                                    </dd>
+                                {% endif %}
+                                {% if field.selection_options %}
+                                    <dt class="col-sm-3">Selections</dt>
+                                    <dd class="col-sm-9">
+                                        <ul class="mb-0 ps-3">
+                                            {% for option in field.selection_options %}
+                                                <li>{{ option }}</li>
+                                            {% endfor %}
+                                        </ul>
+                                    </dd>
+                                {% endif %}
                             </dl>
                         </li>
                     {% endif %}
@@ -98,7 +114,10 @@
             </ul>
         </dd>
     </dl>
-    <form action="" method="post" class="pt-3">
+    <form action=""
+          method="post"
+          class="pt-3"
+          data-prevent-enter-submit="true">
         {% csrf_token %}
         {{ wizard.management_form }}
         {% crispy form %}
diff --git a/radis/extractions/tests/test_forms.py b/radis/extractions/tests/test_forms.py
new file mode 100644
index 00000000..d91fc909
--- /dev/null
+++ b/radis/extractions/tests/test_forms.py
@@ -0,0 +1,186 @@
+import json
+
+import pytest
+from django.core.exceptions import ValidationError
+
+from radis.extractions.factories import ExtractionJobFactory
+from radis.extractions.forms import OutputFieldForm
+from radis.extractions.models import OutputField, OutputType
+
+
+@pytest.mark.django_db
+def test_output_field_form_accepts_selection_options():
+    job = ExtractionJobFactory.create()
+    form = OutputFieldForm(
+        data={
+            "name": "tumor_grade",
+            "description": "Classified tumor grade.",
+            "output_type": OutputType.SELECTION,
+            "selection_options": json.dumps(["Grade 1", "Grade 2"]),
+            "is_array": "false",
+        },
+        instance=OutputField(job=job),
+    )
+
+    assert form.is_valid()
+    instance = form.save(commit=False)
+
+    assert instance.selection_options == ["Grade 1", "Grade 2"]
+
+
+@pytest.mark.django_db
+def test_output_field_form_requires_options_for_selection():
+    job = ExtractionJobFactory.create()
+    form = OutputFieldForm(
+        data={
+            "name": "tumor_grade",
+            "description": "Classified tumor grade.",
+            "output_type": OutputType.SELECTION,
+            "selection_options": json.dumps([]),
+            "is_array": "false",
+        },
+        instance=OutputField(job=job),
+    )
+
+    assert not form.is_valid()
+    assert "selection_options" in form.errors
+
+
+@pytest.mark.django_db
+def test_output_field_form_rejects_options_for_non_selection():
+    job = ExtractionJobFactory.create()
+    form = OutputFieldForm(
+        data={
+            "name": "tumor_grade",
+            "description": "Classified tumor grade.",
+            "output_type": OutputType.TEXT,
+            "selection_options": json.dumps(["Grade 1"]),
+            "is_array": "false",
+        },
+        instance=OutputField(job=job),
+    )
+
+    assert not form.is_valid()
+    assert "selection_options" in form.errors
+
+
+@pytest.mark.django_db
+def test_output_field_clean_trims_selection_options():
+    job = ExtractionJobFactory.create()
+    field = OutputField(
+        job=job,
+        name="tumor_grade",
+        description="Classified tumor grade.",
+        output_type=OutputType.SELECTION,
+        selection_options=["  Grade 1 ", "Grade 2  "],
+    )
+
+    field.full_clean()
+
+    assert field.selection_options == ["Grade 1", "Grade 2"]
+
+
+@pytest.mark.django_db
+def test_output_field_clean_rejects_selection_options_for_other_types():
+    job = ExtractionJobFactory.create()
+    field = OutputField(
+        job=job,
+        name="tumor_grade",
+        description="Classified tumor grade.",
+        output_type=OutputType.TEXT,
+        selection_options=["Grade 1"],
+    )
+
+    with pytest.raises(ValidationError):
+        field.full_clean()
+
+
+@pytest.mark.django_db
+def test_output_field_form_handles_array_toggle():
+    job = ExtractionJobFactory.create()
+    form = OutputFieldForm(
+        data={
+            "name": "measurements",
+            "description": "Multiple numeric values.",
+            "output_type": OutputType.NUMERIC,
+            "selection_options": json.dumps([]),
+            "is_array": "true",
+        },
+        instance=OutputField(job=job),
+    )
+
+    assert form.is_valid()
+    instance = form.save(commit=False)
+    assert instance.is_array is True
+
+
+@pytest.mark.django_db
+def test_output_field_form_rejects_duplicate_selection_options():
+    job = ExtractionJobFactory.create()
+    form = OutputFieldForm(
+        data={
+            "name": "tumor_grade",
+            "description": "Classified tumor grade.",
+            "output_type": OutputType.SELECTION,
+            "selection_options": json.dumps(["Grade 1", "Grade 1"]),
+            "is_array": "false",
+        },
+        instance=OutputField(job=job),
+    )
+
+    assert not form.is_valid()
+    assert "selection_options" in form.errors
+
+
+@pytest.mark.django_db
+def test_output_field_clean_rejects_duplicate_selection_options():
+    job = ExtractionJobFactory.create()
+    field = OutputField(
+        job=job,
+        name="tumor_grade",
+        description="Classified tumor grade.",
+        output_type=OutputType.SELECTION,
+        selection_options=["Grade 1", "Grade 1"],
+    )
+
+    with pytest.raises(ValidationError):
+        field.full_clean()
+
+
+@pytest.mark.django_db
+def test_output_field_form_rejects_whitespace_only_selection_option():
+    job = ExtractionJobFactory.create()
+    form = OutputFieldForm(
+        data={
+            "name": "tumor_grade",
+            "description": "Classified tumor grade.",
+            "output_type": OutputType.SELECTION,
+            "selection_options": json.dumps(["Grade 1", "   "]),
+            "is_array": "false",
+        },
+        instance=OutputField(job=job),
+    )
+
+    assert not form.is_valid()
+    assert "selection_options" in form.errors
+
+
+@pytest.mark.django_db
+def test_output_field_form_accepts_unicode_and_long_selection_options():
+    job = ExtractionJobFactory.create()
+    unicode_option = "Grädé αβγ測試"
+    long_option = "Grade " + ("X" * 150)
+    form = OutputFieldForm(
+        data={
+            "name": "tumor_grade",
+            "description": "Classified tumor grade.",
+            "output_type": OutputType.SELECTION,
+            "selection_options": json.dumps([unicode_option, long_option]),
+            "is_array": "false",
+        },
+        instance=OutputField(job=job),
+    )
+
+    assert form.is_valid()
+    instance = form.save(commit=False)
+    assert instance.selection_options == [unicode_option.strip(), long_option.strip()]
diff --git a/radis/extractions/tests/unit/test_processor_utils.py b/radis/extractions/tests/unit/test_processor_utils.py
new file mode 100644
index 00000000..4f71b127
--- /dev/null
+++ b/radis/extractions/tests/unit/test_processor_utils.py
@@ -0,0 +1,48 @@
+from typing import Literal, get_args, get_origin
+
+import pytest
+
+from radis.extractions.factories import ExtractionJobFactory, OutputFieldFactory
+from radis.extractions.models import OutputType
+from radis.extractions.utils.processor_utils import generate_output_fields_schema
+
+
+@pytest.mark.django_db
+def test_generate_output_fields_schema_uses_literal_for_selection_fields():
+    job = ExtractionJobFactory.create()
+    field = OutputFieldFactory(
+        job=job,
+        name="grade",
+        output_type=OutputType.SELECTION,
+    )
+    field.selection_options = ["Grade 1", "Grade 2"]
+    field.save()
+
+    schema = generate_output_fields_schema(job.output_fields)
+
+    grade_field = schema.model_fields["grade"]
+    annotation = grade_field.annotation
+    assert get_origin(annotation) is Literal
+    assert set(get_args(annotation)) == {"Grade 1", "Grade 2"}
+
+
+@pytest.mark.django_db
+def test_generate_output_fields_schema_wraps_literal_in_list_for_array_selections():
+    job = ExtractionJobFactory.create()
+    field = OutputFieldFactory(
+        job=job,
+        name="grade_multi",
+        output_type=OutputType.SELECTION,
+    )
+    field.selection_options = ["High", "Low"]
+    field.is_array = True
+    field.save()
+
+    schema = generate_output_fields_schema(job.output_fields)
+
+    grade_field = schema.model_fields["grade_multi"]
+    annotation = grade_field.annotation
+    assert get_origin(annotation) is list
+    (inner_annotation,) = get_args(annotation)
+    assert get_origin(inner_annotation) is Literal
+    assert set(get_args(inner_annotation)) == {"High", "Low"}
diff --git a/radis/extractions/utils/processor_utils.py b/radis/extractions/utils/processor_utils.py
index e7f12bca..7fd2e907 100644
--- a/radis/extractions/utils/processor_utils.py
+++ b/radis/extractions/utils/processor_utils.py
@@ -1,4 +1,4 @@
-from typing import Any
+from typing import Any, Literal
 
 from django.db.models import QuerySet
 from pydantic import BaseModel, create_model
@@ -7,27 +7,55 @@
 
 type Numeric = float | int
 
+"""Build a Pydantic model that describes the structure the extractor must output"""
+
 
 def generate_output_fields_schema(fields: QuerySet[OutputField]) -> type[BaseModel]:
     field_definitions: dict[str, Any] = {}
     for field in fields.all():
-        if field.output_type == OutputType.TEXT:
+        field_type = OutputType(field.output_type)
+        if field_type == OutputType.TEXT:
             output_type = str
-        elif field.output_type == OutputType.NUMERIC:
+        elif field_type == OutputType.NUMERIC:
             output_type = Numeric
-        elif field.output_type == OutputType.BOOLEAN:
+        elif field_type == OutputType.BOOLEAN:
             output_type = bool
+        elif field_type == OutputType.SELECTION:
+            # Grab the saved selection_options list and convert it to a tuple.
+            options = tuple(field.selection_options)
+            if not options:
+                raise ValueError("Selection output requires at least one option.")
+            # Set the output type to a Literal of the options so that only these values
+            # can be accepted.
+            output_type = Literal[*options]
         else:
             raise ValueError(f"Unknown data type: {field.output_type}")
 
+        if field.is_array:
+            # If the field stores multiple values, use a list[...] of the base type above.
+            output_type = list[output_type]
+
         field_definitions[field.name] = (output_type, ...)
 
     return create_model("OutputFieldsModel", **field_definitions)
 
 
 def generate_output_fields_prompt(fields: QuerySet[OutputField]) -> str:
+    # Build a human-readable prompt that mirrors the same selection/array rules.
     prompt = ""
     for field in fields.all():
-        prompt += f"{field.name}: {field.description}\n"
+        description = field.description
+        if OutputType(field.output_type) == OutputType.SELECTION and field.selection_options:
+            # Tell the processor exactly which selections it may return.
+            description = (
+                f"{description} (Allowed selections: {', '.join(field.selection_options)})"
+            )
+        if field.is_array:
+            # Remind the processor to return an array, not a single value.
+            description = (
+                f"{description} (Return an array of "
+                f"{field.get_output_type_display().lower()} values.)"
+            )
+        prompt += f"{field.name}: {description}\n"
 
     return prompt

From a90591419113649b9b6d7c721508936f4ce3804a Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 8 Dec 2025 23:32:06 +0000
Subject: [PATCH 57/93] handler based approach for starting subscription jobs

---
 radis/settings/base.py          |  2 +-
 radis/subscriptions/apps.py     | 16 ++++++++++++++++
 radis/subscriptions/handlers.py | 21 +++++++++++++++++++++
 radis/subscriptions/tasks.py    |  2 +-
 4 files changed, 39 insertions(+), 2 deletions(-)
 create mode 100644 radis/subscriptions/handlers.py

diff --git a/radis/settings/base.py b/radis/settings/base.py
index 492566fd..a80dbcbb 100644
--- a/radis/settings/base.py
+++ b/radis/settings/base.py
@@ -426,7 +426,7 @@
 # Subscription
 SUBSCRIPTION_DEFAULT_PRIORITY = 3
 SUBSCRIPTION_URGENT_PRIORITY = 4
-SUBSCRIPTION_CRON = "* * * * *"
+SUBSCRIPTION_CRON = "* * * * *"  # Disabled - using handler-based approach
 SUBSCRIPTION_REFRESH_TASK_BATCH_SIZE = 100
 
 # The priority for stalled jobs that are retried.
diff --git a/radis/subscriptions/apps.py b/radis/subscriptions/apps.py
index dee690a6..c44cb681 100644
--- a/radis/subscriptions/apps.py
+++ b/radis/subscriptions/apps.py
@@ -1,12 +1,17 @@
 from django.apps import AppConfig
 from django.db.models.signals import post_migrate
 
+from radis.reports.site import ReportsCreatedHandler, reports_created_handlers
+
+from .handlers import handle_reports_created
+
 
 class SubscriptionsConfig(AppConfig):
     name = "radis.subscriptions"
 
     def ready(self):
         register_app()
+        register_reports_handler()
 
         # Put calls to db stuff in this signal handler
         post_migrate.connect(init_db, sender=self)
@@ -28,3 +33,14 @@ def init_db(**kwargs):
 
     if not SubscriptionsAppSettings.objects.exists():
         SubscriptionsAppSettings.objects.create()
+
+
+def register_reports_handler():
+    """Register handler to trigger subscriptions when reports are created."""
+
+    reports_created_handlers.append(
+        ReportsCreatedHandler(
+            name="subscription_launcher",
+            handle=handle_reports_created,
+        )
+    )
diff --git a/radis/subscriptions/handlers.py b/radis/subscriptions/handlers.py
new file mode 100644
index 00000000..c0bd70b1
--- /dev/null
+++ b/radis/subscriptions/handlers.py
@@ -0,0 +1,21 @@
+import logging
+import time
+
+from radis.reports.models import Report
+from radis.subscriptions.tasks import subscription_launcher
+
+logger = logging.getLogger(__name__)
+
+
+def handle_reports_created(reports: list[Report]) -> None:
+    """
+    Handler called when reports are created via API.
+    Triggers subscription processing for bulk imports.
+    """
+    if not reports:
+        return
+
+    # Trigger subscriptions for any report creation
+    # The subscription_launcher will filter by last_refreshed timestamp
+    logger.info(f"Triggering subscription processing for {len(reports)} new report(s)")
+    subscription_launcher.defer(timestamp=int(time.time()))
diff --git a/radis/subscriptions/tasks.py b/radis/subscriptions/tasks.py
index ab7465aa..60e0b147 100644
--- a/radis/subscriptions/tasks.py
+++ b/radis/subscriptions/tasks.py
@@ -105,7 +105,7 @@ def process_subscription_job(job_id: int) -> None:
     job.save()
 
 
-@app.periodic(cron=settings.SUBSCRIPTION_CRON)
+# @app.periodic(cron=settings.SUBSCRIPTION_CRON)
 @app.task()
 def subscription_launcher(timestamp: int):
     logger.info("Launching SubscriptionJobs (Timestamp %s)", datetime.fromtimestamp(timestamp))

From 1c8db1cda90148ae65e724f156735f1423e2605c Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 9 Dec 2025 23:28:41 +0000
Subject: [PATCH 58/93] move necessary imports inside register reports handler

---
 radis/subscriptions/apps.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/radis/subscriptions/apps.py b/radis/subscriptions/apps.py
index c44cb681..28680c4d 100644
--- a/radis/subscriptions/apps.py
+++ b/radis/subscriptions/apps.py
@@ -1,10 +1,6 @@
 from django.apps import AppConfig
 from django.db.models.signals import post_migrate
 
-from radis.reports.site import ReportsCreatedHandler, reports_created_handlers
-
-from .handlers import handle_reports_created
-
 
 class SubscriptionsConfig(AppConfig):
     name = "radis.subscriptions"
@@ -37,6 +33,9 @@ def init_db(**kwargs):
 
 def register_reports_handler():
     """Register handler to trigger subscriptions when reports are created."""
+    from radis.reports.site import ReportsCreatedHandler, reports_created_handlers
+
+    from .handlers import handle_reports_created
 
     reports_created_handlers.append(
         ReportsCreatedHandler(

From 417cee12ccd1d339a1f8cdaf2555e424a459b15c Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Wed, 10 Dec 2025 22:41:53 +0000
Subject: [PATCH 59/93] generate query from extraction fields

---
 radis/chats/utils/chat_client.py              |  30 ++
 radis/extractions/forms.py                    |  26 +-
 .../extractions/tests/test_query_generator.py | 262 ++++++++++++++++
 radis/extractions/utils/query_generator.py    | 285 ++++++++++++++++++
 radis/extractions/views.py                    |  48 ++-
 radis/settings/base.py                        |  34 +++
 radis/subscriptions/forms.py                  |   5 +-
 7 files changed, 680 insertions(+), 10 deletions(-)
 create mode 100644 radis/extractions/tests/test_query_generator.py
 create mode 100644 radis/extractions/utils/query_generator.py

diff --git a/radis/chats/utils/chat_client.py b/radis/chats/utils/chat_client.py
index b310247a..7d6bfedc 100644
--- a/radis/chats/utils/chat_client.py
+++ b/radis/chats/utils/chat_client.py
@@ -52,6 +52,36 @@ def __init__(self) -> None:
         self._client = openai.OpenAI(base_url=base_url, api_key=api_key)
         self._llm_model_name = settings.LLM_MODEL_NAME
 
+    def chat(
+        self,
+        messages: Iterable[ChatCompletionMessageParam],
+        max_completion_tokens: int | None = None,
+    ) -> str:
+        """
+        Send messages to LLM and return the response text.
+
+        Args:
+            messages: List of message dictionaries with 'role' and 'content'
+            max_completion_tokens: Optional maximum tokens to generate
+
+        Returns:
+            The LLM's response as a string
+        """
+        logger.debug(f"Sending messages to LLM for chat:\n{messages}")
+
+        request = {
+            "model": self._llm_model_name,
+            "messages": messages,
+        }
+        if max_completion_tokens is not None:
+            request["max_completion_tokens"] = max_completion_tokens
+
+        completion = self._client.chat.completions.create(**request)
+        answer = completion.choices[0].message.content
+        assert answer is not None
+        logger.debug("Received from LLM: %s", answer)
+        return answer
+
     def extract_data(self, prompt: str, schema: type[BaseModel]) -> BaseModel:
         logger.debug("Sending prompt and schema to LLM to extract data.")
         logger.debug("Prompt:\n%s", prompt)
diff --git a/radis/extractions/forms.py b/radis/extractions/forms.py
index 7a1d559d..72bb01fb 100644
--- a/radis/extractions/forms.py
+++ b/radis/extractions/forms.py
@@ -35,7 +35,10 @@ class Meta:
         ]
         help_texts = {
             "title": "Title of the extraction job",
-            "query": "A query to find reports for further analysis",
+            "query": "Search query to filter reports (leave empty to auto-generate from extraction fields)",
+        }
+        widgets = {
+            "query": forms.TextInput(attrs={"placeholder": "Optional - auto-generated if empty"}),
         }
 
     def __init__(self, *args, **kwargs):
@@ -107,10 +110,13 @@ def build_layout(self):
         )
 
     def clean_query(self) -> str:
-        query = self.cleaned_data["query"]
+        query = self.cleaned_data["query"].strip()
+        if not query:
+            # Query is empty - will be auto-generated from output fields
+            return ""
         query_node, _ = QueryParser().parse(query)
         if query_node is None:
-            raise forms.ValidationError("Invalid empty query")
+            raise forms.ValidationError("Invalid query syntax")
         return query
 
     def clean(self) -> dict[str, Any] | None:
@@ -122,7 +128,17 @@ def clean(self) -> dict[str, Any] | None:
         language = cast(Language, cleaned_data["language"])
         modalities = cast(QuerySet[Modality], cleaned_data["modalities"])
 
-        query_node, fixes = QueryParser().parse(cleaned_data["query"])
+        query = cleaned_data.get("query", "").strip()
+
+        # Check if query generation is needed
+        if not query:
+            # Mark for auto-generation from output fields
+            cleaned_data["requires_query_generation"] = True
+            cleaned_data["retrieval_count"] = 0
+            return cleaned_data
+
+        # Validate manual query
+        query_node, fixes = QueryParser().parse(query)
         assert query_node
 
         if len(fixes) > 0:
@@ -166,6 +182,8 @@ def clean(self) -> dict[str, Any] | None:
                 "provider can handle. Please refine your search."
             )
 
+        cleaned_data["requires_query_generation"] = False
+
         return cleaned_data
 
 
diff --git a/radis/extractions/tests/test_query_generator.py b/radis/extractions/tests/test_query_generator.py
new file mode 100644
index 00000000..1492d707
--- /dev/null
+++ b/radis/extractions/tests/test_query_generator.py
@@ -0,0 +1,262 @@
+"""Unit tests for the QueryGenerator class."""
+
+from unittest.mock import Mock, patch
+
+from django.test import TestCase, override_settings
+
+from radis.extractions.models import OutputField, OutputType
+from radis.extractions.utils.query_generator import QueryGenerator
+
+
+class QueryGeneratorTest(TestCase):
+    """Test cases for QueryGenerator."""
+
+    def setUp(self):
+        """Set up test fixtures."""
+        self.generator = QueryGenerator()
+
+    def test_generate_from_empty_fields(self):
+        """Test query generation with no fields returns wildcard."""
+        fields = []
+        query, metadata = self.generator.generate_from_fields(fields)
+
+        assert query == "*"
+        assert metadata["generation_method"] == "wildcard"
+        assert metadata["field_count"] == 0
+        assert metadata["success"] is True
+
+    def test_keyword_fallback_single_field(self):
+        """Test keyword extraction fallback with a single field."""
+        field = OutputField(
+            name="lung_nodule",
+            description="size of lung nodule in millimeters",
+            output_type=OutputType.NUMERIC,
+        )
+
+        with override_settings(ENABLE_AUTO_QUERY_GENERATION=False):
+            query, metadata = self.generator.generate_from_fields([field])
+
+        assert query != ""
+        assert metadata["generation_method"] == "keyword_fallback"
+        assert metadata["success"] is True
+        assert metadata["field_count"] == 1
+        # Should contain keywords from name and description
+        assert "lung" in query.lower() or "nodule" in query.lower()
+
+    def test_keyword_fallback_multiple_fields(self):
+        """Test keyword extraction with multiple fields."""
+        fields = [
+            OutputField(
+                name="fracture_type",
+                description="type of bone fracture",
+                output_type=OutputType.TEXT,
+            ),
+            OutputField(
+                name="bone_name",
+                description="which bone is fractured",
+                output_type=OutputType.TEXT,
+            ),
+        ]
+
+        with override_settings(ENABLE_AUTO_QUERY_GENERATION=False):
+            query, metadata = self.generator.generate_from_fields(fields)
+
+        assert query != ""
+        assert metadata["generation_method"] == "keyword_fallback"
+        assert metadata["success"] is True
+        assert metadata["field_count"] == 2
+        # Should use OR operator
+        assert " OR " in query
+
+    @patch("radis.extractions.utils.query_generator.ChatClient")
+    def test_llm_generation_success(self, mock_chat_client_class):
+        """Test successful LLM query generation."""
+        # Mock the LLM response
+        mock_client = Mock()
+        mock_client.chat.return_value = '("lung nodule" OR "pulmonary nodule") AND size'
+        mock_chat_client_class.return_value = mock_client
+
+        fields = [
+            OutputField(
+                name="nodule_size",
+                description="size of lung nodule in millimeters",
+                output_type=OutputType.NUMERIC,
+            )
+        ]
+
+        with override_settings(ENABLE_AUTO_QUERY_GENERATION=True):
+            generator = QueryGenerator()  # Create new instance with mocked client
+            query, metadata = generator.generate_from_fields(fields)
+
+        assert query != ""
+        assert metadata["generation_method"] == "llm"
+        assert metadata["success"] is True
+        assert "nodule" in query.lower()
+
+    @patch("radis.extractions.utils.query_generator.ChatClient")
+    def test_llm_generation_failure_fallback(self, mock_chat_client_class):
+        """Test that LLM failure triggers keyword fallback."""
+        # Mock LLM to raise an exception
+        mock_client = Mock()
+        mock_client.chat.side_effect = Exception("LLM service unavailable")
+        mock_chat_client_class.return_value = mock_client
+
+        fields = [
+            OutputField(
+                name="tumor_size",
+                description="size of the tumor in centimeters",
+                output_type=OutputType.NUMERIC,
+            )
+        ]
+
+        with override_settings(ENABLE_AUTO_QUERY_GENERATION=True):
+            generator = QueryGenerator()
+            query, metadata = generator.generate_from_fields(fields)
+
+        # Should fall back to keyword extraction
+        assert query != ""
+        assert metadata["generation_method"] == "keyword_fallback"
+        assert metadata["success"] is True
+        assert metadata["error"] is not None
+
+    def test_validate_and_fix_valid_query(self):
+        """Test validation of a valid query."""
+        query = "lung AND nodule"
+        fixed_query, fixes = self.generator.validate_and_fix_query(query)
+
+        assert fixed_query == query
+        assert len(fixes) == 0
+
+    def test_validate_and_fix_empty_query(self):
+        """Test validation of an empty query."""
+        query = ""
+        fixed_query, fixes = self.generator.validate_and_fix_query(query)
+
+        assert fixed_query == ""
+        assert len(fixes) == 0
+
+    def test_validate_and_fix_query_with_quotes(self):
+        """Test validation of a query with quoted phrases."""
+        query = '"lung nodule" AND size'
+        fixed_query, fixes = self.generator.validate_and_fix_query(query)
+
+        assert fixed_query != ""
+        assert "lung nodule" in fixed_query or "lung" in fixed_query
+
+    def test_extract_keywords_from_camel_case(self):
+        """Test keyword extraction from camelCase field names."""
+        field = OutputField(
+            name="patientAge",
+            description="age of the patient in years",
+            output_type=OutputType.NUMERIC,
+        )
+
+        with override_settings(ENABLE_AUTO_QUERY_GENERATION=False):
+            query, metadata = self.generator.generate_from_fields([field])
+
+        assert query != ""
+        # Should split camelCase into separate words
+        query_lower = query.lower()
+        assert "patient" in query_lower or "age" in query_lower
+
+    def test_extract_keywords_filters_stopwords(self):
+        """Test that common stopwords are filtered from keyword extraction."""
+        field = OutputField(
+            name="finding",
+            description="the finding in the report",
+            output_type=OutputType.TEXT,
+        )
+
+        keywords = self.generator._extract_keywords_fallback([field])
+
+        # Stopwords like "the", "in" should not be in the query
+        assert "the" not in keywords.split()
+        assert "in" not in keywords.split()
+
+    def test_format_fields_for_prompt(self):
+        """Test formatting of fields for LLM prompt."""
+        fields = [
+            OutputField(
+                name="test_field",
+                description="test description",
+                output_type=OutputType.TEXT,
+            )
+        ]
+
+        formatted = self.generator._format_fields_for_prompt(fields)
+
+        assert "test_field" in formatted
+        assert "test description" in formatted
+        assert "Text" in formatted
+
+    def test_extract_query_from_response_simple(self):
+        """Test extraction of query from simple LLM response."""
+        response = "lung AND nodule"
+        query = self.generator._extract_query_from_response(response)
+
+        assert query == "lung AND nodule"
+
+    def test_extract_query_from_response_with_prefix(self):
+        """Test extraction when LLM response has a prefix."""
+        response = "Query: lung AND nodule"
+        query = self.generator._extract_query_from_response(response)
+
+        assert query == "lung AND nodule"
+
+    def test_extract_query_from_response_with_quotes(self):
+        """Test extraction when response is wrapped in quotes."""
+        response = '"lung AND nodule"'
+        query = self.generator._extract_query_from_response(response)
+
+        assert query == "lung AND nodule"
+
+    def test_extract_query_from_response_multiline(self):
+        """Test extraction when LLM adds explanation on additional lines."""
+        response = "lung AND nodule\n\nThis query will find..."
+        query = self.generator._extract_query_from_response(response)
+
+        # Should only take the first line
+        assert query == "lung AND nodule"
+        assert "This query" not in query
+
+    def test_query_generation_with_boolean_field(self):
+        """Test query generation with boolean output field."""
+        field = OutputField(
+            name="has_fracture",
+            description="whether a fracture is present",
+            output_type=OutputType.BOOLEAN,
+        )
+
+        with override_settings(ENABLE_AUTO_QUERY_GENERATION=False):
+            query, metadata = self.generator.generate_from_fields([field])
+
+        assert query != ""
+        assert metadata["success"] is True
+        assert "fracture" in query.lower()
+
+    def test_query_generation_with_mixed_field_types(self):
+        """Test query generation with multiple field types."""
+        fields = [
+            OutputField(
+                name="nodule_present",
+                description="is a nodule present",
+                output_type=OutputType.BOOLEAN,
+            ),
+            OutputField(
+                name="nodule_size",
+                description="size in mm",
+                output_type=OutputType.NUMERIC,
+            ),
+            OutputField(
+                name="nodule_location",
+                description="anatomical location",
+                output_type=OutputType.TEXT,
+            ),
+        ]
+
+        with override_settings(ENABLE_AUTO_QUERY_GENERATION=False):
+            query, metadata = self.generator.generate_from_fields(fields)
+
+        assert query != ""
+        assert metadata["success"] is True
+        assert metadata["field_count"] == 3
diff --git a/radis/extractions/utils/query_generator.py b/radis/extractions/utils/query_generator.py
new file mode 100644
index 00000000..c8f10aa5
--- /dev/null
+++ b/radis/extractions/utils/query_generator.py
@@ -0,0 +1,285 @@
+"""
+Query Generator for Automated Query Creation from Extraction Fields
+
+This module provides functionality to automatically generate search queries
+from user-defined extraction fields using LLM with fallback strategies.
+"""
+
+import logging
+import re
+from string import Template
+from typing import Iterable
+
+from django.conf import settings
+
+from radis.chats.utils.chat_client import ChatClient
+from radis.extractions.models import OutputField
+from radis.search.utils.query_parser import QueryParser
+
+logger = logging.getLogger(__name__)
+
+
+class QueryGenerator:
+    """Generates search queries from extraction fields using LLM with fallbacks."""
+
+    def __init__(self):
+        """Initialize the query generator with an LLM client."""
+        self.client = ChatClient()
+        self.parser = QueryParser()
+
+    def generate_from_fields(self, fields: Iterable[OutputField]) -> tuple[str, dict[str, any]]:
+        """
+        Generate a search query from extraction fields.
+
+        Args:
+            fields: Iterable of OutputField objects to generate query from
+
+        Returns:
+            Tuple of (query_string, metadata_dict)
+            - query_string: The generated query
+            - metadata: Dict with keys:
+                - generation_method: "llm", "keyword_fallback", or "wildcard"
+                - field_count: Number of fields processed
+                - success: Boolean indicating if generation succeeded
+                - error: Optional error message if failed
+        """
+        fields_list = list(fields)
+        field_count = len(fields_list)
+
+        metadata = {
+            "field_count": field_count,
+            "success": False,
+            "generation_method": None,
+            "error": None,
+        }
+
+        # Handle empty fields
+        if field_count == 0:
+            logger.warning("No fields provided for query generation, using wildcard")
+            metadata["generation_method"] = "wildcard"
+            metadata["success"] = True
+            return "*", metadata
+
+        # Try LLM generation first
+        if settings.ENABLE_AUTO_QUERY_GENERATION:
+            try:
+                query = self._call_llm(fields_list)
+                if query:
+                    # Validate and fix the query
+                    validated_query, fixes = self.validate_and_fix_query(query)
+                    if validated_query:
+                        logger.info(
+                            f"Successfully generated query from {field_count} fields using LLM"
+                        )
+                        metadata["generation_method"] = "llm"
+                        metadata["success"] = True
+                        metadata["fixes_applied"] = len(fixes) > 0
+                        return validated_query, metadata
+                    else:
+                        logger.warning("LLM generated invalid query, trying fallback")
+                        metadata["error"] = "LLM generated invalid query"
+            except Exception as e:
+                logger.error(f"Error during LLM query generation: {e}", exc_info=True)
+                metadata["error"] = str(e)
+
+        # Fallback to keyword extraction
+        try:
+            query = self._extract_keywords_fallback(fields_list)
+            validated_query, fixes = self.validate_and_fix_query(query)
+            if validated_query:
+                logger.info(f"Generated query from {field_count} fields using keyword fallback")
+                metadata["generation_method"] = "keyword_fallback"
+                metadata["success"] = True
+                metadata["fixes_applied"] = len(fixes) > 0
+                return validated_query, metadata
+        except Exception as e:
+            logger.error(f"Error during keyword fallback: {e}", exc_info=True)
+            metadata["error"] = f"Keyword fallback failed: {e}"
+
+        # Final fallback to wildcard
+        logger.warning("All query generation methods failed, using wildcard")
+        metadata["generation_method"] = "wildcard"
+        metadata["success"] = True
+        return "*", metadata
+
+    def _call_llm(self, fields: list[OutputField]) -> str | None:
+        """
+        Call LLM to generate a query from extraction fields.
+
+        Args:
+            fields: List of OutputField objects
+
+        Returns:
+            Generated query string, or None if failed
+        """
+        # Format fields for the prompt
+        fields_formatted = self._format_fields_for_prompt(fields)
+
+        # Build the prompt from template
+        prompt = Template(settings.QUERY_GENERATION_SYSTEM_PROMPT).substitute(
+            fields=fields_formatted
+        )
+
+        try:
+            # Call LLM with timeout
+            response = self.client.chat([{"role": "user", "content": prompt}])
+
+            if not response:
+                logger.warning("LLM returned empty response")
+                return None
+
+            # Extract and clean the query
+            query = self._extract_query_from_response(response)
+
+            logger.debug(f"LLM generated query: {query}")
+            return query
+
+        except Exception as e:
+            logger.error(f"LLM call failed: {e}")
+            return None
+
+    def _format_fields_for_prompt(self, fields: list[OutputField]) -> str:
+        """
+        Format extraction fields for inclusion in LLM prompt.
+
+        Args:
+            fields: List of OutputField objects
+
+        Returns:
+            Formatted string representation of fields
+        """
+        formatted_fields = []
+        for field in fields:
+            field_dict = {
+                "name": field.name,
+                "description": field.description,
+                "type": field.get_output_type_display(),
+            }
+            formatted_fields.append(str(field_dict))
+
+        return "\n".join(formatted_fields)
+
+    def _extract_query_from_response(self, response: str) -> str:
+        """
+        Extract query from LLM response.
+
+        The LLM might return the query with additional text or formatting.
+        This method cleans and extracts just the query.
+
+        Args:
+            response: Raw LLM response
+
+        Returns:
+            Cleaned query string
+        """
+        # Remove common prefixes like "Query:", "Search:", etc.
+        cleaned = re.sub(
+            r"^(query|search|generated query|result):\s*", "", response.strip(), flags=re.IGNORECASE
+        )
+
+        # Remove quotes if the entire response is quoted
+        if cleaned.startswith('"') and cleaned.endswith('"'):
+            cleaned = cleaned[1:-1]
+        elif cleaned.startswith("'") and cleaned.endswith("'"):
+            cleaned = cleaned[1:-1]
+
+        # Take only the first line (in case LLM added explanation)
+        cleaned = cleaned.split("\n")[0].strip()
+
+        return cleaned
+
+    def _extract_keywords_fallback(self, fields: list[OutputField]) -> str:
+        """
+        Fallback method: Extract keywords from field names and descriptions.
+
+        Creates a simple OR query from field names and key terms from descriptions.
+
+        Args:
+            fields: List of OutputField objects
+
+        Returns:
+            Simple keyword-based query string
+        """
+        keywords = set()
+
+        for field in fields:
+            # Add field name (split on underscores and camelCase)
+            name_parts = re.split(r"[_\s]+|(?<=[a-z])(?=[A-Z])", field.name)
+            keywords.update(part.lower() for part in name_parts if len(part) > 2)
+
+            # Extract key terms from description
+            if field.description:
+                # Remove common words
+                stopwords = {
+                    "the",
+                    "a",
+                    "an",
+                    "and",
+                    "or",
+                    "but",
+                    "in",
+                    "on",
+                    "at",
+                    "to",
+                    "for",
+                    "of",
+                    "with",
+                    "is",
+                    "are",
+                    "was",
+                    "were",
+                    "be",
+                    "been",
+                    "being",
+                    "this",
+                    "that",
+                    "these",
+                    "those",
+                }
+
+                # Split description into words
+                desc_words = re.findall(r"\b[a-zA-Z]{3,}\b", field.description.lower())
+                keywords.update(word for word in desc_words if word not in stopwords)
+
+        # Limit to most relevant keywords (max 10)
+        keywords_list = sorted(keywords)[:10]
+
+        if not keywords_list:
+            # If no keywords extracted, return wildcard
+            return "*"
+
+        # Join with OR operator
+        return " OR ".join(keywords_list)
+
+    def validate_and_fix_query(self, query: str) -> tuple[str, list[str]]:
+        """
+        Validate and fix a query using QueryParser.
+
+        Args:
+            query: Query string to validate
+
+        Returns:
+            Tuple of (fixed_query, list_of_fixes_applied)
+            Returns ("", []) if query is invalid
+        """
+        if not query or not query.strip():
+            return "", []
+
+        try:
+            query_node, fixes = self.parser.parse(query)
+
+            if query_node is None:
+                logger.warning(f"Query validation failed for: {query}")
+                return "", []
+
+            # If fixes were applied, unparse to get the fixed query
+            if len(fixes) > 0:
+                fixed_query = QueryParser.unparse(query_node)
+                logger.debug(f"Applied {len(fixes)} fixes to query: {fixes}")
+                return fixed_query, fixes
+
+            return query, []
+
+        except Exception as e:
+            logger.error(f"Error validating query '{query}': {e}")
+            return "", []
diff --git a/radis/extractions/views.py b/radis/extractions/views.py
index 78f4e3c2..662434f4 100644
--- a/radis/extractions/views.py
+++ b/radis/extractions/views.py
@@ -50,6 +50,7 @@
     ExtractionResultsTable,
     ExtractionTaskTable,
 )
+from .utils.query_generator import QueryGenerator
 
 EXTRACTIONS_SEARCH_PROVIDER = "extractions_search_provider"
 
@@ -101,18 +102,49 @@ def get_context_data(self, form, **kwargs):
             assert data and isinstance(data, dict)
 
             context["fixed_query"] = data.get("fixed_query")
-            context["retrieval_count"] = data["retrieval_count"]
-            self.storage.extra_data["retrieval_count"] = data["retrieval_count"]
+            context["retrieval_count"] = data.get("retrieval_count", 0)
+            context["requires_query_generation"] = data.get("requires_query_generation", False)
+
+            # Store for use in summary step
+            self.storage.extra_data["retrieval_count"] = data.get("retrieval_count", 0)
+            self.storage.extra_data["requires_query_generation"] = data.get(
+                "requires_query_generation", False
+            )
 
         elif self.steps.current == ExtractionJobWizardView.SUMMARY_STEP:
             search_data = self.get_cleaned_data_for_step(ExtractionJobWizardView.SEARCH_STEP)
-            output_fields = self.get_cleaned_data_for_step(
+            output_fields_data = self.get_cleaned_data_for_step(
                 ExtractionJobWizardView.OUTPUT_FIELDS_STEP
             )
 
+            # Auto-generate query if needed
+            if self.storage.extra_data.get("requires_query_generation", False):
+                # Create temporary OutputField objects for query generation
+                from .models import OutputField
+
+                temp_fields = [
+                    OutputField(
+                        name=field_data["name"],
+                        description=field_data["description"],
+                        output_type=field_data["output_type"],
+                    )
+                    for field_data in output_fields_data
+                    if not field_data.get("DELETE", False)
+                ]
+
+                generator = QueryGenerator()
+                generated_query, metadata = generator.generate_from_fields(temp_fields)
+
+                # Store generated query
+                search_data["query"] = generated_query
+                search_data["query_metadata"] = metadata
+                self.storage.extra_data["generated_query"] = generated_query
+                self.storage.extra_data["query_metadata"] = metadata
+
             context["search"] = search_data
-            context["output_fields"] = output_fields
-            context["retrieval_count"] = self.storage.extra_data["retrieval_count"]
+            context["output_fields"] = output_fields_data
+            context["retrieval_count"] = self.storage.extra_data.get("retrieval_count", 0)
+            context["query_metadata"] = self.storage.extra_data.get("query_metadata", {})
 
         return context
 
@@ -137,6 +169,12 @@ def done(
         with transaction.atomic():
             summary_form = form_objs[2]
             job_form = form_objs[0]
+
+            # Use generated query if it was auto-generated
+            if self.storage.extra_data.get("requires_query_generation", False):
+                generated_query = self.storage.extra_data.get("generated_query", "")
+                job_form.instance.query = generated_query
+
             if summary_form.cleaned_data["send_finished_mail"]:
                 job_form.cleaned_data["send_finished_mail"] = True
 
diff --git a/radis/settings/base.py b/radis/settings/base.py
index 492566fd..97c7037d 100644
--- a/radis/settings/base.py
+++ b/radis/settings/base.py
@@ -404,6 +404,40 @@
 $fields
 """
 
+# Query Generation from Extraction Fields
+ENABLE_AUTO_QUERY_GENERATION = env.bool("ENABLE_AUTO_QUERY_GENERATION", default=True)
+QUERY_GENERATION_TIMEOUT = env.int("QUERY_GENERATION_TIMEOUT", default=10)
+QUERY_GENERATION_MAX_RETRIES = env.int("QUERY_GENERATION_MAX_RETRIES", default=2)
+
+QUERY_GENERATION_SYSTEM_PROMPT = """You are an AI assistant specialized in medical informatics and radiology report retrieval.
+
+Your task is to generate an effective search query to find radiology reports that would contain information relevant to the specified data extraction fields.
+
+Given extraction fields with their descriptions and types, generate a boolean search query that will retrieve reports likely to contain the requested information.
+
+Guidelines:
+1. Use medical terminology and common synonyms
+2. Prefer broader terms to avoid missing relevant reports
+3. Use boolean operators: AND, OR, NOT
+4. Use quotes for exact phrases: "lung nodule"
+5. Consider anatomical variations and medical abbreviations
+6. Keep the query concise but comprehensive (max 150 characters recommended)
+7. Focus on key concepts that would appear in reports containing this data
+
+Output format: Return ONLY the search query as a single line, without explanation.
+
+Examples:
+Fields: [{"name": "nodule_size", "description": "size of lung nodule in millimeters", "type": "NUMERIC"}]
+Query: ("lung nodule" OR "pulmonary nodule") AND (size OR measurement OR diameter)
+
+Fields: [{"name": "fracture_type", "description": "type of bone fracture", "type": "TEXT"}, {"name": "bone", "description": "which bone is fractured", "type": "TEXT"}]
+Query: fracture AND bone
+
+Extraction Fields:
+$fields
+
+Generate the search query:"""
+
 # The maximum number of reports that can be extracted by one extraction job.
 EXTRACTION_MAXIMUM_REPORTS_COUNT = 25000
 
diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index 959b0c9d..27d173f6 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -31,7 +31,10 @@ class Meta:
         labels = {"patient_id": "Patient ID"}
         help_texts = {
             "name": "Name of the Subscription",
-            "query": "A query to filter reports",
+            "query": "Search query to filter reports (leave empty to auto-generate from extraction fields)",
+        }
+        widgets = {
+            "query": forms.TextInput(attrs={"placeholder": "Optional - auto-generated if empty"}),
         }
 
     def __init__(self, *args, **kwargs):

From 34a0cbc6b0a0a2a8c73f1e48ee01654165ef85e4 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Wed, 10 Dec 2025 23:34:16 +0000
Subject: [PATCH 60/93] generate query after form validation

---
 radis/subscriptions/views.py | 62 ++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/radis/subscriptions/views.py b/radis/subscriptions/views.py
index 15638021..983e0a89 100644
--- a/radis/subscriptions/views.py
+++ b/radis/subscriptions/views.py
@@ -97,6 +97,37 @@ def form_valid(self, form) -> HttpResponse:
             active_group = user.active_group
             form.instance.group = active_group
 
+            # Auto-generate query if needed
+            if not form.cleaned_data.get("query", "").strip():
+                # Collect output fields from formset
+                temp_fields = []
+                for field_form in output_formset:
+                    if (
+                        field_form.cleaned_data
+                        and not field_form.cleaned_data.get("DELETE", False)
+                        and field_form.cleaned_data.get("name")
+                    ):
+                        from radis.extractions.models import OutputField
+
+                        temp_fields.append(
+                            OutputField(
+                                name=field_form.cleaned_data["name"],
+                                description=field_form.cleaned_data["description"],
+                                output_type=field_form.cleaned_data["output_type"],
+                            )
+                        )
+
+                if temp_fields:
+                    from radis.extractions.utils.query_generator import QueryGenerator
+
+                    generator = QueryGenerator()
+                    generated_query, metadata = generator.generate_from_fields(temp_fields)
+                    form.instance.query = generated_query
+                    logger.info(
+                        f"Auto-generated query for subscription: {generated_query} "
+                        f"(method: {metadata.get('generation_method')})"
+                    )
+
             try:
                 self.object: Subscription = form.save()
             except IntegrityError as e:
@@ -149,6 +180,37 @@ def form_valid(self, form) -> HttpResponse:
         filter_formset = ctx["filter_formset"]
         output_formset = ctx["output_formset"]
         if filter_formset.is_valid() and output_formset.is_valid():
+            # Auto-generate query if needed
+            if not form.cleaned_data.get("query", "").strip():
+                # Collect output fields from formset
+                temp_fields = []
+                for field_form in output_formset:
+                    if (
+                        field_form.cleaned_data
+                        and not field_form.cleaned_data.get("DELETE", False)
+                        and field_form.cleaned_data.get("name")
+                    ):
+                        from radis.extractions.models import OutputField
+
+                        temp_fields.append(
+                            OutputField(
+                                name=field_form.cleaned_data["name"],
+                                description=field_form.cleaned_data["description"],
+                                output_type=field_form.cleaned_data["output_type"],
+                            )
+                        )
+
+                if temp_fields:
+                    from radis.extractions.utils.query_generator import QueryGenerator
+
+                    generator = QueryGenerator()
+                    generated_query, metadata = generator.generate_from_fields(temp_fields)
+                    form.instance.query = generated_query
+                    logger.info(
+                        f"Auto-generated query for subscription update: {generated_query} "
+                        f"(method: {metadata.get('generation_method')})"
+                    )
+
             try:
                 self.object = form.save()
             except IntegrityError as e:

From b71285415febe2db7f4125c21d5f28de63e7c270 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Thu, 11 Dec 2025 10:43:55 +0000
Subject: [PATCH 61/93] ask user for query if llm generation fails

---
 radis/extractions/utils/query_generator.py | 32 ++++++++++++----------
 radis/extractions/views.py                 | 13 +++++++++
 radis/subscriptions/views.py               | 20 ++++++++++++++
 3 files changed, 51 insertions(+), 14 deletions(-)

diff --git a/radis/extractions/utils/query_generator.py b/radis/extractions/utils/query_generator.py
index c8f10aa5..c17a1ae5 100644
--- a/radis/extractions/utils/query_generator.py
+++ b/radis/extractions/utils/query_generator.py
@@ -27,7 +27,7 @@ def __init__(self):
         self.client = ChatClient()
         self.parser = QueryParser()
 
-    def generate_from_fields(self, fields: Iterable[OutputField]) -> tuple[str, dict[str, any]]:
+    def generate_from_fields(self, fields: Iterable[OutputField]) -> tuple[str | None, dict[str, any]]:
         """
         Generate a search query from extraction fields.
 
@@ -55,10 +55,10 @@ def generate_from_fields(self, fields: Iterable[OutputField]) -> tuple[str, dict
 
         # Handle empty fields
         if field_count == 0:
-            logger.warning("No fields provided for query generation, using wildcard")
-            metadata["generation_method"] = "wildcard"
-            metadata["success"] = True
-            return "*", metadata
+            logger.warning("No fields provided for query generation")
+            metadata["error"] = "No fields provided"
+            metadata["success"] = False
+            return None, metadata
 
         # Try LLM generation first
         if settings.ENABLE_AUTO_QUERY_GENERATION:
@@ -96,11 +96,14 @@ def generate_from_fields(self, fields: Iterable[OutputField]) -> tuple[str, dict
             logger.error(f"Error during keyword fallback: {e}", exc_info=True)
             metadata["error"] = f"Keyword fallback failed: {e}"
 
-        # Final fallback to wildcard
-        logger.warning("All query generation methods failed, using wildcard")
-        metadata["generation_method"] = "wildcard"
-        metadata["success"] = True
-        return "*", metadata
+        # All methods failed - return None
+        logger.warning(
+            f"All query generation methods failed for {field_count} fields. "
+            "User must manually enter a query."
+        )
+        metadata["error"] = metadata.get("error") or "All generation methods failed"
+        metadata["success"] = False
+        return None, metadata
 
     def _call_llm(self, fields: list[OutputField]) -> str | None:
         """
@@ -188,7 +191,7 @@ def _extract_query_from_response(self, response: str) -> str:
 
         return cleaned
 
-    def _extract_keywords_fallback(self, fields: list[OutputField]) -> str:
+    def _extract_keywords_fallback(self, fields: list[OutputField]) -> str | None:
         """
         Fallback method: Extract keywords from field names and descriptions.
 
@@ -198,7 +201,7 @@ def _extract_keywords_fallback(self, fields: list[OutputField]) -> str:
             fields: List of OutputField objects
 
         Returns:
-            Simple keyword-based query string
+            Simple keyword-based query string, or None if no keywords extracted
         """
         keywords = set()
 
@@ -245,8 +248,9 @@ def _extract_keywords_fallback(self, fields: list[OutputField]) -> str:
         keywords_list = sorted(keywords)[:10]
 
         if not keywords_list:
-            # If no keywords extracted, return wildcard
-            return "*"
+            # If no keywords extracted, return None
+            logger.warning("No keywords could be extracted from fields")
+            return None
 
         # Join with OR operator
         return " OR ".join(keywords_list)
diff --git a/radis/extractions/views.py b/radis/extractions/views.py
index 662434f4..ffa8b435 100644
--- a/radis/extractions/views.py
+++ b/radis/extractions/views.py
@@ -135,6 +135,19 @@ def get_context_data(self, form, **kwargs):
                 generator = QueryGenerator()
                 generated_query, metadata = generator.generate_from_fields(temp_fields)
 
+                # Check if generation failed
+                if generated_query is None or not metadata.get("success"):
+                    from django.contrib import messages
+
+                    messages.error(
+                        self.request,
+                        "Unable to automatically generate a query from your extraction fields. "
+                        "Please go back to Step 1 and manually enter a search query.",
+                    )
+                    # Redirect back to step 0
+                    self.storage.current_step = self.steps.first
+                    return self.render_goto_step(self.steps.first)
+
                 # Store generated query
                 search_data["query"] = generated_query
                 search_data["query_metadata"] = metadata
diff --git a/radis/subscriptions/views.py b/radis/subscriptions/views.py
index 983e0a89..ec05844e 100644
--- a/radis/subscriptions/views.py
+++ b/radis/subscriptions/views.py
@@ -122,6 +122,16 @@ def form_valid(self, form) -> HttpResponse:
 
                     generator = QueryGenerator()
                     generated_query, metadata = generator.generate_from_fields(temp_fields)
+
+                    # Check if generation failed
+                    if generated_query is None or not metadata.get("success"):
+                        form.add_error(
+                            "query",
+                            "Unable to automatically generate a query from your extraction fields. "
+                            "Please manually enter a search query.",
+                        )
+                        return self.form_invalid(form)
+
                     form.instance.query = generated_query
                     logger.info(
                         f"Auto-generated query for subscription: {generated_query} "
@@ -205,6 +215,16 @@ def form_valid(self, form) -> HttpResponse:
 
                     generator = QueryGenerator()
                     generated_query, metadata = generator.generate_from_fields(temp_fields)
+
+                    # Check if generation failed
+                    if generated_query is None or not metadata.get("success"):
+                        form.add_error(
+                            "query",
+                            "Unable to automatically generate a query from your extraction fields. "
+                            "Please manually enter a search query.",
+                        )
+                        return self.form_invalid(form)
+
                     form.instance.query = generated_query
                     logger.info(
                         f"Auto-generated query for subscription update: {generated_query} "

From b367820fcfed0d78a32c593fa83487b8fd2b23bd Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Thu, 11 Dec 2025 21:07:41 +0000
Subject: [PATCH 62/93] remove keyword fallback

---
 radis/extractions/forms.py                    |   8 +-
 .../0007_alter_extractionjob_query.py         |  18 +++
 radis/extractions/models.py                   |   2 +-
 .../extraction_job_output_fields_form.html    |   6 +-
 .../extractions/tests/test_query_generator.py | 110 ++----------------
 radis/extractions/utils/query_generator.py    |  87 ++------------
 radis/extractions/views.py                    |  82 ++++++++++++-
 radis/settings/base.py                        |  16 +--
 radis/subscriptions/forms.py                  |   5 +-
 .../subscriptions/subscription_inbox.html     |   9 +-
 10 files changed, 139 insertions(+), 204 deletions(-)
 create mode 100644 radis/extractions/migrations/0007_alter_extractionjob_query.py

diff --git a/radis/extractions/forms.py b/radis/extractions/forms.py
index 72bb01fb..912386a6 100644
--- a/radis/extractions/forms.py
+++ b/radis/extractions/forms.py
@@ -35,7 +35,10 @@ class Meta:
         ]
         help_texts = {
             "title": "Title of the extraction job",
-            "query": "Search query to filter reports (leave empty to auto-generate from extraction fields)",
+            "query": (
+                "Search query to filter reports "
+                "(leave empty to auto-generate from extraction fields)"
+            ),
         }
         widgets = {
             "query": forms.TextInput(attrs={"placeholder": "Optional - auto-generated if empty"}),
@@ -46,6 +49,7 @@ def __init__(self, *args, **kwargs):
 
         super().__init__(*args, **kwargs)
 
+        self.fields["query"].required = False
         self.fields["language"].choices = [  # type: ignore
             (language.pk, LANGUAGE_LABELS[language.code])
             for language in Language.objects.order_by("code")
@@ -134,7 +138,7 @@ def clean(self) -> dict[str, Any] | None:
         if not query:
             # Mark for auto-generation from output fields
             cleaned_data["requires_query_generation"] = True
-            cleaned_data["retrieval_count"] = 0
+            cleaned_data["retrieval_count"] = None  # Will be calculated after query generation
             return cleaned_data
 
         # Validate manual query
diff --git a/radis/extractions/migrations/0007_alter_extractionjob_query.py b/radis/extractions/migrations/0007_alter_extractionjob_query.py
new file mode 100644
index 00000000..13cd4833
--- /dev/null
+++ b/radis/extractions/migrations/0007_alter_extractionjob_query.py
@@ -0,0 +1,18 @@
+# Generated by Django 6.0 on 2025-12-11 20:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("extractions", "0006_outputfield_subscription_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="extractionjob",
+            name="query",
+            field=models.CharField(blank=True, max_length=200),
+        ),
+    ]
diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index 0fdf07a2..855c2abc 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -35,7 +35,7 @@ class ExtractionJob(AnalysisJob):
     )
     title = models.CharField(max_length=100)
     group = models.ForeignKey[Group](Group, on_delete=models.CASCADE)
-    query = models.CharField(max_length=200)
+    query = models.CharField(max_length=200, blank=True)
     language = models.ForeignKey[Language](Language, on_delete=models.CASCADE)
     modalities = models.ManyToManyField(Modality, blank=True)
     study_date_from = models.DateField(null=True, blank=True)
diff --git a/radis/extractions/templates/extractions/extraction_job_output_fields_form.html b/radis/extractions/templates/extractions/extraction_job_output_fields_form.html
index 4ab54deb..89b707dd 100644
--- a/radis/extractions/templates/extractions/extraction_job_output_fields_form.html
+++ b/radis/extractions/templates/extractions/extraction_job_output_fields_form.html
@@ -44,7 +44,11 @@
                 </div>
             {% endif %}
             <p>
-                Your search will hit approximately <strong>{{ retrieval_count }} report{{ retrieval_count|pluralize }}</strong>.
+                {% if retrieval_count is None %}
+                    Your search will hit approximately <strong>TBD reports</strong> (to be determined after query generation).
+                {% else %}
+                    Your search will hit approximately <strong>{{ retrieval_count }} report{{ retrieval_count|pluralize }}</strong>.
+                {% endif %}
                 The more reports you analyze the longer it will take. If you like to refine your search you can go back to
                 the previous step and adjust the query and/or filters.
             </p>
diff --git a/radis/extractions/tests/test_query_generator.py b/radis/extractions/tests/test_query_generator.py
index 1492d707..43aa5f99 100644
--- a/radis/extractions/tests/test_query_generator.py
+++ b/radis/extractions/tests/test_query_generator.py
@@ -16,57 +16,15 @@ def setUp(self):
         self.generator = QueryGenerator()
 
     def test_generate_from_empty_fields(self):
-        """Test query generation with no fields returns wildcard."""
+        """Test query generation with no fields returns None."""
         fields = []
         query, metadata = self.generator.generate_from_fields(fields)
 
-        assert query == "*"
-        assert metadata["generation_method"] == "wildcard"
+        assert query is None
+        assert metadata["generation_method"] is None
         assert metadata["field_count"] == 0
-        assert metadata["success"] is True
-
-    def test_keyword_fallback_single_field(self):
-        """Test keyword extraction fallback with a single field."""
-        field = OutputField(
-            name="lung_nodule",
-            description="size of lung nodule in millimeters",
-            output_type=OutputType.NUMERIC,
-        )
-
-        with override_settings(ENABLE_AUTO_QUERY_GENERATION=False):
-            query, metadata = self.generator.generate_from_fields([field])
-
-        assert query != ""
-        assert metadata["generation_method"] == "keyword_fallback"
-        assert metadata["success"] is True
-        assert metadata["field_count"] == 1
-        # Should contain keywords from name and description
-        assert "lung" in query.lower() or "nodule" in query.lower()
-
-    def test_keyword_fallback_multiple_fields(self):
-        """Test keyword extraction with multiple fields."""
-        fields = [
-            OutputField(
-                name="fracture_type",
-                description="type of bone fracture",
-                output_type=OutputType.TEXT,
-            ),
-            OutputField(
-                name="bone_name",
-                description="which bone is fractured",
-                output_type=OutputType.TEXT,
-            ),
-        ]
-
-        with override_settings(ENABLE_AUTO_QUERY_GENERATION=False):
-            query, metadata = self.generator.generate_from_fields(fields)
-
-        assert query != ""
-        assert metadata["generation_method"] == "keyword_fallback"
-        assert metadata["success"] is True
-        assert metadata["field_count"] == 2
-        # Should use OR operator
-        assert " OR " in query
+        assert metadata["success"] is False
+        assert metadata["error"] == "No fields provided"
 
     @patch("radis.extractions.utils.query_generator.ChatClient")
     def test_llm_generation_success(self, mock_chat_client_class):
@@ -91,34 +49,9 @@ def test_llm_generation_success(self, mock_chat_client_class):
         assert query != ""
         assert metadata["generation_method"] == "llm"
         assert metadata["success"] is True
+        assert query is not None
         assert "nodule" in query.lower()
 
-    @patch("radis.extractions.utils.query_generator.ChatClient")
-    def test_llm_generation_failure_fallback(self, mock_chat_client_class):
-        """Test that LLM failure triggers keyword fallback."""
-        # Mock LLM to raise an exception
-        mock_client = Mock()
-        mock_client.chat.side_effect = Exception("LLM service unavailable")
-        mock_chat_client_class.return_value = mock_client
-
-        fields = [
-            OutputField(
-                name="tumor_size",
-                description="size of the tumor in centimeters",
-                output_type=OutputType.NUMERIC,
-            )
-        ]
-
-        with override_settings(ENABLE_AUTO_QUERY_GENERATION=True):
-            generator = QueryGenerator()
-            query, metadata = generator.generate_from_fields(fields)
-
-        # Should fall back to keyword extraction
-        assert query != ""
-        assert metadata["generation_method"] == "keyword_fallback"
-        assert metadata["success"] is True
-        assert metadata["error"] is not None
-
     def test_validate_and_fix_valid_query(self):
         """Test validation of a valid query."""
         query = "lung AND nodule"
@@ -143,36 +76,6 @@ def test_validate_and_fix_query_with_quotes(self):
         assert fixed_query != ""
         assert "lung nodule" in fixed_query or "lung" in fixed_query
 
-    def test_extract_keywords_from_camel_case(self):
-        """Test keyword extraction from camelCase field names."""
-        field = OutputField(
-            name="patientAge",
-            description="age of the patient in years",
-            output_type=OutputType.NUMERIC,
-        )
-
-        with override_settings(ENABLE_AUTO_QUERY_GENERATION=False):
-            query, metadata = self.generator.generate_from_fields([field])
-
-        assert query != ""
-        # Should split camelCase into separate words
-        query_lower = query.lower()
-        assert "patient" in query_lower or "age" in query_lower
-
-    def test_extract_keywords_filters_stopwords(self):
-        """Test that common stopwords are filtered from keyword extraction."""
-        field = OutputField(
-            name="finding",
-            description="the finding in the report",
-            output_type=OutputType.TEXT,
-        )
-
-        keywords = self.generator._extract_keywords_fallback([field])
-
-        # Stopwords like "the", "in" should not be in the query
-        assert "the" not in keywords.split()
-        assert "in" not in keywords.split()
-
     def test_format_fields_for_prompt(self):
         """Test formatting of fields for LLM prompt."""
         fields = [
@@ -232,6 +135,7 @@ def test_query_generation_with_boolean_field(self):
 
         assert query != ""
         assert metadata["success"] is True
+        assert query is not None
         assert "fracture" in query.lower()
 
     def test_query_generation_with_mixed_field_types(self):
diff --git a/radis/extractions/utils/query_generator.py b/radis/extractions/utils/query_generator.py
index c17a1ae5..4713eac3 100644
--- a/radis/extractions/utils/query_generator.py
+++ b/radis/extractions/utils/query_generator.py
@@ -8,7 +8,7 @@
 import logging
 import re
 from string import Template
-from typing import Iterable
+from typing import Any, Iterable
 
 from django.conf import settings
 
@@ -27,7 +27,9 @@ def __init__(self):
         self.client = ChatClient()
         self.parser = QueryParser()
 
-    def generate_from_fields(self, fields: Iterable[OutputField]) -> tuple[str | None, dict[str, any]]:
+    def generate_from_fields(
+        self, fields: Iterable[OutputField]
+    ) -> tuple[str | None, dict[str, Any]]:
         """
         Generate a search query from extraction fields.
 
@@ -38,7 +40,7 @@ def generate_from_fields(self, fields: Iterable[OutputField]) -> tuple[str | Non
             Tuple of (query_string, metadata_dict)
             - query_string: The generated query
             - metadata: Dict with keys:
-                - generation_method: "llm", "keyword_fallback", or "wildcard"
+                - generation_method: "llm"
                 - field_count: Number of fields processed
                 - success: Boolean indicating if generation succeeded
                 - error: Optional error message if failed
@@ -82,19 +84,8 @@ def generate_from_fields(self, fields: Iterable[OutputField]) -> tuple[str | Non
                 logger.error(f"Error during LLM query generation: {e}", exc_info=True)
                 metadata["error"] = str(e)
 
-        # Fallback to keyword extraction
-        try:
-            query = self._extract_keywords_fallback(fields_list)
-            validated_query, fixes = self.validate_and_fix_query(query)
-            if validated_query:
-                logger.info(f"Generated query from {field_count} fields using keyword fallback")
-                metadata["generation_method"] = "keyword_fallback"
-                metadata["success"] = True
-                metadata["fixes_applied"] = len(fixes) > 0
-                return validated_query, metadata
-        except Exception as e:
-            logger.error(f"Error during keyword fallback: {e}", exc_info=True)
-            metadata["error"] = f"Keyword fallback failed: {e}"
+        # No fallback - return None for LLM failures
+        logger.warning(f"LLM query generation failed for {field_count} fields")
 
         # All methods failed - return None
         logger.warning(
@@ -191,70 +182,6 @@ def _extract_query_from_response(self, response: str) -> str:
 
         return cleaned
 
-    def _extract_keywords_fallback(self, fields: list[OutputField]) -> str | None:
-        """
-        Fallback method: Extract keywords from field names and descriptions.
-
-        Creates a simple OR query from field names and key terms from descriptions.
-
-        Args:
-            fields: List of OutputField objects
-
-        Returns:
-            Simple keyword-based query string, or None if no keywords extracted
-        """
-        keywords = set()
-
-        for field in fields:
-            # Add field name (split on underscores and camelCase)
-            name_parts = re.split(r"[_\s]+|(?<=[a-z])(?=[A-Z])", field.name)
-            keywords.update(part.lower() for part in name_parts if len(part) > 2)
-
-            # Extract key terms from description
-            if field.description:
-                # Remove common words
-                stopwords = {
-                    "the",
-                    "a",
-                    "an",
-                    "and",
-                    "or",
-                    "but",
-                    "in",
-                    "on",
-                    "at",
-                    "to",
-                    "for",
-                    "of",
-                    "with",
-                    "is",
-                    "are",
-                    "was",
-                    "were",
-                    "be",
-                    "been",
-                    "being",
-                    "this",
-                    "that",
-                    "these",
-                    "those",
-                }
-
-                # Split description into words
-                desc_words = re.findall(r"\b[a-zA-Z]{3,}\b", field.description.lower())
-                keywords.update(word for word in desc_words if word not in stopwords)
-
-        # Limit to most relevant keywords (max 10)
-        keywords_list = sorted(keywords)[:10]
-
-        if not keywords_list:
-            # If no keywords extracted, return None
-            logger.warning("No keywords could be extracted from fields")
-            return None
-
-        # Join with OR operator
-        return " OR ".join(keywords_list)
-
     def validate_and_fix_query(self, query: str) -> tuple[str, list[str]]:
         """
         Validate and fix a query using QueryParser.
diff --git a/radis/extractions/views.py b/radis/extractions/views.py
index ffa8b435..d1aefec8 100644
--- a/radis/extractions/views.py
+++ b/radis/extractions/views.py
@@ -1,5 +1,7 @@
-from typing import Any, Type, cast
+from datetime import date
+from typing import Any, Literal, Type, cast
 
+from adit_radis_shared.accounts.models import User
 from adit_radis_shared.common.mixins import (
     PageSizeSelectMixin,
 )
@@ -34,6 +36,8 @@
     AnalysisTaskResetView,
     BaseUpdatePreferencesView,
 )
+from radis.reports.models import Language, Modality
+from radis.search.site import Search, SearchFilters
 from radis.search.utils.query_parser import QueryParser
 
 from .filters import ExtractionInstanceFilter, ExtractionJobFilter, ExtractionTaskFilter
@@ -44,6 +48,7 @@
 )
 from .mixins import ExtractionsLockedMixin
 from .models import ExtractionInstance, ExtractionJob, ExtractionTask
+from .site import extraction_retrieval_provider
 from .tables import (
     ExtractionInstanceTable,
     ExtractionJobTable,
@@ -116,7 +121,8 @@ def get_context_data(self, form, **kwargs):
             output_fields_data = self.get_cleaned_data_for_step(
                 ExtractionJobWizardView.OUTPUT_FIELDS_STEP
             )
-
+            assert search_data and isinstance(search_data, dict)
+            assert output_fields_data and isinstance(output_fields_data, list)
             # Auto-generate query if needed
             if self.storage.extra_data.get("requires_query_generation", False):
                 # Create temporary OutputField objects for query generation
@@ -154,6 +160,78 @@ def get_context_data(self, form, **kwargs):
                 self.storage.extra_data["generated_query"] = generated_query
                 self.storage.extra_data["query_metadata"] = metadata
 
+                # Calculate retrieval count for auto-generated query
+                # Parse the generated query
+                query_node, _ = QueryParser().parse(generated_query)
+                assert query_node  # Should be valid since it was just generated and validated
+
+                # Build search with all filters from search_data
+                user = cast(User, self.request.user)
+                active_group = user.active_group
+                assert active_group
+
+                language = cast(Language, search_data.get("language"))
+                modalities = cast(QuerySet[Modality], search_data.get("modalities"))
+
+                search = Search(
+                    query=query_node,
+                    offset=0,
+                    limit=0,
+                    filters=SearchFilters(
+                        group=active_group.pk,
+                        language=language.code,
+                        modalities=list(modalities.values_list("code", flat=True))
+                        if modalities
+                        else [],
+                        study_date_from=cast("date | None", search_data.get("study_date_from")),
+                        study_date_till=cast("date | None", search_data.get("study_date_till")),
+                        study_description=cast(str, search_data.get("study_description")) or "",
+                        patient_sex=cast(Literal["M", "F"], search_data.get("patient_sex")) or None,
+                        patient_age_from=cast("int | None", search_data.get("age_from")),
+                        patient_age_till=cast("int | None", search_data.get("age_till")),
+                    ),
+                )
+
+                # Get the actual count
+                if extraction_retrieval_provider is None:
+                    from django.contrib import messages
+
+                    messages.error(self.request, "Extraction retrieval provider is not configured.")
+                    self.storage.current_step = self.steps.first
+                    return self.render_goto_step(self.steps.first)
+
+                retrieval_count = extraction_retrieval_provider.count(search)
+                self.storage.extra_data["retrieval_count"] = retrieval_count
+
+                # Validate against limits
+                if retrieval_count > settings.EXTRACTION_MAXIMUM_REPORTS_COUNT:
+                    from django.contrib import messages
+
+                    messages.error(
+                        self.request,
+                        f"Your auto-generated query returned {retrieval_count} results, "
+                        "which exceeds the maximum limit of "
+                        f"{settings.EXTRACTION_MAXIMUM_REPORTS_COUNT}. "
+                        "Please go back and refine your extraction fields or add a manual query.",
+                    )
+                    self.storage.current_step = self.steps.first
+                    return self.render_goto_step(self.steps.first)
+
+                if (
+                    extraction_retrieval_provider.max_results
+                    and retrieval_count > extraction_retrieval_provider.max_results
+                ):
+                    from django.contrib import messages
+
+                    messages.error(
+                        self.request,
+                        f"Your auto-generated query returned {retrieval_count} results, "
+                        "which exceeds the provider's limit. "
+                        "Please refine your extraction fields or add a manual query.",
+                    )
+                    self.storage.current_step = self.steps.first
+                    return self.render_goto_step(self.steps.first)
+
             context["search"] = search_data
             context["output_fields"] = output_fields_data
             context["retrieval_count"] = self.storage.extra_data.get("retrieval_count", 0)
diff --git a/radis/settings/base.py b/radis/settings/base.py
index 97c7037d..61d9a1e5 100644
--- a/radis/settings/base.py
+++ b/radis/settings/base.py
@@ -409,11 +409,11 @@
 QUERY_GENERATION_TIMEOUT = env.int("QUERY_GENERATION_TIMEOUT", default=10)
 QUERY_GENERATION_MAX_RETRIES = env.int("QUERY_GENERATION_MAX_RETRIES", default=2)
 
-QUERY_GENERATION_SYSTEM_PROMPT = """You are an AI assistant specialized in medical informatics and radiology report retrieval.
-
-Your task is to generate an effective search query to find radiology reports that would contain information relevant to the specified data extraction fields.
-
-Given extraction fields with their descriptions and types, generate a boolean search query that will retrieve reports likely to contain the requested information.
+QUERY_GENERATION_SYSTEM_PROMPT = """You are an AI assistant specialized in medical informatics and 
+radiology report retrieval. Your task is to generate an effective search query to find radiology 
+reports that would contain information relevant to the specified data extraction fields.
+Given extraction fields with their descriptions and types, generate a boolean search query that 
+will retrieve reports likely to contain the requested information.
 
 Guidelines:
 1. Use medical terminology and common synonyms
@@ -427,10 +427,12 @@
 Output format: Return ONLY the search query as a single line, without explanation.
 
 Examples:
-Fields: [{"name": "nodule_size", "description": "size of lung nodule in millimeters", "type": "NUMERIC"}]
+Fields: [{"name": "nodule_size", "description": "size of lung nodule in millimeters", 
+"type": "NUMERIC"}]
 Query: ("lung nodule" OR "pulmonary nodule") AND (size OR measurement OR diameter)
 
-Fields: [{"name": "fracture_type", "description": "type of bone fracture", "type": "TEXT"}, {"name": "bone", "description": "which bone is fractured", "type": "TEXT"}]
+Fields: [{"name": "fracture_type", "description": "type of bone fracture", "type": "TEXT"}, 
+{"name": "bone", "description": "which bone is fractured", "type": "TEXT"}]
 Query: fracture AND bone
 
 Extraction Fields:
diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index 27d173f6..78ef0194 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -31,7 +31,10 @@ class Meta:
         labels = {"patient_id": "Patient ID"}
         help_texts = {
             "name": "Name of the Subscription",
-            "query": "Search query to filter reports (leave empty to auto-generate from extraction fields)",
+            "query": (
+                "Search query to filter reports "
+                "(leave empty to auto-generate from extraction fields)"
+            ),
         }
         widgets = {
             "query": forms.TextInput(attrs={"placeholder": "Optional - auto-generated if empty"}),
diff --git a/radis/subscriptions/templates/subscriptions/subscription_inbox.html b/radis/subscriptions/templates/subscriptions/subscription_inbox.html
index f2f2f27b..cc360be0 100644
--- a/radis/subscriptions/templates/subscriptions/subscription_inbox.html
+++ b/radis/subscriptions/templates/subscriptions/subscription_inbox.html
@@ -1,6 +1,5 @@
 {% extends 'subscriptions/subscription_layout.html' %}
 {% load crispy from crispy_forms_tags %}
-
 {% block title %}
     Subscription Inbox
 {% endblock title %}
@@ -35,9 +34,7 @@
             {% for subscribed_item in object_list %}
                 {% include "subscriptions/_subscribed_item_preview.html" %}
             {% empty %}
-                <div class="alert alert-light" role="alert">
-                    No items found
-                </div>
+                <div class="alert alert-light" role="alert">No items found</div>
             {% endfor %}
             {# Pagination Controls #}
             <div class="d-flex justify-content-around align-items-center mt-3">
@@ -62,9 +59,7 @@ <h5 class="card-title">Filters</h5>
                         <input type="hidden"
                                name="sort_by"
                                value="{{ request.GET.sort_by|default:'' }}">
-                        <input type="hidden"
-                               name="order"
-                               value="{{ request.GET.order|default:'' }}">
+                        <input type="hidden" name="order" value="{{ request.GET.order|default:'' }}">
                         {% crispy filter.form %}
                     </div>
                 </div>

From bca5fad193f661eca7a7645ef12d7dd5b69c4219 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Thu, 11 Dec 2025 21:35:19 +0000
Subject: [PATCH 63/93] remove provider field from subscription detail template

---
 .../templates/subscriptions/subscription_detail.html      | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/radis/subscriptions/templates/subscriptions/subscription_detail.html b/radis/subscriptions/templates/subscriptions/subscription_detail.html
index 61170e02..ac9d884d 100644
--- a/radis/subscriptions/templates/subscriptions/subscription_detail.html
+++ b/radis/subscriptions/templates/subscriptions/subscription_detail.html
@@ -46,14 +46,6 @@ <h4>General</h4>
     </dl>
     <h4>Search Details</h4>
     <dl class="row">
-        <dt class="col-sm-3">Search Provider</dt>
-        <dd class="col-sm-9">
-            {% if subscription.provider %}
-                {{ subscription.provider }}
-            {% else %}
-                –
-            {% endif %}
-        </dd>
         <dt class="col-sm-3">Search Query</dt>
         <dd class="col-sm-9">
             {% if subscription.query %}

From 3fe6fd6ca72625b6274bc80774d391c5fed99876 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Thu, 11 Dec 2025 22:07:45 +0000
Subject: [PATCH 64/93] resolve n+1 queries for extractions and subscriptions

---
 radis/extractions/processors.py            |  5 +--
 radis/extractions/utils/processor_utils.py | 10 +++---
 radis/subscriptions/processors.py          | 39 ++++++++++++----------
 3 files changed, 30 insertions(+), 24 deletions(-)

diff --git a/radis/extractions/processors.py b/radis/extractions/processors.py
index e4c7d886..ee2e7a31 100644
--- a/radis/extractions/processors.py
+++ b/radis/extractions/processors.py
@@ -46,11 +46,12 @@ def process_instance(self, instance: ExtractionInstance) -> None:
 
     def process_output_fields(self, instance: ExtractionInstance) -> None:
         job = instance.task.job
-        Schema = generate_output_fields_schema(job.output_fields)
+        output_fields = list(job.output_fields.order_by("pk"))
+        Schema = generate_output_fields_schema(output_fields)
         prompt = Template(settings.OUTPUT_FIELDS_SYSTEM_PROMPT).substitute(
             {
                 "report": instance.text,
-                "fields": generate_output_fields_prompt(job.output_fields),
+                "fields": generate_output_fields_prompt(output_fields),
             }
         )
         result = self.client.extract_data(prompt.strip(), Schema)
diff --git a/radis/extractions/utils/processor_utils.py b/radis/extractions/utils/processor_utils.py
index e7f12bca..0c88febd 100644
--- a/radis/extractions/utils/processor_utils.py
+++ b/radis/extractions/utils/processor_utils.py
@@ -1,6 +1,6 @@
+from collections.abc import Iterable
 from typing import Any
 
-from django.db.models import QuerySet
 from pydantic import BaseModel, create_model
 
 from ..models import OutputField, OutputType
@@ -8,9 +8,9 @@
 type Numeric = float | int
 
 
-def generate_output_fields_schema(fields: QuerySet[OutputField]) -> type[BaseModel]:
+def generate_output_fields_schema(fields: Iterable[OutputField]) -> type[BaseModel]:
     field_definitions: dict[str, Any] = {}
-    for field in fields.all():
+    for field in fields:
         if field.output_type == OutputType.TEXT:
             output_type = str
         elif field.output_type == OutputType.NUMERIC:
@@ -25,9 +25,9 @@ def generate_output_fields_schema(fields: QuerySet[OutputField]) -> type[BaseMod
     return create_model("OutputFieldsModel", **field_definitions)
 
 
-def generate_output_fields_prompt(fields: QuerySet[OutputField]) -> str:
+def generate_output_fields_prompt(fields: Iterable[OutputField]) -> str:
     prompt = ""
-    for field in fields.all():
+    for field in fields:
         prompt += f"{field.name}: {field.description}\n"
 
     return prompt
diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index d35287a5..3bb189e9 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -68,23 +68,28 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
                 }
             )
             filter_schema = generate_filter_questions_schema(filter_questions)
-            filter_response = self.client.extract_data(filter_prompt, filter_schema)
-
-            for question in filter_questions:
-                field_name = get_filter_question_field_name(question)
-                answer = getattr(filter_response, field_name, None)
-                if answer is None:
-                    logger.debug(
-                        f"LLM returned None for question {question.pk} on report {report.pk}"
-                    )
-                    is_accepted = False
-                    break
-                else:
-                    answer_bool = bool(answer)
-                    filter_results[str(question.pk)] = answer_bool
-                    if answer_bool != question.expected_answer_bool:
+
+            try:
+                filter_response = self.client.extract_data(filter_prompt, filter_schema)
+
+                for question in filter_questions:
+                    field_name = get_filter_question_field_name(question)
+                    answer = getattr(filter_response, field_name, None)
+                    if answer is None:
+                        logger.debug(
+                            f"LLM returned None for question {question.pk} on report {report.pk}"
+                        )
                         is_accepted = False
                         break
+                    else:
+                        answer_bool = bool(answer)
+                        filter_results[str(question.pk)] = answer_bool
+                        if answer_bool != question.expected_answer_bool:
+                            is_accepted = False
+                            break
+            except Exception as e:
+                logger.error(f"LLM extraction failed for report {report.pk}: {e}")
+                return
         else:
             logger.debug(
                 "Subscription %s has no filter questions; accepting report %s by default",
@@ -97,9 +102,9 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
             return
 
         extraction_results: dict[str, Any] = {}
-        output_fields = subscription.output_fields.order_by("pk")
+        output_fields = list(subscription.output_fields.order_by("pk"))
 
-        if output_fields.first():
+        if output_fields:
             extraction_prompt = Template(settings.SUBSCRIPTION_EXTRACTION_PROMPT).substitute(
                 {
                     "report": report.body,

From 5334420a0b48f26fd1120cdc8f35924548fa2f7e Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Thu, 11 Dec 2025 22:25:59 +0000
Subject: [PATCH 65/93] narrow exception handling

---
 radis/subscriptions/processors.py | 33 ++++++++++++++++++++++++-------
 1 file changed, 26 insertions(+), 7 deletions(-)

diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index 3bb189e9..6fc13224 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -6,6 +6,8 @@
 from adit_radis_shared.common.types import User
 from django import db
 from django.conf import settings
+from openai import APIError
+from pydantic import ValidationError
 
 from radis.chats.utils.chat_client import ChatClient
 from radis.core.processors import AnalysisTaskProcessor
@@ -87,8 +89,14 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
                         if answer_bool != question.expected_answer_bool:
                             is_accepted = False
                             break
-            except Exception as e:
-                logger.error(f"LLM extraction failed for report {report.pk}: {e}")
+            except APIError as e:
+                logger.error(f"LLM API error filtering report {report.pk}: {e}")
+                return
+            except ValidationError as e:
+                logger.error(f"Response validation failed filtering report {report.pk}: {e}")
+                return
+            except AssertionError:
+                logger.error(f"No parsed response received filtering report {report.pk}")
                 return
         else:
             logger.debug(
@@ -112,12 +120,23 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
                 }
             )
             extraction_schema = generate_output_fields_schema(output_fields)
-            extraction_response = self.client.extract_data(extraction_prompt, extraction_schema)
 
-            for field in output_fields:
-                extraction_results[str(field.pk)] = getattr(
-                    extraction_response, get_output_field_name(field), None
-                )
+            try:
+                extraction_response = self.client.extract_data(extraction_prompt, extraction_schema)
+
+                for field in output_fields:
+                    extraction_results[str(field.pk)] = getattr(
+                        extraction_response, get_output_field_name(field), None
+                    )
+            except APIError as e:
+                logger.error(f"LLM API error extracting from report {report.pk}: {e}")
+                return
+            except ValidationError as e:
+                logger.error(f"Response validation failed extracting from report {report.pk}: {e}")
+                return
+            except AssertionError:
+                logger.error(f"No parsed response received extracting from report {report.pk}")
+                return
 
         SubscribedItem.objects.create(
             subscription=task.job.subscription,

From 3ad21a29f3ede910170b65acd8a03a125ddd4c2d Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Fri, 12 Dec 2025 00:02:45 +0000
Subject: [PATCH 66/93] remove optional from outputfield

---
 radis/extractions/models.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index 4a41a71b..b40f9816 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -90,7 +90,6 @@ class OutputField(models.Model):
         max_length=1, choices=OutputType.choices, default=OutputType.TEXT
     )
     get_output_type_display: Callable[[], str]
-    optional = models.BooleanField(default=False)
     # For Selection output type, stores the list of allowed options.
     # Only used if output_type == OutputType.SELECTION.
     selection_options = models.JSONField(default=list, blank=True)

From 645f64df731ae085f3b60c4e6c583f4130448c81 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Fri, 12 Dec 2025 00:29:11 +0000
Subject: [PATCH 67/93] update extraction feature in subscriptions with new
 changes from ritwik

---
 radis/subscriptions/forms.py                  | 139 +++++++++++++++++-
 .../subscriptions/subscription_detail.html    |  41 +++++-
 2 files changed, 168 insertions(+), 12 deletions(-)

diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index 959b0c9d..8ed9cd34 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -1,12 +1,14 @@
+import json
 from typing import Any
 
 from crispy_forms.helper import FormHelper
-from crispy_forms.layout import Column, Div, Field, Layout, Row
+from crispy_forms.layout import HTML, Column, Div, Field, Layout, Row
 from django import forms
 
 from radis.core.constants import LANGUAGE_LABELS
 from radis.core.layouts import Formset, RangeSlider
-from radis.extractions.models import OutputField
+from radis.extractions.constants import MAX_SELECTION_OPTIONS
+from radis.extractions.models import OutputField, OutputType
 from radis.reports.models import Language, Modality
 from radis.search.forms import AGE_STEP, MAX_AGE, MIN_AGE
 
@@ -180,27 +182,89 @@ def clean(self) -> dict[str, Any]:
 
 
 class OutputFieldForm(forms.ModelForm):
+    """Hidden field to store selection options and array flag as JSON string.
+    This is done because the selection options are dynamic and the array toggle
+    is an alpine component that needs to be re-rendered on every change."""
+
+    selection_options = forms.CharField(
+        required=False,
+        widget=forms.HiddenInput(),
+    )
+    is_array = forms.CharField(
+        required=False,
+        widget=forms.HiddenInput(),
+    )
+
     class Meta:
         model = OutputField
-        fields = ["name", "description", "output_type"]
+        fields = [
+            "name",
+            "description",
+            "output_type",
+            "selection_options",
+            "is_array",
+        ]
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
         self.fields["name"].required = True
         self.fields["description"].required = True
+        self.fields["description"].widget = forms.Textarea(attrs={"rows": 3})
+
+        # Stamp data attributes so the selection-options widget can locate these fields.
+        self.fields["selection_options"].widget.attrs.update(
+            {
+                "data-selection-input": "true",
+                "data-max-selection-options": str(MAX_SELECTION_OPTIONS),
+            }
+        )
+        self.fields["is_array"].widget.attrs.update(
+            {
+                "data-array-input": "true",
+            }
+        )
+
+        initial_options = self.instance.selection_options if self.instance.pk else []
+
+        # Prepopulate the hidden fields so existing options/toggle state show up in the widget.
+        self.initial["selection_options"] = json.dumps(initial_options)
+        self.initial["is_array"] = "true" if self.instance.is_array else "false"
 
         self.helper = FormHelper()
         self.helper.form_tag = False
         self.helper.disable_csrf = True
 
+        # Build the layout for selection options and array toggle button using crispy.
         fields = [
             Field("id", type="hidden"),
             Row(
-                Column("name", css_class="col-6"),
-                Column("output_type", css_class="col-4"),
+                Column("name", css_class="col-md-7 col-12"),
+                Column("output_type", css_class="col-md-4 col-10"),
+                Column(
+                    HTML(
+                        (
+                            '<button type="button" '
+                            'class="btn btn-outline-secondary btn-sm array-toggle-btn '
+                            'form-array-toggle" '
+                            'data-array-toggle="true" '
+                            'aria-pressed="false" '
+                            'title="Toggle array output">[ ]</button>'
+                        )
+                    ),
+                    css_class=(
+                        "col-md-1 col-2 d-flex align-items-center "
+                        "justify-content-end array-toggle-field"
+                    ),
+                ),
+                css_class="g-3 align-items-center",
             ),
             "description",
+            # Include the selection options widget partial template here.
+            Div(
+                HTML('{% include "extractions/_selection_options_field.html" %}'),
+                css_class="selection-options-wrapper",
+            ),
         ]
 
         if "DELETE" in self.fields:
@@ -208,6 +272,71 @@ def __init__(self, *args, **kwargs):
 
         self.helper.layout = Layout(Div(*fields))
 
+    # Parse and sanitize the JSON payload (trim strings, enforce uniqueness and max count).
+    # Called automatically by Django form validation.
+    def clean_selection_options(self) -> list[str]:
+        raw_value = self.cleaned_data.get("selection_options") or ""
+        raw_value = raw_value.strip()
+        if raw_value == "":
+            return []
+
+        try:
+            parsed = json.loads(raw_value)
+        except json.JSONDecodeError as exc:
+            raise forms.ValidationError("Invalid selection data.") from exc
+
+        if not isinstance(parsed, list):
+            raise forms.ValidationError("Selection data must be a list.")
+
+        cleaned: list[str] = []
+        for item in parsed:
+            if not isinstance(item, str):
+                raise forms.ValidationError("Selection options must be text.")
+            value = item.strip()
+            if not value:
+                raise forms.ValidationError("Selection options cannot be empty.")
+            cleaned.append(value)
+
+        if len(cleaned) > MAX_SELECTION_OPTIONS:
+            raise forms.ValidationError(
+                f"Provide at most {MAX_SELECTION_OPTIONS} selection options."
+            )
+        if len(set(cleaned)) != len(cleaned):
+            raise forms.ValidationError("Selection options must be unique.")
+
+        return cleaned
+
+    # not exactly needed because we set the value ourselves, but more as a double check
+    def clean_is_array(self) -> bool:
+        raw_value = (self.cleaned_data.get("is_array") or "").strip().lower()
+        if raw_value in {"1", "true", "on"}:
+            return True
+        return False
+
+    def clean(self):
+        cleaned_data = super().clean()
+        if not cleaned_data:
+            return cleaned_data
+
+        output_type = cleaned_data.get("output_type")
+        selection_options: list[str] = cleaned_data.get("selection_options") or []
+
+        if output_type == OutputType.SELECTION:
+            if not selection_options:
+                self.add_error(
+                    "selection_options",
+                    "Add at least one selection to use the Selection type.",
+                )
+        else:
+            if selection_options:
+                self.add_error(
+                    "selection_options",
+                    "Selections are only allowed when Output Type is Selection.",
+                )
+                cleaned_data["selection_options"] = []
+
+        return cleaned_data
+
 
 FilterQuestionFormSet = forms.inlineformset_factory(
     Subscription,
diff --git a/radis/subscriptions/templates/subscriptions/subscription_detail.html b/radis/subscriptions/templates/subscriptions/subscription_detail.html
index ac9d884d..1edc995d 100644
--- a/radis/subscriptions/templates/subscriptions/subscription_detail.html
+++ b/radis/subscriptions/templates/subscriptions/subscription_detail.html
@@ -128,15 +128,42 @@ <h4>Filter Questions</h4>
     <h4 class="mt-4">Extraction Fields</h4>
     {% with fields=subscription.output_fields.all %}
         {% if fields %}
-            <dl class="row">
+            <ul>
                 {% for field in fields %}
-                    <dt class="col-sm-3">{{ field.name }}</dt>
-                    <dd class="col-sm-9">
-                        {{ field.description|default:"No description provided." }}
-                        <div class="text-muted small">Type: {{ field.get_output_type_display }}</div>
-                    </dd>
+                    <li>
+                        <dl class="row mb-0">
+                            <dt class="col-sm-3">Name</dt>
+                            <dd class="col-sm-9">
+                                {{ field.name }}
+                            </dd>
+                            <dt class="col-sm-3">Description</dt>
+                            <dd class="col-sm-9">
+                                {{ field.description }}
+                            </dd>
+                            <dt class="col-sm-3">Data Type</dt>
+                            <dd class="col-sm-9">
+                                {{ field.get_output_type_display }}
+                            </dd>
+                            {% if field.is_array %}
+                                <dt class="col-sm-3">Array Output</dt>
+                                <dd class="col-sm-9">
+                                    Yes — return multiple {{ field.get_output_type_display|lower }} values
+                                </dd>
+                            {% endif %}
+                            {% if field.selection_options %}
+                                <dt class="col-sm-3">Selections</dt>
+                                <dd class="col-sm-9">
+                                    <ul class="mb-0 ps-3">
+                                        {% for option in field.selection_options %}
+                                            <li>{{ option }}</li>
+                                        {% endfor %}
+                                    </ul>
+                                </dd>
+                            {% endif %}
+                        </dl>
+                    </li>
                 {% endfor %}
-            </dl>
+            </ul>
         {% else %}
             <p class="text-muted mb-0">No extraction fields configured.</p>
         {% endif %}

From 879ac15e490d59d7f5de5b3e36be07967b4567ae Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Fri, 12 Dec 2025 02:03:18 +0000
Subject: [PATCH 68/93] removed query from subscriptions form

---
 radis/core/templates/cotton/formset.html      |  4 +-
 radis/subscriptions/factories.py              |  1 -
 radis/subscriptions/forms.py                  |  3 --
 .../0014_remove_subscription_query.py         | 17 ++++++++
 radis/subscriptions/models.py                 |  1 -
 radis/subscriptions/tables.py                 |  2 +-
 radis/subscriptions/tasks.py                  | 42 ++++---------------
 .../subscriptions/subscription_detail.html    | 13 +-----
 radis/subscriptions/tests/test_views.py       |  4 --
 9 files changed, 29 insertions(+), 58 deletions(-)
 create mode 100644 radis/subscriptions/migrations/0014_remove_subscription_query.py

diff --git a/radis/core/templates/cotton/formset.html b/radis/core/templates/cotton/formset.html
index 904473a1..984df98d 100644
--- a/radis/core/templates/cotton/formset.html
+++ b/radis/core/templates/cotton/formset.html
@@ -4,10 +4,10 @@
     {% if legend %}<legend>{{ legend }}</legend>{% endif %}
     {{ formset.management_form }}
 <template>
-    <c-formset-form>{% crispy formset.empty_form %}</c-formset-form>
+    <c-formset-form class="formset-form">{% crispy formset.empty_form %}</c-formset-form>
 </template>
 <div class="formset-forms">
-    {% for form in formset %}<c-formset-form>{% crispy form %}</c-formset-form>{% endfor %}
+    {% for form in formset %}<c-formset-form class="formset-form">{% crispy form %}</c-formset-form>{% endfor %}
 </div>
     {% if add_form_label %}
         <div class="d-flex flex-row-reverse">
diff --git a/radis/subscriptions/factories.py b/radis/subscriptions/factories.py
index 503dd58b..da57a0c7 100644
--- a/radis/subscriptions/factories.py
+++ b/radis/subscriptions/factories.py
@@ -15,7 +15,6 @@ class Meta:
     owner = factory.SubFactory(UserFactory)
     group = factory.SubFactory(GroupFactory)
     patient_id = factory.Faker("numerify", text="##########")
-    query = factory.Faker("sentence", nb_words=3)
     language = factory.SubFactory(LanguageFactory, code="en")
     study_description = factory.Faker("sentence", nb_words=4)
     patient_sex = factory.Faker("random_element", elements=["M", "F", ""])
diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index 8ed9cd34..d6b3d008 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -20,7 +20,6 @@ class Meta:
         model = Subscription
         fields = [
             "name",
-            "query",
             "language",
             "modalities",
             "study_description",
@@ -33,7 +32,6 @@ class Meta:
         labels = {"patient_id": "Patient ID"}
         help_texts = {
             "name": "Name of the Subscription",
-            "query": "A query to filter reports",
         }
 
     def __init__(self, *args, **kwargs):
@@ -84,7 +82,6 @@ def build_layout(self):
             Row(
                 Column(
                     "name",
-                    "query",
                     "send_finished_mail",
                     Formset(
                         "filter_formset",
diff --git a/radis/subscriptions/migrations/0014_remove_subscription_query.py b/radis/subscriptions/migrations/0014_remove_subscription_query.py
new file mode 100644
index 00000000..9bea7412
--- /dev/null
+++ b/radis/subscriptions/migrations/0014_remove_subscription_query.py
@@ -0,0 +1,17 @@
+# Generated by Django 5.2.8 on 2025-12-12 01:49
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("subscriptions", "0013_subscription_last_viewed_at"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="subscription",
+            name="query",
+        ),
+    ]
diff --git a/radis/subscriptions/models.py b/radis/subscriptions/models.py
index 8462ae78..abdad5fd 100644
--- a/radis/subscriptions/models.py
+++ b/radis/subscriptions/models.py
@@ -32,7 +32,6 @@ class Subscription(models.Model):
 
     group = models.ForeignKey[Group](Group, on_delete=models.CASCADE, related_name="+")
     patient_id = models.CharField(max_length=100, blank=True)
-    query = models.CharField(max_length=200, blank=True)
     language = models.ForeignKey[Language](
         Language, on_delete=models.SET_NULL, blank=True, null=True, related_name="+"
     )
diff --git a/radis/subscriptions/tables.py b/radis/subscriptions/tables.py
index 9c2de3e4..8a0aa58d 100644
--- a/radis/subscriptions/tables.py
+++ b/radis/subscriptions/tables.py
@@ -27,7 +27,7 @@ def render_num_reports(self, value, record):
 
         if num_new > 0:
             return format_html(
-                '<a href="{}">{} <span class="badge bg-primary">{} new</span></a>',
+                '<a href="{}">{}<span class="badge bg-primary ms-2">{} new</span></a>',
                 url,
                 value,
                 num_new,
diff --git a/radis/subscriptions/tasks.py b/radis/subscriptions/tasks.py
index c6248ca5..ec17866c 100644
--- a/radis/subscriptions/tasks.py
+++ b/radis/subscriptions/tasks.py
@@ -9,8 +9,7 @@
 from procrastinate.contrib.django import app
 
 from radis.reports.models import Report
-from radis.search.site import Search, SearchFilters
-from radis.search.utils.query_parser import QueryParser
+from radis.search.site import SearchFilters
 
 from . import site
 from .models import Subscription, SubscriptionJob, SubscriptionTask
@@ -40,7 +39,7 @@ def process_subscription_job(job_id: int) -> None:
     logger.debug("Collecting tasks for job %s", job)
 
     language_code = ""
-    if job.subscription.language and job.subscription.query != "":
+    if job.subscription.language:
         language_code = job.subscription.language.code
 
     filters = SearchFilters(
@@ -54,38 +53,13 @@ def process_subscription_job(job_id: int) -> None:
         created_after=job.subscription.last_refreshed,
     )
 
-    if job.subscription.query != "":
-        logger.debug("Searching new reports with query and filters for job %s", job)
+    logger.debug("Searching new reports with filters for job %s", job)
 
-        if site.subscription_retrieval_provider is None:
-            logger.error("Subscription retrieval provider is not configured for job %s", job)
-            raise ImproperlyConfigured("Subscription retrieval provider is not configured.")
-        retrieval_provider = site.subscription_retrieval_provider
-
-        query_node, fixes = QueryParser().parse(job.subscription.query)
-
-        if query_node is None:
-            raise ValueError(f"Not a valid query (evaluated as empty): {job.subscription.query}")
-
-        if len(fixes) > 0:
-            logger.info(f"The following fixes were applied to the query:\n{'\n - '.join(fixes)}")
-
-        search = Search(
-            query=query_node,
-            offset=0,
-            filters=filters,
-        )
-
-        new_document_ids = retrieval_provider.retrieve(search)
-
-    else:
-        logger.debug("Searching new reports with filters for job %s", job)
-
-        if site.subscription_filter_provider is None:
-            logger.error("Subscription filter provider is not configured for job %s", job)
-            raise ImproperlyConfigured("Subscription filter provider is not configured.")
-        filter_provider = site.subscription_filter_provider
-        new_document_ids = filter_provider.filter(filters)
+    if site.subscription_filter_provider is None:
+        logger.error("Subscription filter provider is not configured for job %s", job)
+        raise ImproperlyConfigured("Subscription filter provider is not configured.")
+    filter_provider = site.subscription_filter_provider
+    new_document_ids = filter_provider.filter(filters)
 
     for document_ids in batched(new_document_ids, settings.SUBSCRIPTION_REFRESH_TASK_BATCH_SIZE):
         logger.debug("Creating SubscriptionTask for document IDs: %s", document_ids)
diff --git a/radis/subscriptions/templates/subscriptions/subscription_detail.html b/radis/subscriptions/templates/subscriptions/subscription_detail.html
index 1edc995d..01e1ab58 100644
--- a/radis/subscriptions/templates/subscriptions/subscription_detail.html
+++ b/radis/subscriptions/templates/subscriptions/subscription_detail.html
@@ -44,16 +44,8 @@ <h4>General</h4>
             {{ subscription.last_refreshed }}
         </dd>
     </dl>
-    <h4>Search Details</h4>
+    <h4>Filter Details</h4>
     <dl class="row">
-        <dt class="col-sm-3">Search Query</dt>
-        <dd class="col-sm-9">
-            {% if subscription.query %}
-                {{ subscription.query }}
-            {% else %}
-                –
-            {% endif %}
-        </dd>
         <dt class="col-sm-3">Language</dt>
         <dd class="col-sm-9">
             {% if subscription.language %}
@@ -62,9 +54,6 @@ <h4>Search Details</h4>
                 –
             {% endif %}
         </dd>
-    </dl>
-    <h4>Filter Details</h4>
-    <dl class="row">
         <dt class="col-sm-3">Modalities</dt>
         <dd class="col-sm-9">
             {{ subscription.modalities.all|join:", "|default:"–" }}
diff --git a/radis/subscriptions/tests/test_views.py b/radis/subscriptions/tests/test_views.py
index 822e037e..3b7c446a 100644
--- a/radis/subscriptions/tests/test_views.py
+++ b/radis/subscriptions/tests/test_views.py
@@ -94,7 +94,6 @@ def test_subscription_create_view_post_valid(client: Client):
 
     data = {
         "name": "Test Subscription",
-        "query": "test query",
         "language": language.pk,
         "modalities": [modality.pk],
         "study_description": "Test study",
@@ -140,7 +139,6 @@ def test_subscription_create_view_ignores_empty_filter_question(client: Client):
         "name": "Subscription Without Filter",
         "provider": "test_provider",
         "language": language.pk,
-        "query": "",
         "filter_questions-TOTAL_FORMS": "1",
         "filter_questions-INITIAL_FORMS": "0",
         "filter_questions-MIN_NUM_FORMS": "0",
@@ -178,7 +176,6 @@ def test_subscription_create_view_post_duplicate_name(client: Client):
         "name": "Duplicate Name",
         "provider": "test_provider",
         "language": language.pk,
-        "query": "",
         "filter_questions-TOTAL_FORMS": "0",
         "filter_questions-INITIAL_FORMS": "0",
         "filter_questions-MIN_NUM_FORMS": "0",
@@ -252,7 +249,6 @@ def test_subscription_update_view_post_valid(client: Client):
 
     data = {
         "name": "Updated Name",
-        "query": "updated query",
         "study_description": "Updated study",
         "patient_sex": "F",
         "send_finished_mail": False,

From ea639a4a6413c0cf001de19d9b173f61e80600a7 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Fri, 12 Dec 2025 12:03:12 +0000
Subject: [PATCH 69/93] add extractions export to subscription feature

---
 .../subscriptions/subscription_inbox.html     |  21 ++--
 radis/subscriptions/urls.py                   |   6 +
 radis/subscriptions/utils/csv_export.py       |  82 +++++++++++++
 radis/subscriptions/views.py                  | 115 +++++++++++++++++-
 4 files changed, 214 insertions(+), 10 deletions(-)
 create mode 100644 radis/subscriptions/utils/csv_export.py

diff --git a/radis/subscriptions/templates/subscriptions/subscription_inbox.html b/radis/subscriptions/templates/subscriptions/subscription_inbox.html
index f2f2f27b..35a2f5ee 100644
--- a/radis/subscriptions/templates/subscriptions/subscription_inbox.html
+++ b/radis/subscriptions/templates/subscriptions/subscription_inbox.html
@@ -1,6 +1,6 @@
 {% extends 'subscriptions/subscription_layout.html' %}
 {% load crispy from crispy_forms_tags %}
-
+{% load bootstrap_icon from common_extras %}
 {% block title %}
     Subscription Inbox
 {% endblock title %}
@@ -10,8 +10,8 @@
 {% block content %}
     <div class="row">
         <div class="col">
-            {# Sorting Controls #}
-            <div class="mb-3">
+            {# Sorting Controls and Download Button #}
+            <div class="mb-3 d-flex justify-content-between align-items-center">
                 <div class="btn-group btn-group-sm" role="group" aria-label="Sort options">
                     <a href="?sort_by=created_at&order=desc{% if request.GET.per_page %}&per_page={{ request.GET.per_page }}{% endif %}{% for key, value in request.GET.items %}{% if key != 'sort_by' and key != 'order' and key != 'page' and key != 'per_page' %}&{{ key }}={{ value }}{% endif %}{% endfor %}"
                        class="btn btn-outline-secondary {% if current_sort_by == 'created_at' and current_order == 'desc' %}active{% endif %}">
@@ -30,14 +30,19 @@
                         Study Date (Oldest)
                     </a>
                 </div>
+                {# Download Button #}
+                <a href="{% url 'subscription_inbox_download' object.pk %}?{{ request.GET.urlencode }}"
+                   class="btn btn-sm btn-success"
+                   {% if not object_list %}disabled{% endif %}>
+                    {% bootstrap_icon "download" %}
+                    Download Extractions as CSV
+                </a>
             </div>
             {# Card List #}
             {% for subscribed_item in object_list %}
                 {% include "subscriptions/_subscribed_item_preview.html" %}
             {% empty %}
-                <div class="alert alert-light" role="alert">
-                    No items found
-                </div>
+                <div class="alert alert-light" role="alert">No items found</div>
             {% endfor %}
             {# Pagination Controls #}
             <div class="d-flex justify-content-around align-items-center mt-3">
@@ -62,9 +67,7 @@ <h5 class="card-title">Filters</h5>
                         <input type="hidden"
                                name="sort_by"
                                value="{{ request.GET.sort_by|default:'' }}">
-                        <input type="hidden"
-                               name="order"
-                               value="{{ request.GET.order|default:'' }}">
+                        <input type="hidden" name="order" value="{{ request.GET.order|default:'' }}">
                         {% crispy filter.form %}
                     </div>
                 </div>
diff --git a/radis/subscriptions/urls.py b/radis/subscriptions/urls.py
index 48631378..419269a8 100644
--- a/radis/subscriptions/urls.py
+++ b/radis/subscriptions/urls.py
@@ -5,6 +5,7 @@
     SubscriptionCreateView,
     SubscriptionDeleteView,
     SubscriptionDetailView,
+    SubscriptionInboxDownloadView,
     SubscriptionInboxView,
     SubscriptionListView,
     SubscriptionUpdateView,
@@ -22,4 +23,9 @@
         name="subscription_help",
     ),
     path("<int:pk>/inbox/", SubscriptionInboxView.as_view(), name="subscription_inbox"),
+    path(
+        "<int:pk>/inbox/download/",
+        SubscriptionInboxDownloadView.as_view(),
+        name="subscription_inbox_download",
+    ),
 ]
diff --git a/radis/subscriptions/utils/csv_export.py b/radis/subscriptions/utils/csv_export.py
new file mode 100644
index 00000000..e36e2ec1
--- /dev/null
+++ b/radis/subscriptions/utils/csv_export.py
@@ -0,0 +1,82 @@
+"""Helpers for exporting subscription inbox items in CSV format."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from typing import Any
+
+from django.db.models import QuerySet
+
+from radis.subscriptions.models import SubscribedItem, Subscription
+
+
+def _format_cell(value: Any) -> str:
+    """Format a single output value for CSV export."""
+    if value is None:
+        return ""
+    if isinstance(value, bool):
+        return "yes" if value else "no"
+    return str(value)
+
+
+def iter_subscribed_item_rows(
+    subscription: Subscription, queryset: QuerySet[SubscribedItem]
+) -> Iterable[Sequence[str]]:
+    """Yield rows for the subscription inbox CSV.
+
+    Args:
+        subscription: The subscription whose items should be exported.
+        queryset: Pre-filtered queryset of SubscribedItems to export.
+
+    Yields:
+        Sequences of stringified cell values suitable for csv.writer.
+    """
+    # Get output field names in PK order (to match dict keys)
+    field_names: list[str] = list(
+        subscription.output_fields.order_by("pk").values_list("name", flat=True)
+    )
+
+    # Header row
+    header = [
+        "subscribed_item_id",
+        "report_id",
+        "patient_id",
+        "study_date",
+        "study_description",
+        "modalities",
+    ]
+    header.extend(field_names)
+    yield header
+
+    # Data rows - prefetch related fields for efficiency
+    items = queryset.select_related("report").prefetch_related(
+        "report__modalities", "subscription__output_fields"
+    )
+
+    for item in items.iterator(chunk_size=1000):
+        # Format modalities as comma-separated codes
+        modality_codes = ",".join(
+            item.report.modalities.order_by("code").values_list("code", flat=True)
+        )
+
+        # Format study date
+        study_date = ""
+        if item.report.study_datetime:
+            study_date = item.report.study_datetime.strftime("%Y-%m-%d")
+
+        row = [
+            str(item.pk),
+            str(item.report.pk),
+            item.report.patient_id or "",
+            study_date,
+            item.report.study_description or "",
+            modality_codes,
+        ]
+
+        # Add extraction results (keyed by field PK as string)
+        extraction_results: dict[str, Any] = item.extraction_results or {}
+        for field_pk in subscription.output_fields.order_by("pk").values_list("pk", flat=True):
+            value = extraction_results.get(str(field_pk))
+            row.append(_format_cell(value))
+
+        yield row
diff --git a/radis/subscriptions/views.py b/radis/subscriptions/views.py
index 15638021..de865922 100644
--- a/radis/subscriptions/views.py
+++ b/radis/subscriptions/views.py
@@ -1,3 +1,5 @@
+import csv
+from collections.abc import Generator
 from logging import getLogger
 from typing import Any, Type, cast
 
@@ -13,9 +15,10 @@
 from django.db import IntegrityError
 from django.db.models import Count, F, Q, QuerySet
 from django.forms.models import BaseInlineFormSet
-from django.http import HttpResponse, HttpResponseRedirect
+from django.http import HttpResponse, HttpResponseRedirect, StreamingHttpResponse
 from django.urls import reverse, reverse_lazy
 from django.utils import timezone
+from django.utils.text import slugify
 from django.views.generic import CreateView, DeleteView, DetailView, UpdateView
 from django_tables2 import SingleTableView
 
@@ -28,6 +31,7 @@
     SubscriptionForm,
 )
 from .models import SubscribedItem, Subscription
+from .utils.csv_export import iter_subscribed_item_rows
 
 logger = getLogger(__name__)
 
@@ -265,3 +269,112 @@ def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
         subscription.save(update_fields=["last_viewed_at"])
 
         return context
+
+
+class _Echo:
+    """Lightweight write-only buffer for csv.writer."""
+
+    def write(self, value: str) -> str:
+        return value
+
+
+class SubscriptionInboxDownloadView(LoginRequiredMixin, RelatedFilterMixin, DetailView):
+    """Stream subscription inbox items as a CSV download.
+
+    Applies the same filters as SubscriptionInboxView to ensure users
+    download exactly what they see (respecting filters but ignoring pagination).
+    """
+
+    model = Subscription
+    filterset_class = SubscribedItemFilter
+    request: AuthenticatedHttpRequest
+
+    def get_queryset(self) -> QuerySet[Subscription]:
+        """Return only subscriptions owned by the current user."""
+        assert self.model
+        model = cast(Type[Subscription], self.model)
+        user = cast(User, self.request.user)
+        if user.is_staff:
+            return model.objects.all()
+        return model.objects.filter(owner=self.request.user)
+
+    def get_ordering(self) -> str:
+        """Get the ordering from query parameters (same logic as SubscriptionInboxView)."""
+        sort_by = self.request.GET.get("sort_by", "created_at")
+        order = self.request.GET.get("order", "desc")
+
+        # Define allowed sort fields to prevent injection
+        allowed_fields = {
+            "created_at": "created_at",
+            "study_date": "report__study_datetime",
+        }
+
+        # Validate sort_by parameter
+        if sort_by not in allowed_fields:
+            sort_by = "created_at"
+
+        # Validate order parameter
+        if order not in ["asc", "desc"]:
+            order = "desc"
+
+        field = allowed_fields[sort_by]
+        return field if order == "asc" else f"-{field}"
+
+    def get_related_queryset(self) -> QuerySet[SubscribedItem]:
+        """Build queryset matching the inbox view (for filtering)."""
+        subscription = cast(Subscription, self.get_object())
+        ordering = self.get_ordering()
+        return (
+            SubscribedItem.objects.filter(subscription_id=subscription.pk)
+            .exclude(extraction_results__isnull=True)  # Only items with results
+            .exclude(extraction_results={})  # Only items with non-empty results
+            .select_related("subscription")
+            .prefetch_related(
+                "report",
+                "report__modalities",
+                "subscription__output_fields",
+            )
+            .order_by(ordering)
+        )
+
+    def get_filter_queryset(self) -> QuerySet[SubscribedItem]:
+        """Required by RelatedFilterMixin."""
+        return self.get_related_queryset()
+
+    def get(self, request: AuthenticatedHttpRequest, *args, **kwargs) -> StreamingHttpResponse:
+        """Stream the CSV file response."""
+        subscription = cast(Subscription, self.get_object())
+
+        # Manually instantiate the filterset to apply filters
+        # (RelatedFilterMixin doesn't provide get_filtered_queryset())
+        filterset_class = self.get_filterset_class()
+        filterset_kwargs = self.get_filterset_kwargs(filterset_class)
+        assert filterset_class is not None
+        filterset = filterset_class(**filterset_kwargs)
+
+        # Get the filtered queryset from filterset.qs
+        filtered_items = filterset.qs
+
+        filename = self._build_filename(subscription)
+
+        response = StreamingHttpResponse(
+            self._stream_rows(subscription, filtered_items),
+            content_type="text/csv",
+        )
+        response["Content-Disposition"] = f'attachment; filename="{filename}"'
+        return response
+
+    def _stream_rows(
+        self, subscription: Subscription, items: QuerySet[SubscribedItem]
+    ) -> Generator[str, None, None]:
+        """Yield serialized CSV rows for the response."""
+        pseudo_buffer = _Echo()
+        writer = csv.writer(pseudo_buffer)
+        yield "\ufeff"  # UTF-8 BOM for Excel compatibility
+        for row in iter_subscribed_item_rows(subscription, items):
+            yield writer.writerow(row)
+
+    def _build_filename(self, subscription: Subscription) -> str:
+        """Generate a descriptive CSV filename for the subscription."""
+        slug = slugify(subscription.name) or "inbox"
+        return f"subscription_{subscription.pk}_{slug}.csv"

From 7eb365fd0861d2b974f3b02fc31ad0fbc7a12eff Mon Sep 17 00:00:00 2001
From: Ritwik <ritwikghosh765@gmail.com>
Date: Fri, 12 Dec 2025 13:12:38 +0000
Subject: [PATCH 70/93] Fixing form load without hidden variables error

---
 radis/core/static/core/core.js                | 10 +--
 radis/core/templates/cotton/formset.html      |  4 +-
 radis/extractions/factories.py                | 15 +++++
 .../test_output_fields_selection_options.py   | 65 +++++++++++++++++++
 radis/extractions/utils/testing_helpers.py    |  3 +-
 radis/urls.py                                 | 16 +++--
 6 files changed, 100 insertions(+), 13 deletions(-)
 create mode 100644 radis/extractions/tests/acceptance/test_output_fields_selection_options.py

diff --git a/radis/core/static/core/core.js b/radis/core/static/core/core.js
index 060567d4..7c8665ec 100644
--- a/radis/core/static/core/core.js
+++ b/radis/core/static/core/core.js
@@ -84,14 +84,14 @@ function FormSet(rootEl) {
     minForms: parseInt(minForms.value),
     maxForms: parseInt(maxForms.value),
     init() {
-      console.log(this.formCount);
     },
     addForm() {
-      const newForm = template.content.cloneNode(true);
+      if (!template) {
+        return;
+      }
       const idx = totalForms.value;
-      container.append(newForm);
-      const lastForm = container.querySelector(".formset-form:last-child");
-      lastForm.innerHTML = lastForm.innerHTML.replace(/__prefix__/g, idx);
+      const html = template.innerHTML.replace(/__prefix__/g, idx);
+      container.insertAdjacentHTML("beforeend", html);
       totalForms.value = (parseInt(idx) + 1).toString();
       this.formCount = parseInt(totalForms.value);
     },
diff --git a/radis/core/templates/cotton/formset.html b/radis/core/templates/cotton/formset.html
index 904473a1..f33c1a10 100644
--- a/radis/core/templates/cotton/formset.html
+++ b/radis/core/templates/cotton/formset.html
@@ -4,7 +4,9 @@
     {% if legend %}<legend>{{ legend }}</legend>{% endif %}
     {{ formset.management_form }}
 <template>
-    <c-formset-form>{% crispy formset.empty_form %}</c-formset-form>
+    {% with form=formset.empty_form %}
+        <c-formset-form>{% crispy formset.empty_form %}</c-formset-form>
+    {% endwith %}
 </template>
 <div class="formset-forms">
     {% for form in formset %}<c-formset-form>{% crispy form %}</c-formset-form>{% endfor %}
diff --git a/radis/extractions/factories.py b/radis/extractions/factories.py
index 6533d9fa..d326235d 100644
--- a/radis/extractions/factories.py
+++ b/radis/extractions/factories.py
@@ -1,6 +1,7 @@
 import factory
 from faker import Faker
 
+from adit_radis_shared.accounts.factories import UserFactory
 from radis.reports.factories import ModalityFactory
 
 from .models import (
@@ -27,6 +28,7 @@ class ExtractionJobFactory(BaseDjangoModelFactory):
     class Meta:
         model = ExtractionJob
 
+    owner = factory.SubFactory(UserFactory)
     title = factory.Faker("sentence", nb_words=3)
     provider = factory.Faker("random_element", elements=list(extraction_retrieval_providers.keys()))
     group = factory.SubFactory("adit_radis_shared.accounts.factories.GroupFactory")
@@ -58,6 +60,16 @@ def modalities(self, create, extracted, **kwargs):
             # django_get_or_create would not be respected then
             self.modalities.add(ModalityFactory(code=modality))  # type: ignore
 
+    @factory.post_generation
+    def ensure_owner_in_group(self, create, extracted, **kwargs):
+        if not create:
+            return
+
+        self.owner.groups.add(self.group)
+        if self.owner.active_group_id is None:
+            self.owner.active_group = self.group
+            self.owner.save(update_fields=["active_group"])
+
 
 class OutputFieldFactory(BaseDjangoModelFactory[OutputField]):
     class Meta:
@@ -67,6 +79,9 @@ class Meta:
     name = factory.Sequence(lambda n: f"output_field_{n}")
     description = factory.Faker("sentence", nb_words=10)
     output_type = factory.Faker("random_element", elements=[a[0] for a in OutputType.choices])
+    selection_options = factory.LazyAttribute(
+        lambda obj: ["Option 1", "Option 2"] if obj.output_type == OutputType.SELECTION else []
+    )
 
 
 class ExtractionTaskFactory(BaseDjangoModelFactory[ExtractionTask]):
diff --git a/radis/extractions/tests/acceptance/test_output_fields_selection_options.py b/radis/extractions/tests/acceptance/test_output_fields_selection_options.py
new file mode 100644
index 00000000..fcd145c3
--- /dev/null
+++ b/radis/extractions/tests/acceptance/test_output_fields_selection_options.py
@@ -0,0 +1,65 @@
+from urllib.parse import urlparse
+
+import pytest
+from adit_radis_shared.accounts.factories import GroupFactory, UserFactory
+from django.contrib.auth.models import Permission
+from django.test import Client
+from playwright.sync_api import Page, expect
+from pytest_django.live_server_helper import LiveServer
+
+from radis.reports.models import Language, Modality
+
+
+@pytest.mark.acceptance
+@pytest.mark.order("last")
+@pytest.mark.django_db(transaction=True)
+def test_selection_options_work_for_added_output_field(live_server: LiveServer, page: Page):
+    group = GroupFactory.create()
+    user = UserFactory.create(is_active=True)
+    user.groups.add(group)
+    user.active_group = group
+    user.user_permissions.add(Permission.objects.get(codename="add_extractionjob"))
+    user.save()
+
+    language = Language.objects.create(code="en")
+    Modality.objects.create(code="CT", filterable=True)
+
+    client = Client()
+    client.force_login(user)
+
+    hostname = urlparse(live_server.url).hostname
+    assert hostname
+    page.context.add_cookies(
+        [
+            {
+                "name": "sessionid",
+                "value": client.cookies["sessionid"].value,
+                "domain": hostname,
+                "path": "/",
+            }
+        ]
+    )
+
+    page.goto(f"{live_server.url}/extractions/jobs/new/")
+    page.locator('input[name="0-title"]').fill("Test Extraction")
+    page.locator('select[name="0-provider"]').select_option("PG Search")
+    page.locator('input[name="0-query"]').fill("test")
+    page.locator('select[name="0-language"]').select_option(str(language.pk))
+    page.locator('input[type="submit"][value*="Next Step"]').click()
+
+    page.get_by_role("button", name="Add Field").click()
+
+    forms = page.locator(".formset-form")
+    expect(forms).to_have_count(2)
+
+    second_form = forms.nth(1)
+    selection_input = second_form.locator("[data-selection-input]")
+    expect(selection_input).to_have_count(1)
+    expect(selection_input).to_have_attribute("data-max-selection-options", "7")
+    second_form.locator('select[name$="-output_type"]').select_option("S")
+    expect(second_form.locator('select[name$="-output_type"]')).to_have_value("S")
+    add_option_button = second_form.get_by_role("button", name="Enter a selection")
+    expect(add_option_button).to_be_enabled()
+    add_option_button.click()
+
+    expect(second_form.locator('input[placeholder="Selection 1"]')).to_be_visible()
diff --git a/radis/extractions/utils/testing_helpers.py b/radis/extractions/utils/testing_helpers.py
index 5e864c3b..443af8b9 100644
--- a/radis/extractions/utils/testing_helpers.py
+++ b/radis/extractions/utils/testing_helpers.py
@@ -25,7 +25,8 @@ def create_extraction_task(
     add_user_to_group(user, group)
     job = ExtractionJobFactory.create(
         status=ExtractionJob.Status.PENDING,
-        owner_id=user.id,
+        owner=user,
+        group=group,
         language=language,
     )
 
diff --git a/radis/urls.py b/radis/urls.py
index a131cee5..64ce6bf6 100644
--- a/radis/urls.py
+++ b/radis/urls.py
@@ -14,9 +14,9 @@
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
 
-from django.conf import settings
 from django.contrib import admin
 from django.urls import include, path
+from django.apps import apps
 
 urlpatterns = [
     path("django-admin/", include("loginas.urls")),
@@ -35,11 +35,15 @@
     path("subscriptions/", include("radis.subscriptions.urls")),
 ]
 
-# Debug Toolbar in Debug mode only
-if settings.DEBUG:
-    import debug_toolbar
-
+# Some Django test runners force `DEBUG=False` even if the settings module enables it.
+# If these apps/middlewares are installed, we must still include their URLs so
+# templates can reverse them without raising `NoReverseMatch`.
+if apps.is_installed("django_browser_reload"):
     urlpatterns = [
         path("__reload__/", include("django_browser_reload.urls")),
-        path("__debug__/", include(debug_toolbar.urls)),
+    ] + urlpatterns
+
+if apps.is_installed("debug_toolbar"):
+    urlpatterns = [
+        path("__debug__/", include("debug_toolbar.urls")),
     ] + urlpatterns

From 8926d049653293ced1d41de03927ace55775afcf Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Sun, 14 Dec 2025 22:56:53 +0000
Subject: [PATCH 71/93] lint fix

---
 radis/extractions/factories.py                           | 2 +-
 .../templates/extractions/extraction_job_detail.html     | 4 +---
 .../extractions/extraction_job_wizard_summary.html       | 9 ++-------
 radis/extractions/utils/csv_export.py                    | 4 +---
 radis/extractions/utils/processor_utils.py               | 4 ++--
 .../templates/subscriptions/subscription_detail.html     | 4 +---
 radis/urls.py                                            | 2 +-
 7 files changed, 9 insertions(+), 20 deletions(-)

diff --git a/radis/extractions/factories.py b/radis/extractions/factories.py
index 469cb0e2..ce62d6e3 100644
--- a/radis/extractions/factories.py
+++ b/radis/extractions/factories.py
@@ -1,7 +1,7 @@
 import factory
+from adit_radis_shared.accounts.factories import UserFactory
 from faker import Faker
 
-from adit_radis_shared.accounts.factories import UserFactory
 from radis.reports.factories import ModalityFactory
 
 from .models import (
diff --git a/radis/extractions/templates/extractions/extraction_job_detail.html b/radis/extractions/templates/extractions/extraction_job_detail.html
index 6d3816eb..3a083766 100644
--- a/radis/extractions/templates/extractions/extraction_job_detail.html
+++ b/radis/extractions/templates/extractions/extraction_job_detail.html
@@ -124,9 +124,7 @@ <h5>Output Fields</h5>
                         <dt class="col-sm-3">Selections</dt>
                         <dd class="col-sm-9">
                             <ul class="mb-0 ps-3">
-                                {% for option in field.selection_options %}
-                                    <li>{{ option }}</li>
-                                {% endfor %}
+                                {% for option in field.selection_options %}<li>{{ option }}</li>{% endfor %}
                             </ul>
                         </dd>
                     {% endif %}
diff --git a/radis/extractions/templates/extractions/extraction_job_wizard_summary.html b/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
index a9580b82..32e24b43 100644
--- a/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
+++ b/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
@@ -101,9 +101,7 @@
                                     <dt class="col-sm-3">Selections</dt>
                                     <dd class="col-sm-9">
                                         <ul class="mb-0 ps-3">
-                                            {% for option in field.selection_options %}
-                                                <li>{{ option }}</li>
-                                            {% endfor %}
+                                            {% for option in field.selection_options %}<li>{{ option }}</li>{% endfor %}
                                         </ul>
                                     </dd>
                                 {% endif %}
@@ -114,10 +112,7 @@
             </ul>
         </dd>
     </dl>
-    <form action=""
-          method="post"
-          class="pt-3"
-          data-prevent-enter-submit="true">
+    <form action="" method="post" class="pt-3" data-prevent-enter-submit="true">
         {% csrf_token %}
         {{ wizard.management_form }}
         {% crispy form %}
diff --git a/radis/extractions/utils/csv_export.py b/radis/extractions/utils/csv_export.py
index 6759c050..e51da187 100644
--- a/radis/extractions/utils/csv_export.py
+++ b/radis/extractions/utils/csv_export.py
@@ -27,9 +27,7 @@ def iter_extraction_result_rows(job: ExtractionJob) -> Iterable[Sequence[str]]:
         Sequences of stringified cell values suitable for csv.writer.
     """
 
-    field_names: list[str] = list(
-        job.output_fields.order_by("pk").values_list("name", flat=True)
-    )
+    field_names: list[str] = list(job.output_fields.order_by("pk").values_list("name", flat=True))
 
     header = ["instance_id", "report_id", "is_processed"]
     header.extend(field_names)
diff --git a/radis/extractions/utils/processor_utils.py b/radis/extractions/utils/processor_utils.py
index 4a6b92e0..3e338351 100644
--- a/radis/extractions/utils/processor_utils.py
+++ b/radis/extractions/utils/processor_utils.py
@@ -15,9 +15,9 @@ def generate_output_fields_schema(fields: Iterable[OutputField]) -> type[BaseMod
     for field in fields:
         if field.output_type == OutputType.TEXT:
             output_type = str
-        elif field_type == OutputType.NUMERIC:
+        elif field.output_type == OutputType.NUMERIC:
             output_type = Numeric
-        elif field_type == OutputType.BOOLEAN:
+        elif field.output_type == OutputType.BOOLEAN:
             output_type = bool
         elif field.output_type == OutputType.SELECTION:
             options = tuple(field.selection_options)
diff --git a/radis/subscriptions/templates/subscriptions/subscription_detail.html b/radis/subscriptions/templates/subscriptions/subscription_detail.html
index 01e1ab58..d26dabe3 100644
--- a/radis/subscriptions/templates/subscriptions/subscription_detail.html
+++ b/radis/subscriptions/templates/subscriptions/subscription_detail.html
@@ -143,9 +143,7 @@ <h4 class="mt-4">Extraction Fields</h4>
                                 <dt class="col-sm-3">Selections</dt>
                                 <dd class="col-sm-9">
                                     <ul class="mb-0 ps-3">
-                                        {% for option in field.selection_options %}
-                                            <li>{{ option }}</li>
-                                        {% endfor %}
+                                        {% for option in field.selection_options %}<li>{{ option }}</li>{% endfor %}
                                     </ul>
                                 </dd>
                             {% endif %}
diff --git a/radis/urls.py b/radis/urls.py
index 64ce6bf6..12984602 100644
--- a/radis/urls.py
+++ b/radis/urls.py
@@ -14,9 +14,9 @@
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
 
+from django.apps import apps
 from django.contrib import admin
 from django.urls import include, path
-from django.apps import apps
 
 urlpatterns = [
     path("django-admin/", include("loginas.urls")),

From 3d0bb32149312ba7072aeff71e169c8f20448698 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Sun, 14 Dec 2025 23:24:20 +0000
Subject: [PATCH 72/93] fix attribute error for extractionjobfactory

---
 radis/extractions/factories.py | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/radis/extractions/factories.py b/radis/extractions/factories.py
index ce62d6e3..63a640ee 100644
--- a/radis/extractions/factories.py
+++ b/radis/extractions/factories.py
@@ -1,16 +1,15 @@
+from typing import cast
+
 import factory
-from adit_radis_shared.accounts.factories import UserFactory
+from adit_radis_shared.accounts.factories import GroupFactory, UserFactory
+from adit_radis_shared.accounts.models import User
+from adit_radis_shared.common.utils.testing_helpers import add_user_to_group
+from django.contrib.auth.models import Group
 from faker import Faker
 
 from radis.reports.factories import ModalityFactory
 
-from .models import (
-    ExtractionInstance,
-    ExtractionJob,
-    ExtractionTask,
-    OutputField,
-    OutputType,
-)
+from .models import ExtractionInstance, ExtractionJob, ExtractionTask, OutputField, OutputType
 
 fake = Faker()
 
@@ -29,7 +28,7 @@ class Meta:
 
     owner = factory.SubFactory(UserFactory)
     title = factory.Faker("sentence", nb_words=3)
-    group = factory.SubFactory("adit_radis_shared.accounts.factories.GroupFactory")
+    group = factory.SubFactory(GroupFactory)
     query = factory.Faker("word")
     language = factory.SubFactory("radis.reports.factories.LanguageFactory")
     study_date_from = factory.Faker("date")
@@ -59,14 +58,14 @@ def modalities(self, create, extracted, **kwargs):
             self.modalities.add(ModalityFactory(code=modality))  # type: ignore
 
     @factory.post_generation
-    def ensure_owner_in_group(self, create, extracted, **kwargs):
+    def ensure_owner_in_group(obj, create, extracted, **kwargs):
+        owner = cast(User, obj.owner)
+        group = cast(Group, obj.group)
+
         if not create:
             return
 
-        self.owner.groups.add(self.group)
-        if self.owner.active_group_id is None:
-            self.owner.active_group = self.group
-            self.owner.save(update_fields=["active_group"])
+        add_user_to_group(owner, group)
 
 
 class OutputFieldFactory(BaseDjangoModelFactory[OutputField]):

From 9235cd93a3cb7c2be8227d501a5a99452962b8d6 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Sun, 14 Dec 2025 23:50:24 +0000
Subject: [PATCH 73/93] fix migration order and duplicate migration

---
 .../migrations/0005_outputfield_is_array.py    | 16 ----------------
 .../0009_alter_extractionjob_query.py          | 18 ++++++++++++++++++
 2 files changed, 18 insertions(+), 16 deletions(-)
 create mode 100644 radis/extractions/migrations/0009_alter_extractionjob_query.py

diff --git a/radis/extractions/migrations/0005_outputfield_is_array.py b/radis/extractions/migrations/0005_outputfield_is_array.py
index 1227b60a..568a02c3 100644
--- a/radis/extractions/migrations/0005_outputfield_is_array.py
+++ b/radis/extractions/migrations/0005_outputfield_is_array.py
@@ -1,22 +1,6 @@
 from django.db import migrations, models
 
 
-class Migration(migrations.Migration):
-    dependencies = [
-        ("extractions", "0004_outputfield_selection_options"),
-    ]
-
-    operations = [
-        # Track whether an output field should return a list of values rather than a single value.
-        migrations.AddField(
-            model_name="outputfield",
-            name="is_array",
-            field=models.BooleanField(default=False),
-        ),
-    ]
-from django.db import migrations, models
-
-
 class Migration(migrations.Migration):
     dependencies = [
         ("extractions", "0004_outputfield_selection_options"),
diff --git a/radis/extractions/migrations/0009_alter_extractionjob_query.py b/radis/extractions/migrations/0009_alter_extractionjob_query.py
new file mode 100644
index 00000000..cdb335b5
--- /dev/null
+++ b/radis/extractions/migrations/0009_alter_extractionjob_query.py
@@ -0,0 +1,18 @@
+# Generated by Django 6.0 on 2025-12-11 20:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("extractions", "0008_outputfield_subscription_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="extractionjob",
+            name="query",
+            field=models.CharField(blank=True, max_length=200),
+        ),
+    ]

From 10c5ed0f7d2c5ca90d80f10c8eca64edffcceb9c Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 15 Dec 2025 00:22:17 +0000
Subject: [PATCH 74/93] fix remaining merge conflicts

---
 .../0007_alter_extractionjob_query.py          | 18 ------------------
 radis/extractions/models.py                    |  1 -
 2 files changed, 19 deletions(-)
 delete mode 100644 radis/extractions/migrations/0007_alter_extractionjob_query.py

diff --git a/radis/extractions/migrations/0007_alter_extractionjob_query.py b/radis/extractions/migrations/0007_alter_extractionjob_query.py
deleted file mode 100644
index 13cd4833..00000000
--- a/radis/extractions/migrations/0007_alter_extractionjob_query.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 6.0 on 2025-12-11 20:03
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ("extractions", "0006_outputfield_subscription_and_more"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="extractionjob",
-            name="query",
-            field=models.CharField(blank=True, max_length=200),
-        ),
-    ]
diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index 819f6304..e5b52fa7 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -88,7 +88,6 @@ class OutputField(models.Model):
         max_length=1, choices=OutputType.choices, default=OutputType.TEXT
     )
     get_output_type_display: Callable[[], str]
-    optional = models.BooleanField(default=False)
     selection_options = models.JSONField(default=list, blank=True)
     is_array = models.BooleanField(default=False)
     job = models.ForeignKey[ExtractionJob](

From 77208b3d82a3d3e0b0cd1bea38955ce9d6e2445d Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 15 Dec 2025 19:09:03 +0000
Subject: [PATCH 75/93] constrain autobahn (daphne dependency) to <25.11.1

---
 .github/renovate.json |   5 ++
 pyproject.toml        |   1 +
 uv.lock               | 175 ++++--------------------------------------
 3 files changed, 20 insertions(+), 161 deletions(-)

diff --git a/.github/renovate.json b/.github/renovate.json
index 3f979441..8343b6fa 100644
--- a/.github/renovate.json
+++ b/.github/renovate.json
@@ -22,6 +22,11 @@
       "managers": ["pep621"],
       "matchPackageNames": ["factory-boy"],
       "allowedVersions": "<=3.3.2"
+    },
+    {
+      "managers": ["pep621"],
+      "matchPackageNames": ["autobahn"],
+      "allowedVersions": "<25.11.1"
     }
   ]
 }
diff --git a/pyproject.toml b/pyproject.toml
index b220df59..17e7fb98 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -88,6 +88,7 @@ cli = "cli:app"
 
 [tool.uv]
 default-groups = ["dev", "client"]
+constraint-dependencies = ["autobahn<25.11.1"]
 
 [tool.uv.sources]
 radis-client = { path = "./radis-client", editable = true }
diff --git a/uv.lock b/uv.lock
index fd6db43c..2a59f6d7 100644
--- a/uv.lock
+++ b/uv.lock
@@ -2,6 +2,9 @@ version = 1
 revision = 3
 requires-python = ">=3.12, <4.0"
 
+[manifest]
+constraints = [{ name = "autobahn", specifier = "<25.11.1" }]
+
 [[package]]
 name = "adit-radis-shared"
 version = "0.0.0"
@@ -137,30 +140,24 @@ wheels = [
 
 [[package]]
 name = "autobahn"
-version = "25.12.1"
+version = "25.10.2"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "cbor2" },
-    { name = "cffi" },
     { name = "cryptography" },
     { name = "hyperlink" },
-    { name = "msgpack", marker = "platform_python_implementation == 'CPython'" },
-    { name = "py-ubjson" },
     { name = "txaio" },
-    { name = "u-msgpack-python", marker = "platform_python_implementation != 'CPython'" },
-    { name = "ujson" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/e2/1a/ac15ba19cb6a5d2a3371fec8d5ffa7a92104973aeb60d65ca85c69a17c29/autobahn-25.12.1.tar.gz", hash = "sha256:a13e49f762e97a291136bb1c4e39e6b026c4275593fb1d9ef9b73e7ef22e559d", size = 13851285, upload-time = "2025-12-10T02:43:46.543Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/5c/5d/095541ec46347cdb6d94b1cde7b0236eee7dcdaadb2daad45232d74eeff1/autobahn-25.10.2.tar.gz", hash = "sha256:173d5d836789dffc4292473ea359dcb7f708456a0ff82dcb8ca938d6ccadb12f", size = 375689, upload-time = "2025-10-22T23:34:56.017Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/f6/1a/5c63c0ea2f227d813a0edf6f623d957415da5dae548eccd0ca533818f408/autobahn-25.12.1-cp312-cp312-manylinux1_x86_64.manylinux_2_34_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:72f8f108f7a63199196105e91c7711e11d916533bc1ee5ab02cbc670a0967e84", size = 630537, upload-time = "2025-12-10T02:43:26.801Z" },
-    { url = "https://files.pythonhosted.org/packages/2a/bb/df4c1b33aca882c7329ad52be831099484cfb48b9fce475a9ea86e6eed54/autobahn-25.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:133712e97ff0122b0e2eb4675ef889702fbd0f6fb06ecfd9e5a99512d1886d38", size = 692163, upload-time = "2025-12-10T02:43:27.981Z" },
-    { url = "https://files.pythonhosted.org/packages/a2/a8/3a889c8481d7ee51ec1edf9592c370613999e6399f0e23e3993cdfca2aa7/autobahn-25.12.1-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:9daf8e3ae2eafcd9763e56405822274528df3ea08ed34fd8ef00d42b68c23027", size = 577108, upload-time = "2025-12-10T02:43:29.555Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/1c/5332e2a9b40273aadf04bbdbde7f3cd5317fc481e76d6f9a8cbab65c8858/autobahn-25.12.1-cp313-cp313-manylinux1_x86_64.manylinux_2_34_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c546454801b61db36728675f933e9f333d8015a278ff451f60c4dd7a124eddf9", size = 630516, upload-time = "2025-12-10T02:43:31.025Z" },
-    { url = "https://files.pythonhosted.org/packages/0a/ac/5c7dce7f48d4a980a6ae57ad1f1c1316f374a0ceea7581548f027d5cbf30/autobahn-25.12.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0f3f723e1aa44acef323a7c011b65add6226ead2faa283926680a2315e40655", size = 606210, upload-time = "2025-12-10T02:43:32.408Z" },
-    { url = "https://files.pythonhosted.org/packages/65/fb/2b7703fbf5c19ff58a1209dcced9ec0c5fccac7a714942de8f33d681e7ed/autobahn-25.12.1-cp313-cp313-win_amd64.whl", hash = "sha256:de2fe7e19914b4133c67b12b4fae8010ae9c2db91ffcea7390f0af7a875b3f63", size = 687043, upload-time = "2025-12-10T02:43:33.728Z" },
-    { url = "https://files.pythonhosted.org/packages/60/c9/5e3e8f893eb11415dfa9ecbf0d26ff032aa874c729f01a213f1b87f1637c/autobahn-25.12.1-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:872f569272e569fa65f3d27c0c54291dc1607b9da590588f9ef234f2d5031735", size = 577123, upload-time = "2025-12-10T02:43:34.775Z" },
-    { url = "https://files.pythonhosted.org/packages/e8/66/9ad1fd3c9baa1c16a268bfc358462182c04e05434db00e2bf5c34f0b378b/autobahn-25.12.1-cp314-cp314-manylinux1_x86_64.manylinux_2_34_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:76cee7e4d818bf7428bb740698a7debba4a0f56f88ce2695786b05a4ac55aa6b", size = 630753, upload-time = "2025-12-10T02:43:37.253Z" },
-    { url = "https://files.pythonhosted.org/packages/f5/a0/9abc505dc73dee8ccbd0b34ad57589f31a1a16e660d74e3164b545f935f0/autobahn-25.12.1-cp314-cp314-win_amd64.whl", hash = "sha256:fdf5660e5287e33bea64fd1fa62e2c3355bd2a3681ae4a751b6925e6342f03dd", size = 686646, upload-time = "2025-12-10T02:43:39.077Z" },
+    { url = "https://files.pythonhosted.org/packages/38/83/acf3a2bdd99cd06472788dee243ee017f45e12583afec7554c734d430cf4/autobahn-25.10.2-cp312-cp312-manylinux1_x86_64.manylinux_2_34_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:055611e97deb463741f398f014fb902c8a7ccb6911550a671aab3ca329453d18", size = 569863, upload-time = "2025-10-22T23:34:37.769Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/4c/8bb1185755e6eb6f4dfbe6ee0d77cc937972400dbe4479fa83cefa0b82f6/autobahn-25.10.2-cp312-cp312-win_amd64.whl", hash = "sha256:14f9b7bf938c491045ed39d76b6d300f6021f7db4189b3b5f36aeaea40ba7bca", size = 525982, upload-time = "2025-10-22T23:34:38.776Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/f6/601e87ab8e314e211850570325acc59dad27fc78268b919a8b8344e61785/autobahn-25.10.2-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:7b503e282082b823e85e963d00b49ed72aeae79d4d3b3929a997cad0780ffdb1", size = 517000, upload-time = "2025-10-22T23:34:40.069Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/d1/e8f42bbbcbac63ff7041322523479de78138cf527ff43a814b363dc9609a/autobahn-25.10.2-cp313-cp313-manylinux1_x86_64.manylinux_2_34_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:764f01bf3778a4e3fd1f6d3df9bfbf079772181c5e4915af5a588f7aa92d3d96", size = 569847, upload-time = "2025-10-22T23:34:41.103Z" },
+    { url = "https://files.pythonhosted.org/packages/36/04/8cd5eac3be9f5e61d8d7299f0813da0d420eb49964d1884394fa67f7d84f/autobahn-25.10.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a66252583ee2b00e55d4f311618f87e7f0c0d1f837bbeb40b9277fc86ea027fd", size = 545484, upload-time = "2025-10-22T23:34:42.304Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/c5/1512d54744a4565e2c3e1a16782d72241ed8a3800b12b9e097b5d4041d8d/autobahn-25.10.2-cp313-cp313-win_amd64.whl", hash = "sha256:684fe4200ede3840b4973da8c04f0026459c8189d6bc10df31b1d933b5f01b7e", size = 525963, upload-time = "2025-10-22T23:34:43.8Z" },
+    { url = "https://files.pythonhosted.org/packages/29/1b/135994077860a59b9c565f58b578ce02b7738034d573ebab989f19f06dd9/autobahn-25.10.2-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:2d957e30bc83edd71844a1c97e675c3bf970fccb0814669f374913665758782b", size = 517015, upload-time = "2025-10-22T23:34:45.918Z" },
+    { url = "https://files.pythonhosted.org/packages/23/65/5e94daac80b258a4192e79550a2f8e67fca031ca273a7b4ca13a3c9fc01b/autobahn-25.10.2-cp314-cp314-manylinux1_x86_64.manylinux_2_34_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4cd3965b58c557ee5f61949c2b660d70f4fda2dce05296c5fdf6a1516bc9b27d", size = 570072, upload-time = "2025-10-22T23:34:47.338Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/de/fd542547d685feff66287e984df5f67c3db588cce2e0b82b7c63c811d205/autobahn-25.10.2-cp314-cp314-win_amd64.whl", hash = "sha256:115101d665773da38cba23274d4b36ec10608aa1f84cede0011be0518dbd66e9", size = 526434, upload-time = "2025-10-22T23:34:49.553Z" },
 ]
 
 [[package]]
@@ -189,39 +186,6 @@ css = [
     { name = "tinycss2" },
 ]
 
-[[package]]
-name = "cbor2"
-version = "5.7.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/a2/b8/c0f6a7d46f816cb18b1fda61a2fe648abe16039f1ff93ea720a6e9fb3cee/cbor2-5.7.1.tar.gz", hash = "sha256:7a405a1d7c8230ee9acf240aad48ae947ef584e8af05f169f3c1bde8f01f8b71", size = 102467, upload-time = "2025-10-24T09:23:06.569Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/56/54/48426472f0c051982c647331441aed09b271a0500356ae0b7054c813d174/cbor2-5.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bd5ca44891c06f6b85d440836c967187dc1d30b15f86f315d55c675d3a841078", size = 69031, upload-time = "2025-10-24T09:22:25.438Z" },
-    { url = "https://files.pythonhosted.org/packages/d3/68/1dd58c7706e9752188358223db58c83f3c48e07f728aa84221ffd244652f/cbor2-5.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:537d73ef930ccc1a7b6a2e8d2cbf81407d270deb18e40cda5eb511bd70f71078", size = 68825, upload-time = "2025-10-24T09:22:26.497Z" },
-    { url = "https://files.pythonhosted.org/packages/09/4e/380562fe9f9995a1875fb5ec26fd041e19d61f4630cb690a98c5195945fc/cbor2-5.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:edbf814dd7763b6eda27a5770199f6ccd55bd78be8f4367092460261bfbf19d0", size = 286222, upload-time = "2025-10-24T09:22:27.546Z" },
-    { url = "https://files.pythonhosted.org/packages/7c/bb/9eccdc1ea3c4d5c7cdb2e49b9de49534039616be5455ce69bd64c0b2efe2/cbor2-5.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9fc81da8c0e09beb42923e455e477b36ff14a03b9ca18a8a2e9b462de9a953e8", size = 285688, upload-time = "2025-10-24T09:22:28.651Z" },
-    { url = "https://files.pythonhosted.org/packages/59/8c/4696d82f5bd04b3d45d9a64ec037fa242630c134e3218d6c252b4f59b909/cbor2-5.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e4a7d660d428911a3aadb7105e94438d7671ab977356fdf647a91aab751033bd", size = 277063, upload-time = "2025-10-24T09:22:29.775Z" },
-    { url = "https://files.pythonhosted.org/packages/95/50/6538e44ca970caaad2fa376b81701d073d84bf597aac07a59d0a253b1a7f/cbor2-5.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:228e0af9c0a9ddf6375b6ae010eaa1942a1901d403f134ac9ee6a76a322483f9", size = 278334, upload-time = "2025-10-24T09:22:30.904Z" },
-    { url = "https://files.pythonhosted.org/packages/64/a9/156ccd2207fb26b5b61d23728b4dbdc595d1600125aa79683a4a8ddc9313/cbor2-5.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:2d08a6c0d9ed778448e185508d870f4160ba74f59bb17a966abd0d14d0ff4dd3", size = 68404, upload-time = "2025-10-24T09:22:32.108Z" },
-    { url = "https://files.pythonhosted.org/packages/4f/49/adc53615e9dd32c4421f6935dfa2235013532c6e6b28ee515bbdd92618be/cbor2-5.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:752506cfe72da0f4014b468b30191470ee8919a64a0772bd3b36a4fccf5fcefc", size = 64047, upload-time = "2025-10-24T09:22:33.147Z" },
-    { url = "https://files.pythonhosted.org/packages/16/b1/51fb868fe38d893c570bb90b38d365ff0f00421402c1ae8f63b31b25d665/cbor2-5.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:59d5da59fffe89692d5bd1530eef4d26e4eb7aa794aaa1f4e192614786409009", size = 69068, upload-time = "2025-10-24T09:22:34.464Z" },
-    { url = "https://files.pythonhosted.org/packages/b9/db/5abc62ec456f552f617aac3359a5d7114b23be9c4d886169592cd5f074b9/cbor2-5.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:533117918d518e01348f8cd0331271c207e7224b9a1ed492a0ff00847f28edc8", size = 68927, upload-time = "2025-10-24T09:22:35.458Z" },
-    { url = "https://files.pythonhosted.org/packages/9a/c2/58d787395c99874d2a2395b3a22c9d48a3cfc5a7dcd5817bf74764998b75/cbor2-5.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8d6d9436ff3c3323ea5863ecf7ae1139590991685b44b9eb6b7bb1734a594af6", size = 285185, upload-time = "2025-10-24T09:22:36.867Z" },
-    { url = "https://files.pythonhosted.org/packages/d0/9c/b680b264a8f4b9aa59c95e166c816275a13138cbee92dd2917f58bca47b9/cbor2-5.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:661b871ca754a619fcd98c13a38b4696b2b57dab8b24235c00b0ba322c040d24", size = 284440, upload-time = "2025-10-24T09:22:38.08Z" },
-    { url = "https://files.pythonhosted.org/packages/1f/59/68183c655d6226d0eee10027f52516882837802a8d5746317a88362ed686/cbor2-5.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d8065aa90d715fd9bb28727b2d774ee16e695a0e1627ae76e54bf19f9d99d63f", size = 276876, upload-time = "2025-10-24T09:22:39.561Z" },
-    { url = "https://files.pythonhosted.org/packages/ee/a2/1964e0a569d2b81e8f4862753fee7701ae5773c22e45492a26f92f62e75a/cbor2-5.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb1b7047d73590cfe8e373e2c804fa99be47e55b1b6186602d0f86f384cecec1", size = 278216, upload-time = "2025-10-24T09:22:41.132Z" },
-    { url = "https://files.pythonhosted.org/packages/00/78/9b566d68cb88bb1ecebe354765625161c9d6060a16e55008006d6359f776/cbor2-5.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:31d511df7ebd6624fdb4cecdafb4ffb9a205f9ff8c8d98edd1bef0d27f944d74", size = 68451, upload-time = "2025-10-24T09:22:42.227Z" },
-    { url = "https://files.pythonhosted.org/packages/db/85/7a6a922d147d027fd5d8fd5224b39e8eaf152a42e8cf16351458096d3d62/cbor2-5.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:f5d37f7b0f84394d2995bd8722cb01c86a885c4821a864a34b7b4d9950c5e26e", size = 64111, upload-time = "2025-10-24T09:22:43.213Z" },
-    { url = "https://files.pythonhosted.org/packages/5f/f0/f220222a57371e33434ba7bdc25de31d611cbc0ade2a868e03c3553305e7/cbor2-5.7.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e5826e4fa4c33661960073f99cf67c82783895524fb66f3ebdd635c19b5a7d68", size = 69002, upload-time = "2025-10-24T09:22:44.316Z" },
-    { url = "https://files.pythonhosted.org/packages/c7/3c/34b62ba5173541659f248f005d13373530f02fb997b78fde00bf01ede4f4/cbor2-5.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f19a00d6ac9a77cb611073250b06bf4494b41ba78a1716704f7008e0927d9366", size = 69177, upload-time = "2025-10-24T09:22:45.711Z" },
-    { url = "https://files.pythonhosted.org/packages/77/fd/2400d820d9733df00a5c18aa74201e51d710fb91588687eb594f4a7688ea/cbor2-5.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d2113aea044cd172f199da3520bc4401af69eae96c5180ca7eb660941928cb89", size = 284259, upload-time = "2025-10-24T09:22:46.749Z" },
-    { url = "https://files.pythonhosted.org/packages/42/65/280488ef196c1d71ba123cd406ea47727bb3a0e057767a733d9793fcc428/cbor2-5.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f17eacea2d28fecf28ac413c1d7927cde0a11957487d2630655d6b5c9c46a0b", size = 281958, upload-time = "2025-10-24T09:22:48.876Z" },
-    { url = "https://files.pythonhosted.org/packages/42/82/bcdd3fdc73bd5f4194fdb08c808112010add9530bae1dcfdb1e2b2ceae19/cbor2-5.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d65deea39cae533a629561e7da672402c46731122b6129ed7c8eaa1efe04efce", size = 276025, upload-time = "2025-10-24T09:22:50.147Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/a8/a6065dd6a157b877d7d8f3fe96f410fb191a2db1e6588f4d20b5f9a507c2/cbor2-5.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:57d8cc29ec1fd20500748e0e767ff88c13afcee839081ba4478c41fcda6ee18b", size = 275978, upload-time = "2025-10-24T09:22:51.873Z" },
-    { url = "https://files.pythonhosted.org/packages/62/f4/37934045174af9e4253a340b43f07197af54002070cb80fae82d878f1f14/cbor2-5.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:94fb939d0946f80c49ba45105ca3a3e13e598fc9abd63efc6661b02d4b4d2c50", size = 70269, upload-time = "2025-10-24T09:22:53.275Z" },
-    { url = "https://files.pythonhosted.org/packages/0b/fd/933416643e7f5540ae818691fb23fa4189010c6efa39a12c4f59d825da28/cbor2-5.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4fd7225ac820bbb9f03bd16bc1a7efb6c4d1c451f22c0a153ff4ec46495c59c5", size = 66182, upload-time = "2025-10-24T09:22:54.697Z" },
-    { url = "https://files.pythonhosted.org/packages/d5/7d/383bafeabb54c17fe5b6d5aca4e863e6b7df10bcc833b34aa169e9dfce1a/cbor2-5.7.1-py3-none-any.whl", hash = "sha256:68834e4eff2f56629ce6422b0634bc3f74c5a4269de5363f5265fe452c706ba7", size = 23829, upload-time = "2025-10-24T09:23:05.54Z" },
-]
-
 [[package]]
 name = "certifi"
 version = "2025.11.12"
@@ -1434,50 +1398,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
 ]
 
-[[package]]
-name = "msgpack"
-version = "1.1.2"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/4d/f2/bfb55a6236ed8725a96b0aa3acbd0ec17588e6a2c3b62a93eb513ed8783f/msgpack-1.1.2.tar.gz", hash = "sha256:3b60763c1373dd60f398488069bcdc703cd08a711477b5d480eecc9f9626f47e", size = 173581, upload-time = "2025-10-08T09:15:56.596Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ad/bd/8b0d01c756203fbab65d265859749860682ccd2a59594609aeec3a144efa/msgpack-1.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:70a0dff9d1f8da25179ffcf880e10cf1aad55fdb63cd59c9a49a1b82290062aa", size = 81939, upload-time = "2025-10-08T09:15:01.472Z" },
-    { url = "https://files.pythonhosted.org/packages/34/68/ba4f155f793a74c1483d4bdef136e1023f7bcba557f0db4ef3db3c665cf1/msgpack-1.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:446abdd8b94b55c800ac34b102dffd2f6aa0ce643c55dfc017ad89347db3dbdb", size = 85064, upload-time = "2025-10-08T09:15:03.764Z" },
-    { url = "https://files.pythonhosted.org/packages/f2/60/a064b0345fc36c4c3d2c743c82d9100c40388d77f0b48b2f04d6041dbec1/msgpack-1.1.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c63eea553c69ab05b6747901b97d620bb2a690633c77f23feb0c6a947a8a7b8f", size = 417131, upload-time = "2025-10-08T09:15:05.136Z" },
-    { url = "https://files.pythonhosted.org/packages/65/92/a5100f7185a800a5d29f8d14041f61475b9de465ffcc0f3b9fba606e4505/msgpack-1.1.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:372839311ccf6bdaf39b00b61288e0557916c3729529b301c52c2d88842add42", size = 427556, upload-time = "2025-10-08T09:15:06.837Z" },
-    { url = "https://files.pythonhosted.org/packages/f5/87/ffe21d1bf7d9991354ad93949286f643b2bb6ddbeab66373922b44c3b8cc/msgpack-1.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2929af52106ca73fcb28576218476ffbb531a036c2adbcf54a3664de124303e9", size = 404920, upload-time = "2025-10-08T09:15:08.179Z" },
-    { url = "https://files.pythonhosted.org/packages/ff/41/8543ed2b8604f7c0d89ce066f42007faac1eaa7d79a81555f206a5cdb889/msgpack-1.1.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:be52a8fc79e45b0364210eef5234a7cf8d330836d0a64dfbb878efa903d84620", size = 415013, upload-time = "2025-10-08T09:15:09.83Z" },
-    { url = "https://files.pythonhosted.org/packages/41/0d/2ddfaa8b7e1cee6c490d46cb0a39742b19e2481600a7a0e96537e9c22f43/msgpack-1.1.2-cp312-cp312-win32.whl", hash = "sha256:1fff3d825d7859ac888b0fbda39a42d59193543920eda9d9bea44d958a878029", size = 65096, upload-time = "2025-10-08T09:15:11.11Z" },
-    { url = "https://files.pythonhosted.org/packages/8c/ec/d431eb7941fb55a31dd6ca3404d41fbb52d99172df2e7707754488390910/msgpack-1.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:1de460f0403172cff81169a30b9a92b260cb809c4cb7e2fc79ae8d0510c78b6b", size = 72708, upload-time = "2025-10-08T09:15:12.554Z" },
-    { url = "https://files.pythonhosted.org/packages/c5/31/5b1a1f70eb0e87d1678e9624908f86317787b536060641d6798e3cf70ace/msgpack-1.1.2-cp312-cp312-win_arm64.whl", hash = "sha256:be5980f3ee0e6bd44f3a9e9dea01054f175b50c3e6cdb692bc9424c0bbb8bf69", size = 64119, upload-time = "2025-10-08T09:15:13.589Z" },
-    { url = "https://files.pythonhosted.org/packages/6b/31/b46518ecc604d7edf3a4f94cb3bf021fc62aa301f0cb849936968164ef23/msgpack-1.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4efd7b5979ccb539c221a4c4e16aac1a533efc97f3b759bb5a5ac9f6d10383bf", size = 81212, upload-time = "2025-10-08T09:15:14.552Z" },
-    { url = "https://files.pythonhosted.org/packages/92/dc/c385f38f2c2433333345a82926c6bfa5ecfff3ef787201614317b58dd8be/msgpack-1.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:42eefe2c3e2af97ed470eec850facbe1b5ad1d6eacdbadc42ec98e7dcf68b4b7", size = 84315, upload-time = "2025-10-08T09:15:15.543Z" },
-    { url = "https://files.pythonhosted.org/packages/d3/68/93180dce57f684a61a88a45ed13047558ded2be46f03acb8dec6d7c513af/msgpack-1.1.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1fdf7d83102bf09e7ce3357de96c59b627395352a4024f6e2458501f158bf999", size = 412721, upload-time = "2025-10-08T09:15:16.567Z" },
-    { url = "https://files.pythonhosted.org/packages/5d/ba/459f18c16f2b3fc1a1ca871f72f07d70c07bf768ad0a507a698b8052ac58/msgpack-1.1.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fac4be746328f90caa3cd4bc67e6fe36ca2bf61d5c6eb6d895b6527e3f05071e", size = 424657, upload-time = "2025-10-08T09:15:17.825Z" },
-    { url = "https://files.pythonhosted.org/packages/38/f8/4398c46863b093252fe67368b44edc6c13b17f4e6b0e4929dbf0bdb13f23/msgpack-1.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fffee09044073e69f2bad787071aeec727183e7580443dfeb8556cbf1978d162", size = 402668, upload-time = "2025-10-08T09:15:19.003Z" },
-    { url = "https://files.pythonhosted.org/packages/28/ce/698c1eff75626e4124b4d78e21cca0b4cc90043afb80a507626ea354ab52/msgpack-1.1.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5928604de9b032bc17f5099496417f113c45bc6bc21b5c6920caf34b3c428794", size = 419040, upload-time = "2025-10-08T09:15:20.183Z" },
-    { url = "https://files.pythonhosted.org/packages/67/32/f3cd1667028424fa7001d82e10ee35386eea1408b93d399b09fb0aa7875f/msgpack-1.1.2-cp313-cp313-win32.whl", hash = "sha256:a7787d353595c7c7e145e2331abf8b7ff1e6673a6b974ded96e6d4ec09f00c8c", size = 65037, upload-time = "2025-10-08T09:15:21.416Z" },
-    { url = "https://files.pythonhosted.org/packages/74/07/1ed8277f8653c40ebc65985180b007879f6a836c525b3885dcc6448ae6cb/msgpack-1.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:a465f0dceb8e13a487e54c07d04ae3ba131c7c5b95e2612596eafde1dccf64a9", size = 72631, upload-time = "2025-10-08T09:15:22.431Z" },
-    { url = "https://files.pythonhosted.org/packages/e5/db/0314e4e2db56ebcf450f277904ffd84a7988b9e5da8d0d61ab2d057df2b6/msgpack-1.1.2-cp313-cp313-win_arm64.whl", hash = "sha256:e69b39f8c0aa5ec24b57737ebee40be647035158f14ed4b40e6f150077e21a84", size = 64118, upload-time = "2025-10-08T09:15:23.402Z" },
-    { url = "https://files.pythonhosted.org/packages/22/71/201105712d0a2ff07b7873ed3c220292fb2ea5120603c00c4b634bcdafb3/msgpack-1.1.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e23ce8d5f7aa6ea6d2a2b326b4ba46c985dbb204523759984430db7114f8aa00", size = 81127, upload-time = "2025-10-08T09:15:24.408Z" },
-    { url = "https://files.pythonhosted.org/packages/1b/9f/38ff9e57a2eade7bf9dfee5eae17f39fc0e998658050279cbb14d97d36d9/msgpack-1.1.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6c15b7d74c939ebe620dd8e559384be806204d73b4f9356320632d783d1f7939", size = 84981, upload-time = "2025-10-08T09:15:25.812Z" },
-    { url = "https://files.pythonhosted.org/packages/8e/a9/3536e385167b88c2cc8f4424c49e28d49a6fc35206d4a8060f136e71f94c/msgpack-1.1.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:99e2cb7b9031568a2a5c73aa077180f93dd2e95b4f8d3b8e14a73ae94a9e667e", size = 411885, upload-time = "2025-10-08T09:15:27.22Z" },
-    { url = "https://files.pythonhosted.org/packages/2f/40/dc34d1a8d5f1e51fc64640b62b191684da52ca469da9cd74e84936ffa4a6/msgpack-1.1.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:180759d89a057eab503cf62eeec0aa61c4ea1200dee709f3a8e9397dbb3b6931", size = 419658, upload-time = "2025-10-08T09:15:28.4Z" },
-    { url = "https://files.pythonhosted.org/packages/3b/ef/2b92e286366500a09a67e03496ee8b8ba00562797a52f3c117aa2b29514b/msgpack-1.1.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:04fb995247a6e83830b62f0b07bf36540c213f6eac8e851166d8d86d83cbd014", size = 403290, upload-time = "2025-10-08T09:15:29.764Z" },
-    { url = "https://files.pythonhosted.org/packages/78/90/e0ea7990abea5764e4655b8177aa7c63cdfa89945b6e7641055800f6c16b/msgpack-1.1.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8e22ab046fa7ede9e36eeb4cfad44d46450f37bb05d5ec482b02868f451c95e2", size = 415234, upload-time = "2025-10-08T09:15:31.022Z" },
-    { url = "https://files.pythonhosted.org/packages/72/4e/9390aed5db983a2310818cd7d3ec0aecad45e1f7007e0cda79c79507bb0d/msgpack-1.1.2-cp314-cp314-win32.whl", hash = "sha256:80a0ff7d4abf5fecb995fcf235d4064b9a9a8a40a3ab80999e6ac1e30b702717", size = 66391, upload-time = "2025-10-08T09:15:32.265Z" },
-    { url = "https://files.pythonhosted.org/packages/6e/f1/abd09c2ae91228c5f3998dbd7f41353def9eac64253de3c8105efa2082f7/msgpack-1.1.2-cp314-cp314-win_amd64.whl", hash = "sha256:9ade919fac6a3e7260b7f64cea89df6bec59104987cbea34d34a2fa15d74310b", size = 73787, upload-time = "2025-10-08T09:15:33.219Z" },
-    { url = "https://files.pythonhosted.org/packages/6a/b0/9d9f667ab48b16ad4115c1935d94023b82b3198064cb84a123e97f7466c1/msgpack-1.1.2-cp314-cp314-win_arm64.whl", hash = "sha256:59415c6076b1e30e563eb732e23b994a61c159cec44deaf584e5cc1dd662f2af", size = 66453, upload-time = "2025-10-08T09:15:34.225Z" },
-    { url = "https://files.pythonhosted.org/packages/16/67/93f80545eb1792b61a217fa7f06d5e5cb9e0055bed867f43e2b8e012e137/msgpack-1.1.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:897c478140877e5307760b0ea66e0932738879e7aa68144d9b78ea4c8302a84a", size = 85264, upload-time = "2025-10-08T09:15:35.61Z" },
-    { url = "https://files.pythonhosted.org/packages/87/1c/33c8a24959cf193966ef11a6f6a2995a65eb066bd681fd085afd519a57ce/msgpack-1.1.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a668204fa43e6d02f89dbe79a30b0d67238d9ec4c5bd8a940fc3a004a47b721b", size = 89076, upload-time = "2025-10-08T09:15:36.619Z" },
-    { url = "https://files.pythonhosted.org/packages/fc/6b/62e85ff7193663fbea5c0254ef32f0c77134b4059f8da89b958beb7696f3/msgpack-1.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5559d03930d3aa0f3aacb4c42c776af1a2ace2611871c84a75afe436695e6245", size = 435242, upload-time = "2025-10-08T09:15:37.647Z" },
-    { url = "https://files.pythonhosted.org/packages/c1/47/5c74ecb4cc277cf09f64e913947871682ffa82b3b93c8dad68083112f412/msgpack-1.1.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:70c5a7a9fea7f036b716191c29047374c10721c389c21e9ffafad04df8c52c90", size = 432509, upload-time = "2025-10-08T09:15:38.794Z" },
-    { url = "https://files.pythonhosted.org/packages/24/a4/e98ccdb56dc4e98c929a3f150de1799831c0a800583cde9fa022fa90602d/msgpack-1.1.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f2cb069d8b981abc72b41aea1c580ce92d57c673ec61af4c500153a626cb9e20", size = 415957, upload-time = "2025-10-08T09:15:40.238Z" },
-    { url = "https://files.pythonhosted.org/packages/da/28/6951f7fb67bc0a4e184a6b38ab71a92d9ba58080b27a77d3e2fb0be5998f/msgpack-1.1.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d62ce1f483f355f61adb5433ebfd8868c5f078d1a52d042b0a998682b4fa8c27", size = 422910, upload-time = "2025-10-08T09:15:41.505Z" },
-    { url = "https://files.pythonhosted.org/packages/f0/03/42106dcded51f0a0b5284d3ce30a671e7bd3f7318d122b2ead66ad289fed/msgpack-1.1.2-cp314-cp314t-win32.whl", hash = "sha256:1d1418482b1ee984625d88aa9585db570180c286d942da463533b238b98b812b", size = 75197, upload-time = "2025-10-08T09:15:42.954Z" },
-    { url = "https://files.pythonhosted.org/packages/15/86/d0071e94987f8db59d4eeb386ddc64d0bb9b10820a8d82bcd3e53eeb2da6/msgpack-1.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:5a46bf7e831d09470ad92dff02b8b1ac92175ca36b087f904a0519857c6be3ff", size = 85772, upload-time = "2025-10-08T09:15:43.954Z" },
-    { url = "https://files.pythonhosted.org/packages/81/f2/08ace4142eb281c12701fc3b93a10795e4d4dc7f753911d836675050f886/msgpack-1.1.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d99ef64f349d5ec3293688e91486c5fdb925ed03807f64d98d205d2713c60b46", size = 70868, upload-time = "2025-10-08T09:15:44.959Z" },
-]
-
 [[package]]
 name = "nest-asyncio"
 version = "1.6.0"
@@ -1873,12 +1793,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" },
 ]
 
-[[package]]
-name = "py-ubjson"
-version = "0.16.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/1d/c7/28220d37e041fe1df03e857fe48f768dcd30cd151480bf6f00da8713214a/py-ubjson-0.16.1.tar.gz", hash = "sha256:b9bfb8695a1c7e3632e800fb83c943bf67ed45ddd87cd0344851610c69a5a482", size = 50316, upload-time = "2020-04-18T15:05:57.698Z" }
-
 [[package]]
 name = "pyasn1"
 version = "0.6.1"
@@ -2988,67 +2902,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1", size = 348521, upload-time = "2025-12-13T17:45:33.889Z" },
 ]
 
-[[package]]
-name = "u-msgpack-python"
-version = "2.8.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/36/9d/a40411a475e7d4838994b7f6bcc6bfca9acc5b119ce3a7503608c4428b49/u-msgpack-python-2.8.0.tar.gz", hash = "sha256:b801a83d6ed75e6df41e44518b4f2a9c221dc2da4bcd5380e3a0feda520bc61a", size = 18167, upload-time = "2023-05-18T09:28:12.187Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/b1/5e/512aeb40fd819f4660d00f96f5c7371ee36fc8c6b605128c5ee59e0b28c6/u_msgpack_python-2.8.0-py2.py3-none-any.whl", hash = "sha256:1d853d33e78b72c4228a2025b4db28cda81214076e5b0422ed0ae1b1b2bb586a", size = 10590, upload-time = "2023-05-18T09:28:10.323Z" },
-]
-
-[[package]]
-name = "ujson"
-version = "5.11.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/43/d9/3f17e3c5773fb4941c68d9a37a47b1a79c9649d6c56aefbed87cc409d18a/ujson-5.11.0.tar.gz", hash = "sha256:e204ae6f909f099ba6b6b942131cee359ddda2b6e4ea39c12eb8b991fe2010e0", size = 7156583, upload-time = "2025-08-20T11:57:02.452Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/b9/ef/a9cb1fce38f699123ff012161599fb9f2ff3f8d482b4b18c43a2dc35073f/ujson-5.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7895f0d2d53bd6aea11743bd56e3cb82d729980636cd0ed9b89418bf66591702", size = 55434, upload-time = "2025-08-20T11:55:34.987Z" },
-    { url = "https://files.pythonhosted.org/packages/b1/05/dba51a00eb30bd947791b173766cbed3492269c150a7771d2750000c965f/ujson-5.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12b5e7e22a1fe01058000d1b317d3b65cc3daf61bd2ea7a2b76721fe160fa74d", size = 53190, upload-time = "2025-08-20T11:55:36.384Z" },
-    { url = "https://files.pythonhosted.org/packages/03/3c/fd11a224f73fbffa299fb9644e425f38b38b30231f7923a088dd513aabb4/ujson-5.11.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0180a480a7d099082501cad1fe85252e4d4bf926b40960fb3d9e87a3a6fbbc80", size = 57600, upload-time = "2025-08-20T11:55:37.692Z" },
-    { url = "https://files.pythonhosted.org/packages/55/b9/405103cae24899df688a3431c776e00528bd4799e7d68820e7ebcf824f92/ujson-5.11.0-cp312-cp312-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:fa79fdb47701942c2132a9dd2297a1a85941d966d8c87bfd9e29b0cf423f26cc", size = 59791, upload-time = "2025-08-20T11:55:38.877Z" },
-    { url = "https://files.pythonhosted.org/packages/17/7b/2dcbc2bbfdbf68f2368fb21ab0f6735e872290bb604c75f6e06b81edcb3f/ujson-5.11.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8254e858437c00f17cb72e7a644fc42dad0ebb21ea981b71df6e84b1072aaa7c", size = 57356, upload-time = "2025-08-20T11:55:40.036Z" },
-    { url = "https://files.pythonhosted.org/packages/d1/71/fea2ca18986a366c750767b694430d5ded6b20b6985fddca72f74af38a4c/ujson-5.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1aa8a2ab482f09f6c10fba37112af5f957689a79ea598399c85009f2f29898b5", size = 1036313, upload-time = "2025-08-20T11:55:41.408Z" },
-    { url = "https://files.pythonhosted.org/packages/a3/bb/d4220bd7532eac6288d8115db51710fa2d7d271250797b0bfba9f1e755af/ujson-5.11.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a638425d3c6eed0318df663df44480f4a40dc87cc7c6da44d221418312f6413b", size = 1195782, upload-time = "2025-08-20T11:55:43.357Z" },
-    { url = "https://files.pythonhosted.org/packages/80/47/226e540aa38878ce1194454385701d82df538ccb5ff8db2cf1641dde849a/ujson-5.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7e3cff632c1d78023b15f7e3a81c3745cd3f94c044d1e8fa8efbd6b161997bbc", size = 1088817, upload-time = "2025-08-20T11:55:45.262Z" },
-    { url = "https://files.pythonhosted.org/packages/7e/81/546042f0b23c9040d61d46ea5ca76f0cc5e0d399180ddfb2ae976ebff5b5/ujson-5.11.0-cp312-cp312-win32.whl", hash = "sha256:be6b0eaf92cae8cdee4d4c9e074bde43ef1c590ed5ba037ea26c9632fb479c88", size = 39757, upload-time = "2025-08-20T11:55:46.522Z" },
-    { url = "https://files.pythonhosted.org/packages/44/1b/27c05dc8c9728f44875d74b5bfa948ce91f6c33349232619279f35c6e817/ujson-5.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:b7b136cc6abc7619124fd897ef75f8e63105298b5ca9bdf43ebd0e1fa0ee105f", size = 43859, upload-time = "2025-08-20T11:55:47.987Z" },
-    { url = "https://files.pythonhosted.org/packages/22/2d/37b6557c97c3409c202c838aa9c960ca3896843b4295c4b7bb2bbd260664/ujson-5.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:6cd2df62f24c506a0ba322d5e4fe4466d47a9467b57e881ee15a31f7ecf68ff6", size = 38361, upload-time = "2025-08-20T11:55:49.122Z" },
-    { url = "https://files.pythonhosted.org/packages/1c/ec/2de9dd371d52c377abc05d2b725645326c4562fc87296a8907c7bcdf2db7/ujson-5.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:109f59885041b14ee9569bf0bb3f98579c3fa0652317b355669939e5fc5ede53", size = 55435, upload-time = "2025-08-20T11:55:50.243Z" },
-    { url = "https://files.pythonhosted.org/packages/5b/a4/f611f816eac3a581d8a4372f6967c3ed41eddbae4008d1d77f223f1a4e0a/ujson-5.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a31c6b8004438e8c20fc55ac1c0e07dad42941db24176fe9acf2815971f8e752", size = 53193, upload-time = "2025-08-20T11:55:51.373Z" },
-    { url = "https://files.pythonhosted.org/packages/e9/c5/c161940967184de96f5cbbbcce45b562a4bf851d60f4c677704b1770136d/ujson-5.11.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78c684fb21255b9b90320ba7e199780f653e03f6c2528663768965f4126a5b50", size = 57603, upload-time = "2025-08-20T11:55:52.583Z" },
-    { url = "https://files.pythonhosted.org/packages/2b/d6/c7b2444238f5b2e2d0e3dab300b9ddc3606e4b1f0e4bed5a48157cebc792/ujson-5.11.0-cp313-cp313-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:4c9f5d6a27d035dd90a146f7761c2272cf7103de5127c9ab9c4cd39ea61e878a", size = 59794, upload-time = "2025-08-20T11:55:53.69Z" },
-    { url = "https://files.pythonhosted.org/packages/fe/a3/292551f936d3d02d9af148f53e1bc04306b00a7cf1fcbb86fa0d1c887242/ujson-5.11.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:837da4d27fed5fdc1b630bd18f519744b23a0b5ada1bbde1a36ba463f2900c03", size = 57363, upload-time = "2025-08-20T11:55:54.843Z" },
-    { url = "https://files.pythonhosted.org/packages/90/a6/82cfa70448831b1a9e73f882225980b5c689bf539ec6400b31656a60ea46/ujson-5.11.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:787aff4a84da301b7f3bac09bc696e2e5670df829c6f8ecf39916b4e7e24e701", size = 1036311, upload-time = "2025-08-20T11:55:56.197Z" },
-    { url = "https://files.pythonhosted.org/packages/84/5c/96e2266be50f21e9b27acaee8ca8f23ea0b85cb998c33d4f53147687839b/ujson-5.11.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6dd703c3e86dc6f7044c5ac0b3ae079ed96bf297974598116aa5fb7f655c3a60", size = 1195783, upload-time = "2025-08-20T11:55:58.081Z" },
-    { url = "https://files.pythonhosted.org/packages/8d/20/78abe3d808cf3bb3e76f71fca46cd208317bf461c905d79f0d26b9df20f1/ujson-5.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3772e4fe6b0c1e025ba3c50841a0ca4786825a4894c8411bf8d3afe3a8061328", size = 1088822, upload-time = "2025-08-20T11:55:59.469Z" },
-    { url = "https://files.pythonhosted.org/packages/d8/50/8856e24bec5e2fc7f775d867aeb7a3f137359356200ac44658f1f2c834b2/ujson-5.11.0-cp313-cp313-win32.whl", hash = "sha256:8fa2af7c1459204b7a42e98263b069bd535ea0cd978b4d6982f35af5a04a4241", size = 39753, upload-time = "2025-08-20T11:56:01.345Z" },
-    { url = "https://files.pythonhosted.org/packages/5b/d8/1baee0f4179a4d0f5ce086832147b6cc9b7731c24ca08e14a3fdb8d39c32/ujson-5.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:34032aeca4510a7c7102bd5933f59a37f63891f30a0706fb46487ab6f0edf8f0", size = 43866, upload-time = "2025-08-20T11:56:02.552Z" },
-    { url = "https://files.pythonhosted.org/packages/a9/8c/6d85ef5be82c6d66adced3ec5ef23353ed710a11f70b0b6a836878396334/ujson-5.11.0-cp313-cp313-win_arm64.whl", hash = "sha256:ce076f2df2e1aa62b685086fbad67f2b1d3048369664b4cdccc50707325401f9", size = 38363, upload-time = "2025-08-20T11:56:03.688Z" },
-    { url = "https://files.pythonhosted.org/packages/28/08/4518146f4984d112764b1dfa6fb7bad691c44a401adadaa5e23ccd930053/ujson-5.11.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:65724738c73645db88f70ba1f2e6fb678f913281804d5da2fd02c8c5839af302", size = 55462, upload-time = "2025-08-20T11:56:04.873Z" },
-    { url = "https://files.pythonhosted.org/packages/29/37/2107b9a62168867a692654d8766b81bd2fd1e1ba13e2ec90555861e02b0c/ujson-5.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:29113c003ca33ab71b1b480bde952fbab2a0b6b03a4ee4c3d71687cdcbd1a29d", size = 53246, upload-time = "2025-08-20T11:56:06.054Z" },
-    { url = "https://files.pythonhosted.org/packages/9b/f8/25583c70f83788edbe3ca62ce6c1b79eff465d78dec5eb2b2b56b3e98b33/ujson-5.11.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c44c703842024d796b4c78542a6fcd5c3cb948b9fc2a73ee65b9c86a22ee3638", size = 57631, upload-time = "2025-08-20T11:56:07.374Z" },
-    { url = "https://files.pythonhosted.org/packages/ed/ca/19b3a632933a09d696f10dc1b0dfa1d692e65ad507d12340116ce4f67967/ujson-5.11.0-cp314-cp314-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:e750c436fb90edf85585f5c62a35b35082502383840962c6983403d1bd96a02c", size = 59877, upload-time = "2025-08-20T11:56:08.534Z" },
-    { url = "https://files.pythonhosted.org/packages/55/7a/4572af5324ad4b2bfdd2321e898a527050290147b4ea337a79a0e4e87ec7/ujson-5.11.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f278b31a7c52eb0947b2db55a5133fbc46b6f0ef49972cd1a80843b72e135aba", size = 57363, upload-time = "2025-08-20T11:56:09.758Z" },
-    { url = "https://files.pythonhosted.org/packages/7b/71/a2b8c19cf4e1efe53cf439cdf7198ac60ae15471d2f1040b490c1f0f831f/ujson-5.11.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ab2cb8351d976e788669c8281465d44d4e94413718af497b4e7342d7b2f78018", size = 1036394, upload-time = "2025-08-20T11:56:11.168Z" },
-    { url = "https://files.pythonhosted.org/packages/7a/3e/7b98668cba3bb3735929c31b999b374ebc02c19dfa98dfebaeeb5c8597ca/ujson-5.11.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:090b4d11b380ae25453100b722d0609d5051ffe98f80ec52853ccf8249dfd840", size = 1195837, upload-time = "2025-08-20T11:56:12.6Z" },
-    { url = "https://files.pythonhosted.org/packages/a1/ea/8870f208c20b43571a5c409ebb2fe9b9dba5f494e9e60f9314ac01ea8f78/ujson-5.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:80017e870d882d5517d28995b62e4e518a894f932f1e242cbc802a2fd64d365c", size = 1088837, upload-time = "2025-08-20T11:56:14.15Z" },
-    { url = "https://files.pythonhosted.org/packages/63/b6/c0e6607e37fa47929920a685a968c6b990a802dec65e9c5181e97845985d/ujson-5.11.0-cp314-cp314-win32.whl", hash = "sha256:1d663b96eb34c93392e9caae19c099ec4133ba21654b081956613327f0e973ac", size = 41022, upload-time = "2025-08-20T11:56:15.509Z" },
-    { url = "https://files.pythonhosted.org/packages/4e/56/f4fe86b4c9000affd63e9219e59b222dc48b01c534533093e798bf617a7e/ujson-5.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:849e65b696f0d242833f1df4182096cedc50d414215d1371fca85c541fbff629", size = 45111, upload-time = "2025-08-20T11:56:16.597Z" },
-    { url = "https://files.pythonhosted.org/packages/0a/f3/669437f0280308db4783b12a6d88c00730b394327d8334cc7a32ef218e64/ujson-5.11.0-cp314-cp314-win_arm64.whl", hash = "sha256:e73df8648c9470af2b6a6bf5250d4744ad2cf3d774dcf8c6e31f018bdd04d764", size = 39682, upload-time = "2025-08-20T11:56:17.763Z" },
-    { url = "https://files.pythonhosted.org/packages/6e/cd/e9809b064a89fe5c4184649adeb13c1b98652db3f8518980b04227358574/ujson-5.11.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:de6e88f62796372fba1de973c11138f197d3e0e1d80bcb2b8aae1e826096d433", size = 55759, upload-time = "2025-08-20T11:56:18.882Z" },
-    { url = "https://files.pythonhosted.org/packages/1b/be/ae26a6321179ebbb3a2e2685b9007c71bcda41ad7a77bbbe164005e956fc/ujson-5.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:49e56ef8066f11b80d620985ae36869a3ff7e4b74c3b6129182ec5d1df0255f3", size = 53634, upload-time = "2025-08-20T11:56:20.012Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/e9/fb4a220ee6939db099f4cfeeae796ecb91e7584ad4d445d4ca7f994a9135/ujson-5.11.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1a325fd2c3a056cf6c8e023f74a0c478dd282a93141356ae7f16d5309f5ff823", size = 58547, upload-time = "2025-08-20T11:56:21.175Z" },
-    { url = "https://files.pythonhosted.org/packages/bd/f8/fc4b952b8f5fea09ea3397a0bd0ad019e474b204cabcb947cead5d4d1ffc/ujson-5.11.0-cp314-cp314t-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:a0af6574fc1d9d53f4ff371f58c96673e6d988ed2b5bf666a6143c782fa007e9", size = 60489, upload-time = "2025-08-20T11:56:22.342Z" },
-    { url = "https://files.pythonhosted.org/packages/2e/e5/af5491dfda4f8b77e24cf3da68ee0d1552f99a13e5c622f4cef1380925c3/ujson-5.11.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10f29e71ecf4ecd93a6610bd8efa8e7b6467454a363c3d6416db65de883eb076", size = 58035, upload-time = "2025-08-20T11:56:23.92Z" },
-    { url = "https://files.pythonhosted.org/packages/c4/09/0945349dd41f25cc8c38d78ace49f14c5052c5bbb7257d2f466fa7bdb533/ujson-5.11.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1a0a9b76a89827a592656fe12e000cf4f12da9692f51a841a4a07aa4c7ecc41c", size = 1037212, upload-time = "2025-08-20T11:56:25.274Z" },
-    { url = "https://files.pythonhosted.org/packages/49/44/8e04496acb3d5a1cbee3a54828d9652f67a37523efa3d3b18a347339680a/ujson-5.11.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:b16930f6a0753cdc7d637b33b4e8f10d5e351e1fb83872ba6375f1e87be39746", size = 1196500, upload-time = "2025-08-20T11:56:27.517Z" },
-    { url = "https://files.pythonhosted.org/packages/64/ae/4bc825860d679a0f208a19af2f39206dfd804ace2403330fdc3170334a2f/ujson-5.11.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:04c41afc195fd477a59db3a84d5b83a871bd648ef371cf8c6f43072d89144eef", size = 1089487, upload-time = "2025-08-20T11:56:29.07Z" },
-    { url = "https://files.pythonhosted.org/packages/30/ed/5a057199fb0a5deabe0957073a1c1c1c02a3e99476cd03daee98ea21fa57/ujson-5.11.0-cp314-cp314t-win32.whl", hash = "sha256:aa6d7a5e09217ff93234e050e3e380da62b084e26b9f2e277d2606406a2fc2e5", size = 41859, upload-time = "2025-08-20T11:56:30.495Z" },
-    { url = "https://files.pythonhosted.org/packages/aa/03/b19c6176bdf1dc13ed84b886e99677a52764861b6cc023d5e7b6ebda249d/ujson-5.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:48055e1061c1bb1f79e75b4ac39e821f3f35a9b82de17fce92c3140149009bec", size = 46183, upload-time = "2025-08-20T11:56:31.574Z" },
-    { url = "https://files.pythonhosted.org/packages/5d/ca/a0413a3874b2dc1708b8796ca895bf363292f9c70b2e8ca482b7dbc0259d/ujson-5.11.0-cp314-cp314t-win_arm64.whl", hash = "sha256:1194b943e951092db611011cb8dbdb6cf94a3b816ed07906e14d3bc6ce0e90ab", size = 40264, upload-time = "2025-08-20T11:56:32.773Z" },
-]
-
 [[package]]
 name = "urllib3"
 version = "2.6.2"

From 9903b855937bc2d5a6fbc842a3149b5970858b07 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 15 Dec 2025 22:51:07 +0000
Subject: [PATCH 76/93] fix merge issues

---
 radis/extractions/factories.py                | 27 +++++++++----------
 .../migrations/0005_outputfield_is_array.py   | 16 -----------
 radis/extractions/models.py                   |  1 -
 radis/urls.py                                 |  2 +-
 4 files changed, 14 insertions(+), 32 deletions(-)

diff --git a/radis/extractions/factories.py b/radis/extractions/factories.py
index 469cb0e2..63a640ee 100644
--- a/radis/extractions/factories.py
+++ b/radis/extractions/factories.py
@@ -1,16 +1,15 @@
+from typing import cast
+
 import factory
+from adit_radis_shared.accounts.factories import GroupFactory, UserFactory
+from adit_radis_shared.accounts.models import User
+from adit_radis_shared.common.utils.testing_helpers import add_user_to_group
+from django.contrib.auth.models import Group
 from faker import Faker
 
-from adit_radis_shared.accounts.factories import UserFactory
 from radis.reports.factories import ModalityFactory
 
-from .models import (
-    ExtractionInstance,
-    ExtractionJob,
-    ExtractionTask,
-    OutputField,
-    OutputType,
-)
+from .models import ExtractionInstance, ExtractionJob, ExtractionTask, OutputField, OutputType
 
 fake = Faker()
 
@@ -29,7 +28,7 @@ class Meta:
 
     owner = factory.SubFactory(UserFactory)
     title = factory.Faker("sentence", nb_words=3)
-    group = factory.SubFactory("adit_radis_shared.accounts.factories.GroupFactory")
+    group = factory.SubFactory(GroupFactory)
     query = factory.Faker("word")
     language = factory.SubFactory("radis.reports.factories.LanguageFactory")
     study_date_from = factory.Faker("date")
@@ -59,14 +58,14 @@ def modalities(self, create, extracted, **kwargs):
             self.modalities.add(ModalityFactory(code=modality))  # type: ignore
 
     @factory.post_generation
-    def ensure_owner_in_group(self, create, extracted, **kwargs):
+    def ensure_owner_in_group(obj, create, extracted, **kwargs):
+        owner = cast(User, obj.owner)
+        group = cast(Group, obj.group)
+
         if not create:
             return
 
-        self.owner.groups.add(self.group)
-        if self.owner.active_group_id is None:
-            self.owner.active_group = self.group
-            self.owner.save(update_fields=["active_group"])
+        add_user_to_group(owner, group)
 
 
 class OutputFieldFactory(BaseDjangoModelFactory[OutputField]):
diff --git a/radis/extractions/migrations/0005_outputfield_is_array.py b/radis/extractions/migrations/0005_outputfield_is_array.py
index 1227b60a..568a02c3 100644
--- a/radis/extractions/migrations/0005_outputfield_is_array.py
+++ b/radis/extractions/migrations/0005_outputfield_is_array.py
@@ -1,22 +1,6 @@
 from django.db import migrations, models
 
 
-class Migration(migrations.Migration):
-    dependencies = [
-        ("extractions", "0004_outputfield_selection_options"),
-    ]
-
-    operations = [
-        # Track whether an output field should return a list of values rather than a single value.
-        migrations.AddField(
-            model_name="outputfield",
-            name="is_array",
-            field=models.BooleanField(default=False),
-        ),
-    ]
-from django.db import migrations, models
-
-
 class Migration(migrations.Migration):
     dependencies = [
         ("extractions", "0004_outputfield_selection_options"),
diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index 13e33018..8d8b024a 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -88,7 +88,6 @@ class OutputField(models.Model):
         max_length=1, choices=OutputType.choices, default=OutputType.TEXT
     )
     get_output_type_display: Callable[[], str]
-    optional = models.BooleanField(default=False)
     selection_options = models.JSONField(default=list, blank=True)
     is_array = models.BooleanField(default=False)
     job = models.ForeignKey[ExtractionJob](
diff --git a/radis/urls.py b/radis/urls.py
index 64ce6bf6..12984602 100644
--- a/radis/urls.py
+++ b/radis/urls.py
@@ -14,9 +14,9 @@
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
 
+from django.apps import apps
 from django.contrib import admin
 from django.urls import include, path
-from django.apps import apps
 
 urlpatterns = [
     path("django-admin/", include("loginas.urls")),

From 85fc8bdf45b823b82a620b4f8fc6461d3effdab0 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 15 Dec 2025 22:51:48 +0000
Subject: [PATCH 77/93] fix syntax error

---
 radis/extractions/utils/processor_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/radis/extractions/utils/processor_utils.py b/radis/extractions/utils/processor_utils.py
index 4a6b92e0..3e338351 100644
--- a/radis/extractions/utils/processor_utils.py
+++ b/radis/extractions/utils/processor_utils.py
@@ -15,9 +15,9 @@ def generate_output_fields_schema(fields: Iterable[OutputField]) -> type[BaseMod
     for field in fields:
         if field.output_type == OutputType.TEXT:
             output_type = str
-        elif field_type == OutputType.NUMERIC:
+        elif field.output_type == OutputType.NUMERIC:
             output_type = Numeric
-        elif field_type == OutputType.BOOLEAN:
+        elif field.output_type == OutputType.BOOLEAN:
             output_type = bool
         elif field.output_type == OutputType.SELECTION:
             options = tuple(field.selection_options)

From 775b9c2067cf19d1f5524cc9b4cd01f3a8d287bd Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 16 Dec 2025 09:31:21 +0000
Subject: [PATCH 78/93] swap search and extraction step in extractionjobwizard

---
 radis/extractions/forms.py                    |  36 +--
 .../extraction_job_output_fields_form.html    |  28 +-
 .../extraction_job_search_form.html           |  25 +-
 .../extraction_job_wizard_summary.html        |  12 +-
 radis/extractions/views.py                    | 251 +++++++-----------
 5 files changed, 165 insertions(+), 187 deletions(-)

diff --git a/radis/extractions/forms.py b/radis/extractions/forms.py
index 9d6b8378..b8a760f1 100644
--- a/radis/extractions/forms.py
+++ b/radis/extractions/forms.py
@@ -38,12 +38,12 @@ class Meta:
         help_texts = {
             "title": "Title of the extraction job",
             "query": (
-                "Search query to filter reports "
-                "(leave empty to auto-generate from extraction fields)"
+                "Search query to filter reports. "
+                "This query was auto-generated from your extraction fields - you can edit or refine it."
             ),
         }
         widgets = {
-            "query": forms.TextInput(attrs={"placeholder": "Optional - auto-generated if empty"}),
+            "query": forms.TextInput(attrs={"placeholder": "Auto-generated query (editable)"}),
         }
 
     def __init__(self, *args, **kwargs):
@@ -51,7 +51,7 @@ def __init__(self, *args, **kwargs):
 
         super().__init__(*args, **kwargs)
 
-        self.fields["query"].required = False
+        self.fields["query"].required = True
         self.fields["language"].choices = [  # type: ignore
             (language.pk, LANGUAGE_LABELS[language.code])
             for language in Language.objects.order_by("code")
@@ -99,7 +99,7 @@ def build_layout(self):
                 Column(
                     "title",
                     "query",
-                    Submit("next", "Next Step (Output Fields)", css_class="btn-primary"),
+                    Submit("next", "Next Step (Summary)", css_class="btn-primary"),
                 ),
                 Column(
                     "language",
@@ -118,8 +118,9 @@ def build_layout(self):
     def clean_query(self) -> str:
         query = self.cleaned_data["query"].strip()
         if not query:
-            # Query is empty - will be auto-generated from output fields
-            return ""
+            raise forms.ValidationError(
+                "A search query is required. Please enter a query or go back to regenerate from fields."
+            )
         query_node, _ = QueryParser().parse(query)
         if query_node is None:
             raise forms.ValidationError("Invalid query syntax")
@@ -136,20 +137,21 @@ def clean(self) -> dict[str, Any] | None:
 
         query = cleaned_data.get("query", "").strip()
 
-        # Check if query generation is needed
+        # Query should always be present now (either generated or user-edited)
         if not query:
-            # Mark for auto-generation from output fields
-            cleaned_data["requires_query_generation"] = True
-            cleaned_data["retrieval_count"] = None  # Will be calculated after query generation
-            return cleaned_data
+            raise forms.ValidationError(
+                "A search query is required. Please enter a query or go back to regenerate from fields."
+            )
 
-        # Validate manual query
+        # Validate the query
         query_node, fixes = QueryParser().parse(query)
-        assert query_node
+        if query_node is None:
+            raise forms.ValidationError("Invalid query syntax")
 
         if len(fixes) > 0:
             cleaned_data["fixed_query"] = QueryParser.unparse(query_node)
 
+        # Calculate retrieval count
         search = Search(
             query=query_node,
             offset=0,
@@ -169,14 +171,16 @@ def clean(self) -> dict[str, Any] | None:
 
         if extraction_retrieval_provider is None:
             raise forms.ValidationError("Extraction retrieval provider is not configured.")
+
         retrieval_count = extraction_retrieval_provider.count(search)
         cleaned_data["retrieval_count"] = retrieval_count
 
+        # Validate against limits
         if retrieval_count > settings.EXTRACTION_MAXIMUM_REPORTS_COUNT:
             raise forms.ValidationError(
                 f"Your search returned more results ({retrieval_count}) than the extraction "
                 f"pipeline can handle (max. {settings.EXTRACTION_MAXIMUM_REPORTS_COUNT}). "
-                "Please refine your search."
+                "Please refine your search query."
             )
 
         if (
@@ -188,8 +192,6 @@ def clean(self) -> dict[str, Any] | None:
                 "provider can handle. Please refine your search."
             )
 
-        cleaned_data["requires_query_generation"] = False
-
         return cleaned_data
 
 
diff --git a/radis/extractions/templates/extractions/extraction_job_output_fields_form.html b/radis/extractions/templates/extractions/extraction_job_output_fields_form.html
index 89b707dd..83fbca03 100644
--- a/radis/extractions/templates/extractions/extraction_job_output_fields_form.html
+++ b/radis/extractions/templates/extractions/extraction_job_output_fields_form.html
@@ -5,7 +5,7 @@
     New Extraction Job
 {% endblock title %}
 {% block heading %}
-    <c-page-heading title="New Extraction Job - Output Fields">
+    <c-page-heading title="New Extraction Job - Define Fields (Step 1 of 3)">
     <c-slot name="left">
     <button type="button"
             class="btn btn-info"
@@ -29,33 +29,25 @@
                         <button type="submit"
                                 name="wizard_goto_step"
                                 value="{{ wizard.steps.prev }}"
-                                class="btn btn-secondary">Previous Step (Search)</button>
-                        <button type="submit" class="btn btn-primary">Next Step (Summary)</button>
+                                class="btn btn-secondary"
+                                disabled>Previous Step</button>
+                        <button type="submit" class="btn btn-primary">Next Step (Search Query)</button>
                     </div>
                 </div>
             </form>
         </div>
         <div class="col-4">
-            {% if fixed_query %}
-                <div class="alert alert-warning" role="alert">
-                    Fixed invalid query, instead using:
-                    <br />
-                    <span class="font-monospace">{{ fixed_query }}</span>
-                </div>
-            {% endif %}
             <p>
-                {% if retrieval_count is None %}
-                    Your search will hit approximately <strong>TBD reports</strong> (to be determined after query generation).
-                {% else %}
-                    Your search will hit approximately <strong>{{ retrieval_count }} report{{ retrieval_count|pluralize }}</strong>.
-                {% endif %}
-                The more reports you analyze the longer it will take. If you like to refine your search you can go back to
-                the previous step and adjust the query and/or filters.
+                <strong>Step 1:</strong> Define the fields you want to extract from medical reports.
             </p>
             <p>
-                If the result count sounds reasonable then provide the fields to extract. Choose a short and concise name and more detailed description.
+                Choose a short and concise name and more detailed description for each field.
                 You can add up to 5 fields.
             </p>
+            <p>
+                In the next step, we will automatically generate a search query based on these fields,
+                which you can then review and refine.
+            </p>
             <p>
                 Example for an output field:
                 <br />
diff --git a/radis/extractions/templates/extractions/extraction_job_search_form.html b/radis/extractions/templates/extractions/extraction_job_search_form.html
index dc59fd95..8a22d2e2 100644
--- a/radis/extractions/templates/extractions/extraction_job_search_form.html
+++ b/radis/extractions/templates/extractions/extraction_job_search_form.html
@@ -5,7 +5,7 @@
     New Extraction Job
 {% endblock title %}
 {% block heading %}
-    <c-page-heading title="New Extraction Job - Search">
+    <c-page-heading title="New Extraction Job - Search Query (Step 2 of 3)">
     <c-slot name="left">
     <button type="button"
             class="btn btn-info"
@@ -24,9 +24,32 @@
     </c-page-heading>
 {% endblock heading %}
 {% block content %}
+    {% if generated_query %}
+        <div class="alert alert-info mb-4" role="alert">
+            <h5 class="alert-heading">
+                {% bootstrap_icon "magic" %} Auto-Generated Query
+            </h5>
+            <p>
+                Based on your {{ output_fields_count }} extraction field{{ output_fields_count|pluralize }},
+                we generated the following search query:
+            </p>
+            <code class="d-block bg-light p-2 mb-2">{{ generated_query }}</code>
+            <p class="mb-0">
+                <small>
+                    You can edit this query below to refine your search.
+                    The query will help find relevant medical reports for your extraction.
+                </small>
+            </p>
+        </div>
+    {% endif %}
     <form action="" method="post">
         {% csrf_token %}
         {{ wizard.management_form }}
         {% crispy form %}
+        {% if retrieval_count is not None %}
+            <div class="alert alert-success mt-3" role="alert">
+                Your search will retrieve approximately <strong>{{ retrieval_count }} report{{ retrieval_count|pluralize }}</strong>.
+            </div>
+        {% endif %}
     </form>
 {% endblock content %}
diff --git a/radis/extractions/templates/extractions/extraction_job_wizard_summary.html b/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
index a9580b82..5607b3a7 100644
--- a/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
+++ b/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
@@ -6,7 +6,7 @@
     New Extraction Job
 {% endblock title %}
 {% block heading %}
-    <c-page-heading title="New Extraction Job - Summary">
+    <c-page-heading title="New Extraction Job - Review & Submit (Step 3 of 3)">
     <c-slot name="left">
     <button type="button"
             class="btn btn-info"
@@ -27,6 +27,14 @@
         <dt class="col-sm-3">Search Query</dt>
         <dd class="col-sm-9">
             {{ search.query }}
+            {% if query_metadata.success %}
+                <br />
+                <small class="text-muted">
+                    {% bootstrap_icon "info-circle" %}
+                    Auto-generated from extraction fields
+                    {% if query_metadata.fixes_applied %}(with automatic fixes applied){% endif %}
+                </small>
+            {% endif %}
         </dd>
         <dt class="col-sm-3">Language</dt>
         <dd class="col-sm-9">
@@ -125,7 +133,7 @@
             <button type="submit"
                     name="wizard_goto_step"
                     value="{{ wizard.steps.prev }}"
-                    class="btn btn-secondary">Previous Step (Output Fields)</button>
+                    class="btn btn-secondary">Previous Step (Search Query)</button>
             <button type="submit" class="btn btn-primary">Create Extraction Job</button>
         </div>
     </form>
diff --git a/radis/extractions/views.py b/radis/extractions/views.py
index 3769b5e6..3aad83a3 100644
--- a/radis/extractions/views.py
+++ b/radis/extractions/views.py
@@ -1,9 +1,7 @@
 import csv
 from collections.abc import Generator
-from datetime import date
-from typing import Any, Literal, Type, cast
+from typing import Any, Type, cast
 
-from adit_radis_shared.accounts.models import User
 from adit_radis_shared.common.mixins import (
     PageSizeSelectMixin,
 )
@@ -40,8 +38,6 @@
     AnalysisTaskResetView,
     BaseUpdatePreferencesView,
 )
-from radis.reports.models import Language, Modality
-from radis.search.site import Search, SearchFilters
 from radis.search.utils.query_parser import QueryParser
 
 from .filters import ExtractionInstanceFilter, ExtractionJobFilter, ExtractionTaskFilter
@@ -52,7 +48,6 @@
 )
 from .mixins import ExtractionsLockedMixin
 from .models import ExtractionInstance, ExtractionJob, ExtractionTask
-from .site import extraction_retrieval_provider
 from .tables import (
     ExtractionInstanceTable,
     ExtractionJobTable,
@@ -81,11 +76,11 @@ class ExtractionJobListView(ExtractionsLockedMixin, AnalysisJobListView):
 class ExtractionJobWizardView(
     LoginRequiredMixin, PermissionRequiredMixin, UserPassesTestMixin, SessionWizardView
 ):
-    SEARCH_STEP = "0"
-    OUTPUT_FIELDS_STEP = "1"
+    OUTPUT_FIELDS_STEP = "0"
+    SEARCH_STEP = "1"
     SUMMARY_STEP = "2"
 
-    form_list = [SearchForm, OutputFieldFormSet, SummaryForm]
+    form_list = [OutputFieldFormSet, SearchForm, SummaryForm]
     permission_required = "extractions.add_extractionjob"
     permission_denied_message = "You must be logged in and have an active group"
     request: AuthenticatedHttpRequest
@@ -99,146 +94,107 @@ def get_form_kwargs(self, step=None):
             kwargs["user"] = self.request.user
         return kwargs
 
+    def process_step(self, form):
+        """Process validated form data and trigger query generation after output fields step."""
+        step_data = self.get_form_step_data(form)
+
+        # After output fields are submitted, generate query
+        if self.steps.current == ExtractionJobWizardView.OUTPUT_FIELDS_STEP:
+            # Extract output fields from formset
+            from .models import OutputField
+
+            formset_data = []
+            if hasattr(form, "cleaned_data"):
+                formset_data = form.cleaned_data
+
+            temp_fields = [
+                OutputField(
+                    name=field_data["name"],
+                    description=field_data["description"],
+                    output_type=field_data["output_type"],
+                )
+                for field_data in formset_data
+                if not field_data.get("DELETE", False)
+            ]
+
+            # Generate query from fields
+            generator = QueryGenerator()
+            generated_query, metadata = generator.generate_from_fields(temp_fields)
+
+            # Store in wizard storage for use in next step
+            self.storage.extra_data["generated_query"] = generated_query or ""
+            self.storage.extra_data["query_metadata"] = metadata
+            self.storage.extra_data["query_generation_attempted"] = True
+
+            # Check if generation failed
+            if generated_query is None or not metadata.get("success"):
+                from django.contrib import messages
+
+                messages.warning(
+                    self.request,
+                    "Unable to automatically generate a query from your extraction fields. "
+                    "You can manually enter a search query in the next step.",
+                )
+
+        return step_data
+
+    def get_form_initial(self, step=None):
+        """Provide initial data for forms, including generated query for search step."""
+        initial = super().get_form_initial(step)
+
+        if step == ExtractionJobWizardView.SEARCH_STEP:
+            # Pre-populate query field with generated query
+            generated_query = self.storage.extra_data.get("generated_query", "")
+            if generated_query:
+                initial["query"] = generated_query
+
+        return initial
+
     def get_context_data(self, form, **kwargs):
         context = super().get_context_data(form, **kwargs)
 
-        if self.steps.current == ExtractionJobWizardView.SEARCH_STEP:
-            pass
-
-        elif self.steps.current == ExtractionJobWizardView.OUTPUT_FIELDS_STEP:
+        if self.steps.current == ExtractionJobWizardView.OUTPUT_FIELDS_STEP:
+            # First step - just show the formset
             context["formset"] = form
 
-            data = self.get_cleaned_data_for_step(ExtractionJobWizardView.SEARCH_STEP)
-            assert data and isinstance(data, dict)
-
-            context["fixed_query"] = data.get("fixed_query")
-            context["retrieval_count"] = data.get("retrieval_count", 0)
-            context["requires_query_generation"] = data.get("requires_query_generation", False)
+        elif self.steps.current == ExtractionJobWizardView.SEARCH_STEP:
+            # Second step - show generated query info and retrieval count
+            context["generated_query"] = self.storage.extra_data.get("generated_query", "")
+            context["query_metadata"] = self.storage.extra_data.get("query_metadata", {})
 
-            # Store for use in summary step
-            self.storage.extra_data["retrieval_count"] = data.get("retrieval_count", 0)
-            self.storage.extra_data["requires_query_generation"] = data.get(
-                "requires_query_generation", False
+            # Get output fields data to show context
+            output_fields_data = self.get_cleaned_data_for_step(
+                ExtractionJobWizardView.OUTPUT_FIELDS_STEP
             )
+            if output_fields_data:
+                context["output_fields_count"] = len(
+                    [
+                        f
+                        for f in output_fields_data
+                        if not cast(dict[str, Any], f).get("DELETE", False)
+                    ]
+                )
+
+            # If query is populated (either generated or user-edited), show estimated count
+            if form.is_bound and hasattr(form, "cleaned_data") and form.cleaned_data:
+                retrieval_count = form.cleaned_data.get("retrieval_count")
+                if retrieval_count is not None:
+                    context["retrieval_count"] = retrieval_count
+                    # Store for summary step
+                    self.storage.extra_data["retrieval_count"] = retrieval_count
 
         elif self.steps.current == ExtractionJobWizardView.SUMMARY_STEP:
-            search_data = self.get_cleaned_data_for_step(ExtractionJobWizardView.SEARCH_STEP)
+            # Final step - show everything for review
             output_fields_data = self.get_cleaned_data_for_step(
                 ExtractionJobWizardView.OUTPUT_FIELDS_STEP
             )
-            assert search_data and isinstance(search_data, dict)
-            assert output_fields_data and isinstance(output_fields_data, list)
-            # Auto-generate query if needed
-            if self.storage.extra_data.get("requires_query_generation", False):
-                # Create temporary OutputField objects for query generation
-                from .models import OutputField
-
-                temp_fields = [
-                    OutputField(
-                        name=field_data["name"],
-                        description=field_data["description"],
-                        output_type=field_data["output_type"],
-                    )
-                    for field_data in output_fields_data
-                    if not field_data.get("DELETE", False)
-                ]
-
-                generator = QueryGenerator()
-                generated_query, metadata = generator.generate_from_fields(temp_fields)
-
-                # Check if generation failed
-                if generated_query is None or not metadata.get("success"):
-                    from django.contrib import messages
-
-                    messages.error(
-                        self.request,
-                        "Unable to automatically generate a query from your extraction fields. "
-                        "Please go back to Step 1 and manually enter a search query.",
-                    )
-                    # Redirect back to step 0
-                    self.storage.current_step = self.steps.first
-                    return self.render_goto_step(self.steps.first)
-
-                # Store generated query
-                search_data["query"] = generated_query
-                search_data["query_metadata"] = metadata
-                self.storage.extra_data["generated_query"] = generated_query
-                self.storage.extra_data["query_metadata"] = metadata
-
-                # Calculate retrieval count for auto-generated query
-                # Parse the generated query
-                query_node, _ = QueryParser().parse(generated_query)
-                assert query_node  # Should be valid since it was just generated and validated
-
-                # Build search with all filters from search_data
-                user = cast(User, self.request.user)
-                active_group = user.active_group
-                assert active_group
-
-                language = cast(Language, search_data.get("language"))
-                modalities = cast(QuerySet[Modality], search_data.get("modalities"))
-
-                search = Search(
-                    query=query_node,
-                    offset=0,
-                    limit=0,
-                    filters=SearchFilters(
-                        group=active_group.pk,
-                        language=language.code,
-                        modalities=list(modalities.values_list("code", flat=True))
-                        if modalities
-                        else [],
-                        study_date_from=cast("date | None", search_data.get("study_date_from")),
-                        study_date_till=cast("date | None", search_data.get("study_date_till")),
-                        study_description=cast(str, search_data.get("study_description")) or "",
-                        patient_sex=cast(Literal["M", "F"], search_data.get("patient_sex")) or None,
-                        patient_age_from=cast("int | None", search_data.get("age_from")),
-                        patient_age_till=cast("int | None", search_data.get("age_till")),
-                    ),
-                )
+            search_data = self.get_cleaned_data_for_step(ExtractionJobWizardView.SEARCH_STEP)
 
-                # Get the actual count
-                if extraction_retrieval_provider is None:
-                    from django.contrib import messages
-
-                    messages.error(self.request, "Extraction retrieval provider is not configured.")
-                    self.storage.current_step = self.steps.first
-                    return self.render_goto_step(self.steps.first)
-
-                retrieval_count = extraction_retrieval_provider.count(search)
-                self.storage.extra_data["retrieval_count"] = retrieval_count
-
-                # Validate against limits
-                if retrieval_count > settings.EXTRACTION_MAXIMUM_REPORTS_COUNT:
-                    from django.contrib import messages
-
-                    messages.error(
-                        self.request,
-                        f"Your auto-generated query returned {retrieval_count} results, "
-                        "which exceeds the maximum limit of "
-                        f"{settings.EXTRACTION_MAXIMUM_REPORTS_COUNT}. "
-                        "Please go back and refine your extraction fields or add a manual query.",
-                    )
-                    self.storage.current_step = self.steps.first
-                    return self.render_goto_step(self.steps.first)
-
-                if (
-                    extraction_retrieval_provider.max_results
-                    and retrieval_count > extraction_retrieval_provider.max_results
-                ):
-                    from django.contrib import messages
-
-                    messages.error(
-                        self.request,
-                        f"Your auto-generated query returned {retrieval_count} results, "
-                        "which exceeds the provider's limit. "
-                        "Please refine your extraction fields or add a manual query.",
-                    )
-                    self.storage.current_step = self.steps.first
-                    return self.render_goto_step(self.steps.first)
+            assert output_fields_data and isinstance(output_fields_data, list)
+            assert search_data and isinstance(search_data, dict)
 
-            context["search"] = search_data
             context["output_fields"] = output_fields_data
+            context["search"] = search_data
             context["retrieval_count"] = self.storage.extra_data.get("retrieval_count", 0)
             context["query_metadata"] = self.storage.extra_data.get("query_metadata", {})
 
@@ -246,10 +202,10 @@ def get_context_data(self, form, **kwargs):
 
     def get_template_names(self) -> list[str]:
         step = self.steps.current
-        if step == ExtractionJobWizardView.SEARCH_STEP:
-            return ["extractions/extraction_job_search_form.html"]
-        elif step == ExtractionJobWizardView.OUTPUT_FIELDS_STEP:
+        if step == ExtractionJobWizardView.OUTPUT_FIELDS_STEP:
             return ["extractions/extraction_job_output_fields_form.html"]
+        elif step == ExtractionJobWizardView.SEARCH_STEP:
+            return ["extractions/extraction_job_search_form.html"]
         elif step == ExtractionJobWizardView.SUMMARY_STEP:
             return ["extractions/extraction_job_wizard_summary.html"]
         else:
@@ -257,29 +213,26 @@ def get_template_names(self) -> list[str]:
 
     def done(
         self,
-        form_objs: tuple[SearchForm, BaseInlineFormSet, SummaryForm],
+        form_objs: tuple[BaseInlineFormSet, SearchForm, SummaryForm],
         **kwargs,
     ):
         user = self.request.user
 
         with transaction.atomic():
+            output_fields_formset = form_objs[0]
+            search_form = form_objs[1]
             summary_form = form_objs[2]
-            job_form = form_objs[0]
-
-            # Use generated query if it was auto-generated
-            if self.storage.extra_data.get("requires_query_generation", False):
-                generated_query = self.storage.extra_data.get("generated_query", "")
-                job_form.instance.query = generated_query
 
+            # The query is always in search_form now (no conditional logic needed)
             if summary_form.cleaned_data["send_finished_mail"]:
-                job_form.cleaned_data["send_finished_mail"] = True
+                search_form.cleaned_data["send_finished_mail"] = True
 
-            job: ExtractionJob = job_form.save(commit=False)
+            job: ExtractionJob = search_form.save(commit=False)
 
+            # Parse and normalize the query
             query = job.query
             query_node, fixes = QueryParser().parse(query)
             if len(fixes) > 0:
-                # The query was already validated that it is not empty by the form
                 assert query_node
                 job.query = QueryParser.unparse(query_node)
 
@@ -291,8 +244,8 @@ def done(
             job.save()
 
             # Save output fields
-            form_objs[1].instance = job
-            form_objs[1].save()
+            output_fields_formset.instance = job
+            output_fields_formset.save()
 
             if user.is_staff or settings.START_EXTRACTION_JOB_UNVERIFIED:
                 job.status = ExtractionJob.Status.PENDING

From 1f07595b8b2ac62763516051365ecaa5d7358853 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 16 Dec 2025 09:34:10 +0000
Subject: [PATCH 79/93] fix migration order

---
 ...extractionjob_query.py => 0009_alter_extractionjob_query.py} | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename radis/extractions/migrations/{0007_alter_extractionjob_query.py => 0009_alter_extractionjob_query.py} (84%)

diff --git a/radis/extractions/migrations/0007_alter_extractionjob_query.py b/radis/extractions/migrations/0009_alter_extractionjob_query.py
similarity index 84%
rename from radis/extractions/migrations/0007_alter_extractionjob_query.py
rename to radis/extractions/migrations/0009_alter_extractionjob_query.py
index 13cd4833..cdb335b5 100644
--- a/radis/extractions/migrations/0007_alter_extractionjob_query.py
+++ b/radis/extractions/migrations/0009_alter_extractionjob_query.py
@@ -6,7 +6,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ("extractions", "0006_outputfield_subscription_and_more"),
+        ("extractions", "0008_outputfield_subscription_and_more"),
     ]
 
     operations = [

From b378d9b4db17c71bb8881aa5f5f2cdb8b32eda3a Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 16 Dec 2025 15:11:32 +0000
Subject: [PATCH 80/93] live update report count and search link

---
 radis/extractions/forms.py                    |   9 +-
 .../extractions/_search_preview.html          |  36 ++++
 .../extraction_job_search_form.html           |  81 ++++++++-
 radis/extractions/urls.py                     |   6 +
 radis/extractions/views.py                    | 169 +++++++++++++++++-
 radis/search/site.py                          |   2 +-
 6 files changed, 287 insertions(+), 16 deletions(-)
 create mode 100644 radis/extractions/templates/extractions/_search_preview.html

diff --git a/radis/extractions/forms.py b/radis/extractions/forms.py
index b8a760f1..c462f247 100644
--- a/radis/extractions/forms.py
+++ b/radis/extractions/forms.py
@@ -39,7 +39,8 @@ class Meta:
             "title": "Title of the extraction job",
             "query": (
                 "Search query to filter reports. "
-                "This query was auto-generated from your extraction fields - you can edit or refine it."
+                "This query was auto-generated from your extraction fields"
+                " - you can edit or refine it."
             ),
         }
         widgets = {
@@ -119,7 +120,8 @@ def clean_query(self) -> str:
         query = self.cleaned_data["query"].strip()
         if not query:
             raise forms.ValidationError(
-                "A search query is required. Please enter a query or go back to regenerate from fields."
+                "A search query is required. "
+                "Please enter a query or go back to regenerate from fields."
             )
         query_node, _ = QueryParser().parse(query)
         if query_node is None:
@@ -140,7 +142,8 @@ def clean(self) -> dict[str, Any] | None:
         # Query should always be present now (either generated or user-edited)
         if not query:
             raise forms.ValidationError(
-                "A search query is required. Please enter a query or go back to regenerate from fields."
+                "A search query is required. "
+                "Please enter a query or go back to regenerate from fields."
             )
 
         # Validate the query
diff --git a/radis/extractions/templates/extractions/_search_preview.html b/radis/extractions/templates/extractions/_search_preview.html
new file mode 100644
index 00000000..d5b4354e
--- /dev/null
+++ b/radis/extractions/templates/extractions/_search_preview.html
@@ -0,0 +1,36 @@
+{% load bootstrap_icon from common_extras %}
+
+{% if error %}
+    <div class="alert alert-warning" role="alert">
+        {% bootstrap_icon "exclamation-triangle" %}
+        <strong>{{ error }}</strong>
+    </div>
+{% elif count is None %}
+    <div class="alert alert-secondary" role="alert">
+        {% bootstrap_icon "info-circle" %}
+        Enter a query to see the number of reports that will be retrieved.
+    </div>
+{% else %}
+    <div class="alert alert-success d-flex justify-content-between align-items-center" role="alert">
+        <div>
+            {% bootstrap_icon "check-circle" %}
+            <strong>{{ count }} report{{ count|pluralize }}</strong> will be retrieved
+            {% if count > max_reports_limit %}
+                <br />
+                <small class="text-danger">
+                    {% bootstrap_icon "exclamation-triangle" %}
+                    This exceeds the maximum limit of {{ max_reports_limit }} reports. Please refine your query.
+                </small>
+            {% endif %}
+        </div>
+        {% if search_url %}
+            <a href="{{ search_url }}"
+               target="_blank"
+               class="btn btn-sm btn-outline-primary"
+               rel="noopener noreferrer">
+                {% bootstrap_icon "box-arrow-up-right" %}
+                Preview Search Results
+            </a>
+        {% endif %}
+    </div>
+{% endif %}
diff --git a/radis/extractions/templates/extractions/extraction_job_search_form.html b/radis/extractions/templates/extractions/extraction_job_search_form.html
index 8a22d2e2..7d42ad99 100644
--- a/radis/extractions/templates/extractions/extraction_job_search_form.html
+++ b/radis/extractions/templates/extractions/extraction_job_search_form.html
@@ -26,9 +26,7 @@
 {% block content %}
     {% if generated_query %}
         <div class="alert alert-info mb-4" role="alert">
-            <h5 class="alert-heading">
-                {% bootstrap_icon "magic" %} Auto-Generated Query
-            </h5>
+            <h5 class="alert-heading">{% bootstrap_icon "magic" %} Auto-Generated Query</h5>
             <p>
                 Based on your {{ output_fields_count }} extraction field{{ output_fields_count|pluralize }},
                 we generated the following search query:
@@ -42,14 +40,81 @@ <h5 class="alert-heading">
             </p>
         </div>
     {% endif %}
-    <form action="" method="post">
+    <form action=""
+          method="post"
+          x-data="{ isLoading: false }"
+          @input.debounce.500ms=" console.log('[DEBUG] Form input event:', { fieldName: $event.target.name, fieldType: $event.target.type, fieldValue: $event.target.value, target: $event.target }); isLoading = true; console.log('[DEBUG] Triggering updatePreview event'); htmx.trigger('#search-preview', 'updatePreview'); ">
         {% csrf_token %}
         {{ wizard.management_form }}
         {% crispy form %}
-        {% if retrieval_count is not None %}
-            <div class="alert alert-success mt-3" role="alert">
-                Your search will retrieve approximately <strong>{{ retrieval_count }} report{{ retrieval_count|pluralize }}</strong>.
+        <!-- Live Search Preview Container -->
+        <div id="search-preview"
+             class="mt-3"
+             hx-get="{% url 'extraction_search_preview' %}"
+             hx-trigger="updatePreview, load"
+             hx-include="[name='1-query'], [name='1-language'], [name='1-modalities'], [name='1-study_date_from'], [name='1-study_date_till'], [name='1-study_description'], [name='1-patient_sex'], [name='1-age_from'], [name='1-age_till']"
+             hx-target="this"
+             hx-swap="innerHTML"
+             x-on:htmx:after-request="isLoading = false">
+            <!-- Initial placeholder or server-rendered content -->
+            <div class="alert alert-secondary" role="alert">
+                <span class="spinner-border spinner-border-sm me-2"
+                      role="status"
+                      aria-hidden="true"></span>
+                Loading search preview...
             </div>
-        {% endif %}
+        </div>
     </form>
+    <script>
+    // Debug: Log HTMX events
+    document.addEventListener('htmx:beforeRequest', function(evt) {
+        console.log('[DEBUG] HTMX beforeRequest:', {
+            url: evt.detail.pathInfo?.requestPath,
+            target: evt.detail.target,
+            trigger: evt.detail.triggeringEvent
+        });
+    });
+
+    document.addEventListener('htmx:afterRequest', function(evt) {
+        console.log('[DEBUG] HTMX afterRequest:', {
+            status: evt.detail.xhr.status,
+            successful: evt.detail.successful,
+            responseLength: evt.detail.xhr.response?.length
+        });
+    });
+
+    document.addEventListener('htmx:responseError', function(evt) {
+        console.error('[DEBUG] HTMX responseError:', {
+            error: evt.detail,
+            status: evt.detail.xhr?.status
+        });
+    });
+
+    // Debug: Check initial state when DOM loads
+    document.addEventListener('DOMContentLoaded', function() {
+        console.log('[DEBUG] DOMContentLoaded fired');
+        const queryField = document.querySelector('[name="query"]');
+        console.log('[DEBUG] Query field:', {
+            exists: !!queryField,
+            value: queryField?.value,
+            hasValue: queryField?.value?.trim().length > 0,
+            tagName: queryField?.tagName
+        });
+
+        const previewDiv = document.querySelector('#search-preview');
+        console.log('[DEBUG] Preview div:', {
+            exists: !!previewDiv,
+            hxGet: previewDiv?.getAttribute('hx-get'),
+            hxTrigger: previewDiv?.getAttribute('hx-trigger')
+        });
+
+        // Log if htmx is loaded
+        console.log('[DEBUG] HTMX loaded:', typeof htmx !== 'undefined');
+    });
+
+    // Debug: Listen for the custom updatePreview event
+    document.addEventListener('updatePreview', function(evt) {
+        console.log('[DEBUG] Custom updatePreview event received:', evt);
+    });
+    </script>
 {% endblock content %}
diff --git a/radis/extractions/urls.py b/radis/extractions/urls.py
index 71184dc1..42daea38 100644
--- a/radis/extractions/urls.py
+++ b/radis/extractions/urls.py
@@ -14,6 +14,7 @@
     ExtractionJobWizardView,
     ExtractionResultDownloadView,
     ExtractionResultListView,
+    ExtractionSearchPreviewView,
     ExtractionTaskDeleteView,
     ExtractionTaskDetailView,
     ExtractionTaskResetView,
@@ -40,6 +41,11 @@
         ExtractionJobWizardView.as_view(),
         name="extraction_job_create",
     ),
+    path(
+        "jobs/new/search-preview/",
+        ExtractionSearchPreviewView.as_view(),
+        name="extraction_search_preview",
+    ),
     path(
         "jobs/<int:pk>/",
         ExtractionJobDetailView.as_view(),
diff --git a/radis/extractions/views.py b/radis/extractions/views.py
index 3aad83a3..b07c3f4d 100644
--- a/radis/extractions/views.py
+++ b/radis/extractions/views.py
@@ -1,7 +1,10 @@
 import csv
 from collections.abc import Generator
-from typing import Any, Type, cast
+from datetime import datetime
+from typing import Any, Literal, Type, Union, cast
+from urllib.parse import urlencode
 
+from adit_radis_shared.accounts.models import User
 from adit_radis_shared.common.mixins import (
     PageSizeSelectMixin,
 )
@@ -17,10 +20,10 @@
 from django.db.models import QuerySet
 from django.forms import BaseInlineFormSet
 from django.http import StreamingHttpResponse
-from django.shortcuts import redirect
-from django.urls import reverse_lazy
+from django.shortcuts import redirect, render
+from django.urls import reverse, reverse_lazy
 from django.utils.text import slugify
-from django.views.generic import DetailView
+from django.views.generic import DetailView, View
 from django_tables2 import SingleTableMixin, tables
 from formtools.wizard.views import SessionWizardView
 
@@ -38,6 +41,8 @@
     AnalysisTaskResetView,
     BaseUpdatePreferencesView,
 )
+from radis.reports.models import Language, Modality
+from radis.search.site import Search, SearchFilters
 from radis.search.utils.query_parser import QueryParser
 
 from .filters import ExtractionInstanceFilter, ExtractionJobFilter, ExtractionTaskFilter
@@ -48,6 +53,7 @@
 )
 from .mixins import ExtractionsLockedMixin
 from .models import ExtractionInstance, ExtractionJob, ExtractionTask
+from .site import extraction_retrieval_provider
 from .tables import (
     ExtractionInstanceTable,
     ExtractionJobTable,
@@ -256,6 +262,161 @@ def done(
         return redirect(job)
 
 
+class ExtractionSearchPreviewView(LoginRequiredMixin, View):
+    """HTMX endpoint for live search preview: count and link."""
+
+    request: AuthenticatedHttpRequest
+
+    def get(self, request: AuthenticatedHttpRequest):
+        # Extract query and filter parameters from GET
+        query_str = request.GET.get("query", "").strip()
+        language_id = request.GET.get("language")
+        modality_ids = request.GET.getlist("modalities")
+
+        # Get string values from GET parameters
+        study_date_from_str = request.GET.get("study_date_from", "").strip()
+        study_date_till_str = request.GET.get("study_date_till", "").strip()
+        study_description = request.GET.get("study_description", "")
+        patient_sex_str = request.GET.get("patient_sex")
+        patient_sex: Literal["M", "F"] | None = None
+        if patient_sex_str in ("M", "F"):
+            patient_sex = patient_sex_str  # Type narrowed to Literal["M", "F"]
+        age_from_str = request.GET.get("age_from", "").strip()
+        age_till_str = request.GET.get("age_till", "").strip()
+
+        # Parse dates from YYYY-MM-DD format to date objects
+        study_date_from = None
+        study_date_till = None
+        if study_date_from_str:
+            try:
+                study_date_from = datetime.strptime(study_date_from_str, "%Y-%m-%d").date()
+            except ValueError:
+                pass  # Invalid date format, leave as None
+
+        if study_date_till_str:
+            try:
+                study_date_till = datetime.strptime(study_date_till_str, "%Y-%m-%d").date()
+            except ValueError:
+                pass  # Invalid date format, leave as None
+
+        # Parse age values to integers
+        age_from_int = None
+        age_till_int = None
+        if age_from_str:
+            try:
+                age_from_int = int(age_from_str)
+            except ValueError:
+                pass  # Invalid integer, leave as None
+
+        if age_till_str:
+            try:
+                age_till_int = int(age_till_str)
+            except ValueError:
+                pass  # Invalid integer, leave as None
+
+        # Get user's active group
+        user = cast("User", request.user)
+        active_group = user.active_group
+
+        # Validate query syntax
+        if not query_str:
+            context = {
+                "count": None,
+                "search_url": None,
+                "error": None,
+                "max_reports_limit": settings.EXTRACTION_MAXIMUM_REPORTS_COUNT,
+            }
+            return render(request, "extractions/_search_preview.html", context)
+
+        query_node, fixes = QueryParser().parse(query_str)
+        if query_node is None:
+            context = {
+                "count": None,
+                "search_url": None,
+                "error": "Invalid query syntax",
+                "max_reports_limit": settings.EXTRACTION_MAXIMUM_REPORTS_COUNT,
+            }
+            return render(request, "extractions/_search_preview.html", context)
+
+        # Get language and modalities objects
+        try:
+            language = Language.objects.get(pk=language_id) if language_id else None
+            modalities_qs = (
+                Modality.objects.filter(pk__in=modality_ids)
+                if modality_ids
+                else Modality.objects.none()
+            )
+        except (Language.DoesNotExist, ValueError):
+            context = {
+                "count": None,
+                "search_url": None,
+                "error": "Invalid language or modality selection",
+                "max_reports_limit": settings.EXTRACTION_MAXIMUM_REPORTS_COUNT,
+            }
+            return render(request, "extractions/_search_preview.html", context)
+
+        # Build search object (age_from_int and age_till_int already parsed above)
+        search = Search(
+            query=query_node,
+            offset=0,
+            limit=0,
+            filters=SearchFilters(
+                group=active_group.pk if active_group else None,
+                language=language.code if language else "",
+                modalities=list(modalities_qs.values_list("code", flat=True)),
+                study_date_from=study_date_from,
+                study_date_till=study_date_till,
+                study_description=study_description,
+                patient_sex=patient_sex,
+                patient_age_from=age_from_int,
+                patient_age_till=age_till_int,
+            ),
+        )
+
+        # Calculate count
+        if extraction_retrieval_provider is None:
+            context = {
+                "count": None,
+                "search_url": None,
+                "error": "Extraction retrieval provider is not configured",
+                "max_reports_limit": settings.EXTRACTION_MAXIMUM_REPORTS_COUNT,
+            }
+            return render(request, "extractions/_search_preview.html", context)
+
+        retrieval_count = extraction_retrieval_provider.count(search)
+
+        # Generate search URL - use STRING values for URL parameters
+        # Note: modalities can be a list, which urlencode with doseq=True expands
+        search_params: dict[str, Union[str, list[str]]] = {"query": query_str}
+        if language_id:
+            search_params["language"] = language_id
+        if modality_ids:
+            search_params["modalities"] = modality_ids
+        if study_date_from_str:  # Use STRING version for URL
+            search_params["study_date_from"] = study_date_from_str
+        if study_date_till_str:  # Use STRING version for URL
+            search_params["study_date_till"] = study_date_till_str
+        if study_description:
+            search_params["study_description"] = study_description
+        if patient_sex:
+            search_params["patient_sex"] = patient_sex
+        if age_from_str:  # Use STRING version for URL
+            search_params["age_from"] = age_from_str
+        if age_till_str:  # Use STRING version for URL
+            search_params["age_till"] = age_till_str
+
+        # doseq=True ensures modalities list becomes: modalities=DX&modalities=MR
+        search_url = reverse("search") + "?" + urlencode(search_params, doseq=True)
+
+        context = {
+            "count": retrieval_count,
+            "search_url": search_url,
+            "error": None,
+            "max_reports_limit": settings.EXTRACTION_MAXIMUM_REPORTS_COUNT,
+        }
+        return render(request, "extractions/_search_preview.html", context)
+
+
 class ExtractionJobDetailView(AnalysisJobDetailView):
     model = ExtractionJob
     table_class = ExtractionTaskTable
diff --git a/radis/search/site.py b/radis/search/site.py
index 5e0ac4b6..10552260 100644
--- a/radis/search/site.py
+++ b/radis/search/site.py
@@ -45,7 +45,7 @@ class SearchFilters:
         - patient_age_till: Filter only reports where the patient is at most this age
     """
 
-    group: int  # TODO: Rename to group_id
+    group: int | None = None  # TODO: Rename to group_id
     language: str = ""  # TODO: Rename to language_code
     modalities: list[str] = field(default_factory=list)
     study_date_from: date | None = None

From edb7adc1d374422c0959945a393845ea8b6ded23 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Sun, 21 Dec 2025 00:34:19 +0000
Subject: [PATCH 81/93] move live update below query and fix get request for
 the live update

---
 radis/extractions/forms.py                    |  2 +
 .../_search_preview_form_section.html         | 15 +++++++
 .../extraction_job_search_form.html           | 17 --------
 .../extraction_job_wizard_summary.html        | 17 +-------
 radis/extractions/views.py                    | 40 ++++++++++---------
 5 files changed, 40 insertions(+), 51 deletions(-)
 create mode 100644 radis/extractions/templates/extractions/_search_preview_form_section.html

diff --git a/radis/extractions/forms.py b/radis/extractions/forms.py
index c462f247..cfe629b6 100644
--- a/radis/extractions/forms.py
+++ b/radis/extractions/forms.py
@@ -100,6 +100,8 @@ def build_layout(self):
                 Column(
                     "title",
                     "query",
+                    # Preview div from template include
+                    HTML('{% include "extractions/_search_preview_form_section.html" %}'),
                     Submit("next", "Next Step (Summary)", css_class="btn-primary"),
                 ),
                 Column(
diff --git a/radis/extractions/templates/extractions/_search_preview_form_section.html b/radis/extractions/templates/extractions/_search_preview_form_section.html
new file mode 100644
index 00000000..5b73b2a0
--- /dev/null
+++ b/radis/extractions/templates/extractions/_search_preview_form_section.html
@@ -0,0 +1,15 @@
+{% load bootstrap_icon from common_extras %}
+
+<div id="search-preview"
+     class="mt-3"
+     hx-get="{% url 'extraction_search_preview' %}"
+     hx-trigger="updatePreview, load"
+     hx-include="[name='1-query'], [name='1-language'], [name='1-modalities'], [name='1-study_date_from'], [name='1-study_date_till'], [name='1-study_description'], [name='1-patient_sex'], [name='1-age_from'], [name='1-age_till']"
+     hx-target="this"
+     hx-swap="innerHTML"
+     x-on:htmx:after-request="isLoading = false">
+    <div class="alert alert-secondary" role="alert">
+        <span class="spinner-border spinner-border-sm me-2" role="status" aria-hidden="true"></span>
+        Loading search preview...
+    </div>
+</div>
diff --git a/radis/extractions/templates/extractions/extraction_job_search_form.html b/radis/extractions/templates/extractions/extraction_job_search_form.html
index 7d42ad99..cf25abc9 100644
--- a/radis/extractions/templates/extractions/extraction_job_search_form.html
+++ b/radis/extractions/templates/extractions/extraction_job_search_form.html
@@ -47,23 +47,6 @@ <h5 class="alert-heading">{% bootstrap_icon "magic" %} Auto-Generated Query</h5>
         {% csrf_token %}
         {{ wizard.management_form }}
         {% crispy form %}
-        <!-- Live Search Preview Container -->
-        <div id="search-preview"
-             class="mt-3"
-             hx-get="{% url 'extraction_search_preview' %}"
-             hx-trigger="updatePreview, load"
-             hx-include="[name='1-query'], [name='1-language'], [name='1-modalities'], [name='1-study_date_from'], [name='1-study_date_till'], [name='1-study_description'], [name='1-patient_sex'], [name='1-age_from'], [name='1-age_till']"
-             hx-target="this"
-             hx-swap="innerHTML"
-             x-on:htmx:after-request="isLoading = false">
-            <!-- Initial placeholder or server-rendered content -->
-            <div class="alert alert-secondary" role="alert">
-                <span class="spinner-border spinner-border-sm me-2"
-                      role="status"
-                      aria-hidden="true"></span>
-                Loading search preview...
-            </div>
-        </div>
     </form>
     <script>
     // Debug: Log HTMX events
diff --git a/radis/extractions/templates/extractions/extraction_job_wizard_summary.html b/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
index 5607b3a7..4889464b 100644
--- a/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
+++ b/radis/extractions/templates/extractions/extraction_job_wizard_summary.html
@@ -27,14 +27,6 @@
         <dt class="col-sm-3">Search Query</dt>
         <dd class="col-sm-9">
             {{ search.query }}
-            {% if query_metadata.success %}
-                <br />
-                <small class="text-muted">
-                    {% bootstrap_icon "info-circle" %}
-                    Auto-generated from extraction fields
-                    {% if query_metadata.fixes_applied %}(with automatic fixes applied){% endif %}
-                </small>
-            {% endif %}
         </dd>
         <dt class="col-sm-3">Language</dt>
         <dd class="col-sm-9">
@@ -109,9 +101,7 @@
                                     <dt class="col-sm-3">Selections</dt>
                                     <dd class="col-sm-9">
                                         <ul class="mb-0 ps-3">
-                                            {% for option in field.selection_options %}
-                                                <li>{{ option }}</li>
-                                            {% endfor %}
+                                            {% for option in field.selection_options %}<li>{{ option }}</li>{% endfor %}
                                         </ul>
                                     </dd>
                                 {% endif %}
@@ -122,10 +112,7 @@
             </ul>
         </dd>
     </dl>
-    <form action=""
-          method="post"
-          class="pt-3"
-          data-prevent-enter-submit="true">
+    <form action="" method="post" class="pt-3" data-prevent-enter-submit="true">
         {% csrf_token %}
         {{ wizard.management_form }}
         {% crispy form %}
diff --git a/radis/extractions/views.py b/radis/extractions/views.py
index b07c3f4d..3f33601e 100644
--- a/radis/extractions/views.py
+++ b/radis/extractions/views.py
@@ -142,7 +142,14 @@ def process_step(self, form):
                     "You can manually enter a search query in the next step.",
                 )
 
-        return step_data
+        # After search step is submitted, store retrieval count for summary
+        elif self.steps.current == ExtractionJobWizardView.SEARCH_STEP:
+            if hasattr(form, "cleaned_data"):
+                retrieval_count = form.cleaned_data.get("retrieval_count")
+                if retrieval_count is not None:
+                    self.storage.extra_data["retrieval_count"] = retrieval_count
+
+            return step_data
 
     def get_form_initial(self, step=None):
         """Provide initial data for forms, including generated query for search step."""
@@ -181,14 +188,6 @@ def get_context_data(self, form, **kwargs):
                     ]
                 )
 
-            # If query is populated (either generated or user-edited), show estimated count
-            if form.is_bound and hasattr(form, "cleaned_data") and form.cleaned_data:
-                retrieval_count = form.cleaned_data.get("retrieval_count")
-                if retrieval_count is not None:
-                    context["retrieval_count"] = retrieval_count
-                    # Store for summary step
-                    self.storage.extra_data["retrieval_count"] = retrieval_count
-
         elif self.steps.current == ExtractionJobWizardView.SUMMARY_STEP:
             # Final step - show everything for review
             output_fields_data = self.get_cleaned_data_for_step(
@@ -268,21 +267,24 @@ class ExtractionSearchPreviewView(LoginRequiredMixin, View):
     request: AuthenticatedHttpRequest
 
     def get(self, request: AuthenticatedHttpRequest):
+        # Wizard step prefix for form field names
+        WIZARD_STEP_PREFIX = "1-"
+
         # Extract query and filter parameters from GET
-        query_str = request.GET.get("query", "").strip()
-        language_id = request.GET.get("language")
-        modality_ids = request.GET.getlist("modalities")
+        query_str = request.GET.get(f"{WIZARD_STEP_PREFIX}query", "").strip()
+        language_id = request.GET.get(f"{WIZARD_STEP_PREFIX}language")
+        modality_ids = request.GET.getlist(f"{WIZARD_STEP_PREFIX}modalities")
 
         # Get string values from GET parameters
-        study_date_from_str = request.GET.get("study_date_from", "").strip()
-        study_date_till_str = request.GET.get("study_date_till", "").strip()
-        study_description = request.GET.get("study_description", "")
-        patient_sex_str = request.GET.get("patient_sex")
+        study_date_from_str = request.GET.get(f"{WIZARD_STEP_PREFIX}study_date_from", "").strip()
+        study_date_till_str = request.GET.get(f"{WIZARD_STEP_PREFIX}study_date_till", "").strip()
+        study_description = request.GET.get(f"{WIZARD_STEP_PREFIX}study_description", "")
+        patient_sex_str = request.GET.get(f"{WIZARD_STEP_PREFIX}patient_sex")
         patient_sex: Literal["M", "F"] | None = None
         if patient_sex_str in ("M", "F"):
             patient_sex = patient_sex_str  # Type narrowed to Literal["M", "F"]
-        age_from_str = request.GET.get("age_from", "").strip()
-        age_till_str = request.GET.get("age_till", "").strip()
+        age_from_str = request.GET.get(f"{WIZARD_STEP_PREFIX}age_from", "").strip()
+        age_till_str = request.GET.get(f"{WIZARD_STEP_PREFIX}age_till", "").strip()
 
         # Parse dates from YYYY-MM-DD format to date objects
         study_date_from = None
@@ -389,7 +391,7 @@ def get(self, request: AuthenticatedHttpRequest):
         # Note: modalities can be a list, which urlencode with doseq=True expands
         search_params: dict[str, Union[str, list[str]]] = {"query": query_str}
         if language_id:
-            search_params["language"] = language_id
+            search_params["language"] = Language.objects.get(pk=language_id).code
         if modality_ids:
             search_params["modalities"] = modality_ids
         if study_date_from_str:  # Use STRING version for URL

From 61224b9e4ba46c4eadd2ca894c79d7bb8ea2a5f0 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 22 Dec 2025 16:50:03 +0000
Subject: [PATCH 82/93] fix query generator and selection options tests

---
 radis/extractions/factories.py                |  9 +-
 .../test_output_fields_selection_options.py   | 65 ---------------
 .../extractions/tests/test_query_generator.py | 43 ----------
 .../tests/unit/test_processor_utils.py        |  4 +-
 radis/subscriptions/forms.py                  | 10 ---
 radis/subscriptions/tests/test_views.py       |  2 +
 radis/subscriptions/views.py                  | 82 -------------------
 7 files changed, 12 insertions(+), 203 deletions(-)
 delete mode 100644 radis/extractions/tests/acceptance/test_output_fields_selection_options.py

diff --git a/radis/extractions/factories.py b/radis/extractions/factories.py
index 63a640ee..1aa0c931 100644
--- a/radis/extractions/factories.py
+++ b/radis/extractions/factories.py
@@ -5,6 +5,7 @@
 from adit_radis_shared.accounts.models import User
 from adit_radis_shared.common.utils.testing_helpers import add_user_to_group
 from django.contrib.auth.models import Group
+from factory.declarations import SKIP
 from faker import Faker
 
 from radis.reports.factories import ModalityFactory
@@ -72,7 +73,13 @@ class OutputFieldFactory(BaseDjangoModelFactory[OutputField]):
     class Meta:
         model = OutputField
 
-    job = factory.SubFactory("radis.extractions.factories.ExtractionJobFactory")
+    # Use factory.Maybe to conditionally create job only when subscription is None
+    job = factory.Maybe(
+        factory.SelfAttribute("subscription"),
+        yes_declaration=SKIP,  # If subscription exists, skip job creation
+        no_declaration=factory.SubFactory("radis.extractions.factories.ExtractionJobFactory"),  # type: ignore[arg-type]
+    )
+    subscription = None
     name = factory.Sequence(lambda n: f"output_field_{n}")
     description = factory.Faker("sentence", nb_words=10)
     output_type = factory.Faker("random_element", elements=[a[0] for a in OutputType.choices])
diff --git a/radis/extractions/tests/acceptance/test_output_fields_selection_options.py b/radis/extractions/tests/acceptance/test_output_fields_selection_options.py
deleted file mode 100644
index fcd145c3..00000000
--- a/radis/extractions/tests/acceptance/test_output_fields_selection_options.py
+++ /dev/null
@@ -1,65 +0,0 @@
-from urllib.parse import urlparse
-
-import pytest
-from adit_radis_shared.accounts.factories import GroupFactory, UserFactory
-from django.contrib.auth.models import Permission
-from django.test import Client
-from playwright.sync_api import Page, expect
-from pytest_django.live_server_helper import LiveServer
-
-from radis.reports.models import Language, Modality
-
-
-@pytest.mark.acceptance
-@pytest.mark.order("last")
-@pytest.mark.django_db(transaction=True)
-def test_selection_options_work_for_added_output_field(live_server: LiveServer, page: Page):
-    group = GroupFactory.create()
-    user = UserFactory.create(is_active=True)
-    user.groups.add(group)
-    user.active_group = group
-    user.user_permissions.add(Permission.objects.get(codename="add_extractionjob"))
-    user.save()
-
-    language = Language.objects.create(code="en")
-    Modality.objects.create(code="CT", filterable=True)
-
-    client = Client()
-    client.force_login(user)
-
-    hostname = urlparse(live_server.url).hostname
-    assert hostname
-    page.context.add_cookies(
-        [
-            {
-                "name": "sessionid",
-                "value": client.cookies["sessionid"].value,
-                "domain": hostname,
-                "path": "/",
-            }
-        ]
-    )
-
-    page.goto(f"{live_server.url}/extractions/jobs/new/")
-    page.locator('input[name="0-title"]').fill("Test Extraction")
-    page.locator('select[name="0-provider"]').select_option("PG Search")
-    page.locator('input[name="0-query"]').fill("test")
-    page.locator('select[name="0-language"]').select_option(str(language.pk))
-    page.locator('input[type="submit"][value*="Next Step"]').click()
-
-    page.get_by_role("button", name="Add Field").click()
-
-    forms = page.locator(".formset-form")
-    expect(forms).to_have_count(2)
-
-    second_form = forms.nth(1)
-    selection_input = second_form.locator("[data-selection-input]")
-    expect(selection_input).to_have_count(1)
-    expect(selection_input).to_have_attribute("data-max-selection-options", "7")
-    second_form.locator('select[name$="-output_type"]').select_option("S")
-    expect(second_form.locator('select[name$="-output_type"]')).to_have_value("S")
-    add_option_button = second_form.get_by_role("button", name="Enter a selection")
-    expect(add_option_button).to_be_enabled()
-    add_option_button.click()
-
-    expect(second_form.locator('input[placeholder="Selection 1"]')).to_be_visible()
diff --git a/radis/extractions/tests/test_query_generator.py b/radis/extractions/tests/test_query_generator.py
index 43aa5f99..a4c664ed 100644
--- a/radis/extractions/tests/test_query_generator.py
+++ b/radis/extractions/tests/test_query_generator.py
@@ -121,46 +121,3 @@ def test_extract_query_from_response_multiline(self):
         # Should only take the first line
         assert query == "lung AND nodule"
         assert "This query" not in query
-
-    def test_query_generation_with_boolean_field(self):
-        """Test query generation with boolean output field."""
-        field = OutputField(
-            name="has_fracture",
-            description="whether a fracture is present",
-            output_type=OutputType.BOOLEAN,
-        )
-
-        with override_settings(ENABLE_AUTO_QUERY_GENERATION=False):
-            query, metadata = self.generator.generate_from_fields([field])
-
-        assert query != ""
-        assert metadata["success"] is True
-        assert query is not None
-        assert "fracture" in query.lower()
-
-    def test_query_generation_with_mixed_field_types(self):
-        """Test query generation with multiple field types."""
-        fields = [
-            OutputField(
-                name="nodule_present",
-                description="is a nodule present",
-                output_type=OutputType.BOOLEAN,
-            ),
-            OutputField(
-                name="nodule_size",
-                description="size in mm",
-                output_type=OutputType.NUMERIC,
-            ),
-            OutputField(
-                name="nodule_location",
-                description="anatomical location",
-                output_type=OutputType.TEXT,
-            ),
-        ]
-
-        with override_settings(ENABLE_AUTO_QUERY_GENERATION=False):
-            query, metadata = self.generator.generate_from_fields(fields)
-
-        assert query != ""
-        assert metadata["success"] is True
-        assert metadata["field_count"] == 3
diff --git a/radis/extractions/tests/unit/test_processor_utils.py b/radis/extractions/tests/unit/test_processor_utils.py
index 4f71b127..45fbfbb1 100644
--- a/radis/extractions/tests/unit/test_processor_utils.py
+++ b/radis/extractions/tests/unit/test_processor_utils.py
@@ -18,7 +18,7 @@ def test_generate_output_fields_schema_uses_literal_for_selection_fields():
     field.selection_options = ["Grade 1", "Grade 2"]
     field.save()
 
-    schema = generate_output_fields_schema(job.output_fields)
+    schema = generate_output_fields_schema(job.output_fields.all())
 
     grade_field = schema.model_fields["grade"]
     annotation = grade_field.annotation
@@ -38,7 +38,7 @@ def test_generate_output_fields_schema_wraps_literal_in_list_for_array_selection
     field.is_array = True
     field.save()
 
-    schema = generate_output_fields_schema(job.output_fields)
+    schema = generate_output_fields_schema(job.output_fields.all())
 
     grade_field = schema.model_fields["grade_multi"]
     annotation = grade_field.annotation
diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index cfc4175f..8f60cb44 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -30,16 +30,6 @@ class Meta:
             "send_finished_mail",
         ]
         labels = {"patient_id": "Patient ID"}
-        help_texts = {
-            "name": "Name of the Subscription",
-            "query": (
-                "Search query to filter reports "
-                "(leave empty to auto-generate from extraction fields)"
-            ),
-        }
-        widgets = {
-            "query": forms.TextInput(attrs={"placeholder": "Optional - auto-generated if empty"}),
-        }
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
diff --git a/radis/subscriptions/tests/test_views.py b/radis/subscriptions/tests/test_views.py
index 3b7c446a..f84e8163 100644
--- a/radis/subscriptions/tests/test_views.py
+++ b/radis/subscriptions/tests/test_views.py
@@ -267,6 +267,8 @@ def test_subscription_update_view_post_valid(client: Client):
         "output_fields-0-name": "Volume",
         "output_fields-0-description": "Volume description",
         "output_fields-0-output_type": "N",
+        "output_fields-0-selection_options": "",
+        "output_fields-0-is_array": "false",
     }
 
     response = client.post(f"/subscriptions/{subscription.pk}/update/", data)
diff --git a/radis/subscriptions/views.py b/radis/subscriptions/views.py
index d2b904ec..de865922 100644
--- a/radis/subscriptions/views.py
+++ b/radis/subscriptions/views.py
@@ -101,47 +101,6 @@ def form_valid(self, form) -> HttpResponse:
             active_group = user.active_group
             form.instance.group = active_group
 
-            # Auto-generate query if needed
-            if not form.cleaned_data.get("query", "").strip():
-                # Collect output fields from formset
-                temp_fields = []
-                for field_form in output_formset:
-                    if (
-                        field_form.cleaned_data
-                        and not field_form.cleaned_data.get("DELETE", False)
-                        and field_form.cleaned_data.get("name")
-                    ):
-                        from radis.extractions.models import OutputField
-
-                        temp_fields.append(
-                            OutputField(
-                                name=field_form.cleaned_data["name"],
-                                description=field_form.cleaned_data["description"],
-                                output_type=field_form.cleaned_data["output_type"],
-                            )
-                        )
-
-                if temp_fields:
-                    from radis.extractions.utils.query_generator import QueryGenerator
-
-                    generator = QueryGenerator()
-                    generated_query, metadata = generator.generate_from_fields(temp_fields)
-
-                    # Check if generation failed
-                    if generated_query is None or not metadata.get("success"):
-                        form.add_error(
-                            "query",
-                            "Unable to automatically generate a query from your extraction fields. "
-                            "Please manually enter a search query.",
-                        )
-                        return self.form_invalid(form)
-
-                    form.instance.query = generated_query
-                    logger.info(
-                        f"Auto-generated query for subscription: {generated_query} "
-                        f"(method: {metadata.get('generation_method')})"
-                    )
-
             try:
                 self.object: Subscription = form.save()
             except IntegrityError as e:
@@ -194,47 +153,6 @@ def form_valid(self, form) -> HttpResponse:
         filter_formset = ctx["filter_formset"]
         output_formset = ctx["output_formset"]
         if filter_formset.is_valid() and output_formset.is_valid():
-            # Auto-generate query if needed
-            if not form.cleaned_data.get("query", "").strip():
-                # Collect output fields from formset
-                temp_fields = []
-                for field_form in output_formset:
-                    if (
-                        field_form.cleaned_data
-                        and not field_form.cleaned_data.get("DELETE", False)
-                        and field_form.cleaned_data.get("name")
-                    ):
-                        from radis.extractions.models import OutputField
-
-                        temp_fields.append(
-                            OutputField(
-                                name=field_form.cleaned_data["name"],
-                                description=field_form.cleaned_data["description"],
-                                output_type=field_form.cleaned_data["output_type"],
-                            )
-                        )
-
-                if temp_fields:
-                    from radis.extractions.utils.query_generator import QueryGenerator
-
-                    generator = QueryGenerator()
-                    generated_query, metadata = generator.generate_from_fields(temp_fields)
-
-                    # Check if generation failed
-                    if generated_query is None or not metadata.get("success"):
-                        form.add_error(
-                            "query",
-                            "Unable to automatically generate a query from your extraction fields. "
-                            "Please manually enter a search query.",
-                        )
-                        return self.form_invalid(form)
-
-                    form.instance.query = generated_query
-                    logger.info(
-                        f"Auto-generated query for subscription update: {generated_query} "
-                        f"(method: {metadata.get('generation_method')})"
-                    )
-
             try:
                 self.object = form.save()
             except IntegrityError as e:

From 91fa5adbd5a6c4dfe683d6c3c10d9e306cc6a7e1 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 30 Dec 2025 00:03:55 +0000
Subject: [PATCH 83/93] move selection options js/css from core to extractions.
 remove extraction assertions

---
 radis/core/static/core/core.css               |  33 +---
 radis/core/static/core/core.js                | 142 ------------------
 .../static/extractions/extractions.css        |  35 +++++
 .../static/extractions/extractions.js         | 141 +++++++++++++++++
 radis/extractions/views.py                    |  43 +++++-
 .../subscriptions/subscription_layout.html    |   4 +
 6 files changed, 221 insertions(+), 177 deletions(-)

diff --git a/radis/core/static/core/core.css b/radis/core/static/core/core.css
index c86469b7..7b8f795c 100644
--- a/radis/core/static/core/core.css
+++ b/radis/core/static/core/core.css
@@ -38,35 +38,4 @@
   display: -webkit-box;
   -webkit-line-clamp: 3;
   -webkit-box-orient: vertical;
-}
-
-.selection-options-controls,
-.selection-options-actions {
-  gap: 0.5rem;
-}
-
-.array-toggle-field {
-  display: flex;
-  align-items: center;
-  justify-content: flex-end;
-  padding-top: 14px;
-}
-
-.array-toggle-btn {
-  width: calc(2.5rem - 6px);
-  height: calc(2.5rem - 6px);
-  border-radius: 50%;
-  font-weight: 600;
-  display: inline-flex;
-  align-items: center;
-  justify-content: center;
-  padding: 0;
-  line-height: 1;
-  font-size: 1rem;
-}
-
-.array-toggle-btn.active {
-  background-color: var(--bs-primary);
-  border-color: var(--bs-primary);
-  color: #fff;
-}
+}
\ No newline at end of file
diff --git a/radis/core/static/core/core.js b/radis/core/static/core/core.js
index c66879b8..31468d10 100644
--- a/radis/core/static/core/core.js
+++ b/radis/core/static/core/core.js
@@ -105,145 +105,3 @@ function FormSet(rootEl) {
     },
   };
 }
-
-/**
- * Manages the dynamic selection options input for extraction output fields.
- *
- * @param {HTMLElement} rootEl
- * @returns {Object}
- */
-function SelectionOptions(rootEl) {
-  const hiddenInput = rootEl.querySelector("[data-selection-input]");
-  const arrayInput = rootEl.querySelector("[data-array-input]");
-  const formContainer =
-    rootEl.closest(".formset-form") ?? rootEl.closest("form") ?? rootEl;
-  const outputTypeField =
-    formContainer.querySelector('select[name$="-output_type"]') ??
-    formContainer.querySelector('select[name="output_type"]');
-  const arrayToggleButton =
-    formContainer.querySelector("[data-array-toggle]") ?? null;
-  const parseArrayValue = (value) => {
-    if (!value) {
-      return false;
-    }
-    const normalized = value.trim().toLowerCase();
-    return normalized === "true" || normalized === "1" || normalized === "on";
-  };
-  const parseMaxOptions = () => {
-    const datasetValue =
-      hiddenInput?.dataset.maxSelectionOptions ??
-      rootEl.dataset.maxSelectionOptions ??
-      "";
-    const parsed = Number.parseInt(datasetValue, 10);
-    return Number.isNaN(parsed) ? 0 : parsed;
-  };
-  const initialMaxOptions = parseMaxOptions();
-
-  return {
-    options: [],
-    maxOptions: initialMaxOptions,
-    supportsSelection: false,
-    isArray: parseArrayValue(arrayInput?.value),
-    lastSelectionOptions: [],
-    init() {
-      this.options = this.parseOptions(hiddenInput?.value);
-      this.isArray = parseArrayValue(arrayInput?.value);
-      this.lastSelectionOptions = [...this.options];
-      this.updateSupports();
-      if (arrayToggleButton) {
-        arrayToggleButton.addEventListener("click", (event) => {
-          event.preventDefault();
-          this.toggleArray();
-        });
-        this.updateArrayButton();
-      }
-      if (outputTypeField) {
-        outputTypeField.addEventListener("change", () => {
-          const wasSelection = this.supportsSelection;
-          this.updateSupports();
-          if (!this.supportsSelection) {
-            this.lastSelectionOptions = [...this.options];
-            this.options = [];
-          } else if (!wasSelection && this.options.length === 0) {
-            if (this.lastSelectionOptions.length > 0) {
-              this.options = [...this.lastSelectionOptions];
-            } else {
-              this.options = this.parseOptions(hiddenInput?.value);
-            }
-          }
-        });
-      }
-    },
-    parseOptions(value) {
-      if (!value) {
-        return [];
-      }
-      try {
-        const parsed = JSON.parse(value);
-        if (Array.isArray(parsed)) {
-          return parsed
-            .map((opt) => (typeof opt === "string" ? opt : ""))
-            .filter((opt) => opt !== "");
-        }
-      } catch (err) {
-        console.warn("Invalid selection options payload", err);
-      }
-      return [];
-    },
-    updateSupports() {
-      this.supportsSelection = outputTypeField
-        ? outputTypeField.value === "S"
-        : false;
-    },
-    syncOptions() {
-      if (!hiddenInput) {
-        return;
-      }
-      const sanitized = this.options
-        .map((opt) => (typeof opt === "string" ? opt.trim() : ""))
-        .filter((opt) => opt !== "");
-      hiddenInput.value = JSON.stringify(sanitized);
-      this.lastSelectionOptions = [...sanitized];
-    },
-    syncArray() {
-      if (!arrayInput) {
-        return;
-      }
-      arrayInput.value = this.isArray ? "true" : "false";
-    },
-    syncState() {
-      this.syncOptions();
-      this.syncArray();
-      this.updateArrayButton();
-    },
-    addOption() {
-      if (!this.supportsSelection || this.options.length >= this.maxOptions) {
-        return;
-      }
-      this.options.push("");
-      this.$nextTick(() => {
-        const inputs = rootEl.querySelectorAll("[data-selection-option-input]");
-        const lastInput = inputs[inputs.length - 1];
-        if (lastInput instanceof HTMLInputElement) {
-          lastInput.focus();
-        }
-      });
-    },
-    removeOption(index) {
-      this.options.splice(index, 1);
-    },
-    toggleArray() {
-      this.isArray = !this.isArray;
-    },
-    updateArrayButton() {
-      if (!arrayToggleButton) {
-        return;
-      }
-      arrayToggleButton.classList.toggle("active", this.isArray);
-      arrayToggleButton.setAttribute(
-        "aria-pressed",
-        this.isArray ? "true" : "false"
-      );
-    },
-  };
-}
diff --git a/radis/extractions/static/extractions/extractions.css b/radis/extractions/static/extractions/extractions.css
index 17c43b36..c2164fdc 100644
--- a/radis/extractions/static/extractions/extractions.css
+++ b/radis/extractions/static/extractions/extractions.css
@@ -1,3 +1,38 @@
 #filters .asteriskField {
   display: none;
 }
+
+/*
+ * Selection Options Component
+ */
+
+.selection-options-controls,
+.selection-options-actions {
+  gap: 0.5rem;
+}
+
+.array-toggle-field {
+  display: flex;
+  align-items: center;
+  justify-content: flex-end;
+  padding-top: 14px;
+}
+
+.array-toggle-btn {
+  width: calc(2.5rem - 6px);
+  height: calc(2.5rem - 6px);
+  border-radius: 50%;
+  font-weight: 600;
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  padding: 0;
+  line-height: 1;
+  font-size: 1rem;
+}
+
+.array-toggle-btn.active {
+  background-color: var(--bs-primary);
+  border-color: var(--bs-primary);
+  color: #fff;
+}
diff --git a/radis/extractions/static/extractions/extractions.js b/radis/extractions/static/extractions/extractions.js
index e69de29b..0d400738 100644
--- a/radis/extractions/static/extractions/extractions.js
+++ b/radis/extractions/static/extractions/extractions.js
@@ -0,0 +1,141 @@
+/**
+ * Manages the dynamic selection options input for extraction output fields.
+ *
+ * @param {HTMLElement} rootEl
+ * @returns {Object}
+ */
+function SelectionOptions(rootEl) {
+  const hiddenInput = rootEl.querySelector("[data-selection-input]");
+  const arrayInput = rootEl.querySelector("[data-array-input]");
+  const formContainer =
+    rootEl.closest(".formset-form") ?? rootEl.closest("form") ?? rootEl;
+  const outputTypeField =
+    formContainer.querySelector('select[name$="-output_type"]') ??
+    formContainer.querySelector('select[name="output_type"]');
+  const arrayToggleButton =
+    formContainer.querySelector("[data-array-toggle]") ?? null;
+  const parseArrayValue = (value) => {
+    if (!value) {
+      return false;
+    }
+    const normalized = value.trim().toLowerCase();
+    return normalized === "true" || normalized === "1" || normalized === "on";
+  };
+  const parseMaxOptions = () => {
+    const datasetValue =
+      hiddenInput?.dataset.maxSelectionOptions ??
+      rootEl.dataset.maxSelectionOptions ??
+      "";
+    const parsed = Number.parseInt(datasetValue, 10);
+    return Number.isNaN(parsed) ? 0 : parsed;
+  };
+  const initialMaxOptions = parseMaxOptions();
+
+  return {
+    options: [],
+    maxOptions: initialMaxOptions,
+    supportsSelection: false,
+    isArray: parseArrayValue(arrayInput?.value),
+    lastSelectionOptions: [],
+    init() {
+      this.options = this.parseOptions(hiddenInput?.value);
+      this.isArray = parseArrayValue(arrayInput?.value);
+      this.lastSelectionOptions = [...this.options];
+      this.updateSupports();
+      if (arrayToggleButton) {
+        arrayToggleButton.addEventListener("click", (event) => {
+          event.preventDefault();
+          this.toggleArray();
+        });
+        this.updateArrayButton();
+      }
+      if (outputTypeField) {
+        outputTypeField.addEventListener("change", () => {
+          const wasSelection = this.supportsSelection;
+          this.updateSupports();
+          if (!this.supportsSelection) {
+            this.lastSelectionOptions = [...this.options];
+            this.options = [];
+          } else if (!wasSelection && this.options.length === 0) {
+            if (this.lastSelectionOptions.length > 0) {
+              this.options = [...this.lastSelectionOptions];
+            } else {
+              this.options = this.parseOptions(hiddenInput?.value);
+            }
+          }
+        });
+      }
+    },
+    parseOptions(value) {
+      if (!value) {
+        return [];
+      }
+      try {
+        const parsed = JSON.parse(value);
+        if (Array.isArray(parsed)) {
+          return parsed
+            .map((opt) => (typeof opt === "string" ? opt : ""))
+            .filter((opt) => opt !== "");
+        }
+      } catch (err) {
+        console.warn("Invalid selection options payload", err);
+      }
+      return [];
+    },
+    updateSupports() {
+      this.supportsSelection = outputTypeField
+        ? outputTypeField.value === "S"
+        : false;
+    },
+    syncOptions() {
+      if (!hiddenInput) {
+        return;
+      }
+      const sanitized = this.options
+        .map((opt) => (typeof opt === "string" ? opt.trim() : ""))
+        .filter((opt) => opt !== "");
+      hiddenInput.value = JSON.stringify(sanitized);
+      this.lastSelectionOptions = [...sanitized];
+    },
+    syncArray() {
+      if (!arrayInput) {
+        return;
+      }
+      arrayInput.value = this.isArray ? "true" : "false";
+    },
+    syncState() {
+      this.syncOptions();
+      this.syncArray();
+      this.updateArrayButton();
+    },
+    addOption() {
+      if (!this.supportsSelection || this.options.length >= this.maxOptions) {
+        return;
+      }
+      this.options.push("");
+      this.$nextTick(() => {
+        const inputs = rootEl.querySelectorAll("[data-selection-option-input]");
+        const lastInput = inputs[inputs.length - 1];
+        if (lastInput instanceof HTMLInputElement) {
+          lastInput.focus();
+        }
+      });
+    },
+    removeOption(index) {
+      this.options.splice(index, 1);
+    },
+    toggleArray() {
+      this.isArray = !this.isArray;
+    },
+    updateArrayButton() {
+      if (!arrayToggleButton) {
+        return;
+      }
+      arrayToggleButton.classList.toggle("active", this.isArray);
+      arrayToggleButton.setAttribute(
+        "aria-pressed",
+        this.isArray ? "true" : "false"
+      );
+    },
+  };
+}
diff --git a/radis/extractions/views.py b/radis/extractions/views.py
index 3f33601e..88cc18c3 100644
--- a/radis/extractions/views.py
+++ b/radis/extractions/views.py
@@ -142,6 +142,8 @@ def process_step(self, form):
                     "You can manually enter a search query in the next step.",
                 )
 
+            return step_data
+
         # After search step is submitted, store retrieval count for summary
         elif self.steps.current == ExtractionJobWizardView.SEARCH_STEP:
             if hasattr(form, "cleaned_data"):
@@ -151,6 +153,9 @@ def process_step(self, form):
 
             return step_data
 
+        # For any other steps (SUMMARY_STEP or future additions)
+        return step_data
+
     def get_form_initial(self, step=None):
         """Provide initial data for forms, including generated query for search step."""
         initial = super().get_form_initial(step)
@@ -163,6 +168,40 @@ def get_form_initial(self, step=None):
 
         return initial
 
+    def render(self, form=None, **kwargs):
+        """Override to validate wizard data integrity before rendering summary step."""
+        # Only validate if we're on the summary step
+        if self.steps.current == ExtractionJobWizardView.SUMMARY_STEP:
+            output_fields_data = self.get_cleaned_data_for_step(
+                ExtractionJobWizardView.OUTPUT_FIELDS_STEP
+            )
+            search_data = self.get_cleaned_data_for_step(ExtractionJobWizardView.SEARCH_STEP)
+
+            # If output fields data is missing or invalid, restart wizard
+            if not output_fields_data or not isinstance(output_fields_data, list):
+                from django.contrib import messages
+
+                self.storage.reset()
+                self.storage.current_step = self.steps.first
+                messages.error(
+                    self.request,
+                    "Wizard data was lost or corrupted. Please start over from step 1.",
+                )
+                return redirect(reverse("extraction_job_create"))
+
+            # If search data is missing, go back to search step
+            if not search_data or not isinstance(search_data, dict):
+                from django.contrib import messages
+
+                self.storage.current_step = ExtractionJobWizardView.SEARCH_STEP
+                messages.error(
+                    self.request,
+                    "Search data is missing. Please complete step 2.",
+                )
+                return redirect(reverse("extraction_job_create"))
+
+        return super().render(form, **kwargs)
+
     def get_context_data(self, form, **kwargs):
         context = super().get_context_data(form, **kwargs)
 
@@ -190,14 +229,12 @@ def get_context_data(self, form, **kwargs):
 
         elif self.steps.current == ExtractionJobWizardView.SUMMARY_STEP:
             # Final step - show everything for review
+            # Data integrity validated in render() method
             output_fields_data = self.get_cleaned_data_for_step(
                 ExtractionJobWizardView.OUTPUT_FIELDS_STEP
             )
             search_data = self.get_cleaned_data_for_step(ExtractionJobWizardView.SEARCH_STEP)
 
-            assert output_fields_data and isinstance(output_fields_data, list)
-            assert search_data and isinstance(search_data, dict)
-
             context["output_fields"] = output_fields_data
             context["search"] = search_data
             context["retrieval_count"] = self.storage.extra_data.get("retrieval_count", 0)
diff --git a/radis/subscriptions/templates/subscriptions/subscription_layout.html b/radis/subscriptions/templates/subscriptions/subscription_layout.html
index 0b1b86c0..a0fdad2b 100644
--- a/radis/subscriptions/templates/subscriptions/subscription_layout.html
+++ b/radis/subscriptions/templates/subscriptions/subscription_layout.html
@@ -5,6 +5,9 @@
     <link rel="stylesheet"
           type="text/css"
           href="{% static 'search/search.css' %}" />
+    <link rel="stylesheet"
+          type="text/css"
+          href="{% static 'extractions/extractions.css' %}" />
     <link rel="stylesheet"
           type="text/css"
           href="{% static 'subscriptions/subscriptions.css' %}" />
@@ -12,5 +15,6 @@
 {% block script %}
     {{ block.super }}
     <script src="{% static 'search/search.js' %}"></script>
+    <script src="{% static 'extractions/extractions.js' %}"></script>
     <script src="{% static 'subscriptions/subscriptions.js' %}"></script>
 {% endblock script %}

From 8e916fc54d6114d86dd80fe9b0651feb1ab2f2c4 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 30 Dec 2025 01:18:11 +0000
Subject: [PATCH 84/93] add error handling to chat client, remove optional from
 query field, close db connections

---
 radis/chats/utils/chat_client.py              | 56 +++++++++++++++----
 radis/core/static/core/core.css               |  2 +-
 .../0009_alter_extractionjob_query.py         | 18 ------
 radis/extractions/models.py                   |  2 +-
 radis/subscriptions/processors.py             |  8 +++
 5 files changed, 56 insertions(+), 30 deletions(-)
 delete mode 100644 radis/extractions/migrations/0009_alter_extractionjob_query.py

diff --git a/radis/chats/utils/chat_client.py b/radis/chats/utils/chat_client.py
index 7d6bfedc..9073ba50 100644
--- a/radis/chats/utils/chat_client.py
+++ b/radis/chats/utils/chat_client.py
@@ -37,9 +37,21 @@ async def chat(
         if max_completion_tokens is not None:
             request["max_completion_tokens"] = max_completion_tokens
 
-        completion = await self._client.chat.completions.create(**request)
+        try:
+            completion = await self._client.chat.completions.create(**request)
+        except openai.APIError as e:
+            logger.error(f"OpenAI API error during chat: {e}")
+            raise RuntimeError(f"Failed to communicate with LLM service: {e}") from e
+
+        if not completion.choices:
+            logger.error("LLM returned empty choices list")
+            raise ValueError("LLM returned no response choices")
+
         answer = completion.choices[0].message.content
-        assert answer is not None
+        if answer is None:
+            logger.error("LLM returned None for message content")
+            raise ValueError("LLM returned empty response content")
+
         logger.debug("Received from LLM: %s", answer)
         return answer
 
@@ -76,9 +88,21 @@ def chat(
         if max_completion_tokens is not None:
             request["max_completion_tokens"] = max_completion_tokens
 
-        completion = self._client.chat.completions.create(**request)
+        try:
+            completion = self._client.chat.completions.create(**request)
+        except openai.APIError as e:
+            logger.error(f"OpenAI API error during chat: {e}")
+            raise RuntimeError(f"Failed to communicate with LLM service: {e}") from e
+
+        if not completion.choices:
+            logger.error("LLM returned empty choices list")
+            raise ValueError("LLM returned no response choices")
+
         answer = completion.choices[0].message.content
-        assert answer is not None
+        if answer is None:
+            logger.error("LLM returned None for message content")
+            raise ValueError("LLM returned empty response content")
+
         logger.debug("Received from LLM: %s", answer)
         return answer
 
@@ -87,12 +111,24 @@ def extract_data(self, prompt: str, schema: type[BaseModel]) -> BaseModel:
         logger.debug("Prompt:\n%s", prompt)
         logger.debug("Schema:\n%s", schema.model_json_schema())
 
-        completion = self._client.beta.chat.completions.parse(
-            model=self._llm_model_name,
-            messages=[{"role": "system", "content": prompt}],
-            response_format=schema,
-        )
+        try:
+            completion = self._client.beta.chat.completions.parse(
+                model=self._llm_model_name,
+                messages=[{"role": "system", "content": prompt}],
+                response_format=schema,
+            )
+        except openai.APIError as e:
+            logger.error(f"OpenAI API error during data extraction: {e}")
+            raise RuntimeError(f"Failed to communicate with LLM service: {e}") from e
+
+        if not completion.choices:
+            logger.error("LLM returned empty choices list")
+            raise ValueError("LLM returned no response choices")
+
         event = completion.choices[0].message.parsed
-        assert event
+        if event is None:
+            logger.error("LLM returned None for parsed message")
+            raise ValueError("LLM returned empty parsed response")
+
         logger.debug("Received from LLM: %s", event)
         return event
diff --git a/radis/core/static/core/core.css b/radis/core/static/core/core.css
index 7b8f795c..0dba50c7 100644
--- a/radis/core/static/core/core.css
+++ b/radis/core/static/core/core.css
@@ -38,4 +38,4 @@
   display: -webkit-box;
   -webkit-line-clamp: 3;
   -webkit-box-orient: vertical;
-}
\ No newline at end of file
+}
diff --git a/radis/extractions/migrations/0009_alter_extractionjob_query.py b/radis/extractions/migrations/0009_alter_extractionjob_query.py
deleted file mode 100644
index cdb335b5..00000000
--- a/radis/extractions/migrations/0009_alter_extractionjob_query.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 6.0 on 2025-12-11 20:03
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ("extractions", "0008_outputfield_subscription_and_more"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="extractionjob",
-            name="query",
-            field=models.CharField(blank=True, max_length=200),
-        ),
-    ]
diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index e5b52fa7..8d8b024a 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -37,7 +37,7 @@ class ExtractionJob(AnalysisJob):
     )
     title = models.CharField(max_length=100)
     group = models.ForeignKey[Group](Group, on_delete=models.CASCADE)
-    query = models.CharField(max_length=200, blank=True)
+    query = models.CharField(max_length=200)
     language = models.ForeignKey[Language](Language, on_delete=models.CASCADE)
     modalities = models.ManyToManyField(Modality, blank=True)
     study_date_from = models.DateField(null=True, blank=True)
diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index 6fc13224..b7c6b25d 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -91,12 +91,15 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
                             break
             except APIError as e:
                 logger.error(f"LLM API error filtering report {report.pk}: {e}")
+                db.close_old_connections()
                 return
             except ValidationError as e:
                 logger.error(f"Response validation failed filtering report {report.pk}: {e}")
+                db.close_old_connections()
                 return
             except AssertionError:
                 logger.error(f"No parsed response received filtering report {report.pk}")
+                db.close_old_connections()
                 return
         else:
             logger.debug(
@@ -107,6 +110,7 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
 
         if not is_accepted:
             logger.debug(f"Report {report.pk} was rejected by subscription {subscription.pk}")
+            db.close_old_connections()
             return
 
         extraction_results: dict[str, Any] = {}
@@ -130,12 +134,15 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
                     )
             except APIError as e:
                 logger.error(f"LLM API error extracting from report {report.pk}: {e}")
+                db.close_old_connections()
                 return
             except ValidationError as e:
                 logger.error(f"Response validation failed extracting from report {report.pk}: {e}")
+                db.close_old_connections()
                 return
             except AssertionError:
                 logger.error(f"No parsed response received extracting from report {report.pk}")
+                db.close_old_connections()
                 return
 
         SubscribedItem.objects.create(
@@ -146,3 +153,4 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
             extraction_results=extraction_results or None,
         )
         logger.debug(f"Report {report.pk} was accepted by subscription {subscription.pk}")
+        db.close_old_connections()

From e453cad27eb88672ac8ef559a698cd7d6d1e096a Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 30 Dec 2025 01:31:27 +0000
Subject: [PATCH 85/93] use try finally for closing db connection in threads

---
 radis/subscriptions/processors.py | 185 +++++++++++++++---------------
 1 file changed, 92 insertions(+), 93 deletions(-)

diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index b7c6b25d..2e3a87aa 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -55,102 +55,101 @@ def process_task(self, task: SubscriptionTask) -> None:
                 db.close_old_connections()
 
     def process_report(self, report: Report, task: SubscriptionTask) -> None:
-        subscription: Subscription = task.job.subscription
-
-        filter_results: dict[str, bool] = {}
-        is_accepted = True
-
-        filter_questions = list(subscription.filter_questions.order_by("pk"))
-
-        if filter_questions:
-            filter_prompt = Template(settings.SUBSCRIPTION_FILTER_PROMPT).substitute(
-                {
-                    "report": report.body,
-                    "questions": generate_filter_questions_prompt(filter_questions),
-                }
-            )
-            filter_schema = generate_filter_questions_schema(filter_questions)
-
-            try:
-                filter_response = self.client.extract_data(filter_prompt, filter_schema)
-
-                for question in filter_questions:
-                    field_name = get_filter_question_field_name(question)
-                    answer = getattr(filter_response, field_name, None)
-                    if answer is None:
-                        logger.debug(
-                            f"LLM returned None for question {question.pk} on report {report.pk}"
-                        )
-                        is_accepted = False
-                        break
-                    else:
-                        answer_bool = bool(answer)
-                        filter_results[str(question.pk)] = answer_bool
-                        if answer_bool != question.expected_answer_bool:
+        try:
+            subscription: Subscription = task.job.subscription
+
+            filter_results: dict[str, bool] = {}
+            is_accepted = True
+
+            filter_questions = list(subscription.filter_questions.order_by("pk"))
+
+            if filter_questions:
+                filter_prompt = Template(settings.SUBSCRIPTION_FILTER_PROMPT).substitute(
+                    {
+                        "report": report.body,
+                        "questions": generate_filter_questions_prompt(filter_questions),
+                    }
+                )
+                filter_schema = generate_filter_questions_schema(filter_questions)
+
+                try:
+                    filter_response = self.client.extract_data(filter_prompt, filter_schema)
+
+                    for question in filter_questions:
+                        field_name = get_filter_question_field_name(question)
+                        answer = getattr(filter_response, field_name, None)
+                        if answer is None:
+                            logger.debug(
+                                f"LLM returned None for question {question.pk} on report {report.pk}"
+                            )
                             is_accepted = False
                             break
-            except APIError as e:
-                logger.error(f"LLM API error filtering report {report.pk}: {e}")
-                db.close_old_connections()
+                        else:
+                            answer_bool = bool(answer)
+                            filter_results[str(question.pk)] = answer_bool
+                            if answer_bool != question.expected_answer_bool:
+                                is_accepted = False
+                                break
+                except APIError as e:
+                    logger.error(f"LLM API error filtering report {report.pk}: {e}")
+                    return
+                except ValidationError as e:
+                    logger.error(f"Response validation failed filtering report {report.pk}: {e}")
+                    return
+                except AssertionError:
+                    logger.error(f"No parsed response received filtering report {report.pk}")
+                    return
+            else:
+                logger.debug(
+                    "Subscription %s has no filter questions; accepting report %s by default",
+                    subscription.pk,
+                    report.pk,
+                )
+
+            if not is_accepted:
+                logger.debug(f"Report {report.pk} was rejected by subscription {subscription.pk}")
                 return
-            except ValidationError as e:
-                logger.error(f"Response validation failed filtering report {report.pk}: {e}")
-                db.close_old_connections()
-                return
-            except AssertionError:
-                logger.error(f"No parsed response received filtering report {report.pk}")
-                db.close_old_connections()
-                return
-        else:
-            logger.debug(
-                "Subscription %s has no filter questions; accepting report %s by default",
-                subscription.pk,
-                report.pk,
-            )
-
-        if not is_accepted:
-            logger.debug(f"Report {report.pk} was rejected by subscription {subscription.pk}")
-            db.close_old_connections()
-            return
 
-        extraction_results: dict[str, Any] = {}
-        output_fields = list(subscription.output_fields.order_by("pk"))
-
-        if output_fields:
-            extraction_prompt = Template(settings.SUBSCRIPTION_EXTRACTION_PROMPT).substitute(
-                {
-                    "report": report.body,
-                    "fields": generate_output_fields_prompt(output_fields),
-                }
-            )
-            extraction_schema = generate_output_fields_schema(output_fields)
-
-            try:
-                extraction_response = self.client.extract_data(extraction_prompt, extraction_schema)
-
-                for field in output_fields:
-                    extraction_results[str(field.pk)] = getattr(
-                        extraction_response, get_output_field_name(field), None
+            extraction_results: dict[str, Any] = {}
+            output_fields = list(subscription.output_fields.order_by("pk"))
+
+            if output_fields:
+                extraction_prompt = Template(settings.SUBSCRIPTION_EXTRACTION_PROMPT).substitute(
+                    {
+                        "report": report.body,
+                        "fields": generate_output_fields_prompt(output_fields),
+                    }
+                )
+                extraction_schema = generate_output_fields_schema(output_fields)
+
+                try:
+                    extraction_response = self.client.extract_data(
+                        extraction_prompt, extraction_schema
                     )
-            except APIError as e:
-                logger.error(f"LLM API error extracting from report {report.pk}: {e}")
-                db.close_old_connections()
-                return
-            except ValidationError as e:
-                logger.error(f"Response validation failed extracting from report {report.pk}: {e}")
-                db.close_old_connections()
-                return
-            except AssertionError:
-                logger.error(f"No parsed response received extracting from report {report.pk}")
-                db.close_old_connections()
-                return
 
-        SubscribedItem.objects.create(
-            subscription=task.job.subscription,
-            job=task.job,
-            report=report,
-            filter_results=filter_results or None,
-            extraction_results=extraction_results or None,
-        )
-        logger.debug(f"Report {report.pk} was accepted by subscription {subscription.pk}")
-        db.close_old_connections()
+                    for field in output_fields:
+                        extraction_results[str(field.pk)] = getattr(
+                            extraction_response, get_output_field_name(field), None
+                        )
+                except APIError as e:
+                    logger.error(f"LLM API error extracting from report {report.pk}: {e}")
+                    return
+                except ValidationError as e:
+                    logger.error(
+                        f"Response validation failed extracting from report {report.pk}: {e}"
+                    )
+                    return
+                except AssertionError:
+                    logger.error(f"No parsed response received extracting from report {report.pk}")
+                    return
+
+            SubscribedItem.objects.create(
+                subscription=task.job.subscription,
+                job=task.job,
+                report=report,
+                filter_results=filter_results or None,
+                extraction_results=extraction_results or None,
+            )
+            logger.debug(f"Report {report.pk} was accepted by subscription {subscription.pk}")
+        finally:
+            db.close_old_connections()

From 54a2273ac833dd25d55adf4505814b9250beb326 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 30 Dec 2025 01:45:13 +0000
Subject: [PATCH 86/93] lint fix

---
 radis/subscriptions/processors.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index 2e3a87aa..9a5cfb76 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -80,7 +80,8 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
                         answer = getattr(filter_response, field_name, None)
                         if answer is None:
                             logger.debug(
-                                f"LLM returned None for question {question.pk} on report {report.pk}"
+                                f"LLM returned None for question {question.pk} ",
+                                f"on report {report.pk}",
                             )
                             is_accepted = False
                             break

From 0a4e8c87226b7848a94649360abdd584b78147ce Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 30 Dec 2025 10:54:38 +0000
Subject: [PATCH 87/93] remove error handling code smell, add transaction
 atomicity to subscription saves

---
 radis/subscriptions/processors.py | 9 ++++-----
 radis/subscriptions/views.py      | 4 +++-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/radis/subscriptions/processors.py b/radis/subscriptions/processors.py
index 9a5cfb76..5c76a517 100644
--- a/radis/subscriptions/processors.py
+++ b/radis/subscriptions/processors.py
@@ -6,7 +6,6 @@
 from adit_radis_shared.common.types import User
 from django import db
 from django.conf import settings
-from openai import APIError
 from pydantic import ValidationError
 
 from radis.chats.utils.chat_client import ChatClient
@@ -91,13 +90,13 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
                             if answer_bool != question.expected_answer_bool:
                                 is_accepted = False
                                 break
-                except APIError as e:
+                except RuntimeError as e:
                     logger.error(f"LLM API error filtering report {report.pk}: {e}")
                     return
                 except ValidationError as e:
                     logger.error(f"Response validation failed filtering report {report.pk}: {e}")
                     return
-                except AssertionError:
+                except ValueError:
                     logger.error(f"No parsed response received filtering report {report.pk}")
                     return
             else:
@@ -132,7 +131,7 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
                         extraction_results[str(field.pk)] = getattr(
                             extraction_response, get_output_field_name(field), None
                         )
-                except APIError as e:
+                except RuntimeError as e:
                     logger.error(f"LLM API error extracting from report {report.pk}: {e}")
                     return
                 except ValidationError as e:
@@ -140,7 +139,7 @@ def process_report(self, report: Report, task: SubscriptionTask) -> None:
                         f"Response validation failed extracting from report {report.pk}: {e}"
                     )
                     return
-                except AssertionError:
+                except ValueError:
                     logger.error(f"No parsed response received extracting from report {report.pk}")
                     return
 
diff --git a/radis/subscriptions/views.py b/radis/subscriptions/views.py
index de865922..afe4529e 100644
--- a/radis/subscriptions/views.py
+++ b/radis/subscriptions/views.py
@@ -12,7 +12,7 @@
 from adit_radis_shared.common.types import AuthenticatedHttpRequest
 from django.contrib.auth.mixins import LoginRequiredMixin
 from django.contrib.messages.views import SuccessMessageMixin
-from django.db import IntegrityError
+from django.db import IntegrityError, transaction
 from django.db.models import Count, F, Q, QuerySet
 from django.forms.models import BaseInlineFormSet
 from django.http import HttpResponse, HttpResponseRedirect, StreamingHttpResponse
@@ -91,6 +91,7 @@ def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
             ctx["output_formset"] = OutputFieldFormSet()
         return ctx
 
+    @transaction.atomic()
     def form_valid(self, form) -> HttpResponse:
         ctx = self.get_context_data()
         filter_formset: BaseInlineFormSet = ctx["filter_formset"]
@@ -148,6 +149,7 @@ def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
         ctx["output_formset"].extra = 0
         return ctx
 
+    @transaction.atomic()
     def form_valid(self, form) -> HttpResponse:
         ctx = self.get_context_data()
         filter_formset = ctx["filter_formset"]

From 86539a14fa41b816f3411ed8f13b9beb9078de09 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 30 Dec 2025 11:16:09 +0000
Subject: [PATCH 88/93] add auto escape to protect against xss attacks

---
 .../_subscribed_item_preview.html             | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html b/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
index 47c47eb7..894786f2 100644
--- a/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
+++ b/radis/subscriptions/templates/subscriptions/_subscribed_item_preview.html
@@ -22,14 +22,17 @@
                     <div class="border-top pt-2 mt-1">
                         <div class="fw-semibold small text-muted mb-1">Extracted Fields</div>
                         <dl class="row mb-0 small">
-                            {% for field in subscribed_item.subscription.output_fields.all %}
-                                {% with field_key=field.pk|stringformat:"s" %}
-                                    <dt class="col-sm-4">{{ field.name }}</dt>
-                                    <dd class="col-sm-8">
-                                        {{ subscribed_item.extraction_results|get_item:field_key|default_if_none:"—" }}
-                                    </dd>
-                                {% endwith %}
-                            {% endfor %}
+                            {# LLM-generated content - ensure HTML escaping for XSS protection #}
+                            {% autoescape on %}
+                                {% for field in subscribed_item.subscription.output_fields.all %}
+                                    {% with field_key=field.pk|stringformat:"s" %}
+                                        <dt class="col-sm-4">{{ field.name }}</dt>
+                                        <dd class="col-sm-8">
+                                            {{ subscribed_item.extraction_results|get_item:field_key|default_if_none:"—" }}
+                                        </dd>
+                                    {% endwith %}
+                                {% endfor %}
+                            {% endautoescape %}
                         </dl>
                     </div>
                 {% endif %}

From 4e4d1ea33db5c229b1f0608882b8f543952fe8ea Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 30 Dec 2025 11:28:39 +0000
Subject: [PATCH 89/93] prefetch pks to avoid n+1 query during csv export

---
 radis/subscriptions/utils/csv_export.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/radis/subscriptions/utils/csv_export.py b/radis/subscriptions/utils/csv_export.py
index e36e2ec1..a5e84279 100644
--- a/radis/subscriptions/utils/csv_export.py
+++ b/radis/subscriptions/utils/csv_export.py
@@ -36,6 +36,11 @@ def iter_subscribed_item_rows(
         subscription.output_fields.order_by("pk").values_list("name", flat=True)
     )
 
+    # Pre-fetch field PKs to avoid N+1 query in the loop below
+    field_pks: list[int] = list(
+        subscription.output_fields.order_by("pk").values_list("pk", flat=True)
+    )
+
     # Header row
     header = [
         "subscribed_item_id",
@@ -75,7 +80,7 @@ def iter_subscribed_item_rows(
 
         # Add extraction results (keyed by field PK as string)
         extraction_results: dict[str, Any] = item.extraction_results or {}
-        for field_pk in subscription.output_fields.order_by("pk").values_list("pk", flat=True):
+        for field_pk in field_pks:
             value = extraction_results.get(str(field_pk))
             row.append(_format_cell(value))
 

From 566924ff70bdfd5536151a618a79831d2efac0d9 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 30 Dec 2025 22:44:04 +0000
Subject: [PATCH 90/93] made query generation async

---
 radis/extractions/forms.py                    |   2 +
 .../extractions/_query_generation_result.html |  44 ++++++
 .../_query_generation_section.html            |  21 +++
 .../extraction_job_search_form.html           |  70 +--------
 .../extractions/tests/test_query_generator.py |  29 ++--
 radis/extractions/urls.py                     |   6 +
 radis/extractions/utils/query_generator.py    |  70 +++------
 radis/extractions/views.py                    | 138 ++++++++++++++----
 8 files changed, 219 insertions(+), 161 deletions(-)
 create mode 100644 radis/extractions/templates/extractions/_query_generation_result.html
 create mode 100644 radis/extractions/templates/extractions/_query_generation_section.html

diff --git a/radis/extractions/forms.py b/radis/extractions/forms.py
index cfe629b6..b7d20db8 100644
--- a/radis/extractions/forms.py
+++ b/radis/extractions/forms.py
@@ -99,6 +99,8 @@ def build_layout(self):
             Row(
                 Column(
                     "title",
+                    # Query generation section (async HTMX)
+                    HTML('{% include "extractions/_query_generation_section.html" %}'),
                     "query",
                     # Preview div from template include
                     HTML('{% include "extractions/_search_preview_form_section.html" %}'),
diff --git a/radis/extractions/templates/extractions/_query_generation_result.html b/radis/extractions/templates/extractions/_query_generation_result.html
new file mode 100644
index 00000000..6205dca0
--- /dev/null
+++ b/radis/extractions/templates/extractions/_query_generation_result.html
@@ -0,0 +1,44 @@
+{% load bootstrap_icon from common_extras %}
+{% if error %}
+    <div class="alert alert-warning mb-4" role="alert">
+        <h5 class="alert-heading">{% bootstrap_icon "exclamation-triangle" %} Query Generation Failed</h5>
+        <p>{{ error }}</p>
+        <p class="mb-0">
+            <small>
+                You can manually enter a search query below.
+                The query will help find relevant medical reports for your extraction.
+            </small>
+        </p>
+    </div>
+{% elif generated_query %}
+    <div class="alert alert-success mb-4" role="alert">
+        <h5 class="alert-heading">{% bootstrap_icon "check-circle" %} Query Generated Successfully</h5>
+        <p>
+            Based on your {{ output_fields_count }} extraction field{{ output_fields_count|pluralize }},
+            we generated the following search query:
+        </p>
+        <code class="d-block bg-light p-2 mb-2">{{ generated_query }}</code>
+        <p class="mb-0">
+            <small>
+                You can edit this query below to refine your search.
+                The query will help find relevant medical reports for your extraction.
+            </small>
+        </p>
+    </div>
+    {# Update the query input field using out-of-band swap #}
+    <input type="text"
+           name="1-query"
+           value="{{ generated_query }}"
+           hx-swap-oob="true"
+           id="id_1-query"
+           class="textinput textInput form-control"
+           maxlength="500"
+           placeholder="Auto-generated query (editable)" />
+{% else %}
+    <div class="alert alert-info mb-4" role="alert">
+        <h5 class="alert-heading">{% bootstrap_icon "info-circle" %} No Query Generated</h5>
+        <p class="mb-0">
+            <small>Please manually enter a search query below.</small>
+        </p>
+    </div>
+{% endif %}
diff --git a/radis/extractions/templates/extractions/_query_generation_section.html b/radis/extractions/templates/extractions/_query_generation_section.html
new file mode 100644
index 00000000..1a73b3b2
--- /dev/null
+++ b/radis/extractions/templates/extractions/_query_generation_section.html
@@ -0,0 +1,21 @@
+{% load bootstrap_icon from common_extras %}
+<div id="query-generation-section"
+     hx-post="{% url 'extraction_generate_query' %}"
+     hx-trigger="load"
+     hx-swap="innerHTML"
+     hx-on::after-swap="htmx.trigger('#search-preview', 'updatePreview')">
+    <!-- Loading state shown initially -->
+    <div class="alert alert-info mb-4">
+        <div class="d-flex align-items-center">
+            <span class="spinner-border spinner-border-sm me-3"
+                  role="status"
+                  aria-hidden="true"></span>
+            <div>
+                <h5 class="alert-heading mb-0">{% bootstrap_icon "magic" %} Generating Search Query...</h5>
+                <p class="mb-0 mt-2">
+                    <small>Using AI to create a search query from your extraction fields...</small>
+                </p>
+            </div>
+        </div>
+    </div>
+</div>
diff --git a/radis/extractions/templates/extractions/extraction_job_search_form.html b/radis/extractions/templates/extractions/extraction_job_search_form.html
index cf25abc9..999f752f 100644
--- a/radis/extractions/templates/extractions/extraction_job_search_form.html
+++ b/radis/extractions/templates/extractions/extraction_job_search_form.html
@@ -24,80 +24,12 @@
     </c-page-heading>
 {% endblock heading %}
 {% block content %}
-    {% if generated_query %}
-        <div class="alert alert-info mb-4" role="alert">
-            <h5 class="alert-heading">{% bootstrap_icon "magic" %} Auto-Generated Query</h5>
-            <p>
-                Based on your {{ output_fields_count }} extraction field{{ output_fields_count|pluralize }},
-                we generated the following search query:
-            </p>
-            <code class="d-block bg-light p-2 mb-2">{{ generated_query }}</code>
-            <p class="mb-0">
-                <small>
-                    You can edit this query below to refine your search.
-                    The query will help find relevant medical reports for your extraction.
-                </small>
-            </p>
-        </div>
-    {% endif %}
     <form action=""
           method="post"
           x-data="{ isLoading: false }"
-          @input.debounce.500ms=" console.log('[DEBUG] Form input event:', { fieldName: $event.target.name, fieldType: $event.target.type, fieldValue: $event.target.value, target: $event.target }); isLoading = true; console.log('[DEBUG] Triggering updatePreview event'); htmx.trigger('#search-preview', 'updatePreview'); ">
+          @input.debounce.500ms="isLoading = true; htmx.trigger('#search-preview', 'updatePreview');">
         {% csrf_token %}
         {{ wizard.management_form }}
         {% crispy form %}
     </form>
-    <script>
-    // Debug: Log HTMX events
-    document.addEventListener('htmx:beforeRequest', function(evt) {
-        console.log('[DEBUG] HTMX beforeRequest:', {
-            url: evt.detail.pathInfo?.requestPath,
-            target: evt.detail.target,
-            trigger: evt.detail.triggeringEvent
-        });
-    });
-
-    document.addEventListener('htmx:afterRequest', function(evt) {
-        console.log('[DEBUG] HTMX afterRequest:', {
-            status: evt.detail.xhr.status,
-            successful: evt.detail.successful,
-            responseLength: evt.detail.xhr.response?.length
-        });
-    });
-
-    document.addEventListener('htmx:responseError', function(evt) {
-        console.error('[DEBUG] HTMX responseError:', {
-            error: evt.detail,
-            status: evt.detail.xhr?.status
-        });
-    });
-
-    // Debug: Check initial state when DOM loads
-    document.addEventListener('DOMContentLoaded', function() {
-        console.log('[DEBUG] DOMContentLoaded fired');
-        const queryField = document.querySelector('[name="query"]');
-        console.log('[DEBUG] Query field:', {
-            exists: !!queryField,
-            value: queryField?.value,
-            hasValue: queryField?.value?.trim().length > 0,
-            tagName: queryField?.tagName
-        });
-
-        const previewDiv = document.querySelector('#search-preview');
-        console.log('[DEBUG] Preview div:', {
-            exists: !!previewDiv,
-            hxGet: previewDiv?.getAttribute('hx-get'),
-            hxTrigger: previewDiv?.getAttribute('hx-trigger')
-        });
-
-        // Log if htmx is loaded
-        console.log('[DEBUG] HTMX loaded:', typeof htmx !== 'undefined');
-    });
-
-    // Debug: Listen for the custom updatePreview event
-    document.addEventListener('updatePreview', function(evt) {
-        console.log('[DEBUG] Custom updatePreview event received:', evt);
-    });
-    </script>
 {% endblock content %}
diff --git a/radis/extractions/tests/test_query_generator.py b/radis/extractions/tests/test_query_generator.py
index a4c664ed..7a7808a2 100644
--- a/radis/extractions/tests/test_query_generator.py
+++ b/radis/extractions/tests/test_query_generator.py
@@ -1,24 +1,24 @@
-"""Unit tests for the QueryGenerator class."""
+"""Unit tests for the AsyncQueryGenerator class."""
 
-from unittest.mock import Mock, patch
+from unittest.mock import AsyncMock, patch
 
 from django.test import TestCase, override_settings
 
 from radis.extractions.models import OutputField, OutputType
-from radis.extractions.utils.query_generator import QueryGenerator
+from radis.extractions.utils.query_generator import AsyncQueryGenerator
 
 
-class QueryGeneratorTest(TestCase):
-    """Test cases for QueryGenerator."""
+class AsyncQueryGeneratorTest(TestCase):
+    """Test cases for AsyncQueryGenerator."""
 
     def setUp(self):
         """Set up test fixtures."""
-        self.generator = QueryGenerator()
+        self.generator = AsyncQueryGenerator()
 
-    def test_generate_from_empty_fields(self):
+    async def test_generate_from_empty_fields(self):
         """Test query generation with no fields returns None."""
         fields = []
-        query, metadata = self.generator.generate_from_fields(fields)
+        query, metadata = await self.generator.generate_from_fields(fields)
 
         assert query is None
         assert metadata["generation_method"] is None
@@ -26,11 +26,11 @@ def test_generate_from_empty_fields(self):
         assert metadata["success"] is False
         assert metadata["error"] == "No fields provided"
 
-    @patch("radis.extractions.utils.query_generator.ChatClient")
-    def test_llm_generation_success(self, mock_chat_client_class):
+    @patch("radis.chats.utils.chat_client.AsyncChatClient")
+    async def test_llm_generation_success(self, mock_chat_client_class):
         """Test successful LLM query generation."""
-        # Mock the LLM response
-        mock_client = Mock()
+        # Mock the LLM response using AsyncMock for async methods
+        mock_client = AsyncMock()
         mock_client.chat.return_value = '("lung nodule" OR "pulmonary nodule") AND size'
         mock_chat_client_class.return_value = mock_client
 
@@ -43,11 +43,10 @@ def test_llm_generation_success(self, mock_chat_client_class):
         ]
 
         with override_settings(ENABLE_AUTO_QUERY_GENERATION=True):
-            generator = QueryGenerator()  # Create new instance with mocked client
-            query, metadata = generator.generate_from_fields(fields)
+            generator = AsyncQueryGenerator()  # Create new instance with mocked client
+            query, metadata = await generator.generate_from_fields(fields)
 
         assert query != ""
-        assert metadata["generation_method"] == "llm"
         assert metadata["success"] is True
         assert query is not None
         assert "nodule" in query.lower()
diff --git a/radis/extractions/urls.py b/radis/extractions/urls.py
index 42daea38..6b9ef946 100644
--- a/radis/extractions/urls.py
+++ b/radis/extractions/urls.py
@@ -12,6 +12,7 @@
     ExtractionJobRetryView,
     ExtractionJobVerifyView,
     ExtractionJobWizardView,
+    ExtractionQueryGeneratorView,
     ExtractionResultDownloadView,
     ExtractionResultListView,
     ExtractionSearchPreviewView,
@@ -46,6 +47,11 @@
         ExtractionSearchPreviewView.as_view(),
         name="extraction_search_preview",
     ),
+    path(
+        "jobs/new/generate-query/",
+        ExtractionQueryGeneratorView.as_view(),
+        name="extraction_generate_query",
+    ),
     path(
         "jobs/<int:pk>/",
         ExtractionJobDetailView.as_view(),
diff --git a/radis/extractions/utils/query_generator.py b/radis/extractions/utils/query_generator.py
index 4713eac3..95615fcb 100644
--- a/radis/extractions/utils/query_generator.py
+++ b/radis/extractions/utils/query_generator.py
@@ -12,38 +12,34 @@
 
 from django.conf import settings
 
-from radis.chats.utils.chat_client import ChatClient
 from radis.extractions.models import OutputField
 from radis.search.utils.query_parser import QueryParser
 
 logger = logging.getLogger(__name__)
 
 
-class QueryGenerator:
-    """Generates search queries from extraction fields using LLM with fallbacks."""
+class AsyncQueryGenerator:
+    """Async version of QueryGenerator for use in async views."""
 
     def __init__(self):
-        """Initialize the query generator with an LLM client."""
-        self.client = ChatClient()
+        """Initialize the async query generator with an async LLM client."""
+        from radis.chats.utils.chat_client import AsyncChatClient
+
+        self.client = AsyncChatClient()
         self.parser = QueryParser()
 
-    def generate_from_fields(
+    async def generate_from_fields(
         self, fields: Iterable[OutputField]
     ) -> tuple[str | None, dict[str, Any]]:
         """
-        Generate a search query from extraction fields.
+        Async version of generate_from_fields.
 
         Args:
             fields: Iterable of OutputField objects to generate query from
 
         Returns:
             Tuple of (query_string, metadata_dict)
-            - query_string: The generated query
-            - metadata: Dict with keys:
-                - generation_method: "llm"
-                - field_count: Number of fields processed
-                - success: Boolean indicating if generation succeeded
-                - error: Optional error message if failed
+            Same structure as synchronous version
         """
         fields_list = list(fields)
         field_count = len(fields_list)
@@ -55,19 +51,15 @@ def generate_from_fields(
             "error": None,
         }
 
-        # Handle empty fields
         if field_count == 0:
-            logger.warning("No fields provided for query generation")
+            logger.warning("No fields provided for async query generation")
             metadata["error"] = "No fields provided"
-            metadata["success"] = False
             return None, metadata
 
-        # Try LLM generation first
         if settings.ENABLE_AUTO_QUERY_GENERATION:
             try:
-                query = self._call_llm(fields_list)
+                query = await self._call_llm(fields_list)
                 if query:
-                    # Validate and fix the query
                     validated_query, fixes = self.validate_and_fix_query(query)
                     if validated_query:
                         logger.info(
@@ -78,27 +70,20 @@ def generate_from_fields(
                         metadata["fixes_applied"] = len(fixes) > 0
                         return validated_query, metadata
                     else:
-                        logger.warning("LLM generated invalid query, trying fallback")
+                        logger.warning("LLM generated invalid query")
                         metadata["error"] = "LLM generated invalid query"
             except Exception as e:
-                logger.error(f"Error during LLM query generation: {e}", exc_info=True)
+                logger.error(f"Error during async LLM query generation: {e}", exc_info=True)
                 metadata["error"] = str(e)
 
-        # No fallback - return None for LLM failures
-        logger.warning(f"LLM query generation failed for {field_count} fields")
-
-        # All methods failed - return None
-        logger.warning(
-            f"All query generation methods failed for {field_count} fields. "
-            "User must manually enter a query."
-        )
+        logger.warning(f"Async query generation failed for {field_count} fields")
         metadata["error"] = metadata.get("error") or "All generation methods failed"
         metadata["success"] = False
         return None, metadata
 
-    def _call_llm(self, fields: list[OutputField]) -> str | None:
+    async def _call_llm(self, fields: list[OutputField]) -> str | None:
         """
-        Call LLM to generate a query from extraction fields.
+        Async version of _call_llm.
 
         Args:
             fields: List of OutputField objects
@@ -106,35 +91,31 @@ def _call_llm(self, fields: list[OutputField]) -> str | None:
         Returns:
             Generated query string, or None if failed
         """
-        # Format fields for the prompt
         fields_formatted = self._format_fields_for_prompt(fields)
 
-        # Build the prompt from template
         prompt = Template(settings.QUERY_GENERATION_SYSTEM_PROMPT).substitute(
             fields=fields_formatted
         )
 
         try:
-            # Call LLM with timeout
-            response = self.client.chat([{"role": "user", "content": prompt}])
+            response = await self.client.chat([{"role": "user", "content": prompt}])
 
             if not response:
                 logger.warning("LLM returned empty response")
                 return None
 
-            # Extract and clean the query
             query = self._extract_query_from_response(response)
-
-            logger.debug(f"LLM generated query: {query}")
+            logger.debug(f"Async LLM generated query: {query}")
             return query
 
         except Exception as e:
-            logger.error(f"LLM call failed: {e}")
+            logger.error(f"Async LLM call failed: {e}")
             return None
 
     def _format_fields_for_prompt(self, fields: list[OutputField]) -> str:
         """
         Format extraction fields for inclusion in LLM prompt.
+        Reuses synchronous version logic.
 
         Args:
             fields: List of OutputField objects
@@ -156,9 +137,7 @@ def _format_fields_for_prompt(self, fields: list[OutputField]) -> str:
     def _extract_query_from_response(self, response: str) -> str:
         """
         Extract query from LLM response.
-
-        The LLM might return the query with additional text or formatting.
-        This method cleans and extracts just the query.
+        Reuses synchronous version logic.
 
         Args:
             response: Raw LLM response
@@ -166,32 +145,28 @@ def _extract_query_from_response(self, response: str) -> str:
         Returns:
             Cleaned query string
         """
-        # Remove common prefixes like "Query:", "Search:", etc.
         cleaned = re.sub(
             r"^(query|search|generated query|result):\s*", "", response.strip(), flags=re.IGNORECASE
         )
 
-        # Remove quotes if the entire response is quoted
         if cleaned.startswith('"') and cleaned.endswith('"'):
             cleaned = cleaned[1:-1]
         elif cleaned.startswith("'") and cleaned.endswith("'"):
             cleaned = cleaned[1:-1]
 
-        # Take only the first line (in case LLM added explanation)
         cleaned = cleaned.split("\n")[0].strip()
-
         return cleaned
 
     def validate_and_fix_query(self, query: str) -> tuple[str, list[str]]:
         """
         Validate and fix a query using QueryParser.
+        Reuses synchronous version logic.
 
         Args:
             query: Query string to validate
 
         Returns:
             Tuple of (fixed_query, list_of_fixes_applied)
-            Returns ("", []) if query is invalid
         """
         if not query or not query.strip():
             return "", []
@@ -203,7 +178,6 @@ def validate_and_fix_query(self, query: str) -> tuple[str, list[str]]:
                 logger.warning(f"Query validation failed for: {query}")
                 return "", []
 
-            # If fixes were applied, unparse to get the fixed query
             if len(fixes) > 0:
                 fixed_query = QueryParser.unparse(query_node)
                 logger.debug(f"Applied {len(fixes)} fixes to query: {fixes}")
diff --git a/radis/extractions/views.py b/radis/extractions/views.py
index 88cc18c3..3010e778 100644
--- a/radis/extractions/views.py
+++ b/radis/extractions/views.py
@@ -61,7 +61,6 @@
     ExtractionTaskTable,
 )
 from .utils.csv_export import iter_extraction_result_rows
-from .utils.query_generator import QueryGenerator
 
 EXTRACTIONS_SEARCH_PROVIDER = "extractions_search_provider"
 
@@ -104,43 +103,29 @@ def process_step(self, form):
         """Process validated form data and trigger query generation after output fields step."""
         step_data = self.get_form_step_data(form)
 
-        # After output fields are submitted, generate query
+        # After output fields are submitted, store field data for async query generation
         if self.steps.current == ExtractionJobWizardView.OUTPUT_FIELDS_STEP:
-            # Extract output fields from formset
-            from .models import OutputField
-
+            # Extract and serialize output fields data for async generation
             formset_data = []
             if hasattr(form, "cleaned_data"):
                 formset_data = form.cleaned_data
 
-            temp_fields = [
-                OutputField(
-                    name=field_data["name"],
-                    description=field_data["description"],
-                    output_type=field_data["output_type"],
-                )
+            output_fields_data = [
+                {
+                    "name": field_data["name"],
+                    "description": field_data["description"],
+                    "output_type": field_data["output_type"],
+                }
                 for field_data in formset_data
                 if not field_data.get("DELETE", False)
             ]
 
-            # Generate query from fields
-            generator = QueryGenerator()
-            generated_query, metadata = generator.generate_from_fields(temp_fields)
-
-            # Store in wizard storage for use in next step
-            self.storage.extra_data["generated_query"] = generated_query or ""
-            self.storage.extra_data["query_metadata"] = metadata
-            self.storage.extra_data["query_generation_attempted"] = True
-
-            # Check if generation failed
-            if generated_query is None or not metadata.get("success"):
-                from django.contrib import messages
-
-                messages.warning(
-                    self.request,
-                    "Unable to automatically generate a query from your extraction fields. "
-                    "You can manually enter a search query in the next step.",
-                )
+            # Store serialized data for async generation - query will be generated via HTMX
+            self.storage.extra_data["output_fields_data"] = output_fields_data
+            self.storage.extra_data["query_generation_attempted"] = False
+            # Clear any previous query to ensure fresh generation
+            self.storage.extra_data["generated_query"] = ""
+            self.storage.extra_data["query_metadata"] = {}
 
             return step_data
 
@@ -481,6 +466,101 @@ class ExtractionJobResumeView(ExtractionsLockedMixin, AnalysisJobResumeView):
     model = ExtractionJob
 
 
+class ExtractionQueryGeneratorView(LoginRequiredMixin, View):
+    """HTMX endpoint for async query generation from output fields."""
+
+    request: AuthenticatedHttpRequest
+
+    async def post(self, request: AuthenticatedHttpRequest):
+        """Generate query asynchronously and save to wizard session."""
+        import logging
+
+        logger = logging.getLogger(__name__)
+        logger.info("Query generation endpoint called")
+
+        # Access wizard session storage
+        # Django-formtools stores wizard data in a nested structure:
+        # session['wizard_extraction_job_wizard_view'] = {
+        #     'step': '1',
+        #     'step_data': {...},
+        #     'extra_data': {'output_fields_data': [...], ...}
+        # }
+        wizard_session_key = "wizard_extraction_job_wizard_view"
+        wizard_data = request.session.get(wizard_session_key, {})
+
+        if not wizard_data:
+            logger.error(f"No wizard session data found for key: {wizard_session_key}")
+            logger.error(f"Available session keys: {list(request.session.keys())}")
+
+        # Get extra_data from within the wizard data
+        extra_data = wizard_data.get("extra_data", {})
+        output_fields_data = extra_data.get("output_fields_data", [])
+
+        logger.info(f"Found wizard_data: {bool(wizard_data)}, extra_data: {bool(extra_data)}")
+        logger.info(f"Found {len(output_fields_data)} output fields in session")
+
+        if not output_fields_data:
+            context = {
+                "error": "No output fields found. Please go back to step 1.",
+                "generated_query": "",
+                "query_metadata": {},
+            }
+            return render(request, "extractions/_query_generation_result.html", context)
+
+        # Reconstruct OutputField objects from stored data
+        from .models import OutputField
+
+        temp_fields = [
+            OutputField(
+                name=field_data["name"],
+                description=field_data["description"],
+                output_type=field_data["output_type"],
+            )
+            for field_data in output_fields_data
+        ]
+
+        # Generate query using async query generator
+        from .utils.query_generator import AsyncQueryGenerator
+
+        try:
+            generator = AsyncQueryGenerator()
+            generated_query, metadata = await generator.generate_from_fields(temp_fields)
+
+            # Store in wizard session
+            extra_data["generated_query"] = generated_query or ""
+            extra_data["query_metadata"] = metadata
+            extra_data["query_generation_attempted"] = True
+
+            # Update wizard data and save back to session
+            wizard_data["extra_data"] = extra_data
+            request.session[wizard_session_key] = wizard_data
+            request.session.modified = True
+            logger.info("Saved query to wizard session")
+
+            context = {
+                "generated_query": generated_query,
+                "query_metadata": metadata,
+                "output_fields_count": len(temp_fields),
+                "error": None if metadata.get("success") else "Query generation failed",
+            }
+
+        except Exception as e:
+            logger.error(f"Error during async query generation: {e}", exc_info=True)
+            context = {
+                "error": f"Error generating query: {str(e)}",
+                "generated_query": "",
+                "query_metadata": {"success": False, "error": str(e)},
+            }
+            extra_data["generated_query"] = ""
+            extra_data["query_metadata"] = context["query_metadata"]
+            extra_data["query_generation_attempted"] = True
+            wizard_data["extra_data"] = extra_data
+            request.session[wizard_session_key] = wizard_data
+            request.session.modified = True
+
+        return render(request, "extractions/_query_generation_result.html", context)
+
+
 class ExtractionJobRetryView(ExtractionsLockedMixin, AnalysisJobRetryView):
     model = ExtractionJob
 

From bb70ad4c0f8ccab3972c1ca960827fb355b8e2a7 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Mon, 5 Jan 2026 09:22:21 +0000
Subject: [PATCH 91/93] Remove duplicate OutputFieldForm from subscriptions,
 import from extractions

---
 radis/extractions/forms.py   |  21 ++++-
 radis/subscriptions/forms.py | 164 +----------------------------------
 2 files changed, 21 insertions(+), 164 deletions(-)

diff --git a/radis/extractions/forms.py b/radis/extractions/forms.py
index b7d20db8..9df3e447 100644
--- a/radis/extractions/forms.py
+++ b/radis/extractions/forms.py
@@ -3,7 +3,7 @@
 
 from adit_radis_shared.accounts.models import User
 from crispy_forms.helper import FormHelper
-from crispy_forms.layout import HTML, Column, Div, Layout, Row, Submit
+from crispy_forms.layout import HTML, Column, Div, Field, Layout, Row, Submit
 from django import forms
 from django.conf import settings
 from django.db.models import QuerySet
@@ -203,6 +203,10 @@ def clean(self) -> dict[str, Any] | None:
 
 
 class OutputFieldForm(forms.ModelForm):
+    """Hidden field to store selection options and array flag as JSON string.
+    This is done because the selection options are dynamic and the array toggle
+    is an alpine component that needs to be re-rendered on every change."""
+
     selection_options = forms.CharField(
         required=False,
         widget=forms.HiddenInput(),
@@ -225,6 +229,8 @@ class Meta:
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
+        self.fields["name"].required = True
+        self.fields["description"].required = True
         self.fields["description"].widget = forms.Textarea(attrs={"rows": 3})
         self.fields["selection_options"].widget.attrs.update(
             {
@@ -245,7 +251,10 @@ def __init__(self, *args, **kwargs):
         self.helper = FormHelper()
         self.helper.form_tag = False
         self.helper.disable_csrf = True
-        self.helper.layout = Layout(
+
+        # Build the layout for selection options and array toggle button using crispy.
+        fields = [
+            Field("id", type="hidden"),
             Row(
                 Column("name", css_class="col-md-7 col-12"),
                 Column("output_type", css_class="col-md-4 col-10"),
@@ -268,11 +277,17 @@ def __init__(self, *args, **kwargs):
                 css_class="g-3 align-items-center",
             ),
             "description",
+            # Include the selection options widget partial template here.
             Div(
                 HTML('{% include "extractions/_selection_options_field.html" %}'),
                 css_class="selection-options-wrapper",
             ),
-        )
+        ]
+
+        if "DELETE" in self.fields:
+            fields.insert(1, Field("DELETE", type="hidden"))
+
+        self.helper.layout = Layout(Div(*fields))
 
     def clean_selection_options(self) -> list[str]:
         raw_value = self.cleaned_data.get("selection_options") or ""
diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index 8f60cb44..e99bf937 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -1,14 +1,13 @@
-import json
 from typing import Any
 
 from crispy_forms.helper import FormHelper
-from crispy_forms.layout import HTML, Column, Div, Field, Layout, Row
+from crispy_forms.layout import Column, Div, Field, Layout, Row
 from django import forms
 
 from radis.core.constants import LANGUAGE_LABELS
 from radis.core.layouts import Formset, RangeSlider
-from radis.extractions.constants import MAX_SELECTION_OPTIONS
-from radis.extractions.models import OutputField, OutputType
+from radis.extractions.forms import OutputFieldForm
+from radis.extractions.models import OutputField
 from radis.reports.models import Language, Modality
 from radis.search.forms import AGE_STEP, MAX_AGE, MIN_AGE
 
@@ -175,163 +174,6 @@ def clean(self) -> dict[str, Any]:
         return cleaned_data
 
 
-class OutputFieldForm(forms.ModelForm):
-    """Hidden field to store selection options and array flag as JSON string.
-    This is done because the selection options are dynamic and the array toggle
-    is an alpine component that needs to be re-rendered on every change."""
-
-    selection_options = forms.CharField(
-        required=False,
-        widget=forms.HiddenInput(),
-    )
-    is_array = forms.CharField(
-        required=False,
-        widget=forms.HiddenInput(),
-    )
-
-    class Meta:
-        model = OutputField
-        fields = [
-            "name",
-            "description",
-            "output_type",
-            "selection_options",
-            "is_array",
-        ]
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        self.fields["name"].required = True
-        self.fields["description"].required = True
-        self.fields["description"].widget = forms.Textarea(attrs={"rows": 3})
-
-        # Stamp data attributes so the selection-options widget can locate these fields.
-        self.fields["selection_options"].widget.attrs.update(
-            {
-                "data-selection-input": "true",
-                "data-max-selection-options": str(MAX_SELECTION_OPTIONS),
-            }
-        )
-        self.fields["is_array"].widget.attrs.update(
-            {
-                "data-array-input": "true",
-            }
-        )
-
-        initial_options = self.instance.selection_options if self.instance.pk else []
-
-        # Prepopulate the hidden fields so existing options/toggle state show up in the widget.
-        self.initial["selection_options"] = json.dumps(initial_options)
-        self.initial["is_array"] = "true" if self.instance.is_array else "false"
-
-        self.helper = FormHelper()
-        self.helper.form_tag = False
-        self.helper.disable_csrf = True
-
-        # Build the layout for selection options and array toggle button using crispy.
-        fields = [
-            Field("id", type="hidden"),
-            Row(
-                Column("name", css_class="col-md-7 col-12"),
-                Column("output_type", css_class="col-md-4 col-10"),
-                Column(
-                    HTML(
-                        (
-                            '<button type="button" '
-                            'class="btn btn-outline-secondary btn-sm array-toggle-btn '
-                            'form-array-toggle" '
-                            'data-array-toggle="true" '
-                            'aria-pressed="false" '
-                            'title="Toggle array output">[ ]</button>'
-                        )
-                    ),
-                    css_class=(
-                        "col-md-1 col-2 d-flex align-items-center "
-                        "justify-content-end array-toggle-field"
-                    ),
-                ),
-                css_class="g-3 align-items-center",
-            ),
-            "description",
-            # Include the selection options widget partial template here.
-            Div(
-                HTML('{% include "extractions/_selection_options_field.html" %}'),
-                css_class="selection-options-wrapper",
-            ),
-        ]
-
-        if "DELETE" in self.fields:
-            fields.insert(1, Field("DELETE", type="hidden"))
-
-        self.helper.layout = Layout(Div(*fields))
-
-    # Parse and sanitize the JSON payload (trim strings, enforce uniqueness and max count).
-    # Called automatically by Django form validation.
-    def clean_selection_options(self) -> list[str]:
-        raw_value = self.cleaned_data.get("selection_options") or ""
-        raw_value = raw_value.strip()
-        if raw_value == "":
-            return []
-
-        try:
-            parsed = json.loads(raw_value)
-        except json.JSONDecodeError as exc:
-            raise forms.ValidationError("Invalid selection data.") from exc
-
-        if not isinstance(parsed, list):
-            raise forms.ValidationError("Selection data must be a list.")
-
-        cleaned: list[str] = []
-        for item in parsed:
-            if not isinstance(item, str):
-                raise forms.ValidationError("Selection options must be text.")
-            value = item.strip()
-            if not value:
-                raise forms.ValidationError("Selection options cannot be empty.")
-            cleaned.append(value)
-
-        if len(cleaned) > MAX_SELECTION_OPTIONS:
-            raise forms.ValidationError(
-                f"Provide at most {MAX_SELECTION_OPTIONS} selection options."
-            )
-        if len(set(cleaned)) != len(cleaned):
-            raise forms.ValidationError("Selection options must be unique.")
-
-        return cleaned
-
-    # not exactly needed because we set the value ourselves, but more as a double check
-    def clean_is_array(self) -> bool:
-        raw_value = (self.cleaned_data.get("is_array") or "").strip().lower()
-        if raw_value in {"1", "true", "on"}:
-            return True
-        return False
-
-    def clean(self):
-        cleaned_data = super().clean()
-        if not cleaned_data:
-            return cleaned_data
-
-        output_type = cleaned_data.get("output_type")
-        selection_options: list[str] = cleaned_data.get("selection_options") or []
-
-        if output_type == OutputType.SELECTION:
-            if not selection_options:
-                self.add_error(
-                    "selection_options",
-                    "Add at least one selection to use the Selection type.",
-                )
-        else:
-            if selection_options:
-                self.add_error(
-                    "selection_options",
-                    "Selections are only allowed when Output Type is Selection.",
-                )
-                cleaned_data["selection_options"] = []
-
-        return cleaned_data
-
-
 FilterQuestionFormSet = forms.inlineformset_factory(
     Subscription,
     FilterQuestion,

From 42faad0bea2a51b7c6157f18bc7b806ec3f80176 Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Tue, 6 Jan 2026 22:40:34 +0000
Subject: [PATCH 92/93] refactor search, extractions, subscriptions forms

---
 radis/core/constants.py      |   4 +
 radis/core/form_fields.py    | 177 ++++++++++++++++++++++++++++++++
 radis/extractions/forms.py   |  72 ++++---------
 radis/extractions/views.py   | 189 ++++++++++++++++-------------------
 radis/search/forms.py        |  55 +++-------
 radis/subscriptions/forms.py |  48 ++-------
 6 files changed, 308 insertions(+), 237 deletions(-)
 create mode 100644 radis/core/form_fields.py

diff --git a/radis/core/constants.py b/radis/core/constants.py
index db1b048b..8e1d74bd 100644
--- a/radis/core/constants.py
+++ b/radis/core/constants.py
@@ -2,3 +2,7 @@
     "de": "German",
     "en": "English",
 }
+
+MIN_AGE = 0
+MAX_AGE = 120
+AGE_STEP = 10
diff --git a/radis/core/form_fields.py b/radis/core/form_fields.py
new file mode 100644
index 00000000..1ce2a311
--- /dev/null
+++ b/radis/core/form_fields.py
@@ -0,0 +1,177 @@
+"""
+Reusable form field factories for RADIS forms.
+
+This module provides factory functions for commonly used form fields
+to reduce duplication across the codebase.
+"""
+
+from typing import Literal, overload
+
+from django import forms
+
+from radis.core.constants import AGE_STEP, LANGUAGE_LABELS, MAX_AGE, MIN_AGE
+from radis.reports.models import Language, Modality
+
+
+@overload
+def create_language_field(
+    required: bool = False,
+    empty_label: str | None = None,
+    use_pk: Literal[True] = True,
+) -> forms.ModelChoiceField: ...
+
+
+@overload
+def create_language_field(
+    required: bool = False,
+    empty_label: str | None = None,
+    use_pk: Literal[False] = False,
+) -> forms.ChoiceField: ...
+
+
+def create_language_field(
+    required: bool = False,
+    empty_label: str | None = None,
+    use_pk: bool = True,
+) -> forms.ModelChoiceField | forms.ChoiceField:
+    """
+    Create a language choice field with consistent configuration.
+
+    Args:
+        required: Whether the field is required
+        empty_label: Label for empty option (None = no empty option)
+        use_pk: If True, returns ModelChoiceField with Language objects;
+                if False, returns ChoiceField with code strings
+
+    Returns:
+        ModelChoiceField (if use_pk=True) or ChoiceField (if use_pk=False)
+
+    Example:
+        # For extraction forms (uses ModelChoiceField, returns Language objects)
+        self.fields["language"] = create_language_field()
+
+        # For subscription forms (uses ModelChoiceField, allows "All")
+        self.fields["language"] = create_language_field(empty_label="All")
+
+        # For search forms (uses ChoiceField with codes)
+        self.fields["language"] = create_language_field(use_pk=False)
+    """
+    languages = Language.objects.order_by("code")
+
+    if use_pk:
+        # Return ModelChoiceField - cleaned_data will contain Language objects
+        field = forms.ModelChoiceField(
+            queryset=languages,
+            required=required,
+            empty_label=empty_label,
+        )
+        field.label_from_instance = lambda obj: LANGUAGE_LABELS[obj.code]
+        return field
+    else:
+        # Return ChoiceField - cleaned_data will contain code strings
+        field = forms.ChoiceField(required=required)
+        field.choices = [(lang.code, LANGUAGE_LABELS[lang.code]) for lang in languages]
+        if empty_label is not None:
+            field.empty_label = empty_label  # type: ignore
+        return field
+
+
+@overload
+def create_modality_field(
+    required: bool = False,
+    widget_size: int = 6,
+    use_pk: Literal[True] = True,
+) -> forms.ModelMultipleChoiceField: ...
+
+
+@overload
+def create_modality_field(
+    required: bool = False,
+    widget_size: int = 6,
+    use_pk: Literal[False] = False,
+) -> forms.MultipleChoiceField: ...
+
+
+def create_modality_field(
+    required: bool = False,
+    widget_size: int = 6,
+    use_pk: bool = True,
+) -> forms.ModelMultipleChoiceField | forms.MultipleChoiceField:
+    """
+    Create a modality multiple choice field with consistent configuration.
+
+    Args:
+        required: Whether the field is required
+        widget_size: Height of the select widget
+        use_pk: If True, returns ModelMultipleChoiceField with Modality objects;
+                if False, returns MultipleChoiceField with code strings
+
+    Returns:
+        ModelMultipleChoiceField (if use_pk=True) or MultipleChoiceField (if use_pk=False)
+
+    Example:
+        # For extraction forms (uses ModelMultipleChoiceField, returns Modality objects)
+        self.fields["modalities"] = create_modality_field()
+
+        # For search forms (uses MultipleChoiceField with codes)
+        self.fields["modalities"] = create_modality_field(use_pk=False)
+    """
+    modalities = Modality.objects.filter(filterable=True).order_by("code")
+
+    if use_pk:
+        # Return ModelMultipleChoiceField - cleaned_data will contain Modality objects
+        field = forms.ModelMultipleChoiceField(
+            queryset=modalities,
+            required=required,
+        )
+        # Display just the code for each modality
+        field.label_from_instance = lambda obj: obj.code
+        field.widget.attrs["size"] = widget_size
+        return field
+    else:
+        # Return MultipleChoiceField - cleaned_data will contain code strings
+        field = forms.MultipleChoiceField(required=required)
+        field.choices = [(mod.code, mod.code) for mod in modalities]
+        field.widget.attrs["size"] = widget_size
+        return field
+
+
+def create_age_range_fields() -> tuple[forms.IntegerField, forms.IntegerField]:
+    """
+    Create age_from and age_till fields with consistent configuration.
+
+    Returns:
+        Tuple of (age_from_field, age_till_field)
+
+    Example:
+        age_from, age_till = create_age_range_fields()
+        self.fields["age_from"] = age_from
+        self.fields["age_till"] = age_till
+    """
+    age_from = forms.IntegerField(
+        required=False,
+        min_value=MIN_AGE,
+        max_value=MAX_AGE,
+        widget=forms.NumberInput(
+            attrs={
+                "type": "range",
+                "step": AGE_STEP,
+                "value": MIN_AGE,
+            }
+        ),
+    )
+
+    age_till = forms.IntegerField(
+        required=False,
+        min_value=MIN_AGE,
+        max_value=MAX_AGE,
+        widget=forms.NumberInput(
+            attrs={
+                "type": "range",
+                "step": AGE_STEP,
+                "value": MAX_AGE,
+            }
+        ),
+    )
+
+    return age_from, age_till
diff --git a/radis/extractions/forms.py b/radis/extractions/forms.py
index 9df3e447..93b0a456 100644
--- a/radis/extractions/forms.py
+++ b/radis/extractions/forms.py
@@ -8,10 +8,13 @@
 from django.conf import settings
 from django.db.models import QuerySet
 
-from radis.core.constants import LANGUAGE_LABELS
+from radis.core.form_fields import (
+    create_age_range_fields,
+    create_language_field,
+    create_modality_field,
+)
 from radis.core.layouts import RangeSlider
 from radis.reports.models import Language, Modality
-from radis.search.forms import AGE_STEP, MAX_AGE, MIN_AGE
 from radis.search.site import Search, SearchFilters
 from radis.search.utils.query_parser import QueryParser
 
@@ -53,41 +56,13 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
         self.fields["query"].required = True
-        self.fields["language"].choices = [  # type: ignore
-            (language.pk, LANGUAGE_LABELS[language.code])
-            for language in Language.objects.order_by("code")
-        ]
-        self.fields["modalities"].choices = [  # type: ignore
-            (modality.pk, modality.code)
-            for modality in Modality.objects.filter(filterable=True).order_by("code")
-        ]
-        self.fields["modalities"].widget.attrs["size"] = 6
+        self.fields["language"] = create_language_field()
+        self.fields["modalities"] = create_modality_field()
         self.fields["study_date_from"].widget = forms.DateInput(attrs={"type": "date"})
         self.fields["study_date_till"].widget = forms.DateInput(attrs={"type": "date"})
-        self.fields["age_from"] = forms.IntegerField(
-            required=False,
-            min_value=MIN_AGE,
-            max_value=MAX_AGE,
-            widget=forms.NumberInput(
-                attrs={
-                    "type": "range",
-                    "step": AGE_STEP,
-                    "value": MIN_AGE,
-                }
-            ),
-        )
-        self.fields["age_till"] = forms.IntegerField(
-            required=False,
-            min_value=MIN_AGE,
-            max_value=MAX_AGE,
-            widget=forms.NumberInput(
-                attrs={
-                    "type": "range",
-                    "step": AGE_STEP,
-                    "value": MAX_AGE,
-                }
-            ),
-        )
+        age_from, age_till = create_age_range_fields()
+        self.fields["age_from"] = age_from
+        self.fields["age_till"] = age_till
 
         self.helper = FormHelper()
         self.helper.form_tag = False
@@ -127,9 +102,13 @@ def clean_query(self) -> str:
                 "A search query is required. "
                 "Please enter a query or go back to regenerate from fields."
             )
-        query_node, _ = QueryParser().parse(query)
+        query_node, fixes = QueryParser().parse(query)
         if query_node is None:
             raise forms.ValidationError("Invalid query syntax")
+        else:
+            self.cleaned_data["query_node"] = query_node
+        if len(fixes) > 0:
+            query = QueryParser.unparse(query_node)
         return query
 
     def clean(self) -> dict[str, Any] | None:
@@ -141,26 +120,9 @@ def clean(self) -> dict[str, Any] | None:
         language = cast(Language, cleaned_data["language"])
         modalities = cast(QuerySet[Modality], cleaned_data["modalities"])
 
-        query = cleaned_data.get("query", "").strip()
-
-        # Query should always be present now (either generated or user-edited)
-        if not query:
-            raise forms.ValidationError(
-                "A search query is required. "
-                "Please enter a query or go back to regenerate from fields."
-            )
-
-        # Validate the query
-        query_node, fixes = QueryParser().parse(query)
-        if query_node is None:
-            raise forms.ValidationError("Invalid query syntax")
-
-        if len(fixes) > 0:
-            cleaned_data["fixed_query"] = QueryParser.unparse(query_node)
-
-        # Calculate retrieval count
+        # Calculate retrieval count with inline Search construction
         search = Search(
-            query=query_node,
+            query=cleaned_data["query_node"],
             offset=0,
             limit=0,
             filters=SearchFilters(
diff --git a/radis/extractions/views.py b/radis/extractions/views.py
index 3010e778..a0f2222a 100644
--- a/radis/extractions/views.py
+++ b/radis/extractions/views.py
@@ -1,7 +1,6 @@
 import csv
 from collections.abc import Generator
-from datetime import datetime
-from typing import Any, Literal, Type, Union, cast
+from typing import Any, Type, Union, cast
 from urllib.parse import urlencode
 
 from adit_radis_shared.accounts.models import User
@@ -19,7 +18,7 @@
 from django.db import transaction
 from django.db.models import QuerySet
 from django.forms import BaseInlineFormSet
-from django.http import StreamingHttpResponse
+from django.http import QueryDict, StreamingHttpResponse
 from django.shortcuts import redirect, render
 from django.urls import reverse, reverse_lazy
 from django.utils.text import slugify
@@ -41,7 +40,6 @@
     AnalysisTaskResetView,
     BaseUpdatePreferencesView,
 )
-from radis.reports.models import Language, Modality
 from radis.search.site import Search, SearchFilters
 from radis.search.utils.query_parser import QueryParser
 
@@ -292,108 +290,86 @@ def get(self, request: AuthenticatedHttpRequest):
         # Wizard step prefix for form field names
         WIZARD_STEP_PREFIX = "1-"
 
-        # Extract query and filter parameters from GET
-        query_str = request.GET.get(f"{WIZARD_STEP_PREFIX}query", "").strip()
-        language_id = request.GET.get(f"{WIZARD_STEP_PREFIX}language")
-        modality_ids = request.GET.getlist(f"{WIZARD_STEP_PREFIX}modalities")
-
-        # Get string values from GET parameters
-        study_date_from_str = request.GET.get(f"{WIZARD_STEP_PREFIX}study_date_from", "").strip()
-        study_date_till_str = request.GET.get(f"{WIZARD_STEP_PREFIX}study_date_till", "").strip()
-        study_description = request.GET.get(f"{WIZARD_STEP_PREFIX}study_description", "")
-        patient_sex_str = request.GET.get(f"{WIZARD_STEP_PREFIX}patient_sex")
-        patient_sex: Literal["M", "F"] | None = None
-        if patient_sex_str in ("M", "F"):
-            patient_sex = patient_sex_str  # Type narrowed to Literal["M", "F"]
-        age_from_str = request.GET.get(f"{WIZARD_STEP_PREFIX}age_from", "").strip()
-        age_till_str = request.GET.get(f"{WIZARD_STEP_PREFIX}age_till", "").strip()
-
-        # Parse dates from YYYY-MM-DD format to date objects
-        study_date_from = None
-        study_date_till = None
-        if study_date_from_str:
-            try:
-                study_date_from = datetime.strptime(study_date_from_str, "%Y-%m-%d").date()
-            except ValueError:
-                pass  # Invalid date format, leave as None
-
-        if study_date_till_str:
-            try:
-                study_date_till = datetime.strptime(study_date_till_str, "%Y-%m-%d").date()
-            except ValueError:
-                pass  # Invalid date format, leave as None
-
-        # Parse age values to integers
-        age_from_int = None
-        age_till_int = None
-        if age_from_str:
-            try:
-                age_from_int = int(age_from_str)
-            except ValueError:
-                pass  # Invalid integer, leave as None
-
-        if age_till_str:
-            try:
-                age_till_int = int(age_till_str)
-            except ValueError:
-                pass  # Invalid integer, leave as None
-
-        # Get user's active group
-        user = cast("User", request.user)
-        active_group = user.active_group
+        # Extract wizard data and strip "1-" prefix
+        wizard_data = {
+            key.replace(WIZARD_STEP_PREFIX, "", 1): value
+            for key, value in request.GET.items()
+            if key.startswith(WIZARD_STEP_PREFIX)
+        }
 
-        # Validate query syntax
-        if not query_str:
-            context = {
-                "count": None,
-                "search_url": None,
-                "error": None,
-                "max_reports_limit": settings.EXTRACTION_MAXIMUM_REPORTS_COUNT,
-            }
-            return render(request, "extractions/_search_preview.html", context)
+        # Build QueryDict for SearchForm (supports getlist for modalities)
+        query_dict = QueryDict(mutable=True)
+        for key, value in wizard_data.items():
+            if key == "modalities":
+                # modalities are sent as multiple values
+                query_dict.setlist(key, request.GET.getlist(f"{WIZARD_STEP_PREFIX}{key}"))
+            else:
+                query_dict[key] = str(value)
 
-        query_node, fixes = QueryParser().parse(query_str)
-        if query_node is None:
-            context = {
-                "count": None,
-                "search_url": None,
-                "error": "Invalid query syntax",
-                "max_reports_limit": settings.EXTRACTION_MAXIMUM_REPORTS_COUNT,
-            }
-            return render(request, "extractions/_search_preview.html", context)
+        # Add dummy title (required field but not used in preview)
+        query_dict["title"] = "preview"
 
-        # Get language and modalities objects
-        try:
-            language = Language.objects.get(pk=language_id) if language_id else None
-            modalities_qs = (
-                Modality.objects.filter(pk__in=modality_ids)
-                if modality_ids
-                else Modality.objects.none()
-            )
-        except (Language.DoesNotExist, ValueError):
+        # Validate with SearchForm (pass user kwarg)
+        user = cast("User", request.user)
+        form = SearchForm(query_dict, user=user)
+
+        if not form.is_valid():
+            # Extract error messages
+            error_messages = []
+            for field, errors in form.errors.items():
+                # Skip title field errors (dummy field)
+                if field == "title":
+                    continue
+                for error in errors:
+                    if field == "__all__":
+                        error_messages.append(str(error))
+                    else:
+                        error_messages.append(f"{field}: {error}")
+
+            error = "; ".join(error_messages) if error_messages else "Invalid search parameters"
             context = {
                 "count": None,
                 "search_url": None,
-                "error": "Invalid language or modality selection",
+                "error": error,
                 "max_reports_limit": settings.EXTRACTION_MAXIMUM_REPORTS_COUNT,
             }
             return render(request, "extractions/_search_preview.html", context)
 
-        # Build search object (age_from_int and age_till_int already parsed above)
+        # Extract validated data from form
+        # Note: query already validated by clean_query() including QueryParser!
+        query_str = form.cleaned_data["query"]
+        query_node = form.cleaned_data["query_node"]  # QueryNode object
+        language = form.cleaned_data["language"]  # Language object or None
+        modalities = form.cleaned_data["modalities"]  # QuerySet[Modality]
+        study_date_from = form.cleaned_data["study_date_from"]  # date object or None
+        study_date_till = form.cleaned_data["study_date_till"]  # date object or None
+        study_description = form.cleaned_data["study_description"]
+        patient_sex = form.cleaned_data["patient_sex"]
+        age_from = form.cleaned_data["age_from"]  # int or None
+        age_till = form.cleaned_data["age_till"]  # int or None
+
+        # Convert objects to codes for Search object
+        language_code = language.code if language else ""
+        modality_codes = list(modalities.values_list("code", flat=True))
+
+        # Get active group (user already cast above)
+        active_group = user.active_group
+
+        # Build Search object
         search = Search(
             query=query_node,
             offset=0,
             limit=0,
             filters=SearchFilters(
                 group=active_group.pk if active_group else None,
-                language=language.code if language else "",
-                modalities=list(modalities_qs.values_list("code", flat=True)),
-                study_date_from=study_date_from,
-                study_date_till=study_date_till,
+                language=language_code,  # Converted from Language object
+                modalities=modality_codes,  # Converted from QuerySet
+                study_date_from=study_date_from,  # Already date object from form
+                study_date_till=study_date_till,  # Already date object from form
                 study_description=study_description,
                 patient_sex=patient_sex,
-                patient_age_from=age_from_int,
-                patient_age_till=age_till_int,
+                patient_age_from=age_from,  # Already int from form
+                patient_age_till=age_till,  # Already int from form
             ),
         )
 
@@ -409,29 +385,36 @@ def get(self, request: AuthenticatedHttpRequest):
 
         retrieval_count = extraction_retrieval_provider.count(search)
 
-        # Generate search URL - use STRING values for URL parameters
-        # Note: modalities can be a list, which urlencode with doseq=True expands
+        # Generate search URL with codes (FIX: use codes not PKs!)
         search_params: dict[str, Union[str, list[str]]] = {"query": query_str}
-        if language_id:
-            search_params["language"] = Language.objects.get(pk=language_id).code
-        if modality_ids:
-            search_params["modalities"] = modality_ids
-        if study_date_from_str:  # Use STRING version for URL
-            search_params["study_date_from"] = study_date_from_str
-        if study_date_till_str:  # Use STRING version for URL
-            search_params["study_date_till"] = study_date_till_str
+
+        if language_code:
+            search_params["language"] = language_code
+
+        if modality_codes:
+            search_params["modalities"] = modality_codes  # FIX: Use codes not PKs!
+
+        if study_date_from:
+            search_params["study_date_from"] = study_date_from.strftime("%Y-%m-%d")
+
+        if study_date_till:
+            search_params["study_date_till"] = study_date_till.strftime("%Y-%m-%d")
+
         if study_description:
             search_params["study_description"] = study_description
+
         if patient_sex:
             search_params["patient_sex"] = patient_sex
-        if age_from_str:  # Use STRING version for URL
-            search_params["age_from"] = age_from_str
-        if age_till_str:  # Use STRING version for URL
-            search_params["age_till"] = age_till_str
 
-        # doseq=True ensures modalities list becomes: modalities=DX&modalities=MR
+        if age_from is not None:
+            search_params["age_from"] = str(age_from)
+
+        if age_till is not None:
+            search_params["age_till"] = str(age_till)
+
         search_url = reverse("search") + "?" + urlencode(search_params, doseq=True)
 
+        # Return context
         context = {
             "count": retrieval_count,
             "search_url": search_url,
diff --git a/radis/search/forms.py b/radis/search/forms.py
index 6bef0c2d..3c83f3de 100644
--- a/radis/search/forms.py
+++ b/radis/search/forms.py
@@ -4,23 +4,21 @@
 from crispy_forms.layout import Button, Div, Field
 from django import forms
 
-from radis.core.constants import LANGUAGE_LABELS
+from radis.core.constants import AGE_STEP
+from radis.core.form_fields import (
+    create_age_range_fields,
+    create_language_field,
+    create_modality_field,
+)
 from radis.core.layouts import RangeSlider
-from radis.reports.models import Language, Modality
 
 from .layouts import QueryInput
 
-MIN_AGE = 0
-MAX_AGE = 120
-AGE_STEP = 10
-
 
 class SearchForm(forms.Form):
     # Query fields
     query = forms.CharField(required=False, label=False)  # type: ignore
-    # Filter fields
-    language = forms.ChoiceField(required=False, choices=[])
-    modalities = forms.MultipleChoiceField(required=False, choices=[])
+    # Filter fields - language, modalities, and age fields created in __init__
     study_date_from = forms.DateField(
         required=False, widget=forms.DateInput(attrs={"type": "date"})
     )
@@ -31,43 +29,16 @@ class SearchForm(forms.Form):
     patient_sex = forms.ChoiceField(
         required=False, choices=[("", "All"), ("M", "Male"), ("F", "Female")]
     )
-    age_from = forms.IntegerField(
-        required=False,
-        min_value=MIN_AGE,
-        max_value=MAX_AGE,
-        widget=forms.NumberInput(
-            attrs={
-                "type": "range",
-                "step": AGE_STEP,
-                "value": MIN_AGE,
-            }
-        ),
-    )
-    age_till = forms.IntegerField(
-        required=False,
-        min_value=MIN_AGE,
-        max_value=MAX_AGE,
-        widget=forms.NumberInput(
-            attrs={
-                "type": "range",
-                "step": AGE_STEP,
-                "value": MAX_AGE,
-            }
-        ),
-    )
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        self.fields["language"].choices = [  # type: ignore
-            (language.code, LANGUAGE_LABELS[language.code])
-            for language in Language.objects.order_by("code")
-        ]
-        self.fields["modalities"].choices = [  # type: ignore
-            (modality.code, modality.code)
-            for modality in Modality.objects.filter(filterable=True).order_by("code")
-        ]
-        self.fields["modalities"].widget.attrs["size"] = 6
+        # Create fields using factory functions (use codes, not PKs)
+        self.fields["language"] = create_language_field(use_pk=False)
+        self.fields["modalities"] = create_modality_field(use_pk=False)
+        age_from, age_till = create_age_range_fields()
+        self.fields["age_from"] = age_from
+        self.fields["age_till"] = age_till
 
         self.query_helper = FormHelper()
         self.query_helper.template = "search/form_elements/form_part.html"  # type: ignore
diff --git a/radis/subscriptions/forms.py b/radis/subscriptions/forms.py
index e99bf937..d272df12 100644
--- a/radis/subscriptions/forms.py
+++ b/radis/subscriptions/forms.py
@@ -4,12 +4,15 @@
 from crispy_forms.layout import Column, Div, Field, Layout, Row
 from django import forms
 
-from radis.core.constants import LANGUAGE_LABELS
+from radis.core.constants import AGE_STEP
+from radis.core.form_fields import (
+    create_age_range_fields,
+    create_language_field,
+    create_modality_field,
+)
 from radis.core.layouts import Formset, RangeSlider
 from radis.extractions.forms import OutputFieldForm
 from radis.extractions.models import OutputField
-from radis.reports.models import Language, Modality
-from radis.search.forms import AGE_STEP, MAX_AGE, MIN_AGE
 
 from .models import FilterQuestion, Subscription
 
@@ -33,40 +36,11 @@ class Meta:
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        self.fields["language"].choices = [  # type: ignore
-            (language.pk, LANGUAGE_LABELS[language.code])
-            for language in Language.objects.order_by("code")
-        ]
-        self.fields["language"].empty_label = "All"  # type: ignore
-        self.fields["modalities"].choices = [  # type: ignore
-            (modality.pk, modality.code)
-            for modality in Modality.objects.filter(filterable=True).order_by("code")
-        ]
-        self.fields["modalities"].widget.attrs["size"] = 6
-        self.fields["age_from"] = forms.IntegerField(
-            required=False,
-            min_value=MIN_AGE,
-            max_value=MAX_AGE,
-            widget=forms.NumberInput(
-                attrs={
-                    "type": "range",
-                    "step": AGE_STEP,
-                    "value": MIN_AGE,
-                }
-            ),
-        )
-        self.fields["age_till"] = forms.IntegerField(
-            required=False,
-            min_value=MIN_AGE,
-            max_value=MAX_AGE,
-            widget=forms.NumberInput(
-                attrs={
-                    "type": "range",
-                    "step": AGE_STEP,
-                    "value": MAX_AGE,
-                }
-            ),
-        )
+        self.fields["language"] = create_language_field(empty_label="All")
+        self.fields["modalities"] = create_modality_field()
+        age_from, age_till = create_age_range_fields()
+        self.fields["age_from"] = age_from
+        self.fields["age_till"] = age_till
         self.fields["send_finished_mail"].label = "Notify me via mail of new reports"
 
         self.helper = FormHelper()

From b7d126d05fb5870010234c79dd3ca333940a35ca Mon Sep 17 00:00:00 2001
From: Samuel Kwong <samvictor232@gmail.com>
Date: Wed, 7 Jan 2026 14:30:57 +0000
Subject: [PATCH 93/93] reduce code duplication in chat_client and in
 validating selection options for extractions

---
 radis/chats/utils/chat_client.py         | 168 +++++++++++++++--------
 radis/core/templates/cotton/formset.html |   4 +-
 radis/extractions/forms.py               |  26 +---
 radis/extractions/models.py              |  31 +----
 radis/extractions/utils/validation.py    |  48 +++++++
 5 files changed, 171 insertions(+), 106 deletions(-)
 create mode 100644 radis/extractions/utils/validation.py

diff --git a/radis/chats/utils/chat_client.py b/radis/chats/utils/chat_client.py
index 9073ba50..c9c1f9dd 100644
--- a/radis/chats/utils/chat_client.py
+++ b/radis/chats/utils/chat_client.py
@@ -16,53 +16,129 @@ def _get_base_url() -> str:
     return base_url
 
 
-class AsyncChatClient:
+def _validate_completion_response(completion) -> str:
+    """
+    Validates that the LLM completion response contains valid content.
+
+    Args:
+        completion: The completion response from the LLM
+
+    Returns:
+        The message content as a string
+
+    Raises:
+        ValueError: If the response is empty or invalid
+    """
+    if not completion.choices:
+        logger.error("LLM returned empty choices list")
+        raise ValueError("LLM returned no response choices")
+
+    answer = completion.choices[0].message.content
+    if answer is None:
+        logger.error("LLM returned None for message content")
+        raise ValueError("LLM returned empty response content")
+
+    return answer
+
+
+def _validate_parsed_response(completion) -> BaseModel:
+    """
+    Validates that the LLM completion response contains valid parsed data.
+
+    Args:
+        completion: The completion response from the LLM
+
+    Returns:
+        The parsed BaseModel instance
+
+    Raises:
+        ValueError: If the response is empty or invalid
+    """
+    if not completion.choices:
+        logger.error("LLM returned empty choices list")
+        raise ValueError("LLM returned no response choices")
+
+    parsed = completion.choices[0].message.parsed
+    if parsed is None:
+        logger.error("LLM returned None for parsed message")
+        raise ValueError("LLM returned empty parsed response")
+
+    return parsed
+
+
+def _handle_api_error(error: openai.APIError, operation: str) -> None:
+    """
+    Logs and re-raises API errors with consistent error messages.
+
+    Args:
+        error: The API error from OpenAI
+        operation: Description of the operation that failed (e.g., "chat", "data extraction")
+
+    Raises:
+        RuntimeError: Always raises with a user-friendly error message
+    """
+    logger.error(f"OpenAI API error during {operation}: {error}")
+    raise RuntimeError(f"Failed to communicate with LLM service: {error}") from error
+
+
+class _BaseChatClient:
+    """Base class containing shared chat client logic."""
+
     def __init__(self):
-        base_url = _get_base_url()
-        api_key = settings.EXTERNAL_LLM_PROVIDER_API_KEY
-        self._client = openai.AsyncOpenAI(base_url=base_url, api_key=api_key)
+        self._base_url = _get_base_url()
+        self._api_key = settings.EXTERNAL_LLM_PROVIDER_API_KEY
         self._model_name = settings.LLM_MODEL_NAME
 
-    async def chat(
+    def _build_chat_request(
         self,
         messages: Iterable[ChatCompletionMessageParam],
         max_completion_tokens: int | None = None,
-    ) -> str:
-        logger.debug(f"Sending messages to LLM for chat:\n{messages}")
-
+    ) -> dict:
+        """Build the request dictionary for chat completion."""
         request = {
             "model": self._model_name,
             "messages": messages,
         }
         if max_completion_tokens is not None:
             request["max_completion_tokens"] = max_completion_tokens
+        return request
+
+    def _log_request(self, messages: Iterable[ChatCompletionMessageParam]) -> None:
+        """Log the outgoing request."""
+        logger.debug(f"Sending messages to LLM for chat:\n{messages}")
+
+    def _log_response(self, answer: str) -> None:
+        """Log the response from LLM."""
+        logger.debug("Received from LLM: %s", answer)
+
+
+class AsyncChatClient(_BaseChatClient):
+    def __init__(self):
+        super().__init__()
+        self._client = openai.AsyncOpenAI(base_url=self._base_url, api_key=self._api_key)
+
+    async def chat(
+        self,
+        messages: Iterable[ChatCompletionMessageParam],
+        max_completion_tokens: int | None = None,
+    ) -> str:
+        self._log_request(messages)
+        request = self._build_chat_request(messages, max_completion_tokens)
 
         try:
             completion = await self._client.chat.completions.create(**request)
         except openai.APIError as e:
-            logger.error(f"OpenAI API error during chat: {e}")
-            raise RuntimeError(f"Failed to communicate with LLM service: {e}") from e
+            _handle_api_error(e, "chat")
 
-        if not completion.choices:
-            logger.error("LLM returned empty choices list")
-            raise ValueError("LLM returned no response choices")
-
-        answer = completion.choices[0].message.content
-        if answer is None:
-            logger.error("LLM returned None for message content")
-            raise ValueError("LLM returned empty response content")
-
-        logger.debug("Received from LLM: %s", answer)
+        answer = _validate_completion_response(completion)
+        self._log_response(answer)
         return answer
 
 
-class ChatClient:
+class ChatClient(_BaseChatClient):
     def __init__(self) -> None:
-        base_url = _get_base_url()
-        api_key = settings.EXTERNAL_LLM_PROVIDER_API_KEY
-
-        self._client = openai.OpenAI(base_url=base_url, api_key=api_key)
-        self._llm_model_name = settings.LLM_MODEL_NAME
+        super().__init__()
+        self._client = openai.OpenAI(base_url=self._base_url, api_key=self._api_key)
 
     def chat(
         self,
@@ -79,31 +155,16 @@ def chat(
         Returns:
             The LLM's response as a string
         """
-        logger.debug(f"Sending messages to LLM for chat:\n{messages}")
-
-        request = {
-            "model": self._llm_model_name,
-            "messages": messages,
-        }
-        if max_completion_tokens is not None:
-            request["max_completion_tokens"] = max_completion_tokens
+        self._log_request(messages)
+        request = self._build_chat_request(messages, max_completion_tokens)
 
         try:
             completion = self._client.chat.completions.create(**request)
         except openai.APIError as e:
-            logger.error(f"OpenAI API error during chat: {e}")
-            raise RuntimeError(f"Failed to communicate with LLM service: {e}") from e
-
-        if not completion.choices:
-            logger.error("LLM returned empty choices list")
-            raise ValueError("LLM returned no response choices")
-
-        answer = completion.choices[0].message.content
-        if answer is None:
-            logger.error("LLM returned None for message content")
-            raise ValueError("LLM returned empty response content")
+            _handle_api_error(e, "chat")
 
-        logger.debug("Received from LLM: %s", answer)
+        answer = _validate_completion_response(completion)
+        self._log_response(answer)
         return answer
 
     def extract_data(self, prompt: str, schema: type[BaseModel]) -> BaseModel:
@@ -113,22 +174,13 @@ def extract_data(self, prompt: str, schema: type[BaseModel]) -> BaseModel:
 
         try:
             completion = self._client.beta.chat.completions.parse(
-                model=self._llm_model_name,
+                model=self._model_name,
                 messages=[{"role": "system", "content": prompt}],
                 response_format=schema,
             )
         except openai.APIError as e:
-            logger.error(f"OpenAI API error during data extraction: {e}")
-            raise RuntimeError(f"Failed to communicate with LLM service: {e}") from e
-
-        if not completion.choices:
-            logger.error("LLM returned empty choices list")
-            raise ValueError("LLM returned no response choices")
-
-        event = completion.choices[0].message.parsed
-        if event is None:
-            logger.error("LLM returned None for parsed message")
-            raise ValueError("LLM returned empty parsed response")
+            _handle_api_error(e, "data extraction")
 
+        event = _validate_parsed_response(completion)
         logger.debug("Received from LLM: %s", event)
         return event
diff --git a/radis/core/templates/cotton/formset.html b/radis/core/templates/cotton/formset.html
index cae787c9..95ea87bd 100644
--- a/radis/core/templates/cotton/formset.html
+++ b/radis/core/templates/cotton/formset.html
@@ -4,9 +4,7 @@
     {% if legend %}<legend>{{ legend }}</legend>{% endif %}
     {{ formset.management_form }}
     <template>
-        {% with form=formset.empty_form %}
-            <c-formset-form>{% crispy formset.empty_form %}</c-formset-form>
-        {% endwith %}
+        <c-formset-form>{% crispy formset.empty_form %}</c-formset-form>
     </template>
     <div class="formset-forms">
         {% for form in formset %}
diff --git a/radis/extractions/forms.py b/radis/extractions/forms.py
index 93b0a456..0d9c6162 100644
--- a/radis/extractions/forms.py
+++ b/radis/extractions/forms.py
@@ -21,6 +21,7 @@
 from .constants import MAX_SELECTION_OPTIONS
 from .models import ExtractionJob, OutputField, OutputType
 from .site import extraction_retrieval_provider
+from .utils.validation import validate_selection_options
 
 
 class SearchForm(forms.ModelForm):
@@ -115,6 +116,10 @@ def clean(self) -> dict[str, Any] | None:
         cleaned_data = super().clean()
         assert cleaned_data
 
+        # If query validation failed, query_node won't exist - exit early
+        if "query_node" not in cleaned_data:
+            return cleaned_data
+
         active_group = self.user.active_group
 
         language = cast(Language, cleaned_data["language"])
@@ -262,26 +267,7 @@ def clean_selection_options(self) -> list[str]:
         except json.JSONDecodeError as exc:
             raise forms.ValidationError("Invalid selection data.") from exc
 
-        if not isinstance(parsed, list):
-            raise forms.ValidationError("Selection data must be a list.")
-
-        cleaned: list[str] = []
-        for item in parsed:
-            if not isinstance(item, str):
-                raise forms.ValidationError("Selection options must be text.")
-            value = item.strip()
-            if not value:
-                raise forms.ValidationError("Selection options cannot be empty.")
-            cleaned.append(value)
-
-        if len(cleaned) > MAX_SELECTION_OPTIONS:
-            raise forms.ValidationError(
-                f"Provide at most {MAX_SELECTION_OPTIONS} selection options."
-            )
-        if len(set(cleaned)) != len(cleaned):
-            raise forms.ValidationError("Selection options must be unique.")
-
-        return cleaned
+        return validate_selection_options(parsed)
 
     def clean_is_array(self) -> bool:
         raw_value = (self.cleaned_data.get("is_array") or "").strip().lower()
diff --git a/radis/extractions/models.py b/radis/extractions/models.py
index 8d8b024a..dc0f972e 100644
--- a/radis/extractions/models.py
+++ b/radis/extractions/models.py
@@ -12,8 +12,6 @@
 from radis.core.models import AnalysisJob, AnalysisTask
 from radis.reports.models import Language, Modality, Report
 
-from .constants import MAX_SELECTION_OPTIONS
-
 
 class ExtractionsAppSettings(AppSettings):
     class Meta:
@@ -128,34 +126,17 @@ def __str__(self) -> str:
     def clean(self) -> None:
         from django.core.exceptions import ValidationError
 
+        from radis.extractions.utils.validation import validate_selection_options
+
         super().clean()
 
         if self.output_type == OutputType.SELECTION:
             if not self.selection_options:
                 raise ValidationError({"selection_options": "Add at least one selection option."})
-            if len(self.selection_options) > MAX_SELECTION_OPTIONS:
-                raise ValidationError(
-                    {
-                        "selection_options": (
-                            f"Provide at most {MAX_SELECTION_OPTIONS} selection options."
-                        )
-                    }
-                )
-            cleaned_options = []
-            for option in self.selection_options:
-                if not isinstance(option, str):
-                    raise ValidationError(
-                        {"selection_options": "All selection options must be text."}
-                    )
-                stripped = option.strip()
-                if not stripped:
-                    raise ValidationError(
-                        {"selection_options": "Selection options cannot be empty strings."}
-                    )
-                cleaned_options.append(stripped)
-            if len(set(cleaned_options)) != len(cleaned_options):
-                raise ValidationError({"selection_options": "Selection options must be unique."})
-            self.selection_options = cleaned_options
+            try:
+                self.selection_options = validate_selection_options(self.selection_options)
+            except ValidationError as e:
+                raise ValidationError({"selection_options": e.message})
         else:
             if self.selection_options:
                 raise ValidationError(
diff --git a/radis/extractions/utils/validation.py b/radis/extractions/utils/validation.py
new file mode 100644
index 00000000..c7c8bfbc
--- /dev/null
+++ b/radis/extractions/utils/validation.py
@@ -0,0 +1,48 @@
+"""Shared validation utilities for the extractions app."""
+
+from django.core.exceptions import ValidationError
+
+from radis.extractions.constants import MAX_SELECTION_OPTIONS
+
+
+def validate_selection_options(options: list) -> list[str]:
+    """
+    Validates selection options for output fields.
+
+    Args:
+        options: A list of selection options to validate
+
+    Returns:
+        A list of cleaned (stripped) selection option strings
+
+    Raises:
+        ValidationError: If validation fails for any of these reasons:
+            - Options is not a list
+            - Any option is not a string
+            - Any option is empty after stripping
+            - Too many options (exceeds MAX_SELECTION_OPTIONS)
+            - Options are not unique
+    """
+    if not isinstance(options, list):
+        raise ValidationError("Selection options must be a list.")
+
+    cleaned_options = []
+    for option in options:
+        if not isinstance(option, str):
+            raise ValidationError("All selection options must be text.")
+
+        stripped = option.strip()
+        if not stripped:
+            raise ValidationError("Selection options cannot be empty strings.")
+
+        cleaned_options.append(stripped)
+
+    if len(cleaned_options) > MAX_SELECTION_OPTIONS:
+        raise ValidationError(
+            f"Provide at most {MAX_SELECTION_OPTIONS} selection options."
+        )
+
+    if len(set(cleaned_options)) != len(cleaned_options):
+        raise ValidationError("Selection options must be unique.")
+
+    return cleaned_options