Query is Not Available During Judge Step #13
Description
Running through your initial NON demo, like so:
# Standard API: Create a verification pipeline of ensemble→judge→verifier
pipeline = non.Sequential(operators=[
# 1. Ensemble of 5 model instances running in parallel
non.UniformEnsemble(
num_units=5,
model_name="openai:gpt-4o-mini",
temperature=0.7
),
# 2. Judge to synthesize the ensemble responses
non.JudgeSynthesis(
model_name="anthropic:claude-3-5-sonnet",
temperature=0.2
),
# 3. Verifier for quality control and fact-checking
non.Verifier(
model_name="anthropic:claude-3-5-haiku",
temperature=0.0
)
])When I run, result = pipeline(query="What causes tsunamis?"), the calls to the ensemble work but the judgement call fails with the following error:
2025-04-15 08:32:37,691 [ERROR] ember.core.registry.specification.specification: Missing input for placeholder: 'query'
Here's the full dump:
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
File ~/Development/experiment/ember_explore/ember/src/ember/core/registry/specification/specification.py:118, in Specification.render_prompt(self, inputs)
117 try:
--> 118 prompt: str = self.prompt_template.format(**input_dict)
119 return prompt
KeyError: 'query'
The above exception was the direct cause of the following exception:
InvalidPromptError Traceback (most recent call last)
File ~/Development/experiment/ember_explore/ember/src/ember/core/registry/operator/base/operator_base.py:211, in Operator.__call__(self, inputs, **kwargs)
210 # Execute the core computation
--> 211 operator_output: OutputT = self.forward(inputs=validated_inputs)
213 # Ensure we have a proper model instance for the output
214 # If we got a dict, convert it to the appropriate model
File ~/Development/experiment/ember_explore/ember/src/ember/core/registry/operator/core/synthesis_judge.py:71, in JudgeSynthesisOperator.forward(self, inputs)
67 raise MissingLMModuleError(
68 "No LM module attached to JudgeSynthesisOperator."
69 )
---> 71 rendered_prompt: str = self.specification.render_prompt(inputs=inputs)
72 raw_output: str = self.lm_module(prompt=rendered_prompt).strip()
File ~/Development/experiment/ember_explore/ember/src/ember/core/registry/specification/specification.py:124, in Specification.render_prompt(self, inputs)
123 logger.error(error_msg)
--> 124 raise InvalidPromptError.with_context(
125 error_msg,
126 missing_placeholder=missing_key,
127 available_keys=list(input_dict.keys()),
128 template=self.prompt_template,
129 ) from key_err
131 if self.input_model is not None:
InvalidPromptError: [Error 3040] Missing input for placeholder: 'query' [Recovery: Check prompt format and template variables] [Context: available_keys=['responses'], caller_file='/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py', caller_function='with_context', caller_lineno=205, missing_placeholder="'query'", template='We have multiple advisors who proposed different answers:\n{responses}\nNow, we want to synthesize a single best, final answer to:\n{query}\nExplain your reasoning concisely, then provide the single best final answer.\nFormat:\nReasoning: <your reasoning for synthesizing this answer in this way>\nFinal Answer: <the single best answer>\n']
The above exception was the direct cause of the following exception:
OperatorExecutionError Traceback (most recent call last)
File ~/Development/experiment/ember_explore/ember/src/ember/core/registry/operator/base/operator_base.py:211, in Operator.__call__(self, inputs, **kwargs)
210 # Execute the core computation
--> 211 operator_output: OutputT = self.forward(inputs=validated_inputs)
213 # Ensure we have a proper model instance for the output
214 # If we got a dict, convert it to the appropriate model
File ~/Development/experiment/ember_explore/ember/src/ember/core/non.py:289, in JudgeSynthesis.forward(self, inputs)
288 raise ValueError("JudgeSynthesisOperator not initialized")
--> 289 return self._judge_synthesis_op(inputs=inputs)
File ~/Development/experiment/ember_explore/ember/src/ember/core/registry/operator/base/operator_base.py:234, in Operator.__call__(self, inputs, **kwargs)
233 if not isinstance(e, OperatorSpecificationError):
--> 234 raise OperatorExecutionError.for_operator(
235 operator_name=self.__class__.__name__,
236 message=f"Error executing operator: {str(e)}",
237 cause=e,
238 operator_type=type(self).__module__,
239 ) from e
240 raise
OperatorExecutionError: [Error 2002] Error executing operator: [Error 3040] Missing input for placeholder: 'query' [Recovery: Check prompt format and template variables] [Context: available_keys=['responses'], caller_file='/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py', caller_function='with_context', caller_lineno=205, missing_placeholder="'query'", template='We have multiple advisors who proposed different answers:\n{responses}\nNow, we want to synthesize a single best, final answer to:\n{query}\nExplain your reasoning concisely, then provide the single best final answer.\nFormat:\nReasoning: <your reasoning for synthesizing this answer in this way>\nFinal Answer: <the single best answer>\n'] [Recovery: Check operator inputs and execution environment] [Context: caller_file='/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py', caller_function='for_operator', caller_lineno=646, cause_message='[Error 3040] Missing input for placeholder: \'query\' [Recovery: Check prompt format and template variables] [Context: available_keys=[\'responses\'], caller_file=\'/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py\', caller_function=\'with_context\', caller_lineno=205, missing_placeholder="\'query\'", template=\'We have multiple advisors who proposed different answers:\\n{responses}\\nNow, we want to synthesize a single best, final answer to:\\n{query}\\nExplain your reasoning concisely, then provide the single best final answer.\\nFormat:\\nReasoning: <your reasoning for synthesizing this answer in this way>\\nFinal Answer: <the single best answer>\\n\']', cause_type='InvalidPromptError', operator_name='JudgeSynthesisOperator', operator_type='ember.core.registry.operator.core.synthesis_judge']
The above exception was the direct cause of the following exception:
OperatorExecutionError Traceback (most recent call last)
File ~/Development/experiment/ember_explore/ember/src/ember/core/registry/operator/base/operator_base.py:211, in Operator.__call__(self, inputs, **kwargs)
210 # Execute the core computation
--> 211 operator_output: OutputT = self.forward(inputs=validated_inputs)
213 # Ensure we have a proper model instance for the output
214 # If we got a dict, convert it to the appropriate model
File ~/Development/experiment/ember_explore/ember/src/ember/core/non.py:491, in Sequential.forward(self, inputs)
490 for op in self.operators:
--> 491 current_input = op(inputs=current_input)
492 return cast(OutputT, current_input)
File ~/Development/experiment/ember_explore/ember/src/ember/core/registry/operator/base/operator_base.py:234, in Operator.__call__(self, inputs, **kwargs)
233 if not isinstance(e, OperatorSpecificationError):
--> 234 raise OperatorExecutionError.for_operator(
235 operator_name=self.__class__.__name__,
236 message=f"Error executing operator: {str(e)}",
237 cause=e,
238 operator_type=type(self).__module__,
239 ) from e
240 raise
OperatorExecutionError: [Error 2002] Error executing operator: [Error 2002] Error executing operator: [Error 3040] Missing input for placeholder: 'query' [Recovery: Check prompt format and template variables] [Context: available_keys=['responses'], caller_file='/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py', caller_function='with_context', caller_lineno=205, missing_placeholder="'query'", template='We have multiple advisors who proposed different answers:\n{responses}\nNow, we want to synthesize a single best, final answer to:\n{query}\nExplain your reasoning concisely, then provide the single best final answer.\nFormat:\nReasoning: <your reasoning for synthesizing this answer in this way>\nFinal Answer: <the single best answer>\n'] [Recovery: Check operator inputs and execution environment] [Context: caller_file='/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py', caller_function='for_operator', caller_lineno=646, cause_message='[Error 3040] Missing input for placeholder: \'query\' [Recovery: Check prompt format and template variables] [Context: available_keys=[\'responses\'], caller_file=\'/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py\', caller_function=\'with_context\', caller_lineno=205, missing_placeholder="\'query\'", template=\'We have multiple advisors who proposed different answers:\\n{responses}\\nNow, we want to synthesize a single best, final answer to:\\n{query}\\nExplain your reasoning concisely, then provide the single best final answer.\\nFormat:\\nReasoning: <your reasoning for synthesizing this answer in this way>\\nFinal Answer: <the single best answer>\\n\']', cause_type='InvalidPromptError', operator_name='JudgeSynthesisOperator', operator_type='ember.core.registry.operator.core.synthesis_judge'] [Recovery: Check operator inputs and execution environment] [Context: caller_file='/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py', caller_function='for_operator', caller_lineno=646, cause_message='[Error 2002] Error executing operator: [Error 3040] Missing input for placeholder: \'query\' [Recovery: Check prompt format and template variables] [Context: available_keys=[\'responses\'], caller_file=\'/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py\', caller_function=\'with_context\', caller_lineno=205, missing_placeholder="\'query\'", template=\'We have multiple advisors who proposed different answers:\\n{responses}\\nNow, we want to synthesize a single best, final answer to:\\n{query}\\nExplain your reasoning concisely, then provide the single best final answer.\\nFormat:\\nReasoning: <your reasoning for synthesizing this answer in this way>\\nFinal Answer: <the single best answer>\\n\'] [Recovery: Check operator inputs and execution environment] [Context: caller_file=\'/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py\', caller_function=\'for_operator\', caller_lineno=646, cause_message=\'[Error 3040] Missing input for placeholder: \\\'query\\\' [Recovery: Check prompt format and template variables] [Context: available_keys=[\\\'responses\\\'], caller_file=\\\'/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py\\\', caller_function=\\\'with_context\\\', caller_lineno=205, missing_placeholder="\\\'query\\\'", template=\\\'We have multiple advisors who proposed different answers:\\\\n{responses}\\\\nNow, we want to synthesize a single best, final answer to:\\\\n{query}\\\\nExplain your reasoning concisely, then provide the single best final answer.\\\\nFormat:\\\\nReasoning: <your reasoning for synthesizing this answer in this way>\\\\nFinal Answer: <the single best answer>\\\\n\\\']\', cause_type=\'InvalidPromptError\', operator_name=\'JudgeSynthesisOperator\', operator_type=\'ember.core.registry.operator.core.synthesis_judge\']', cause_type='OperatorExecutionError', operator_name='JudgeSynthesis', operator_type='ember.core.non']
The above exception was the direct cause of the following exception:
OperatorExecutionError Traceback (most recent call last)
Cell In[6], line 2
1 # Execute with a single call
----> 2 result = pipeline(query="What causes tsunamis?")
File ~/Development/experiment/ember_explore/ember/src/ember/core/registry/operator/base/operator_base.py:234, in Operator.__call__(self, inputs, **kwargs)
231 except Exception as e:
232 # Catch any errors during execution and wrap them
233 if not isinstance(e, OperatorSpecificationError):
--> 234 raise OperatorExecutionError.for_operator(
235 operator_name=self.__class__.__name__,
236 message=f"Error executing operator: {str(e)}",
237 cause=e,
238 operator_type=type(self).__module__,
239 ) from e
240 raise
OperatorExecutionError: [Error 2002] Error executing operator: [Error 2002] Error executing operator: [Error 2002] Error executing operator: [Error 3040] Missing input for placeholder: 'query' [Recovery: Check prompt format and template variables] [Context: available_keys=['responses'], caller_file='/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py', caller_function='with_context', caller_lineno=205, missing_placeholder="'query'", template='We have multiple advisors who proposed different answers:\n{responses}\nNow, we want to synthesize a single best, final answer to:\n{query}\nExplain your reasoning concisely, then provide the single best final answer.\nFormat:\nReasoning: <your reasoning for synthesizing this answer in this way>\nFinal Answer: <the single best answer>\n'] [Recovery: Check operator inputs and execution environment] [Context: caller_file='/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py', caller_function='for_operator', caller_lineno=646, cause_message='[Error 3040] Missing input for placeholder: \'query\' [Recovery: Check prompt format and template variables] [Context: available_keys=[\'responses\'], caller_file=\'/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py\', caller_function=\'with_context\', caller_lineno=205, missing_placeholder="\'query\'", template=\'We have multiple advisors who proposed different answers:\\n{responses}\\nNow, we want to synthesize a single best, final answer to:\\n{query}\\nExplain your reasoning concisely, then provide the single best final answer.\\nFormat:\\nReasoning: <your reasoning for synthesizing this answer in this way>\\nFinal Answer: <the single best answer>\\n\']', cause_type='InvalidPromptError', operator_name='JudgeSynthesisOperator', operator_type='ember.core.registry.operator.core.synthesis_judge'] [Recovery: Check operator inputs and execution environment] [Context: caller_file='/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py', caller_function='for_operator', caller_lineno=646, cause_message='[Error 2002] Error executing operator: [Error 3040] Missing input for placeholder: \'query\' [Recovery: Check prompt format and template variables] [Context: available_keys=[\'responses\'], caller_file=\'/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py\', caller_function=\'with_context\', caller_lineno=205, missing_placeholder="\'query\'", template=\'We have multiple advisors who proposed different answers:\\n{responses}\\nNow, we want to synthesize a single best, final answer to:\\n{query}\\nExplain your reasoning concisely, then provide the single best final answer.\\nFormat:\\nReasoning: <your reasoning for synthesizing this answer in this way>\\nFinal Answer: <the single best answer>\\n\'] [Recovery: Check operator inputs and execution environment] [Context: caller_file=\'/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py\', caller_function=\'for_operator\', caller_lineno=646, cause_message=\'[Error 3040] Missing input for placeholder: \\\'query\\\' [Recovery: Check prompt format and template variables] [Context: available_keys=[\\\'responses\\\'], caller_file=\\\'/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py\\\', caller_function=\\\'with_context\\\', caller_lineno=205, missing_placeholder="\\\'query\\\'", template=\\\'We have multiple advisors who proposed different answers:\\\\n{responses}\\\\nNow, we want to synthesize a single best, final answer to:\\\\n{query}\\\\nExplain your reasoning concisely, then provide the single best final answer.\\\\nFormat:\\\\nReasoning: <your reasoning for synthesizing this answer in this way>\\\\nFinal Answer: <the single best answer>\\\\n\\\']\', cause_type=\'InvalidPromptError\', operator_name=\'JudgeSynthesisOperator\', operator_type=\'ember.core.registry.operator.core.synthesis_judge\']', cause_type='OperatorExecutionError', operator_name='JudgeSynthesis', operator_type='ember.core.non'] [Recovery: Check operator inputs and execution environment] [Context: caller_file='/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py', caller_function='for_operator', caller_lineno=646, cause_message='[Error 2002] Error executing operator: [Error 2002] Error executing operator: [Error 3040] Missing input for placeholder: \'query\' [Recovery: Check prompt format and template variables] [Context: available_keys=[\'responses\'], caller_file=\'/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py\', caller_function=\'with_context\', caller_lineno=205, missing_placeholder="\'query\'", template=\'We have multiple advisors who proposed different answers:\\n{responses}\\nNow, we want to synthesize a single best, final answer to:\\n{query}\\nExplain your reasoning concisely, then provide the single best final answer.\\nFormat:\\nReasoning: <your reasoning for synthesizing this answer in this way>\\nFinal Answer: <the single best answer>\\n\'] [Recovery: Check operator inputs and execution environment] [Context: caller_file=\'/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py\', caller_function=\'for_operator\', caller_lineno=646, cause_message=\'[Error 3040] Missing input for placeholder: \\\'query\\\' [Recovery: Check prompt format and template variables] [Context: available_keys=[\\\'responses\\\'], caller_file=\\\'/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py\\\', caller_function=\\\'with_context\\\', caller_lineno=205, missing_placeholder="\\\'query\\\'", template=\\\'We have multiple advisors who proposed different answers:\\\\n{responses}\\\\nNow, we want to synthesize a single best, final answer to:\\\\n{query}\\\\nExplain your reasoning concisely, then provide the single best final answer.\\\\nFormat:\\\\nReasoning: <your reasoning for synthesizing this answer in this way>\\\\nFinal Answer: <the single best answer>\\\\n\\\']\', cause_type=\'InvalidPromptError\', operator_name=\'JudgeSynthesisOperator\', operator_type=\'ember.core.registry.operator.core.synthesis_judge\'] [Recovery: Check operator inputs and execution environment] [Context: caller_file=\'/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py\', caller_function=\'for_operator\', caller_lineno=646, cause_message=\'[Error 2002] Error executing operator: [Error 3040] Missing input for placeholder: \\\'query\\\' [Recovery: Check prompt format and template variables] [Context: available_keys=[\\\'responses\\\'], caller_file=\\\'/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py\\\', caller_function=\\\'with_context\\\', caller_lineno=205, missing_placeholder="\\\'query\\\'", template=\\\'We have multiple advisors who proposed different answers:\\\\n{responses}\\\\nNow, we want to synthesize a single best, final answer to:\\\\n{query}\\\\nExplain your reasoning concisely, then provide the single best final answer.\\\\nFormat:\\\\nReasoning: <your reasoning for synthesizing this answer in this way>\\\\nFinal Answer: <the single best answer>\\\\n\\\'] [Recovery: Check operator inputs and execution environment] [Context: caller_file=\\\'/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py\\\', caller_function=\\\'for_operator\\\', caller_lineno=646, cause_message=\\\'[Error 3040] Missing input for placeholder: \\\\\\\'query\\\\\\\' [Recovery: Check prompt format and template variables] [Context: available_keys=[\\\\\\\'responses\\\\\\\'], caller_file=\\\\\\\'/Users/dbreunig/Development/experiment/ember_explore/ember/src/ember/core/exceptions.py\\\\\\\', caller_function=\\\\\\\'with_context\\\\\\\', caller_lineno=205, missing_placeholder="\\\\\\\'query\\\\\\\'", template=\\\\\\\'We have multiple advisors who proposed different answers:\\\\\\\\n{responses}\\\\\\\\nNow, we want to synthesize a single best, final answer to:\\\\\\\\n{query}\\\\\\\\nExplain your reasoning concisely, then provide the single best final answer.\\\\\\\\nFormat:\\\\\\\\nReasoning: <your reasoning for synthesizing this answer in this way>\\\\\\\\nFinal Answer: <the single best answer>\\\\\\\\n\\\\\\\']\\\', cause_type=\\\'InvalidPromptError\\\', operator_name=\\\'JudgeSynthesisOperator\\\', operator_type=\\\'ember.core.registry.operator.core.synthesis_judge\\\']\', cause_type=\'OperatorExecutionError\', operator_name=\'JudgeSynthesis\', operator_type=\'ember.core.non\']', cause_type='OperatorExecutionError', operator_name='Sequential', operator_type='ember.core.non']