From 0093dfdc30decf6ad1f153fc3c2e0481469096eb Mon Sep 17 00:00:00 2001 From: Lou Kratz <219901029+loukratz-bv@users.noreply.github.com> Date: Wed, 4 Feb 2026 16:39:35 -0500 Subject: [PATCH] feat:Add valdiation checking for variable bindings --- docs/Reference/semantic-validation-rules.md | 14 +++ qtype/semantic/checker.py | 99 +++++++++++++++++++ ...invoke_flow_wrong_input_binding.qtype.yaml | 50 ++++++++++ ...nvoke_flow_wrong_output_binding.qtype.yaml | 50 ++++++++++ ...invoke_tool_wrong_input_binding.qtype.yaml | 40 ++++++++ ...nvoke_tool_wrong_output_binding.qtype.yaml | 40 ++++++++ tests/semantic/test_checker_validation.py | 16 +++ 7 files changed, 309 insertions(+) create mode 100644 tests/semantic/checker-error-specs/invalid_invoke_flow_wrong_input_binding.qtype.yaml create mode 100644 tests/semantic/checker-error-specs/invalid_invoke_flow_wrong_output_binding.qtype.yaml create mode 100644 tests/semantic/checker-error-specs/invalid_invoke_tool_wrong_input_binding.qtype.yaml create mode 100644 tests/semantic/checker-error-specs/invalid_invoke_tool_wrong_output_binding.qtype.yaml diff --git a/docs/Reference/semantic-validation-rules.md b/docs/Reference/semantic-validation-rules.md index 8b641c63..191a8936 100644 --- a/docs/Reference/semantic-validation-rules.md +++ b/docs/Reference/semantic-validation-rules.md @@ -153,6 +153,20 @@ This document lists all semantic validation rules enforced by QType. These rules --- +## InvokeFlow + +- All `input_bindings` keys must match flow input variable IDs +- All `output_bindings` keys must match flow output variable IDs + +--- + +## InvokeTool + +- All `input_bindings` keys must match tool input parameter IDs +- All `output_bindings` keys must match tool output parameter IDs + +--- + ## LLMInference - Must have exactly 1 output diff --git a/qtype/semantic/checker.py b/qtype/semantic/checker.py index 969056ce..e0a51d58 100644 --- a/qtype/semantic/checker.py +++ b/qtype/semantic/checker.py @@ -24,6 +24,8 @@ FieldExtractor, Flow, IndexUpsert, + InvokeFlow, + InvokeTool, ListType, LLMInference, PromptTemplate, @@ -681,6 +683,101 @@ def _validate_bedrock_reranker(reranker: BedrockReranker) -> None: ) +def _validate_bindings( + step_id: str, + step_type: str, + component_id: str, + component_type: str, + input_bindings: dict[str, Any], + output_bindings: dict[str, Any], + valid_input_ids: set[str], + valid_output_ids: set[str], +) -> None: + """Validate input and output bindings match component parameters. + + Args: + step_id: ID of the step being validated + step_type: Type of the step (e.g., 'InvokeTool', 'InvokeFlow') + component_id: ID of the component being invoked + component_type: Type of component (e.g., 'tool', 'flow') + input_bindings: The input_bindings dict to validate + output_bindings: The output_bindings dict to validate + valid_input_ids: Set of valid input parameter IDs + valid_output_ids: Set of valid output parameter IDs + + Raises: + QTypeSemanticError: If any binding keys don't match valid IDs + """ # Check input_bindings + for binding_key in input_bindings.keys(): + if binding_key not in valid_input_ids: + raise QTypeSemanticError( + ( + f"{step_type} step '{step_id}' has input_binding " + f"'{binding_key}' which does not match any {component_type} " + f"parameter. {component_type.capitalize()} '{component_id}' has input " + f"parameters: {sorted(valid_input_ids)}. {component_type.capitalize()} " + f"parameter '{binding_key}' not defined in {component_type}." + ) + ) + + # Check output_bindings + for binding_key in output_bindings.keys(): + if binding_key not in valid_output_ids: + raise QTypeSemanticError( + ( + f"{step_type} step '{step_id}' has output_binding " + f"'{binding_key}' which does not match any {component_type} " + f"parameter. {component_type.capitalize()} '{component_id}' has output " + f"parameters: {sorted(valid_output_ids)}. {component_type.capitalize()} " + f"parameter '{binding_key}' not defined in {component_type}." + ) + ) + + +def _validate_invoke_tool(step: InvokeTool) -> None: + """Validate InvokeTool has bindings that match the tool's parameters. + + Validates: + - All input_bindings keys must match tool input parameter IDs + - All output_bindings keys must match tool output parameter IDs + """ + tool_input_ids = {inp.id for inp in step.tool.inputs} + tool_output_ids = {out.id for out in step.tool.outputs} + + _validate_bindings( + step_id=step.id, + step_type="InvokeTool", + component_id=step.tool.id, + component_type="tool", + input_bindings=step.input_bindings, + output_bindings=step.output_bindings, + valid_input_ids=tool_input_ids, + valid_output_ids=tool_output_ids, + ) + + +def _validate_invoke_flow(step: InvokeFlow) -> None: + """Validate InvokeFlow has bindings that match the flow's parameters. + + Validates: + - All input_bindings keys must match flow input variable IDs + - All output_bindings keys must match flow output variable IDs + """ + flow_input_ids = {inp.id for inp in step.flow.inputs} + flow_output_ids = {out.id for out in step.flow.outputs} + + _validate_bindings( + step_id=step.id, + step_type="InvokeFlow", + component_id=step.flow.id, + component_type="flow", + input_bindings=step.input_bindings, + output_bindings=step.output_bindings, + valid_input_ids=flow_input_ids, + valid_output_ids=flow_output_ids, + ) + + # Mapping of types to their validation functions _VALIDATORS = { Agent: _validate_agent, @@ -700,6 +797,8 @@ def _validate_bedrock_reranker(reranker: BedrockReranker) -> None: FieldExtractor: _validate_field_extractor, Flow: _validate_flow, IndexUpsert: _validate_index_upsert, + InvokeFlow: _validate_invoke_flow, + InvokeTool: _validate_invoke_tool, LLMInference: _validate_llm_inference, PromptTemplate: _validate_prompt_template, SQLSource: _validate_sql_source, diff --git a/tests/semantic/checker-error-specs/invalid_invoke_flow_wrong_input_binding.qtype.yaml b/tests/semantic/checker-error-specs/invalid_invoke_flow_wrong_input_binding.qtype.yaml new file mode 100644 index 00000000..d32d9f70 --- /dev/null +++ b/tests/semantic/checker-error-specs/invalid_invoke_flow_wrong_input_binding.qtype.yaml @@ -0,0 +1,50 @@ +# InvokeFlow must have input_bindings that match the flow's input variable IDs +id: test_app + +models: + - id: test_model + type: Model + provider: openai + +flows: + - id: inner_flow + variables: + - id: inner_input + type: text + - id: inner_output + type: text + inputs: + - inner_input + outputs: + - inner_output + steps: + - id: llm_step + type: LLMInference + model: test_model + inputs: + - inner_input + outputs: + - inner_output + + - id: outer_flow + variables: + - id: outer_input + type: text + - id: outer_output + type: text + inputs: + - outer_input + outputs: + - outer_output + steps: + - id: invoke_step + type: InvokeFlow + flow: inner_flow + inputs: + - outer_input + outputs: + - outer_output + input_bindings: + wrong_param: outer_input # Wrong! Should be 'inner_input' + output_bindings: + inner_output: outer_output diff --git a/tests/semantic/checker-error-specs/invalid_invoke_flow_wrong_output_binding.qtype.yaml b/tests/semantic/checker-error-specs/invalid_invoke_flow_wrong_output_binding.qtype.yaml new file mode 100644 index 00000000..6bb1a41c --- /dev/null +++ b/tests/semantic/checker-error-specs/invalid_invoke_flow_wrong_output_binding.qtype.yaml @@ -0,0 +1,50 @@ +# InvokeFlow must have output_bindings that match the flow's output variable IDs +id: test_app + +models: + - id: test_model + type: Model + provider: openai + +flows: + - id: inner_flow + variables: + - id: inner_input + type: text + - id: inner_output + type: text + inputs: + - inner_input + outputs: + - inner_output + steps: + - id: llm_step + type: LLMInference + model: test_model + inputs: + - inner_input + outputs: + - inner_output + + - id: outer_flow + variables: + - id: outer_input + type: text + - id: outer_output + type: text + inputs: + - outer_input + outputs: + - outer_output + steps: + - id: invoke_step + type: InvokeFlow + flow: inner_flow + inputs: + - outer_input + outputs: + - outer_output + input_bindings: + inner_input: outer_input + output_bindings: + wrong_output: outer_output # Wrong! Should be 'inner_output' diff --git a/tests/semantic/checker-error-specs/invalid_invoke_tool_wrong_input_binding.qtype.yaml b/tests/semantic/checker-error-specs/invalid_invoke_tool_wrong_input_binding.qtype.yaml new file mode 100644 index 00000000..83202eeb --- /dev/null +++ b/tests/semantic/checker-error-specs/invalid_invoke_tool_wrong_input_binding.qtype.yaml @@ -0,0 +1,40 @@ +# InvokeTool must have input_bindings that match the tool's input parameter names +id: test_app + +tools: + - type: PythonFunctionTool + id: test_tool + name: test_function + function_name: test_function + module_path: some.module + description: Test tool + inputs: + - id: input_param + type: text + outputs: + - id: test_function_result + type: text + +flows: + - id: test_flow + variables: + - id: input_var + type: text + - id: output_var + type: text + inputs: + - input_var + outputs: + - output_var + steps: + - id: invoke_step + type: InvokeTool + tool: test_tool + inputs: + - input_var + outputs: + - output_var + input_bindings: + wrong_param: input_var # Wrong! Should be 'input_param' + output_bindings: + test_function_result: output_var diff --git a/tests/semantic/checker-error-specs/invalid_invoke_tool_wrong_output_binding.qtype.yaml b/tests/semantic/checker-error-specs/invalid_invoke_tool_wrong_output_binding.qtype.yaml new file mode 100644 index 00000000..737aa55f --- /dev/null +++ b/tests/semantic/checker-error-specs/invalid_invoke_tool_wrong_output_binding.qtype.yaml @@ -0,0 +1,40 @@ +# InvokeTool must have output_bindings that match the tool's output parameter names +id: test_app + +tools: + - type: PythonFunctionTool + id: test_tool + name: test_function + function_name: test_function + module_path: some.module + description: Test tool + inputs: + - id: input_param + type: text + outputs: + - id: test_function_result + type: text + +flows: + - id: test_flow + variables: + - id: input_var + type: text + - id: output_var + type: text + inputs: + - input_var + outputs: + - output_var + steps: + - id: invoke_step + type: InvokeTool + tool: test_tool + inputs: + - input_var + outputs: + - output_var + input_bindings: + input_param: input_var + output_bindings: + result: output_var # Wrong! Should be 'test_function_result' diff --git a/tests/semantic/test_checker_validation.py b/tests/semantic/test_checker_validation.py index f9ce2982..de3dace6 100644 --- a/tests/semantic/test_checker_validation.py +++ b/tests/semantic/test_checker_validation.py @@ -109,6 +109,22 @@ "invalid_complete_flow_no_text_output.qtype.yaml", "final step 'echo_step' is of type 'Echo' which does not support streaming", ), + ( + "invalid_invoke_tool_wrong_input_binding.qtype.yaml", + "Tool parameter 'wrong_param' not defined in tool", + ), + ( + "invalid_invoke_tool_wrong_output_binding.qtype.yaml", + "Tool parameter 'result' not defined in tool", + ), + ( + "invalid_invoke_flow_wrong_input_binding.qtype.yaml", + "Flow parameter 'wrong_param' not defined in flow", + ), + ( + "invalid_invoke_flow_wrong_output_binding.qtype.yaml", + "Flow parameter 'wrong_output' not defined in flow", + ), ], ) def test_checker_validation_errors(yaml_file, expected_error_fragment):