From eae9298343c7d80d767530f807fb303c286abd59 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Sun, 30 Mar 2025 12:39:14 +0200 Subject: [PATCH 01/58] Addition of solver voting verifiers and two mutators --- fuzz_test_utils/mutators.py | 837 ++++++++++++++++++---- verifiers/__init__.py | 12 +- verifiers/solver_voting_count_verifier.py | 114 +++ verifiers/solver_voting_sat_verifier.py | 115 +++ verifiers/verifier.py | 46 +- 5 files changed, 958 insertions(+), 166 deletions(-) create mode 100644 verifiers/solver_voting_count_verifier.py create mode 100644 verifiers/solver_voting_sat_verifier.py diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index c05a6148..70520863 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1,62 +1,108 @@ import copy import random +import cpmpy +import numpy as np +from cpmpy.expressions.globalfunctions import GlobalFunction, Abs, Minimum, Maximum, Element, Count, Among, NValue, \ + NValueExcept from cpmpy.transformations.negation import push_down_negation from cpmpy.transformations.to_cnf import flat2cnf from cpmpy import intvar, Model -from cpmpy.expressions.core import Operator, Comparison +from cpmpy.expressions.core import Operator, Comparison, BoolVal, Expression from cpmpy.transformations.decompose_global import decompose_in_tree from cpmpy.transformations.get_variables import get_variables from cpmpy.transformations.linearize import linearize_constraint, only_positive_bv, canonical_comparison -from cpmpy.expressions.utils import is_boolexpr, is_any_list +from cpmpy.expressions.utils import is_boolexpr, is_any_list, is_bool, is_int, is_num from cpmpy.transformations.flatten_model import flatten_constraint, normalized_boolexpr, normalized_numexpr, \ flatten_objective, __is_flat_var from cpmpy.transformations.normalize import toplevel_list, simplify_boolean from cpmpy.transformations.reification import only_bv_reifies, reify_rewrite from cpmpy.transformations.comparison import only_numexpr_equality -from cpmpy.expressions.globalconstraints import Xor +from cpmpy.expressions.globalconstraints import Xor, AllDifferent, AllDifferentExceptN, AllEqual, AllEqualExceptN, \ + Circuit, Inverse, Table, NegativeTable, IfThenElse, InDomain, Cumulative, Precedence, NoOverlap, \ + GlobalCardinalityCount, Increasing, Decreasing, IncreasingStrict, DecreasingStrict, LexLess, LexLessEq, \ + LexChainLess, LexChainLessEq, GlobalConstraint +# from cpmpy.expressions.globalfunctions import Abs, Mimimum(GlobalFunction), Maximum +from cpmpy.expressions.variables import boolvar, _IntVarImpl + + +class Function: + def __init__(self, name, func, type_: str, int_args: int, bool_args: int, + bool_return: bool | None, + min_args: int = None, + max_args: int = None, + multiple: int = 1): + """ + type = string that describes the type of function it is + int_args = the amount of args of type int it requires + bool_args = the amount of args of type bool it requires + bool_return = a boolean representing whether it returns a boolean (False means int return type, None means it can be either) + min_args = the minimum amount of args the function takes + max_args = the maximum amount of args the function takes + multiple = the arguments have to be a multiple of this int + """ + self.name = name + self.func = func + self.type = type_ + self.int_args = int_args + self.bool_args = bool_args + self.bool_return = bool_return + self.min_args = min_args + self.max_args = max_args + self.multiple = multiple + + def __repr__(self): + return (f"Operation({self.name}, {self.type}, {self.int_args}, {self.bool_args}, " + f"{self.bool_return}, min_args={self.min_args}, max_args={self.max_args}, multiple={self.multiple})") '''TRUTH TABLE BASED MORPHS''' + + def not_morph(cons): con = random.choice(cons) ncon = ~con return [~ncon] + + def xor_morph(cons): '''morph two constraints with XOR''' - con1, con2 = random.choices(cons,k=2) - #add a random option as per xor truth table + con1, con2 = random.choices(cons, k=2) + # add a random option as per xor truth table return [random.choice(( Xor([con1, ~con2]), Xor([~con1, con2]), ~Xor([~con1, ~con2]), ~Xor([con1, con2])))] + def and_morph(cons): '''morph two constraints with AND''' - con1, con2 = random.choices(cons,k=2) + con1, con2 = random.choices(cons, k=2) return [random.choice(( ~((con1) & (~con2)), ~((~con1) & (~con2)), ~((~con1) & (con2)), ((con1) & (con2))))] + def or_morph(cons): '''morph two constraints with OR''' - con1, con2 = random.choices(cons,k=2) - #add all options as per xor truth table + con1, con2 = random.choices(cons, k=2) + # add all options as per xor truth table return [random.choice(( ((con1) | (~con2)), ~((~con1) | (~con2)), ((~con1) | (con2)), ((con1) | (con2))))] + def implies_morph(cons): '''morph two constraints with ->''' - con1, con2 = random.choices(cons,k=2) + con1, con2 = random.choices(cons, k=2) try: - #add all options as per xor truth table + # add all options as per xor truth table return [random.choice(( ~((con1).implies(~con2)), ((~con1).implies(~con2)), @@ -67,10 +113,12 @@ def implies_morph(cons): ((~con2).implies(con1)), ((con2).implies(con1))))] except Exception as e: - raise MetamorphicError(implies_morph,cons,e) + raise MetamorphicError(implies_morph, cons, e) + '''CPMPY-TRANSFORMATION MORPHS''' + def canonical_comparison_morph(cons): n = random.randint(1, len(cons)) randcons = random.choices(cons, k=n) @@ -79,16 +127,17 @@ def canonical_comparison_morph(cons): except Exception as e: raise MetamorphicError(canonical_comparison, cons, e) + def flatten_morph(cons, flatten_all=False): if flatten_all is False: - n = random.randint(1,len(cons)) - randcons = random.choices(cons,k=n) + n = random.randint(1, len(cons)) + randcons = random.choices(cons, k=n) else: randcons = cons try: return flatten_constraint(randcons) except Exception as e: - raise MetamorphicError(flatten_constraint,randcons, e) + raise MetamorphicError(flatten_constraint, randcons, e) def simplify_boolean_morph(cons): @@ -98,10 +147,10 @@ def simplify_boolean_morph(cons): raise MetamorphicError(simplify_boolean, cons, e) -def only_numexpr_equality_morph(cons,supported=frozenset()): +def only_numexpr_equality_morph(cons, supported=frozenset()): n = random.randint(1, len(cons)) randcons = random.choices(cons, k=n) - flatcons = flatten_morph(randcons, flatten_all=True) # only_numexpr_equality requires flat constraints + flatcons = flatten_morph(randcons, flatten_all=True) # only_numexpr_equality requires flat constraints try: newcons = only_numexpr_equality(flatcons, supported=supported) return newcons @@ -122,6 +171,7 @@ def normalized_boolexpr_morph(cons): else: return cons + def normalized_numexpr_morph(const): try: cons = copy.deepcopy(const) @@ -131,9 +181,9 @@ def normalized_numexpr_morph(const): res = pickaritmetic(con, log=[i]) if res != []: firstcon = random.choice(res) - break #numexpr found + break # numexpr found if firstcon is None: - #no numexpressions found but still call the function to test on all inputs + # no numexpressions found but still call the function to test on all inputs randcon = random.choice(cons) try: con, newcons = normalized_numexpr(randcon) @@ -141,7 +191,7 @@ def normalized_numexpr_morph(const): except Exception as e: raise MetamorphicError(normalized_numexpr, randcon, e) else: - #get the numexpr + # get the numexpr arg = cons[firstcon[0]] newfirst = arg for i in firstcon[1:]: @@ -172,21 +222,22 @@ def normalized_numexpr_morph(const): raise MetamorphicError(normalized_numexpr_morph, cons, e) -def linearize_constraint_morph(cons,linearize_all=False,supported={}): +def linearize_constraint_morph(cons, linearize_all=False, supported={}): if linearize_all: randcons = cons else: n = random.randint(1, len(cons)) randcons = random.choices(cons, k=n) - #only apply linearize after only_bv_reifies - decomcons = decompose_in_tree_morph(randcons,decompose_all=True,supported=supported) + # only apply linearize after only_bv_reifies + decomcons = decompose_in_tree_morph(randcons, decompose_all=True, supported=supported) flatcons = only_bv_reifies_morph(decomcons, morph_all=True) try: return linearize_constraint(flatcons, supported={'mul'}) except Exception as e: raise MetamorphicError(linearize_constraint, flatcons, e) + def reify_rewrite_morph(cons): n = random.randint(1, len(cons)) randcons = random.choices(cons, k=n) @@ -212,14 +263,15 @@ def flatten_objective_morph(objective): except Exception as e: raise MetamorphicError(flatten_objective, objective, e) -def decompose_in_tree_morph(cons,decompose_all=False,supported={}): + +def decompose_in_tree_morph(cons, decompose_all=False, supported={}): try: - return decompose_in_tree(cons,supported=supported) + return decompose_in_tree(cons, supported=supported) except Exception as e: raise MetamorphicError(decompose_in_tree, cons, e) -def only_bv_reifies_morph(cons,morph_all=True): +def only_bv_reifies_morph(cons, morph_all=True): if morph_all: randcons = cons else: @@ -231,22 +283,24 @@ def only_bv_reifies_morph(cons,morph_all=True): except Exception as e: raise MetamorphicError(only_bv_reifies, flatcons, e) + def only_positive_bv_morph(cons): - lincons = linearize_constraint_morph(cons,linearize_all=True,supported={}) + lincons = linearize_constraint_morph(cons, linearize_all=True, supported={}) try: return only_positive_bv(lincons) except Exception as e: raise MetamorphicError(only_positive_bv, lincons, e) - def flat2cnf_morph(cons): - #flatcons = flatten_morph(cons,flatten_all=True) - onlycons = only_bv_reifies_morph(cons,morph_all=True) + # flatcons = flatten_morph(cons,flatten_all=True) + onlycons = only_bv_reifies_morph(cons, morph_all=True) try: return flat2cnf(onlycons) except Exception as e: raise MetamorphicError(flat2cnf, onlycons, e) + + def toplevel_list_morph(cons): try: return toplevel_list(cons) @@ -262,6 +316,7 @@ def add_solution(cons): raise MetamorphicError(add_solution, cons, e) return [var == var.value() for var in vars if var.value() is not None] + def semanticFusion(const): try: firstcon = None @@ -269,28 +324,28 @@ def semanticFusion(const): cons = copy.deepcopy(const) random.shuffle(cons) for i, con in enumerate(cons): - res = pickaritmetic(con,log=[i]) + res = pickaritmetic(con, log=[i]) if res != []: if firstcon == None: firstcon = random.choice(res) elif secondcon == None: secondcon = random.choice(res) - break #stop when 2 constraints found. still random because cons are shuffled + break # stop when 2 constraints found. still random because cons are shuffled if secondcon != None: - #two constraints with aritmetic expressions found, perform semantic fusion on them - #get the expressions to fuse + # two constraints with aritmetic expressions found, perform semantic fusion on them + # get the expressions to fuse arg = cons[firstcon[0]] newfirst = copy.deepcopy(arg) count = 0 for i in firstcon[1:]: count += 1 arg = arg.args[i] - if hasattr(arg,'name'): + if hasattr(arg, 'name'): if arg.name in ['div', 'mod', 'pow']: if len(firstcon) > count + 1: if firstcon[count + 1] == 1: - return [] #we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 + return [] # we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 firstexpr = arg arg = cons[secondcon[0]] @@ -299,25 +354,25 @@ def semanticFusion(const): for i in secondcon[1:]: count += 1 arg = arg.args[i] - if hasattr(arg,'name'): + if hasattr(arg, 'name'): if arg.name in ['div', 'mod', 'pow']: if len(secondcon) > count + 1: if secondcon[count + 1] == 1: - return [] #we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 + return [] # we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 secondexpr = arg - lb,ub = Operator('sum',[firstexpr,secondexpr]).get_bounds() + lb, ub = Operator('sum', [firstexpr, secondexpr]).get_bounds() z = intvar(lb, ub) firstexpr, secondexpr = z - secondexpr, z - firstexpr - #make the new constraints + # make the new constraints arg = newfirst c = 1 firststr = str(firstexpr) for i in firstcon[1:]: if str(arg) in firststr: - return [] #cyclical - c+=1 + return [] # cyclical + c += 1 if c == len(firstcon): if isinstance(arg.args, tuple): arg.args = list(arg.args) @@ -330,7 +385,7 @@ def semanticFusion(const): secondstr = str(secondexpr) for i in secondcon[1:]: if str(arg) in secondstr: - return [] #cyclical + return [] # cyclical c += 1 if c == len(secondcon): if isinstance(arg.args, tuple): @@ -339,15 +394,16 @@ def semanticFusion(const): else: arg = arg.args[i] - return [newfirst,newsecond] + return [newfirst, newsecond] else: - #no expressions found to fuse + # no expressions found to fuse return [] except Exception as e: raise MetamorphicError(semanticFusion, cons, e) + def semanticFusionMinus(const): try: firstcon = None @@ -355,28 +411,28 @@ def semanticFusionMinus(const): cons = copy.deepcopy(const) random.shuffle(cons) for i, con in enumerate(cons): - res = pickaritmetic(con,log=[i]) + res = pickaritmetic(con, log=[i]) if res != []: if firstcon == None: firstcon = random.choice(res) elif secondcon == None: secondcon = random.choice(res) - break #stop when 2 constraints found. still random because cons are shuffled + break # stop when 2 constraints found. still random because cons are shuffled if secondcon != None: - #two constraints with aritmetic expressions found, perform semantic fusion on them - #get the expressions to fuse + # two constraints with aritmetic expressions found, perform semantic fusion on them + # get the expressions to fuse arg = cons[firstcon[0]] newfirst = copy.deepcopy(arg) count = 0 for i in firstcon[1:]: count += 1 arg = arg.args[i] - if hasattr(arg,'name'): + if hasattr(arg, 'name'): if arg.name in ['div', 'mod', 'pow']: if len(firstcon) > count + 1: if firstcon[count + 1] == 1: - return [] #we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 + return [] # we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 firstexpr = arg arg = cons[secondcon[0]] @@ -385,25 +441,25 @@ def semanticFusionMinus(const): for i in secondcon[1:]: count += 1 arg = arg.args[i] - if hasattr(arg,'name'): + if hasattr(arg, 'name'): if arg.name in ['div', 'mod', 'pow']: if len(secondcon) > count + 1: if secondcon[count + 1] == 1: - return [] #we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 + return [] # we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 secondexpr = arg - lb,ub = Operator('sub',[firstexpr,secondexpr]).get_bounds() + lb, ub = Operator('sub', [firstexpr, secondexpr]).get_bounds() z = intvar(lb, ub) firstexpr, secondexpr = z + secondexpr, firstexpr - z - #make the new constraints + # make the new constraints arg = newfirst c = 1 firststr = str(firstexpr) for i in firstcon[1:]: if str(arg) in firststr: - return [] #cyclical - c+=1 + return [] # cyclical + c += 1 if c == len(firstcon): if isinstance(arg.args, tuple): arg.args = list(arg.args) @@ -416,7 +472,7 @@ def semanticFusionMinus(const): secondstr = str(secondexpr) for i in secondcon[1:]: if str(arg) in secondstr: - return [] #cyclical + return [] # cyclical c += 1 if c == len(secondcon): if isinstance(arg.args, tuple): @@ -425,15 +481,16 @@ def semanticFusionMinus(const): else: arg = arg.args[i] - return [newfirst,newsecond] + return [newfirst, newsecond] else: - #no expressions found to fuse + # no expressions found to fuse return [] except Exception as e: raise MetamorphicError(semanticFusionMinus, cons, e) + def semanticFusionwsum(const): try: firstcon = None @@ -441,58 +498,61 @@ def semanticFusionwsum(const): cons = copy.deepcopy(const) random.shuffle(cons) for i, con in enumerate(cons): - res = pickaritmetic(con,log=[i]) + res = pickaritmetic(con, log=[i]) if res != []: if firstcon == None: firstcon = random.choice(res) elif secondcon == None: secondcon = random.choice(res) - break #stop when 2 constraints found. still random because cons are shuffled + break # stop when 2 constraints found. still random because cons are shuffled if secondcon != None: - #two constraints with aritmetic expressions found, perform semantic fusion on them - #get the expressions to fuse + # two constraints with aritmetic expressions found, perform semantic fusion on them + # get the expressions to fuse arg = cons[firstcon[0]] newfirst = copy.deepcopy(arg) count = 0 for i in firstcon[1:]: count += 1 arg = arg.args[i] - if hasattr(arg,'name'): + if hasattr(arg, 'name'): if arg.name in ['div', 'mod', 'pow']: if len(firstcon) > count + 1: if firstcon[count + 1] == 1: - return [] #we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 + return [] # we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 firstexpr = arg arg = cons[secondcon[0]] - #newsecond = copy.deepcopy(arg) + # newsecond = copy.deepcopy(arg) newsecond = (arg) count = 0 for i in secondcon[1:]: count += 1 arg = arg.args[i] - if hasattr(arg,'name'): + if hasattr(arg, 'name'): if arg.name in ['div', 'mod', 'pow']: if len(secondcon) > count + 1: if secondcon[count + 1] == 1: - return [] #we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 + return [] # we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 secondexpr = arg l = random.randint(1, 10) n = random.randint(1, 10) m = random.randint(1, 10) - lb, ub = Operator('wsum',[[l, m, n], [firstexpr, secondexpr, 1]]).get_bounds() + lb, ub = Operator('wsum', [[l, m, n], [firstexpr, secondexpr, 1]]).get_bounds() z = intvar(lb, ub) - firstexpr, secondexpr = Operator('wsum',[[1, -m, -n], [z, secondexpr, 1]]) / l, Operator('wsum',[[1, -l, -n], [z, firstexpr, 1]]) / m - #make the new constraints + firstexpr, secondexpr = Operator('wsum', [[1, -m, -n], [z, secondexpr, 1]]) / l, Operator('wsum', + [[1, -l, -n], + [z, firstexpr, + 1]]) / m + # make the new constraints arg = newfirst c = 1 firststr = str(firstexpr) for i in firstcon[1:]: if str(arg) in firststr: - return [] #cyclical - c+=1 + return [] # cyclical + c += 1 if c == len(firstcon): if isinstance(arg.args, tuple): arg.args = list(arg.args) @@ -505,7 +565,7 @@ def semanticFusionwsum(const): secondstr = str(secondexpr) for i in secondcon[1:]: if str(arg) in secondstr: - return [] #cyclical + return [] # cyclical c += 1 if c == len(secondcon): if isinstance(arg.args, tuple): @@ -517,11 +577,13 @@ def semanticFusionwsum(const): return [newfirst, newsecond] else: - #no expressions found to fuse + # no expressions found to fuse return [] except Exception as e: raise MetamorphicError(semanticFusionwsum, cons, e) + + def semanticFusionCountingwsum(const): try: firstcon = None @@ -529,28 +591,28 @@ def semanticFusionCountingwsum(const): cons = copy.deepcopy(const) random.shuffle(cons) for i, con in enumerate(cons): - res = pickaritmetic(con,log=[i]) + res = pickaritmetic(con, log=[i]) if res != []: if firstcon == None: firstcon = random.choice(res) elif secondcon == None: secondcon = random.choice(res) - break #stop when 2 constraints found. still random because cons are shuffled + break # stop when 2 constraints found. still random because cons are shuffled if secondcon != None: - #two constraints with aritmetic expressions found, perform semantic fusion on them - #get the expressions to fuse + # two constraints with aritmetic expressions found, perform semantic fusion on them + # get the expressions to fuse arg = cons[firstcon[0]] newfirst = copy.deepcopy(arg) count = 0 for i in firstcon[1:]: count += 1 arg = arg.args[i] - if hasattr(arg,'name'): + if hasattr(arg, 'name'): if arg.name in ['div', 'mod', 'pow']: if len(firstcon) > count + 1: if firstcon[count + 1] == 1: - return [] #we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 + return [] # we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 firstexpr = arg arg = cons[secondcon[0]] @@ -559,29 +621,32 @@ def semanticFusionCountingwsum(const): for i in secondcon[1:]: count += 1 arg = arg.args[i] - if hasattr(arg,'name'): + if hasattr(arg, 'name'): if arg.name in ['div', 'mod', 'pow']: if len(secondcon) > count + 1: if secondcon[count + 1] == 1: - return [] #we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 + return [] # we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 secondexpr = arg l = random.randint(1, 10) n = random.randint(1, 10) m = random.randint(1, 10) - lb, ub = Operator('wsum',[[l, m, n], [firstexpr, secondexpr, 1]]).get_bounds() + lb, ub = Operator('wsum', [[l, m, n], [firstexpr, secondexpr, 1]]).get_bounds() z = intvar(lb, ub) thirdcon = z == Operator('wsum', [[l, m, n], [firstexpr, secondexpr, 1]]) - firstexpr, secondexpr = Operator('wsum',[[1, -m, -n], [z, secondexpr, 1]]) / l, Operator('wsum',[[1, -l, -n], [z, firstexpr, 1]]) / m + firstexpr, secondexpr = Operator('wsum', [[1, -m, -n], [z, secondexpr, 1]]) / l, Operator('wsum', + [[1, -l, -n], + [z, firstexpr, + 1]]) / m - #make the new constraints + # make the new constraints arg = newfirst c = 1 firststr = str(firstexpr) for i in firstcon[1:]: if str(arg) in firststr: - return [] #cyclical - c+=1 + return [] # cyclical + c += 1 if c == len(firstcon): if isinstance(arg.args, tuple): arg.args = list(arg.args) @@ -594,7 +659,7 @@ def semanticFusionCountingwsum(const): secondstr = str(secondexpr) for i in secondcon[1:]: if str(arg) in secondstr: - return [] #cyclical + return [] # cyclical c += 1 if c == len(secondcon): if isinstance(arg.args, tuple): @@ -606,7 +671,7 @@ def semanticFusionCountingwsum(const): return [newfirst, newsecond, thirdcon] else: - #no expressions found to fuse + # no expressions found to fuse return [] except Exception as e: @@ -620,28 +685,28 @@ def semanticFusionCounting(const): cons = copy.deepcopy(const) random.shuffle(cons) for i, con in enumerate(cons): - res = pickaritmetic(con,log=[i]) + res = pickaritmetic(con, log=[i]) if res != []: if firstcon == None: firstcon = random.choice(res) elif secondcon == None: secondcon = random.choice(res) - break #stop when 2 constraints found. still random because cons are shuffled + break # stop when 2 constraints found. still random because cons are shuffled if secondcon != None: - #two constraints with aritmetic expressions found, perform semantic fusion on them - #get the expressions to fuse + # two constraints with aritmetic expressions found, perform semantic fusion on them + # get the expressions to fuse arg = cons[firstcon[0]] newfirst = copy.deepcopy(arg) count = 0 for i in firstcon[1:]: count += 1 arg = arg.args[i] - if hasattr(arg,'name'): + if hasattr(arg, 'name'): if arg.name in ['div', 'mod', 'pow']: if len(firstcon) > count + 1: if firstcon[count + 1] == 1: - return [] #we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 + return [] # we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 firstexpr = arg arg = cons[secondcon[0]] @@ -650,26 +715,26 @@ def semanticFusionCounting(const): for i in secondcon[1:]: count += 1 arg = arg.args[i] - if hasattr(arg,'name'): + if hasattr(arg, 'name'): if arg.name in ['div', 'mod', 'pow']: if len(secondcon) > count + 1: if secondcon[count + 1] == 1: - return [] #we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 + return [] # we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 secondexpr = arg - lb,ub = Operator('sum',[firstexpr,secondexpr]).get_bounds() + lb, ub = Operator('sum', [firstexpr, secondexpr]).get_bounds() z = intvar(lb, ub) firstexpr, secondexpr = z - secondexpr, z - firstexpr thirdcon = z == firstexpr + secondexpr - #make the new constraints + # make the new constraints arg = newfirst c = 1 firststr = str(firstexpr) for i in firstcon[1:]: if str(arg) in firststr: - return [] #cyclical - c+=1 + return [] # cyclical + c += 1 if c == len(firstcon): if isinstance(arg.args, tuple): arg.args = list(arg.args) @@ -682,7 +747,7 @@ def semanticFusionCounting(const): secondstr = str(secondexpr) for i in secondcon[1:]: if str(arg) in secondstr: - return [] #cyclical + return [] # cyclical c += 1 if c == len(secondcon): if isinstance(arg.args, tuple): @@ -691,15 +756,16 @@ def semanticFusionCounting(const): else: arg = arg.args[i] - return [newfirst,newsecond, thirdcon] + return [newfirst, newsecond, thirdcon] else: - #no expressions found to fuse + # no expressions found to fuse return [] except Exception as e: raise MetamorphicError(semanticFusionCounting, cons, e) + def semanticFusionCountingMinus(const): try: firstcon = None @@ -707,28 +773,28 @@ def semanticFusionCountingMinus(const): cons = copy.deepcopy(const) random.shuffle(cons) for i, con in enumerate(cons): - res = pickaritmetic(con,log=[i]) + res = pickaritmetic(con, log=[i]) if res != []: if firstcon == None: firstcon = random.choice(res) elif secondcon == None: secondcon = random.choice(res) - break #stop when 2 constraints found. still random because cons are shuffled + break # stop when 2 constraints found. still random because cons are shuffled if secondcon != None: - #two constraints with aritmetic expressions found, perform semantic fusion on them - #get the expressions to fuse + # two constraints with aritmetic expressions found, perform semantic fusion on them + # get the expressions to fuse arg = cons[firstcon[0]] newfirst = copy.deepcopy(arg) count = 0 for i in firstcon[1:]: count += 1 arg = arg.args[i] - if hasattr(arg,'name'): + if hasattr(arg, 'name'): if arg.name in ['div', 'mod', 'pow']: if len(firstcon) > count + 1: if firstcon[count + 1] == 1: - return [] #we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 + return [] # we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 firstexpr = arg arg = cons[secondcon[0]] @@ -737,25 +803,25 @@ def semanticFusionCountingMinus(const): for i in secondcon[1:]: count += 1 arg = arg.args[i] - if hasattr(arg,'name'): + if hasattr(arg, 'name'): if arg.name in ['div', 'mod', 'pow']: if len(secondcon) > count + 1: if secondcon[count + 1] == 1: - return [] #we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 + return [] # we don't want to mess with the divisor of a division, since we can't divide by a domain containing 0 secondexpr = arg - lb,ub = Operator('sub',[firstexpr,secondexpr]).get_bounds() + lb, ub = Operator('sub', [firstexpr, secondexpr]).get_bounds() z = intvar(lb, ub) firstexpr, secondexpr = z + secondexpr, firstexpr - z thirdcon = z == firstexpr - secondexpr - #make the new constraints + # make the new constraints arg = newfirst c = 1 firststr = str(firstexpr) for i in firstcon[1:]: if str(arg) in firststr: - return [] #cyclical - c+=1 + return [] # cyclical + c += 1 if c == len(firstcon): if isinstance(arg.args, tuple): arg.args = list(arg.args) @@ -768,7 +834,7 @@ def semanticFusionCountingMinus(const): secondstr = str(secondexpr) for i in secondcon[1:]: if str(arg) in secondstr: - return [] #cyclical + return [] # cyclical c += 1 if c == len(secondcon): if isinstance(arg.args, tuple): @@ -780,15 +846,13 @@ def semanticFusionCountingMinus(const): return [newfirst, newsecond, thirdcon] else: - #no expressions found to fuse + # no expressions found to fuse return [] except Exception as e: raise MetamorphicError(semanticFusionCountingMinus, cons, e) - - def aritmetic_comparison_morph(const): try: cons = copy.deepcopy(const) @@ -818,14 +882,14 @@ def aritmetic_comparison_morph(const): rhs2 = rhs + 7 lhs3 = lhs - 7 rhs3 = rhs - 7 - lhs, rhs = random.choice([(lhs3,rhs3),(lhs2,rhs2),(lhs1,rhs1)]) - newcon = Comparison(name=firstexpr.name,left=lhs,right=rhs) + lhs, rhs = random.choice([(lhs3, rhs3), (lhs2, rhs2), (lhs1, rhs1)]) + newcon = Comparison(name=firstexpr.name, left=lhs, right=rhs) except Exception as e: raise MetamorphicError(aritmetic_comparison_morph, firstexpr, e) # make the new constraint (newfirst) arg = newfirst - if len(firstcon) == 1: #toplevel comparison + if len(firstcon) == 1: # toplevel comparison return [newcon] c = 1 for i in firstcon[1:]: @@ -844,6 +908,490 @@ def aritmetic_comparison_morph(const): except Exception as e: raise MetamorphicError(aritmetic_comparison_morph, cons, e) + +def type_aware_operator_replacement(constraints): + """ + Replaces a random operator of a random constraint from a list of given constraints. + IMPORTANT: This can change satisfiability of the constraint! Only to be used with verifiers that allow this! + This means it returns a list of ALL constraints and has to be handled accordingly in the 'generate_mutations' + function to swap out the constraints instead of adding them. + + ~ Parameters: + - constraints: a list of all the constraints to possibly be mutated + ~ Return: + - final_cons: a list of the same constraints where one constraint has a mutated operator + + Types of operators ('...' means the amount of arguments is variable): + - Int, Bool -> Bool : == != < <= > >= + - Bool, Int -> Bool : == != < <= > >= + - Int, Int -> Bool : == != < <= > >= + - [Bool, Bool] -> Bool : and or -> + - [Bool, ...] -> Bool : and or + - [Bool] -> Bool : not + - Int, Int -> Int : sum sub mul div mod pow + - [Int, ...] -> Int : sum + - [Int] -> Int : - + - [Int, ...] [Int, ...] (arrays of same len) -> Int : wsum + """ + try: + final_cons = copy.deepcopy(constraints) + cons_set = set(constraints) + # pick a random constraint and calculate their mutable expressions until there is at least 1 + con = random.choice(final_cons) + exprs = get_all_mutable_op_exprs(con) + while len(exprs) < 1: + con = random.choice(list(cons_set)) + exprs = get_all_mutable_op_exprs(con) + cons_set.remove(con) + + # remove the constraint from the constraints + # final_cons = [c for c in final_cons if (c.name != con.name or c.args != con.args)] + final_cons.remove(con) + + # Choose an expression to change + expr = random.choice(exprs) + + mutate_op_expression(expr, con) + + # add the changed constraint back + final_cons.append(con) + return final_cons + + except Exception as e: + return Exception(e) + + +def mutate_op_expression(expr, con): + """ + Mutates the constraint containing the expression by mutating said expression. + Only to be called when the expression is known to be in the constraint. + + ~ Parameters: + - expr: the expression that will be mutated + - con: the constraint containing the expression + ~ No return. Mutates the constraint! + """ + # types that can be converted into each-other + comparisons = {'==', '!=', '<', '<=', '>', '>='} + int_ops = {'sum', 'sub', 'mul', + 'pow', 'mod', 'div'} + logic_ops = {'and', 'or', '->'} + logic_ops_inf_args = {'and', 'or'} + if expr == con: # Found the expression to mutate + # Determine the type of the operator + if expr.name in comparisons: # a, b -> Bool (Comparison() always has two arguments) + possible_replacements = comparisons - {expr.name} + elif expr.name in int_ops and len(expr.args) == 2: # a, a -> Int + possible_replacements = int_ops - {expr.name} + elif expr.name in logic_ops and len(expr.args) == 2: # [Bool, Bool] -> Bool + possible_replacements = logic_ops - {expr.name} + elif expr.name in logic_ops_inf_args: # [Bool, ...] -> Bool + possible_replacements = logic_ops_inf_args - {expr.name} + else: + raise ValueError(f"Unknown operator type: {expr.name}. (You should not be able to get here)") + + new_operator = random.choice(list(possible_replacements)) + expr.name = new_operator + return + + # recursively search in arguments + if hasattr(con, "args"): + for arg in con.args: + mutate_op_expression(expr, arg) + return + + +def get_all_mutable_op_exprs(con): + """ + Returns a list of all expressions inside the given constraint of which the + operator can be mutated into another one. This can be extended with other operators. + + ~ Paremeters: + - con: a single constraint, possibly containing multiple expressions + ~ Return: + - mutable_exprs: all expressions in the constraint that can be mutated (safely) + """ + comparisons = {'==', '!=', '<', '<=', '>', '>='} + int_ops = {'sum', 'sub', 'mul', 'div', 'mod', 'pow'} + logic_ops = {'and', 'or', '->'} + logic_ops_inf_args = {'and', 'or'} + mutable_exprs = [] + for expr in get_all_op_exprs(con): + if expr.name in comparisons: # a, b -> Bool (Comparison() always has two arguments) + mutable_exprs.append(expr) + elif expr.name in int_ops and len(expr.args) == 2: # a, b -> Int + mutable_exprs.append(expr) + elif expr.name in logic_ops and len(expr.args) == 2: # [Bool, Bool] -> Bool + mutable_exprs.append(expr) + elif expr.name in logic_ops_inf_args: # [Bool, ...] -> Bool + mutable_exprs.append(expr) + return mutable_exprs + + +# Helper function to get all expressions WITH an operator in a given constraint +def get_all_op_exprs(con): + if type(con) in {Comparison, Operator}: + return sum((get_all_op_exprs(arg) for arg in con.args), []) + [con] # All subexpressions + current expression + else: + return [] + + +# Helper function to get all expressions WITHOUT an operator in a given constraint +def get_all_non_op_exprs(con): + if hasattr(con, 'args') and con.name != 'boolval': + return sum((get_all_non_op_exprs(arg) for arg in con.args), []) + elif type(con) == list: + if all([is_num(e) for e in con]): # wsum constants + return [] + else: + return [e for e in con] + else: + return [con] + + +# Helper function to get all epxressions in a given constraint (Might be unnecessary but let's use this for now) +def get_all_exprs(con): + return get_all_op_exprs(con)[::-1] + get_all_non_op_exprs(con) + + +def get_all_exprs_mult(cons): + all_exprs = [] + for con in cons: + all_exprs += get_all_exprs(con) + return all_exprs + + +def satisfies_args(func: Function, ints: int, bools: int, values: int, has_bool_return: bool): + """ + returns whether the given function `func` can work with the given amount of integers `ints` + and booleans `bools` and the given return type `has_bool_return` + """ + match func.type: + case 'op' | 'comp': + match func.name: + case 'wsum': + return values >= 1 and ints + bools >= func.min_args and has_bool_return == func.bool_return + case _: + return ((func.int_args == -1 or func.int_args <= ints + bools) and # enough int args + (func.bool_args == -1 or func.bool_args <= bools) and # enough bool args + (ints + bools >= func.min_args) and # enough args in general + (bools >= func.min_args if func.bool_args == -1 else True) and # enough bools + has_bool_return == func.bool_return) # return type matches + case 'gfun': + match func.name: + case 'Abs' | 'NValue' | 'Count': + return ints + bools >= func.min_args and not has_bool_return + case 'Minimum' | 'Maximum' | 'Element': # We make these have only ints, so it always has the same return type + return values >= func.min_args and not has_bool_return + case 'Among' | 'NValueExcept': + return values >= 1 and ints >= func.min_args and not has_bool_return + case 'gcon': + match func.name: + case 'Circuit' | 'Inverse' | 'GlobalCardinalityCount': + return values >= func.min_args and has_bool_return + case 'IfThenElse' | 'Xor': + return bools >= func.min_args and has_bool_return + case _: + return ints + bools >= func.min_args and has_bool_return + + + +def get_new_operator(func: Function, ints, bools, vals): + comb = ints + bools + match func.type: + case 'op' | 'comp': + # Separate logic for wsum + if func.name == 'wsum': # (-1, 0, False, 2, max_args, 2) + amnt_args = random.randint(func.min_args // 2, min(len(vals), func.max_args)//2) + # First take constants + constants = random.sample(vals, amnt_args) + # Then the other expressions + others = random.sample(comb, amnt_args) + return Operator(func.name, [constants, others]) + + # Logic for all other operators and comparisons is the same + if func.int_args == -1: + amnt_args = random.randint(func.min_args, min(len(comb), func.max_args)) # Take at least min_args and at most max_args arguments + args = random.sample(comb, amnt_args) + elif func.int_args > 0: + args = random.sample(comb, func.int_args) + if func.bool_args == -1: + amnt_args = random.randint(func.min_args, min(len(bools), func.max_args)) # Take at least min_args and at most max_args arguments + args = random.sample(bools, amnt_args) + elif func.bool_args > 0: + args = random.sample(bools, func.bool_args) + if func.type == 'op': + return Operator(func.name, args) + if func.type == 'comp': + return Comparison(func.name, *args) + case 'gfun': + match func.name: + case 'Abs': + args = random.choice(comb), + case 'Minimum' | 'Maximum': + amnt_args = random.randint(func.min_args, min(len(vals), func.max_args)) + args = random.sample(vals, amnt_args), + case 'Element': + amnt_args = random.randint(func.min_args, min(len(vals), func.max_args)) + first_arg = random.sample(vals, amnt_args) + idx = random.randint(0, amnt_args - 1) + args = first_arg, idx + case 'NValue': + amnt_args = random.randint(func.min_args, min(len(comb), func.max_args)) + args = random.sample(comb, amnt_args), + case 'Count': + amnt_args = random.randint(func.min_args, min(len(comb), func.max_args)) + first_arg = random.sample(comb, amnt_args - 1) + last_arg = random.choice(comb) + args = first_arg, last_arg + case 'Among': + amnt_fst_arg = random.randint(func.min_args//2, min(len(comb), func.max_args)//2) + amnt_snd_arg = random.randint(func.min_args//2, min(len(vals), func.max_args)//2) + first_arg = random.sample(comb, amnt_fst_arg) + second_arg = random.sample(vals, amnt_snd_arg) + args = first_arg, second_arg + case 'NValueExcept': + amnt_fst_arg = random.randint(func.min_args//2, min(len(comb), func.max_args)//2) + first_arg = random.sample(comb, amnt_fst_arg) + second_arg = random.choice(vals) + args = first_arg, second_arg + return func.func(*args) + case 'gcon': + match func.name: + case 'AllDifferent' | 'AllEqual' | 'Increasing' | 'Decreasing' | 'IncreasingStrict' | 'DecreasingStrict': + amnt_args = random.randint(func.min_args, min(len(comb), func.max_args)) + args = random.sample(comb, amnt_args) + case 'AllDifferentExceptN' | 'AllEqualExceptN': + amnt_fst_args = random.randint(func.min_args//2, min(len(comb), func.max_args)//2) + amnt_snd_args = random.randint(func.min_args//2, min(len(comb), func.max_args - amnt_fst_args)) + args = random.sample(comb, amnt_fst_args), random.sample(comb, amnt_snd_args) + case 'LexLess' | 'LexLessEq': + half_amnt_args = random.randint(func.min_args//2, min(len(comb), func.max_args)//2) + args = random.sample(comb, half_amnt_args), random.sample(comb, half_amnt_args) + case 'LexChainLess' | 'LexChainLessEq': + amnt_args = random.randint(func.min_args, min(len(comb), func.max_args)//2) + all_args = random.sample(comb, amnt_args) + divisors = [i for i in range(1, amnt_args) if amnt_args % i == 0] + fst_dimension = random.choice(divisors) + snd_dimension = int(amnt_args / fst_dimension) + args = [all_args[i * fst_dimension:(i + 1) * fst_dimension] for i in range(snd_dimension)], + case 'Circuit': + amnt_args = random.randint(func.min_args, min(len(vals), func.max_args)) + args = random.sample(vals, amnt_args), + case 'Inverse': + amnt_args = random.randint(func.min_args//2, min(len(vals), func.max_args)//2) + args = random.sample(range(1, amnt_args+1), amnt_args), random.sample(range(1, amnt_args+1), amnt_args) + case 'IfThenElse': + amnt_args = random.randint(func.min_args, min(len(bools), func.max_args)) + args = random.sample(bools, amnt_args) + case 'Xor': + amnt_args = random.randint(func.min_args, min(len(bools), func.max_args)) + args = random.sample(bools, amnt_args), + case 'Table' | 'NegativeTable': + vars = [e for e in comb if hasattr(e, 'value')] + amnt_fst_arg = random.randint(1, min(len(vars), func.max_args//4)) + amnt_snd_args = random.randint(1, min(len(comb), func.max_args-amnt_fst_arg) // amnt_fst_arg) * amnt_fst_arg + fst_args = random.sample(vars, amnt_fst_arg) + snd_args = random.sample(comb, amnt_snd_args) + snd_args_transformed = [snd_args[i * amnt_fst_arg:(i + 1) * amnt_fst_arg] for i in range(int(amnt_snd_args/amnt_fst_arg))] + args = fst_args, snd_args_transformed + case 'InDomain': + amnt_args = random.randint(func.min_args, min(len(comb), func.max_args)) + all_args = random.sample(comb, amnt_args) + args = all_args[0], all_args[1:] + case 'NoOverlap': + amnt_args = random.randint(func.min_args, min(len(comb), func.max_args//3)) + args = random.sample(comb, amnt_args), random.sample(comb, amnt_args), random.sample(comb, amnt_args) + case 'GlobalCardinalityCount': + amnt_fst_args = random.randint(1, min(len(ints), func.max_args - 2)) + amnt_snd_args = random.randint(1, min(len(ints), (func.max_args - amnt_fst_args)//2)) + counts = [random.randint(0, amnt_fst_args) for _ in range(amnt_snd_args)] + args = random.sample(ints, amnt_fst_args), counts, random.sample(ints, amnt_snd_args) + case _: + args = [] + return func.func(*args) + +def get_operator(args, has_bool_return): + """ + Returns a new expression that needs fewer arguments of each type than available in `args`. + It has a boolean return type if `has_bool_return` is True, otherwise it has an int return type. + """ + ints = [e for e in args if not is_boolexpr(e)] + bools = [e for e in args if is_boolexpr(e)] + values = [e for e in args if not hasattr(e, 'value')] # Is this the only way to extract constants only? (e.g. for wsum) + ints_cnt = len(ints) + bools_cnt = len(bools) + vals_cnt = len(values) + max_args = 12 + + # Operators: + ops = { + # name: (type, int_args, bool_args, bool_return, min_args, max_args, multiple) .._args -1 = n-ary, min 2 + name: Function(name, name, *attrs) + for name, attrs in { + 'and': ('op', 0, -1, True, 2, max_args), + 'or': ('op', 0, -1, True, 2, max_args), + '->': ('op', 0, 2, True, 2, 2), + 'not': ('op', 0, 1, True, 1, 1), + 'sum': ('op', -1, 0, False, 2, max_args), + 'wsum': ('op', -1, 0, False, 2, max_args, 2), + 'sub': ('op', 2, 0, False, 2, 2), + 'mul': ('op', 2, 0, False, 2, 2), + 'div': ('op', 2, 0, False, 2, 2), + 'mod': ('op', 2, 0, False, 2, 2), + 'pow': ('op', 2, 0, False, 2, 2), + '-': ('op', 1, 0, False, 1, 1), + }.items() + } + # Comparisons + comps = { + # name: (type, int_args, bool_args, bool_return) .._args -1 = n-ary, min 2 + name: Function(name, name, *attrs) + for name, attrs in { + '==': ('comp', 2, 0, True, 2, 2), + '!=': ('comp', 2, 0, True, 2, 2), + '<=': ('comp', 2, 0, True, 2, 2), + '<': ('comp', 2, 0, True, 2, 2), + '>=': ('comp', 2, 0, True, 2, 2), + '>': ('comp', 2, 0, True, 2, 2) + }.items() + } + # Global functions + global_fns = { + # name: (type, int_args, bool_args, bool_return, min_args, max_args) .._args -1 = n-ary, min 2 + name: Function(name.__name__, name, *attrs) + for name, attrs in { + Abs: ('gfun', 1, 0, False, 1, 1), # expr | (min 1, max 1, /) + Minimum: ('gfun', -1, 0, None, 2, max_args), # [...] | Can return a boolean but this is not known beforehand (min 2, max /, /) + Maximum: ('gfun', -1, 0, None, 2, max_args), # [...] | Can return a boolean but this is not known beforehand (min 2, max /, /) + NValue: ('gfun', -1, 0, False, 2, max_args), # [...] | (min 2, max /, /) + Element: ('gfun', -1, 0, None, 2, max_args), # [...], idx | Can return a boolean but this is not known beforehand (min 2, max /, /) + # Denk best enkel de array vullen en de idx gewoon tussen 0 en len-1 pakken + Count: ('gfun', -1, 0, False, 2, max_args), # [...], expr | (min 2, max /, /) + Among: ('gfun', -1, 0, False, 2, max_args), # [...], [...] | Second array can only have values, no expressions (not even BoolVal()) (min 2, max /, /) + NValueExcept: ('gfun', -1, 0, False, 2, max_args) # [...], val | Second argument can only have values, no expressions (not even BoolVal()) (min 2, max /, /) + }.items() + } + # Global constraints + global_cons = { + # name: (type, int_args, bool_args, bool_return, min_args, max_args, multiple) .._args -1 = n-ary, min 2 + name: Function(name.__name__, name, *attrs) + for name, attrs in { + AllDifferent: ('gcon', -1, 0, True, 2, max_args), # [...] | (min 2, max /, /) + AllDifferentExceptN: ('gcon', -1, 0, True, 2, max_args), # [...], [...] | Second arg can also be a single non-list value (min 2, max /, /) + AllEqual: ('gcon', -1, 0, True, 2, max_args), # [...] | (min 2, max /, /) + AllEqualExceptN: ('gcon', -1, 0, True, 2, max_args), # [...], [...] | Second arg can also be a single non-list value (min 2, max /, /) + Circuit: ('gcon', -1, 0, True, 2, max_args), # [...] | Can only have ints, NO BOOLS! (min 2, max /, /) + Inverse: ('gcon', -1, 0, True, 2, max_args, 2), # [...], [...] | Can only have ints, NO BOOLS! (min 2, max /, 2n) + Table: ('gcon', -1, 0, True, 2, max_args), # [...], [[...],[...],...] | First argument only variables, Second argument should have a multiple amnt of args as the first one (min 2, max /, n + mn?) + NegativeTable: ('gcon', -1, 0, True, 2, max_args), # [...], [[...],[...],...] | First argument only variables, Second argument should have a multiple amnt of args as the first one (min 2, max /, n + mn?) + IfThenElse: ('gcon', 0, 3, True, 3, 3), # arg1, arg2, arg3 (min 3, max 3, /) + InDomain: ('gcon', -1, 0, True, 2, max_args), # val, [...] | (min 2, max /, /) + Xor: ('gcon', 0, -1, True, 1, max_args), # [...] | (min 1, max /, /) + # Cumulative: (-1, 0, True), # st, dur, end, demand, cap (Ingewikkelde constraint) + # Precedence: (?, ?, True), # (Ingewikkelde constraint) + NoOverlap: ('gcon', -1, 0, True, 3, max_args, 3), # [...], [...], [...] | Three lists all have same length (min 3, max /, 3n) + GlobalCardinalityCount: ('gcon', -1, 0, True, 2, max_args), # [...], [...], [...] | The first and last list have to be the same length and + # they all have to be ints, NO BOOLS (min 2, max /, /) + Increasing: ('gcon', -1, 0, True, 2, max_args), # [...] (min 2, max /, /) + Decreasing: ('gcon', -1, 0, True, 2, max_args), # [...] (min 2, max /, /) + IncreasingStrict: ('gcon', -1, 0, True, 2, max_args), # [...] (min 2, max /, /) + DecreasingStrict: ('gcon', -1, 0, True, 2, max_args), # [...] (min 2, max /, /) + LexLess: ('gcon', -1, 0, True, 2, max_args, 2), # [...], [...] | Lists have same length (min 2, max /, 2n) + LexLessEq: ('gcon', -1, 0, True, 2, max_args, 2), # [...], [...] | Lists have same length (min 2, max /, 2n) + LexChainLess: ('gcon', -1, 0, True, 2, max_args), # [...][...] | Rows have same length (min 2, max /, mn) + LexChainLessEq: ('gcon', -1, 0, True, 2, max_args), # [...][...] | Rows have same length (min 2, max /, mn) + }.items() + } + all_ops = ops | comps | global_fns | global_cons + after = {k: v for k, v in all_ops.items() if satisfies_args(v, ints_cnt, bools_cnt, vals_cnt, has_bool_return)} + func = random.choice(list(after.values())) + return get_new_operator(func, ints, bools, values) + +def find_all_occurrences(con, target_node): + """ + Recursively finds all occurrences of `target_node` in the expression `con`. + Returns a list of paths (as tuples) to each occurrence. + """ + occurrences = [] + if con is target_node: + occurrences.append(()) # Current node is the target + if hasattr(con, 'args') and con.name != 'boolval': + for i, arg in enumerate(con.args): + for path in find_all_occurrences(arg, target_node): + occurrences.append((i,) + path) # Add index to the path + elif type(con) == list: + for i, arg in enumerate(con): + for path in find_all_occurrences(arg, target_node): + occurrences.append((i,) + path) + return occurrences + + +def replace_at_path(con, path, new_expr): + if not path: # END OF PATH + return new_expr + if hasattr(con, 'args') and con.name != 'boolval': + args = con.args + args[path[0]] = replace_at_path(args[path[0]], path[1:], new_expr) + if type(con) == Operator: + return Operator(con.name, args) + if type(con) == Comparison: + return Comparison(con.name, args[0], args[1]) + else: # global constraints (of 'max()' toch) + return type(con)(args) + elif type(con) == list: + return [new_expr if i == path[0] else e for i, e in enumerate(con)] + else: + return con + + +def mutate_con(con, old_expr, new_expr): + paths = find_all_occurrences(con, old_expr) + rand_path = random.choice(paths) + return replace_at_path(con, rand_path, new_expr) + + +def type_aware_expression_replacement(constraints): + """ + Replaces a random expression of a random constraint from a list of given constraints. It replaces this expression + by an operator with the same return type that takes as arguments the other expressions from all constraints. + IMPORTANT: This can change satisfiability of the constraint! Only to be used with verifiers that allow this! + This means it returns a list of ALL constraints and has to be handled accordingly in the 'generate_mutations' + function to swap out the constraints instead of adding them. + + ~ Parameters: + - constraints: a list of all the constraints to possibly be mutated + ~ Return: + - final_cons: a list of the same constraints where one constraint has a mutated expression + """ + try: + final_cons = copy.deepcopy(constraints) + # 1. Neem een (random) expression van een (random) constraint en de return type + has_bool_return = False + rand_con = random.choice(final_cons) + all_con_exprs = get_all_exprs(rand_con) + expr = random.choice(all_con_exprs) + has_bool_return = is_boolexpr(expr) + # 2. Tel het aantal resterende params van elk type (van alle constraints of enkel in de constraint zelf?) + all_exprs = get_all_exprs_mult(final_cons) + # 3. Zoek een operator die <= aantal params nodig heeft met zelfde return type + new_expr = get_operator(all_exprs, has_bool_return) + # 4. Vervang expression (+ vervang constraint) + new_con = mutate_con(rand_con, old_expr=expr, new_expr=new_expr) + # 5. Return the new constraints + # final_cons.remove(rand_con) DOES NOT WORK because it uses == instead of 'is' + index = None + for i, constraint in enumerate(final_cons): + if constraint is rand_con: + index = i + break + if index is not None: + del final_cons[index] + final_cons.append(new_con) + return final_cons + except Exception as e: + return Exception(e) + class MetamorphicError(Exception): pass @@ -851,57 +1399,60 @@ class MetamorphicError(Exception): returns a list of aritmetic expressions (as lists of indexes to traverse the expression tree) that occur in the input expression. One (random) candidate is taken from each level of the expression if there exists one ''' -def pickaritmetic(con,log=[], candidates=[]): - if hasattr(con,'name'): +def pickaritmetic(con, log=[], candidates=[]): + if hasattr(con, 'name'): if con.name == 'wsum': - #wsum has lists as arguments so we need a separate case - #wsum is the lowest possible level + # wsum has lists as arguments so we need a separate case + # wsum is the lowest possible level return candidates + [log] - if con.name == "element":# or con.name == "table" or con.name == "cumulative": - #no good way to know if element will return bool or not so ignore it + if con.name == "element": # or con.name == "table" or con.name == "cumulative": + # no good way to know if element will return bool or not so ignore it return candidates if hasattr(con, "args"): iargs = [(j, e) for j, e in enumerate(con.args)] random.shuffle(iargs) for j, arg in iargs: if is_boolexpr(arg): - res = pickaritmetic(arg,log+[j]) + res = pickaritmetic(arg, log + [j]) if res != []: return res elif is_any_list(arg): - return pickaritmetic((arg,log+[j],candidates)) + return pickaritmetic((arg, log + [j], candidates)) else: - return pickaritmetic(arg,log+[j],candidates+[log+[j]]) + return pickaritmetic(arg, log + [j], candidates + [log + [j]]) return candidates + ''' Adapted pickaritmetic that only picks from arithmetic comparisons used for mutators that i.e. multiple both sides with a number returns a list of aritmetic expressions (as lists of indexes to traverse the expression tree) that occur in the input expression. One (random) candidate is taken from each level of the expression if there exists one ''' -def pickaritmeticComparison(con,log=[], candidates=[]): - if hasattr(con,'name'): + + +def pickaritmeticComparison(con, log=[], candidates=[]): + if hasattr(con, 'name'): if con.name == 'wsum': - #wsum has lists as arguments so we need a separate case - #wsum is the lowest possible level + # wsum has lists as arguments so we need a separate case + # wsum is the lowest possible level return candidates if con.name == "element" or con.name == "table" or con.name == 'cumulative': - #no good way to know if element will return bool or not so ignore it (lists and element always return false to isbool) + # no good way to know if element will return bool or not so ignore it (lists and element always return false to isbool) return candidates if hasattr(con, "args"): iargs = [(j, e) for j, e in enumerate(con.args)] random.shuffle(iargs) for j, arg in iargs: if is_boolexpr(arg): - res = pickaritmeticComparison(arg,log+[j], candidates) + res = pickaritmeticComparison(arg, log + [j], candidates) if res != []: return res else: - if isinstance(con,Comparison): - return pickaritmeticComparison(arg,log+[j],candidates+[log]) + if isinstance(con, Comparison): + return pickaritmeticComparison(arg, log + [j], candidates + [log]) else: - return pickaritmeticComparison(arg,log+[j],candidates) + return pickaritmeticComparison(arg, log + [j], candidates) - return candidates \ No newline at end of file + return candidates diff --git a/verifiers/__init__.py b/verifiers/__init__.py index 390208df..13220ba6 100644 --- a/verifiers/__init__.py +++ b/verifiers/__init__.py @@ -13,6 +13,8 @@ from .model_counting_verifier import Model_Count_Verifier from .equivalance_verifier import Equivalance_Verifier from .optimization_verifier import Optimization_Verifier +from .solver_voting_sat_verifier import Solver_Vote_Sat_Verifier +from .solver_voting_count_verifier import Solver_Vote_Count_Verifier from .verifier_runner import run_verifiers, get_all_verifiers @@ -31,7 +33,13 @@ def lookup_verifier(verfier_name: str) -> Verifier: elif verfier_name == "equivalance verifier": return Equivalance_Verifier - - else: + + elif verfier_name == "solver_vote_sat_verifier": + return Solver_Vote_Sat_Verifier + + elif verfier_name == "solver_vote_count_verifier": + return Solver_Vote_Count_Verifier + + else: raise ValueError(f"Error verifier with name {verfier_name} does not exist") return None \ No newline at end of file diff --git a/verifiers/solver_voting_count_verifier.py b/verifiers/solver_voting_count_verifier.py new file mode 100644 index 00000000..31079a9d --- /dev/null +++ b/verifiers/solver_voting_count_verifier.py @@ -0,0 +1,114 @@ +from verifiers import * + +class Solver_Vote_Count_Verifier(Verifier): + """ + The Solver Count Verifier will verify if the satisfiability is the same for all solvers after running multiple mutations + """ + + def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, time_limit: float, seed: int): + self.name = "solver_vote_count_verifier" + self.type = 'sat' + + self.solvers = solver.split(",") # Change the solvers (str) into list of solvers ([str]) + # ALTERNATE: take one (or two) random solver(s) of all possible ones (=> no difference in input) + + self.mutations_per_model = mutations_per_model + self.exclude_dict = exclude_dict + self.time_limit = time_limit + self.seed = seed + self.mm_mutators = [#xor_morph, and_morph, or_morph, implies_morph, not_morph, + # linearize_constraint_morph, + # flatten_morph, + # only_numexpr_equality_morph, + # normalized_numexpr_morph, + # reify_rewrite_morph, + # only_bv_reifies_morph, + # only_positive_bv_morph, + # flat2cnf_morph, + # toplevel_list_morph, + # decompose_in_tree_morph, + # push_down_negation_morph, + # simplify_boolean_morph, + # canonical_comparison_morph, + # aritmetic_comparison_morph, + # semanticFusionCounting, + # semanticFusionCountingMinus, + # semanticFusionCountingwsum, + # semanticFusionCounting, + # semanticFusionCountingMinus, + # semanticFusionCountingwsum, + # type_aware_operator_replacement, + type_aware_expression_replacement] + self.mutators = [] + self.original_model = None + + def initialize_run(self) -> None: + if self.original_model == None: + with open(self.model_file, 'rb') as fpcl: + self.original_model = pickle.loads(fpcl.read()) + self.cons = self.original_model.constraints + assert (len(self.cons) > 0), f"{self.model_file} has no constraints" + self.cons = toplevel_list(self.cons) + + self.sol_count_1 = cp.Model(self.cons).solveAll(solver=self.solvers[0],time_limit=max(1,min(250,self.time_limit-time.time()))) + self.sol_count_2 = cp.Model(self.cons).solveAll(solver=self.solvers[1],time_limit=max(1,min(250,self.time_limit-time.time()))) + assert self.sol_count_1 == self.sol_count_2, f"{self.solvers} don't agree on amount of solutions: {self.sol_count_1} and {self.sol_count_2}" + + self.mutators = [copy.deepcopy( + self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. + + def verify_model(self) -> dict: + try: + model = cp.Model(self.cons) + time_limit = max(1, min(200, + self.time_limit - time.time())) # set the max time limit to the given time limit or to 1 if the self.time_limit-time.time() would be smaller then 1 + + solver_1 = self.solvers[0] + solver_2 = self.solvers[1] + new_count_1 = model.solveAll(solver=solver_1, time_limit=time_limit) + new_count_2 = model.solveAll(solver=solver_2, time_limit=time_limit) + + if model.status().runtime > time_limit - 10: + # timeout, skip + print('T', end='', flush=True) + return None + elif new_count_1 == new_count_2: + # has to be same + print('.', end='', flush=True) + return None + else: + print('X', end='', flush=True) + return dict(type=Fuzz_Test_ErrorTypes.failed_model, + originalmodel_file=self.model_file, + exception=f"Amount of solutions of the two solvers are not equal." + f" #Solutions of {solver_1}: {new_count_1}." + f" #Solutions of {solver_2}: {new_count_2}." + f" Before: {self.sol_count_1} and {self.sol_count_2}", + constraints=self.cons, + mutators=self.mutators, + model=model, + originalmodel=self.original_model + ) + + except Exception as e: + if isinstance(e, (CPMpyException, NotImplementedError)): + # expected error message, ignore + return None + print('E', end='', flush=True) + return dict(type=Fuzz_Test_ErrorTypes.internalcrash, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + mutators=self.mutators, + model=model, + originalmodel=self.original_model + ) + # if you got here, the model failed... + return dict(type=Fuzz_Test_ErrorTypes.failed_model, + originalmodel_file=self.model_file, + constraints=self.cons, + mutators=self.mutators, + model=newModel, + originalmodel=self.original_model + ) diff --git a/verifiers/solver_voting_sat_verifier.py b/verifiers/solver_voting_sat_verifier.py new file mode 100644 index 00000000..ff7f287c --- /dev/null +++ b/verifiers/solver_voting_sat_verifier.py @@ -0,0 +1,115 @@ +from verifiers import * + +class Solver_Vote_Sat_Verifier(Verifier): + """ + The Solver Count Verifier will verify if the satisfiability is the same for all solvers after running multiple mutations + """ + + def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, time_limit: float, seed: int): + self.name = "solver_vote_sat_verifier" + self.type = 'sat' + + self.solvers = solver.split(",") # Change the solvers (str) into list of solvers ([str]) + # ALTERNATE: take one (or two) random solver(s) of all possible ones (=> no difference in input) + + self.mutations_per_model = mutations_per_model + self.exclude_dict = exclude_dict + self.time_limit = time_limit + self.seed = seed + self.mm_mutators = [#xor_morph, and_morph, or_morph, implies_morph, not_morph, + # linearize_constraint_morph, + # flatten_morph, + # only_numexpr_equality_morph, + # normalized_numexpr_morph, + # reify_rewrite_morph, + # only_bv_reifies_morph, + # only_positive_bv_morph, + # flat2cnf_morph, + # toplevel_list_morph, + # decompose_in_tree_morph, + # push_down_negation_morph, + # simplify_boolean_morph, + # canonical_comparison_morph, + # aritmetic_comparison_morph, + # semanticFusionCounting, + # semanticFusionCountingMinus, + # semanticFusionCountingwsum, + # semanticFusionCounting, + # semanticFusionCountingMinus, + # semanticFusionCountingwsum, + # type_aware_operator_replacement, + type_aware_expression_replacement] + self.mutators = [] + self.original_model = None + + def initialize_run(self) -> None: + if self.original_model == None: + with open(self.model_file, 'rb') as fpcl: + self.original_model = pickle.loads(fpcl.read()) + self.cons = self.original_model.constraints + assert (len(self.cons) > 0), f"{self.model_file} has no constraints" + self.cons = toplevel_list(self.cons) + + # No other preparation necessary + + self.mutators = [copy.deepcopy( + self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. + + def verify_model(self) -> dict: + try: + model = cp.Model(self.cons) + time_limit = max(1, min(200, + self.time_limit - time.time())) # set the max time limit to the given time limit or to 1 if the self.time_limit-time.time() would be smaller then 1 + + # choosing the solvers + solver_1 = self.solvers[0] + solver_2 = self.solvers[1] + solver_1_is_sat = model.solve(solver=solver_1, time_limit=time_limit) + solver_2_is_sat = model.solve(solver=solver_2, time_limit=time_limit) + # for prettier exception printing + solver_1_print = "sat" if solver_1_is_sat else "unsat" + solver_2_print = "sat" if solver_2_is_sat else "unsat" + + if model.status().runtime > time_limit - 10: + # timeout, skip + print('T', end='', flush=True) + return None + elif solver_1_is_sat == solver_2_is_sat: + # has to be same + print('.', end='', flush=True) + return None + else: + print('X', end='', flush=True) + return dict(type=Fuzz_Test_ErrorTypes.failed_model, + originalmodel_file=self.model_file, + exception=f"Results of the two solvers are not equal." + f" Result of {solver_1}: {solver_1_print}." + f" Result of {solver_2}: {solver_2_print}.", + constraints=self.cons, + mutators=self.mutators, + model=model, + originalmodel=self.original_model + ) + + except Exception as e: + if isinstance(e, (CPMpyException, NotImplementedError)): + # expected error message, ignore + return None + print('E', end='', flush=True) + return dict(type=Fuzz_Test_ErrorTypes.internalcrash, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + mutators=self.mutators, + model=model, + originalmodel=self.original_model + ) + # if you got here, the model failed... + return dict(type=Fuzz_Test_ErrorTypes.failed_model, + originalmodel_file=self.model_file, + constraints=self.cons, + mutators=self.mutators, + model=newModel, + originalmodel=self.original_model + ) diff --git a/verifiers/verifier.py b/verifiers/verifier.py index e729dd1b..87c23d69 100644 --- a/verifiers/verifier.py +++ b/verifiers/verifier.py @@ -16,26 +16,27 @@ def __init__(self, name: str, type: str, solver: str, mutations_per_model: int, self.time_limit = time_limit self.seed = seed self.mm_mutators = [xor_morph, and_morph, or_morph, implies_morph, not_morph, - linearize_constraint_morph, - flatten_morph, - only_numexpr_equality_morph, - normalized_numexpr_morph, - reify_rewrite_morph, - only_bv_reifies_morph, - only_positive_bv_morph, - flat2cnf_morph, - toplevel_list_morph, - decompose_in_tree_morph, - push_down_negation_morph, - simplify_boolean_morph, - canonical_comparison_morph, - aritmetic_comparison_morph, - semanticFusionCounting, - semanticFusionCountingMinus, - semanticFusionCountingwsum, - semanticFusionCounting, - semanticFusionCountingMinus, - semanticFusionCountingwsum] + linearize_constraint_morph, + flatten_morph, + only_numexpr_equality_morph, + normalized_numexpr_morph, + reify_rewrite_morph, + only_bv_reifies_morph, + only_positive_bv_morph, + flat2cnf_morph, + toplevel_list_morph, + decompose_in_tree_morph, + push_down_negation_morph, + simplify_boolean_morph, + canonical_comparison_morph, + aritmetic_comparison_morph, + semanticFusionCounting, + semanticFusionCountingMinus, + semanticFusionCountingwsum, + semanticFusionCounting, + semanticFusionCountingMinus, + semanticFusionCountingwsum, + type_aware_operator_replacement] self.mutators = [] self.original_model = None @@ -56,7 +57,10 @@ def generate_mutations(self) -> None: # log function and arguments in that case self.mutators += [m] try: - self.cons += m(self.cons) # apply a metamorphic mutation + if m in {type_aware_operator_replacement, type_aware_expression_replacement}: + self.cons = m(self.cons) # apply an operator change and REPLACE constraints + else: + self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints self.mutators += [copy.deepcopy(self.cons)] except MetamorphicError as exc: #add to exclude_dict, to avoid running into the same error From 7b0493bf54270e15e864e905b1e9a137b8d83c25 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 8 Apr 2025 12:13:22 +0200 Subject: [PATCH 02/58] Added solution limit for gurobi and some minor changes for new solver implementation --- verifiers/model_counting_verifier.py | 11 +++- verifiers/solver_voting_count_verifier.py | 69 +++++++++++++---------- verifiers/solver_voting_sat_verifier.py | 47 ++++++++------- verifiers/verifier_runner.py | 7 ++- 4 files changed, 77 insertions(+), 57 deletions(-) diff --git a/verifiers/model_counting_verifier.py b/verifiers/model_counting_verifier.py index 169b1167..ad0aa58b 100644 --- a/verifiers/model_counting_verifier.py +++ b/verifiers/model_counting_verifier.py @@ -34,7 +34,11 @@ def initialize_run(self) -> None: assert (len(self.cons)>0), f"{self.model_file} has no constraints" self.cons = toplevel_list(self.cons) - self.sol_count = cp.Model(self.cons).solveAll(solver=self.solver,time_limit=max(1,min(250,self.time_limit-time.time()))) + if self.solver == 'gurobi': + self.sol_lim = 10000 # TODO: is hardcode best idea? + self.sol_count = cp.Model(self.cons).solveAll(solver=self.solver,time_limit=max(1,min(250,self.time_limit-time.time())), solution_limit=self.sol_lim) + else: + self.sol_count = cp.Model(self.cons).solveAll(solver=self.solver,time_limit=max(1,min(250,self.time_limit-time.time()))) self.mutators = [copy.deepcopy(self.cons)] #keep track of list of cons alternated with mutators that transformed it into the next list of cons. def verify_model(self) -> dict: @@ -42,7 +46,10 @@ def verify_model(self) -> dict: model = cp.Model(self.cons) time_limit=max(1,min(200,self.time_limit-time.time())) # set the max time limit to the given time limit or to 1 if the self.time_limit-time.time() would be smaller then 1 - new_count = model.solveAll(solver=self.solver, time_limit=time_limit) + if hasattr(self, 'sol_lim'): + new_count = model.solveAll(solver=self.solver, time_limit=time_limit, solution_limit=self.sol_lim) + else: + new_count = model.solveAll(solver=self.solver, time_limit=time_limit) if model.status().runtime > time_limit-10: # timeout, skip print('T', end='', flush=True) diff --git a/verifiers/solver_voting_count_verifier.py b/verifiers/solver_voting_count_verifier.py index 31079a9d..2d170ed6 100644 --- a/verifiers/solver_voting_count_verifier.py +++ b/verifiers/solver_voting_count_verifier.py @@ -9,35 +9,34 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti self.name = "solver_vote_count_verifier" self.type = 'sat' - self.solvers = solver.split(",") # Change the solvers (str) into list of solvers ([str]) - # ALTERNATE: take one (or two) random solver(s) of all possible ones (=> no difference in input) + self.solvers = solver self.mutations_per_model = mutations_per_model self.exclude_dict = exclude_dict self.time_limit = time_limit self.seed = seed - self.mm_mutators = [#xor_morph, and_morph, or_morph, implies_morph, not_morph, - # linearize_constraint_morph, - # flatten_morph, - # only_numexpr_equality_morph, - # normalized_numexpr_morph, - # reify_rewrite_morph, - # only_bv_reifies_morph, - # only_positive_bv_morph, - # flat2cnf_morph, - # toplevel_list_morph, - # decompose_in_tree_morph, - # push_down_negation_morph, - # simplify_boolean_morph, - # canonical_comparison_morph, - # aritmetic_comparison_morph, - # semanticFusionCounting, - # semanticFusionCountingMinus, - # semanticFusionCountingwsum, - # semanticFusionCounting, - # semanticFusionCountingMinus, - # semanticFusionCountingwsum, - # type_aware_operator_replacement, + self.mm_mutators = [xor_morph, and_morph, or_morph, implies_morph, not_morph, + linearize_constraint_morph, + flatten_morph, + only_numexpr_equality_morph, + normalized_numexpr_morph, + reify_rewrite_morph, + only_bv_reifies_morph, + only_positive_bv_morph, + flat2cnf_morph, + toplevel_list_morph, + decompose_in_tree_morph, + push_down_negation_morph, + simplify_boolean_morph, + canonical_comparison_morph, + aritmetic_comparison_morph, + semanticFusionCounting, + semanticFusionCountingMinus, + semanticFusionCountingwsum, + semanticFusionCounting, + semanticFusionCountingMinus, + semanticFusionCountingwsum, + type_aware_operator_replacement, type_aware_expression_replacement] self.mutators = [] self.original_model = None @@ -50,9 +49,16 @@ def initialize_run(self) -> None: assert (len(self.cons) > 0), f"{self.model_file} has no constraints" self.cons = toplevel_list(self.cons) - self.sol_count_1 = cp.Model(self.cons).solveAll(solver=self.solvers[0],time_limit=max(1,min(250,self.time_limit-time.time()))) - self.sol_count_2 = cp.Model(self.cons).solveAll(solver=self.solvers[1],time_limit=max(1,min(250,self.time_limit-time.time()))) - assert self.sol_count_1 == self.sol_count_2, f"{self.solvers} don't agree on amount of solutions: {self.sol_count_1} and {self.sol_count_2}" + assert len(self.solvers) == 2, f"2 solvers required, {len(self.solvers)} given." + if 'gurobi' in [s.lower() for s in self.solvers]: + self.sol_lim = 10000 # TODO: is hardcode best idea? + self.sol_count_1 = cp.Model(self.cons).solveAll(solver=self.solvers[0],time_limit=max(1,min(250,self.time_limit-time.time())),solution_limit=self.sol_lim) + self.sol_count_2 = cp.Model(self.cons).solveAll(solver=self.solvers[1],time_limit=max(1,min(250,self.time_limit-time.time())),solution_limit=self.sol_lim) + else: + self.sol_count_1 = cp.Model(self.cons).solveAll(solver=self.solvers[0],time_limit=max(1, min(250, self.time_limit - time.time()))) + self.sol_count_2 = cp.Model(self.cons).solveAll(solver=self.solvers[1],time_limit=max(1, min(250, self.time_limit - time.time()))) + + assert self.sol_count_1 == self.sol_count_2, f"{self.solvers} don't agree on amount of solutions (before mutations): {self.sol_count_1} and {self.sol_count_2}" self.mutators = [copy.deepcopy( self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. @@ -65,8 +71,13 @@ def verify_model(self) -> dict: solver_1 = self.solvers[0] solver_2 = self.solvers[1] - new_count_1 = model.solveAll(solver=solver_1, time_limit=time_limit) - new_count_2 = model.solveAll(solver=solver_2, time_limit=time_limit) + if hasattr(self, 'sol_lim'): + print("this works") + new_count_1 = model.solveAll(solver=solver_1, time_limit=time_limit, solution_limit=self.sol_lim) + new_count_2 = model.solveAll(solver=solver_2, time_limit=time_limit, solution_limit=self.sol_lim) + else: + new_count_1 = model.solveAll(solver=solver_1, time_limit=time_limit) + new_count_2 = model.solveAll(solver=solver_2, time_limit=time_limit) if model.status().runtime > time_limit - 10: # timeout, skip diff --git a/verifiers/solver_voting_sat_verifier.py b/verifiers/solver_voting_sat_verifier.py index ff7f287c..acbd8e88 100644 --- a/verifiers/solver_voting_sat_verifier.py +++ b/verifiers/solver_voting_sat_verifier.py @@ -9,35 +9,34 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti self.name = "solver_vote_sat_verifier" self.type = 'sat' - self.solvers = solver.split(",") # Change the solvers (str) into list of solvers ([str]) - # ALTERNATE: take one (or two) random solver(s) of all possible ones (=> no difference in input) + self.solvers = solver self.mutations_per_model = mutations_per_model self.exclude_dict = exclude_dict self.time_limit = time_limit self.seed = seed - self.mm_mutators = [#xor_morph, and_morph, or_morph, implies_morph, not_morph, - # linearize_constraint_morph, - # flatten_morph, - # only_numexpr_equality_morph, - # normalized_numexpr_morph, - # reify_rewrite_morph, - # only_bv_reifies_morph, - # only_positive_bv_morph, - # flat2cnf_morph, - # toplevel_list_morph, - # decompose_in_tree_morph, - # push_down_negation_morph, - # simplify_boolean_morph, - # canonical_comparison_morph, - # aritmetic_comparison_morph, - # semanticFusionCounting, - # semanticFusionCountingMinus, - # semanticFusionCountingwsum, - # semanticFusionCounting, - # semanticFusionCountingMinus, - # semanticFusionCountingwsum, - # type_aware_operator_replacement, + self.mm_mutators = [xor_morph, and_morph, or_morph, implies_morph, not_morph, + linearize_constraint_morph, + flatten_morph, + only_numexpr_equality_morph, + normalized_numexpr_morph, + reify_rewrite_morph, + only_bv_reifies_morph, + only_positive_bv_morph, + flat2cnf_morph, + toplevel_list_morph, + decompose_in_tree_morph, + push_down_negation_morph, + simplify_boolean_morph, + canonical_comparison_morph, + aritmetic_comparison_morph, + semanticFusionCounting, + semanticFusionCountingMinus, + semanticFusionCountingwsum, + semanticFusionCounting, + semanticFusionCountingMinus, + semanticFusionCountingwsum, + type_aware_operator_replacement, type_aware_expression_replacement] self.mutators = [] self.original_model = None diff --git a/verifiers/verifier_runner.py b/verifiers/verifier_runner.py index 5e737691..248350b3 100644 --- a/verifiers/verifier_runner.py +++ b/verifiers/verifier_runner.py @@ -6,9 +6,11 @@ from verifiers import * from fuzz_test_utils import Fuzz_Test_ErrorTypes def get_all_verifiers() -> list: - return [Solution_Verifier,Optimization_Verifier,Model_Count_Verifier,Metamorphic_Verifier,Equivalance_Verifier] + # TODO: kiezen adhv aantal gegeven solvers + # return [Solution_Verifier,Optimization_Verifier,Model_Count_Verifier,Metamorphic_Verifier,Equivalance_Verifier] + return [Solver_Vote_Count_Verifier, Solver_Vote_Sat_Verifier] -def run_verifiers(current_amount_of_tests, current_amount_of_error, lock, solver: str, mutations_per_model: int, folders: list, max_error_treshold: int, output_dir: str, time_limit: float) -> None: +def run_verifiers(current_amount_of_tests, current_amount_of_error, lock, solver: list[str], mutations_per_model: int, folders: list, max_error_treshold: int, output_dir: str, time_limit: float) -> None: """ This function will be used to run different verifiers @@ -28,6 +30,7 @@ def run_verifiers(current_amount_of_tests, current_amount_of_error, lock, solver exclude_dict = {} random_seed = random.random() random.seed(random_seed) + solver = solver[0] if len(solver) == 1 else solver # Take the solver as a string if there is only one verifier_kwargs = {"solver":solver, "mutations_per_model":mutations_per_model, "exclude_dict":exclude_dict,"time_limit": time_limit, "seed":random_seed} From e5c030cd5cc99d3bed91f2d6ad8ec8839fae471a Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 8 Apr 2025 12:14:26 +0200 Subject: [PATCH 03/58] Changed solver argument to take 1 or more arguments --- fuzz_test.py | 4 ++-- fuzz_test_rerunner.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/fuzz_test.py b/fuzz_test.py index 1d8a7dd6..0b65c76b 100644 --- a/fuzz_test.py +++ b/fuzz_test.py @@ -26,7 +26,7 @@ def check_positive(value): return ivalue parser = argparse.ArgumentParser(description = "A python application to fuzz_test your solver(s)") - parser.add_argument("-s", "--solver", help = "The Solver to use", required = False,type=str,choices=available_solvers, default=available_solvers[0]) + parser.add_argument("-s", "--solver", help = "The Solver to use", required = False,type=str,choices=available_solvers, nargs='+', default=[available_solvers[0]]) parser.add_argument("-m", "--models", help = "The path to load the models", required=False, type=str, default="models") parser.add_argument("-o", "--output-dir", help = "The directory to store the output (will be created if it does not exist).", required=False, type=str, default="output") parser.add_argument("-g", "--skip-global-constraints", help = "Skip the global constraints when testing", required=False, default = False) @@ -49,7 +49,7 @@ def check_positive(value): os.makedirs(args.output_dir, exist_ok=True) # showing the info about the given params to the user - print("\nUsing solver '"+args.solver+"' with models in '"+args.models+"' and writing to '"+args.output_dir+"'." ,flush=True,end="\n\n") + print("\nUsing solver(s): '" + ", ".join(args.solver)+"' with models in '"+args.models+"' and writing to '"+args.output_dir+"'." ,flush=True,end="\n\n") print("Will use "+str(args.amount_of_processes)+ " parallel executions, starting...",flush=True,end="\n\n") # creating the vars for the multiprocessing diff --git a/fuzz_test_rerunner.py b/fuzz_test_rerunner.py index 0cc002a1..17b4e4bd 100644 --- a/fuzz_test_rerunner.py +++ b/fuzz_test_rerunner.py @@ -33,7 +33,11 @@ def rerun_file(failed_model_file,output_dir ): error_data = pickle.loads(fpcl.read()) random.seed(error_data["seed"]) if error_data["error"]["type"].name != "fuzz_test_crash": # if it is a fuzz_test crash error we skip it - verifier_kwargs = {'solver': error_data["solver"], "mutations_per_model": error_data["mutations_per_model"], "exclude_dict": {}, "time_limit": time.time()*3600, "seed": error_data["seed"]} + if type(error_data["solver"]) == str: # old solver voting implementation (e.g. "minizinc,z3") + solver = error_data["solver"].split(',') + else: + solver = error_data["solver"] + verifier_kwargs = {'solver': solver, "mutations_per_model": error_data["mutations_per_model"], "exclude_dict": {}, "time_limit": time.time()*3600, "seed": error_data["seed"]} error = lookup_verifier(error_data["verifier"])(**verifier_kwargs).rerun(error_data["error"]) error_data["error"] = error From 23ede89b335f98c1338de333c96534a006bdb33e Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 8 Apr 2025 12:15:06 +0200 Subject: [PATCH 04/58] addition and bugfixing of new mutators --- fuzz_test_utils/mutators.py | 62 ++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 70520863..cbc83bd6 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -24,7 +24,7 @@ GlobalCardinalityCount, Increasing, Decreasing, IncreasingStrict, DecreasingStrict, LexLess, LexLessEq, \ LexChainLess, LexChainLessEq, GlobalConstraint # from cpmpy.expressions.globalfunctions import Abs, Mimimum(GlobalFunction), Maximum -from cpmpy.expressions.variables import boolvar, _IntVarImpl +from cpmpy.expressions.variables import boolvar, _IntVarImpl, NDVarArray class Function: @@ -937,15 +937,16 @@ def type_aware_operator_replacement(constraints): final_cons = copy.deepcopy(constraints) cons_set = set(constraints) # pick a random constraint and calculate their mutable expressions until there is at least 1 - con = random.choice(final_cons) - exprs = get_all_mutable_op_exprs(con) - while len(exprs) < 1: - con = random.choice(list(cons_set)) + candidates = list(cons_set) # or final_cons + random.shuffle(candidates) + for con in candidates: exprs = get_all_mutable_op_exprs(con) - cons_set.remove(con) + if exprs: + break + else: + return final_cons # remove the constraint from the constraints - # final_cons = [c for c in final_cons if (c.name != con.name or c.args != con.args)] final_cons.remove(con) # Choose an expression to change @@ -1038,7 +1039,7 @@ def get_all_op_exprs(con): # Helper function to get all expressions WITHOUT an operator in a given constraint def get_all_non_op_exprs(con): - if hasattr(con, 'args') and con.name != 'boolval': + if hasattr(con, 'args') and type(con) != NDVarArray and con.name != 'boolval': return sum((get_all_non_op_exprs(arg) for arg in con.args), []) elif type(con) == list: if all([is_num(e) for e in con]): # wsum constants @@ -1216,7 +1217,7 @@ def get_operator(args, has_bool_return): Returns a new expression that needs fewer arguments of each type than available in `args`. It has a boolean return type if `has_bool_return` is True, otherwise it has an int return type. """ - ints = [e for e in args if not is_boolexpr(e)] + ints = [e for e in args if not (is_boolexpr(e) or type(e) == list)] bools = [e for e in args if is_boolexpr(e)] values = [e for e in args if not hasattr(e, 'value')] # Is this the only way to extract constants only? (e.g. for wsum) ints_cnt = len(ints) @@ -1304,8 +1305,12 @@ def get_operator(args, has_bool_return): }.items() } all_ops = ops | comps | global_fns | global_cons + # print(f"ints_cnt={ints_cnt},bools_cnt={bools_cnt},has_bool_return={has_bool_return}") after = {k: v for k, v in all_ops.items() if satisfies_args(v, ints_cnt, bools_cnt, vals_cnt, has_bool_return)} - func = random.choice(list(after.values())) + if after: + func = random.choice(list(after.values())) + else: + return None return get_new_operator(func, ints, bools, values) def find_all_occurrences(con, target_node): @@ -1337,8 +1342,8 @@ def replace_at_path(con, path, new_expr): return Operator(con.name, args) if type(con) == Comparison: return Comparison(con.name, args[0], args[1]) - else: # global constraints (of 'max()' toch) - return type(con)(args) + else: # global constraints + return type(con)(*args) elif type(con) == list: return [new_expr if i == path[0] else e for i, e in enumerate(con)] else: @@ -1364,19 +1369,22 @@ def type_aware_expression_replacement(constraints): ~ Return: - final_cons: a list of the same constraints where one constraint has a mutated expression """ - try: - final_cons = copy.deepcopy(constraints) - # 1. Neem een (random) expression van een (random) constraint en de return type - has_bool_return = False - rand_con = random.choice(final_cons) - all_con_exprs = get_all_exprs(rand_con) - expr = random.choice(all_con_exprs) - has_bool_return = is_boolexpr(expr) - # 2. Tel het aantal resterende params van elk type (van alle constraints of enkel in de constraint zelf?) - all_exprs = get_all_exprs_mult(final_cons) - # 3. Zoek een operator die <= aantal params nodig heeft met zelfde return type - new_expr = get_operator(all_exprs, has_bool_return) - # 4. Vervang expression (+ vervang constraint) + final_cons = copy.deepcopy(constraints) + # print(f"All constraints at the moment: {final_cons}") + # 1. Neem een (random) expression van een (random) constraint en de return type + rand_con = random.choice(final_cons) + all_con_exprs = get_all_exprs(rand_con) + expr = random.choice(all_con_exprs) + has_bool_return = is_boolexpr(expr) + # print(f"Changing constarint: {rand_con}") + # print(f"Old expression: {expr}") + # 2. Tel het aantal resterende params van elk type (van alle constraints of enkel in de constraint zelf?) + all_exprs = get_all_exprs_mult(final_cons) + # 3. Zoek een operator die <= aantal params nodig heeft met zelfde return type + new_expr = get_operator(all_exprs, has_bool_return) + # 4. Vervang expression (+ vervang constraint) + # print(f"New expression: {new_expr}") + if new_expr: new_con = mutate_con(rand_con, old_expr=expr, new_expr=new_expr) # 5. Return the new constraints # final_cons.remove(rand_con) DOES NOT WORK because it uses == instead of 'is' @@ -1388,9 +1396,7 @@ def type_aware_expression_replacement(constraints): if index is not None: del final_cons[index] final_cons.append(new_con) - return final_cons - except Exception as e: - return Exception(e) + return final_cons class MetamorphicError(Exception): pass From e8078678da5029402458088c554762db3f0776ea Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 8 Apr 2025 16:09:22 +0200 Subject: [PATCH 05/58] fixed a bug when adding lexical ordering constraints that allowed non-variables --- fuzz_test_utils/mutators.py | 51 ++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index cbc83bd6..72b1fb5f 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -293,7 +293,7 @@ def only_positive_bv_morph(cons): def flat2cnf_morph(cons): - # flatcons = flatten_morph(cons,flatten_all=True) + # flatcons = flatten_morph(og_cons,flatten_all=True) onlycons = only_bv_reifies_morph(cons, morph_all=True) try: return flat2cnf(onlycons) @@ -330,7 +330,7 @@ def semanticFusion(const): firstcon = random.choice(res) elif secondcon == None: secondcon = random.choice(res) - break # stop when 2 constraints found. still random because cons are shuffled + break # stop when 2 constraints found. still random because og_cons are shuffled if secondcon != None: # two constraints with aritmetic expressions found, perform semantic fusion on them @@ -417,7 +417,7 @@ def semanticFusionMinus(const): firstcon = random.choice(res) elif secondcon == None: secondcon = random.choice(res) - break # stop when 2 constraints found. still random because cons are shuffled + break # stop when 2 constraints found. still random because og_cons are shuffled if secondcon != None: # two constraints with aritmetic expressions found, perform semantic fusion on them @@ -504,7 +504,7 @@ def semanticFusionwsum(const): firstcon = random.choice(res) elif secondcon == None: secondcon = random.choice(res) - break # stop when 2 constraints found. still random because cons are shuffled + break # stop when 2 constraints found. still random because og_cons are shuffled if secondcon != None: # two constraints with aritmetic expressions found, perform semantic fusion on them @@ -597,7 +597,7 @@ def semanticFusionCountingwsum(const): firstcon = random.choice(res) elif secondcon == None: secondcon = random.choice(res) - break # stop when 2 constraints found. still random because cons are shuffled + break # stop when 2 constraints found. still random because og_cons are shuffled if secondcon != None: # two constraints with aritmetic expressions found, perform semantic fusion on them @@ -691,7 +691,7 @@ def semanticFusionCounting(const): firstcon = random.choice(res) elif secondcon == None: secondcon = random.choice(res) - break # stop when 2 constraints found. still random because cons are shuffled + break # stop when 2 constraints found. still random because og_cons are shuffled if secondcon != None: # two constraints with aritmetic expressions found, perform semantic fusion on them @@ -779,7 +779,7 @@ def semanticFusionCountingMinus(const): firstcon = random.choice(res) elif secondcon == None: secondcon = random.choice(res) - break # stop when 2 constraints found. still random because cons are shuffled + break # stop when 2 constraints found. still random because og_cons are shuffled if secondcon != None: # two constraints with aritmetic expressions found, perform semantic fusion on them @@ -1062,7 +1062,7 @@ def get_all_exprs_mult(cons): return all_exprs -def satisfies_args(func: Function, ints: int, bools: int, values: int, has_bool_return: bool): +def satisfies_args(func: Function, ints: int, bools: int, values: int, vars: int, has_bool_return: bool): """ returns whether the given function `func` can work with the given amount of integers `ints` and booleans `bools` and the given return type `has_bool_return` @@ -1085,19 +1085,23 @@ def satisfies_args(func: Function, ints: int, bools: int, values: int, has_bool_ case 'Minimum' | 'Maximum' | 'Element': # We make these have only ints, so it always has the same return type return values >= func.min_args and not has_bool_return case 'Among' | 'NValueExcept': - return values >= 1 and ints >= func.min_args and not has_bool_return + return values >= 1 and ints >= func.min_args - 1 and not has_bool_return case 'gcon': match func.name: case 'Circuit' | 'Inverse' | 'GlobalCardinalityCount': return values >= func.min_args and has_bool_return case 'IfThenElse' | 'Xor': return bools >= func.min_args and has_bool_return + case 'Table' | 'NegativeTable': + return vars >= 1 and ints + bools >= func.min_args - 1 and has_bool_return + case 'LexLess' | 'LexLessEq' | 'LexChainLess' | 'LexChainLessEq': + return vars >= func.min_args and has_bool_return case _: return ints + bools >= func.min_args and has_bool_return -def get_new_operator(func: Function, ints, bools, vals): +def get_new_operator(func: Function, ints, bools, vals, variables): comb = ints + bools match func.type: case 'op' | 'comp': @@ -1167,11 +1171,11 @@ def get_new_operator(func: Function, ints, bools, vals): amnt_snd_args = random.randint(func.min_args//2, min(len(comb), func.max_args - amnt_fst_args)) args = random.sample(comb, amnt_fst_args), random.sample(comb, amnt_snd_args) case 'LexLess' | 'LexLessEq': - half_amnt_args = random.randint(func.min_args//2, min(len(comb), func.max_args)//2) - args = random.sample(comb, half_amnt_args), random.sample(comb, half_amnt_args) + half_amnt_args = random.randint(func.min_args//2, min(len(variables), func.max_args)//2) + args = random.sample(variables, half_amnt_args), random.sample(variables, half_amnt_args) case 'LexChainLess' | 'LexChainLessEq': - amnt_args = random.randint(func.min_args, min(len(comb), func.max_args)//2) - all_args = random.sample(comb, amnt_args) + amnt_args = random.randint(func.min_args, min(len(variables), func.max_args)//2) + all_args = random.sample(variables, amnt_args) divisors = [i for i in range(1, amnt_args) if amnt_args % i == 0] fst_dimension = random.choice(divisors) snd_dimension = int(amnt_args / fst_dimension) @@ -1189,10 +1193,9 @@ def get_new_operator(func: Function, ints, bools, vals): amnt_args = random.randint(func.min_args, min(len(bools), func.max_args)) args = random.sample(bools, amnt_args), case 'Table' | 'NegativeTable': - vars = [e for e in comb if hasattr(e, 'value')] - amnt_fst_arg = random.randint(1, min(len(vars), func.max_args//4)) + amnt_fst_arg = random.randint(1, min(len(variables), func.max_args//4)) amnt_snd_args = random.randint(1, min(len(comb), func.max_args-amnt_fst_arg) // amnt_fst_arg) * amnt_fst_arg - fst_args = random.sample(vars, amnt_fst_arg) + fst_args = random.sample(variables, amnt_fst_arg) snd_args = random.sample(comb, amnt_snd_args) snd_args_transformed = [snd_args[i * amnt_fst_arg:(i + 1) * amnt_fst_arg] for i in range(int(amnt_snd_args/amnt_fst_arg))] args = fst_args, snd_args_transformed @@ -1220,9 +1223,11 @@ def get_operator(args, has_bool_return): ints = [e for e in args if not (is_boolexpr(e) or type(e) == list)] bools = [e for e in args if is_boolexpr(e)] values = [e for e in args if not hasattr(e, 'value')] # Is this the only way to extract constants only? (e.g. for wsum) + variables = get_variables(args) ints_cnt = len(ints) bools_cnt = len(bools) vals_cnt = len(values) + vars_cnt = len(variables) max_args = 12 # Operators: @@ -1298,20 +1303,20 @@ def get_operator(args, has_bool_return): Decreasing: ('gcon', -1, 0, True, 2, max_args), # [...] (min 2, max /, /) IncreasingStrict: ('gcon', -1, 0, True, 2, max_args), # [...] (min 2, max /, /) DecreasingStrict: ('gcon', -1, 0, True, 2, max_args), # [...] (min 2, max /, /) - LexLess: ('gcon', -1, 0, True, 2, max_args, 2), # [...], [...] | Lists have same length (min 2, max /, 2n) - LexLessEq: ('gcon', -1, 0, True, 2, max_args, 2), # [...], [...] | Lists have same length (min 2, max /, 2n) - LexChainLess: ('gcon', -1, 0, True, 2, max_args), # [...][...] | Rows have same length (min 2, max /, mn) - LexChainLessEq: ('gcon', -1, 0, True, 2, max_args), # [...][...] | Rows have same length (min 2, max /, mn) + LexLess: ('gcon', -1, 0, True, 2, max_args, 2), # [...], [...] | Lists have same length (min 2, max /, 2n), ONLY VARS + LexLessEq: ('gcon', -1, 0, True, 2, max_args, 2), # [...], [...] | Lists have same length (min 2, max /, 2n), ONLY VARS + LexChainLess: ('gcon', -1, 0, True, 2, max_args), # [...][...] | Rows have same length (min 2, max /, mn), ONLY VARS + LexChainLessEq: ('gcon', -1, 0, True, 2, max_args), # [...][...] | Rows have same length (min 2, max /, mn), ONLY VARS }.items() } all_ops = ops | comps | global_fns | global_cons # print(f"ints_cnt={ints_cnt},bools_cnt={bools_cnt},has_bool_return={has_bool_return}") - after = {k: v for k, v in all_ops.items() if satisfies_args(v, ints_cnt, bools_cnt, vals_cnt, has_bool_return)} + after = {k: v for k, v in all_ops.items() if satisfies_args(v, ints_cnt, bools_cnt, vals_cnt, vars_cnt, has_bool_return)} if after: func = random.choice(list(after.values())) else: return None - return get_new_operator(func, ints, bools, values) + return get_new_operator(func, ints, bools, values, variables) def find_all_occurrences(con, target_node): """ From 77c5767e01c4ba1b158b02d1e4665b2b33b44a7e Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 8 Apr 2025 16:10:07 +0200 Subject: [PATCH 06/58] added printing of domains when rerunning on certain verifiers (still has to be added for the other ones) --- verifiers/solver_voting_count_verifier.py | 31 +++++++++++++---------- verifiers/solver_voting_sat_verifier.py | 10 +++++--- verifiers/verifier.py | 6 ++--- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/verifiers/solver_voting_count_verifier.py b/verifiers/solver_voting_count_verifier.py index 2d170ed6..19188175 100644 --- a/verifiers/solver_voting_count_verifier.py +++ b/verifiers/solver_voting_count_verifier.py @@ -41,43 +41,46 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti self.mutators = [] self.original_model = None - def initialize_run(self) -> None: + def initialize_run(self, is_rerun=False) -> None: if self.original_model == None: with open(self.model_file, 'rb') as fpcl: self.original_model = pickle.loads(fpcl.read()) self.cons = self.original_model.constraints assert (len(self.cons) > 0), f"{self.model_file} has no constraints" self.cons = toplevel_list(self.cons) + if is_rerun: + print([(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)]) assert len(self.solvers) == 2, f"2 solvers required, {len(self.solvers)} given." if 'gurobi' in [s.lower() for s in self.solvers]: self.sol_lim = 10000 # TODO: is hardcode best idea? - self.sol_count_1 = cp.Model(self.cons).solveAll(solver=self.solvers[0],time_limit=max(1,min(250,self.time_limit-time.time())),solution_limit=self.sol_lim) - self.sol_count_2 = cp.Model(self.cons).solveAll(solver=self.solvers[1],time_limit=max(1,min(250,self.time_limit-time.time())),solution_limit=self.sol_lim) + self.sol_count_1 = cp.Model(self.cons).solveAll(solver=self.solvers[0],solution_limit=self.sol_lim) + self.sol_count_2 = cp.Model(self.cons).solveAll(solver=self.solvers[1],solution_limit=self.sol_lim) else: - self.sol_count_1 = cp.Model(self.cons).solveAll(solver=self.solvers[0],time_limit=max(1, min(250, self.time_limit - time.time()))) - self.sol_count_2 = cp.Model(self.cons).solveAll(solver=self.solvers[1],time_limit=max(1, min(250, self.time_limit - time.time()))) + self.sol_count_1 = cp.Model(self.cons).solveAll(solver=self.solvers[0]) + self.sol_count_2 = cp.Model(self.cons).solveAll(solver=self.solvers[1]) - assert self.sol_count_1 == self.sol_count_2, f"{self.solvers} don't agree on amount of solutions (before mutations): {self.sol_count_1} and {self.sol_count_2}" + # assert self.sol_count_1 == self.sol_count_2, f"{self.solvers} don't agree on amount of solutions (before mutations): {self.sol_count_1} and {self.sol_count_2}" self.mutators = [copy.deepcopy( - self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. + self.cons)] # keep track of list of og_cons alternated with mutators that transformed it into the next list of og_cons. - def verify_model(self) -> dict: + def verify_model(self, is_rerun=False) -> dict: try: model = cp.Model(self.cons) + if is_rerun: + print([(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)]) time_limit = max(1, min(200, self.time_limit - time.time())) # set the max time limit to the given time limit or to 1 if the self.time_limit-time.time() would be smaller then 1 solver_1 = self.solvers[0] solver_2 = self.solvers[1] if hasattr(self, 'sol_lim'): - print("this works") - new_count_1 = model.solveAll(solver=solver_1, time_limit=time_limit, solution_limit=self.sol_lim) - new_count_2 = model.solveAll(solver=solver_2, time_limit=time_limit, solution_limit=self.sol_lim) + new_count_1 = model.solveAll(solver=solver_1, solution_limit=self.sol_lim) + new_count_2 = model.solveAll(solver=solver_2, solution_limit=self.sol_lim) else: - new_count_1 = model.solveAll(solver=solver_1, time_limit=time_limit) - new_count_2 = model.solveAll(solver=solver_2, time_limit=time_limit) + new_count_1 = model.solveAll(solver=solver_1) + new_count_2 = model.solveAll(solver=solver_2) if model.status().runtime > time_limit - 10: # timeout, skip @@ -118,7 +121,7 @@ def verify_model(self) -> dict: # if you got here, the model failed... return dict(type=Fuzz_Test_ErrorTypes.failed_model, originalmodel_file=self.model_file, - constraints=self.cons, + constraints=self.og_cons, mutators=self.mutators, model=newModel, originalmodel=self.original_model diff --git a/verifiers/solver_voting_sat_verifier.py b/verifiers/solver_voting_sat_verifier.py index acbd8e88..26b89992 100644 --- a/verifiers/solver_voting_sat_verifier.py +++ b/verifiers/solver_voting_sat_verifier.py @@ -41,20 +41,22 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti self.mutators = [] self.original_model = None - def initialize_run(self) -> None: + def initialize_run(self, is_rerun=False) -> None: if self.original_model == None: with open(self.model_file, 'rb') as fpcl: self.original_model = pickle.loads(fpcl.read()) self.cons = self.original_model.constraints assert (len(self.cons) > 0), f"{self.model_file} has no constraints" self.cons = toplevel_list(self.cons) + if is_rerun: + print([(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)]) # No other preparation necessary self.mutators = [copy.deepcopy( - self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. + self.cons)] # keep track of list of og_cons alternated with mutators that transformed it into the next list of og_cons. - def verify_model(self) -> dict: + def verify_model(self, is_rerun=False) -> dict: try: model = cp.Model(self.cons) time_limit = max(1, min(200, @@ -107,7 +109,7 @@ def verify_model(self) -> dict: # if you got here, the model failed... return dict(type=Fuzz_Test_ErrorTypes.failed_model, originalmodel_file=self.model_file, - constraints=self.cons, + constraints=self.og_cons, mutators=self.mutators, model=newModel, originalmodel=self.original_model diff --git a/verifiers/verifier.py b/verifiers/verifier.py index 87c23d69..bf64d82d 100644 --- a/verifiers/verifier.py +++ b/verifiers/verifier.py @@ -158,15 +158,15 @@ def rerun(self,error: dict) -> dict: self.model_file = error["originalmodel_file"] self.original_model = error["originalmodel"] self.exclude_dict = {} - self.initialize_run() + self.initialize_run(is_rerun=True) gen_mutations_error = self.generate_mutations() # check if no error occured while generation the mutations if gen_mutations_error == None: - return self.verify_model() + return self.verify_model(is_rerun=True) else: return gen_mutations_error - # self.cons = error["constraints"] + # self.og_cons = error["constraints"] # return self.verify_model() except AssertionError as e: From f3e2c78fd904b08fcdac09d90b4e45a6e3bd068c Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 8 Apr 2025 17:40:36 +0200 Subject: [PATCH 07/58] fixed bug in expression mutator where gcc was allowing non-int variables into the last argument --- fuzz_test_utils/mutators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 72b1fb5f..56cbdeaf 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1207,10 +1207,10 @@ def get_new_operator(func: Function, ints, bools, vals, variables): amnt_args = random.randint(func.min_args, min(len(comb), func.max_args//3)) args = random.sample(comb, amnt_args), random.sample(comb, amnt_args), random.sample(comb, amnt_args) case 'GlobalCardinalityCount': - amnt_fst_args = random.randint(1, min(len(ints), func.max_args - 2)) - amnt_snd_args = random.randint(1, min(len(ints), (func.max_args - amnt_fst_args)//2)) + amnt_fst_args = random.randint(1, min(len(vals), func.max_args - 2)) + amnt_snd_args = random.randint(1, min(len(vals), (func.max_args - amnt_fst_args)//2)) counts = [random.randint(0, amnt_fst_args) for _ in range(amnt_snd_args)] - args = random.sample(ints, amnt_fst_args), counts, random.sample(ints, amnt_snd_args) + args = random.sample(vals, amnt_fst_args), counts, random.sample(vals, amnt_snd_args) case _: args = [] return func.func(*args) From 0d6e8cb41e8aad3c03fa936ad20a96e87af5f783 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 8 Apr 2025 18:52:58 +0200 Subject: [PATCH 08/58] Found bug regarding AllDifferentExcept0 (not fixed, added TODO) --- fuzz_test_utils/mutators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 56cbdeaf..18de55c5 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1348,6 +1348,7 @@ def replace_at_path(con, path, new_expr): if type(con) == Comparison: return Comparison(con.name, args[0], args[1]) else: # global constraints + # TODO: bugfix: AllDifferentExceptN can be of type AllDifferentExcept0 which adds a (meaningless) 0 in the constraint return type(con)(*args) elif type(con) == list: return [new_expr if i == path[0] else e for i, e in enumerate(con)] From 9818c6463d33deef9d2d72065450de0520f94c47 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Wed, 9 Apr 2025 11:05:13 +0200 Subject: [PATCH 09/58] reimplementation of eplace_at_path to fix infinite recursion bug and AllDifferentExcept0 inconsistency. Also cleaned up type(..) == .. --- fuzz_test_utils/mutators.py | 48 +++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 18de55c5..97a2ddd3 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1039,9 +1039,9 @@ def get_all_op_exprs(con): # Helper function to get all expressions WITHOUT an operator in a given constraint def get_all_non_op_exprs(con): - if hasattr(con, 'args') and type(con) != NDVarArray and con.name != 'boolval': + if hasattr(con, 'args') and not isinstance(con, NDVarArray) and con.name != 'boolval': return sum((get_all_non_op_exprs(arg) for arg in con.args), []) - elif type(con) == list: + elif isinstance(con, list): if all([is_num(e) for e in con]): # wsum constants return [] else: @@ -1220,7 +1220,7 @@ def get_operator(args, has_bool_return): Returns a new expression that needs fewer arguments of each type than available in `args`. It has a boolean return type if `has_bool_return` is True, otherwise it has an int return type. """ - ints = [e for e in args if not (is_boolexpr(e) or type(e) == list)] + ints = [e for e in args if not (is_boolexpr(e) or isinstance(e, list))] bools = [e for e in args if is_boolexpr(e)] values = [e for e in args if not hasattr(e, 'value')] # Is this the only way to extract constants only? (e.g. for wsum) variables = get_variables(args) @@ -1330,7 +1330,7 @@ def find_all_occurrences(con, target_node): for i, arg in enumerate(con.args): for path in find_all_occurrences(arg, target_node): occurrences.append((i,) + path) # Add index to the path - elif type(con) == list: + elif isinstance(con, list): for i, arg in enumerate(con): for path in find_all_occurrences(arg, target_node): occurrences.append((i,) + path) @@ -1338,22 +1338,34 @@ def find_all_occurrences(con, target_node): def replace_at_path(con, path, new_expr): + """ + Replace the subexpression at the given `path` in the expression tree `con` with `new_expr`. + + Parameters: + ~ con: The constraint in which we will replace an expression + ~ path: The path to the expression we will mutate (e.g. y has path (0, 1) in (x > y) -> p) + ~ new_expr: The new expression which will be at the specified `path` + Returns: + ~ con: The mutated constraint + """ if not path: # END OF PATH return new_expr - if hasattr(con, 'args') and con.name != 'boolval': - args = con.args - args[path[0]] = replace_at_path(args[path[0]], path[1:], new_expr) - if type(con) == Operator: - return Operator(con.name, args) - if type(con) == Comparison: - return Comparison(con.name, args[0], args[1]) - else: # global constraints - # TODO: bugfix: AllDifferentExceptN can be of type AllDifferentExcept0 which adds a (meaningless) 0 in the constraint - return type(con)(*args) - elif type(con) == list: - return [new_expr if i == path[0] else e for i, e in enumerate(con)] - else: - return con + new_expr = copy.deepcopy(new_expr) # Important to avoid infinite recursion! + parent = con + # Traverse the parents until we have the final parent + for idx in path[:-1]: + if hasattr(parent, 'args') and parent.name != 'boolval': + parent = parent.args[idx] + elif isinstance(parent, list): + parent = parent[idx] + # Change the arguments of the parent + if hasattr(parent, 'args') and parent.name != 'boolval': + args = list(parent.args) + args[path[-1]] = new_expr + parent.update_args(args) + elif isinstance(parent, list): + parent[path[-1]] = new_expr + return con def mutate_con(con, old_expr, new_expr): From 18a16f6d6d9d24154ce9d3a69b137c9b50cd15b1 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Fri, 11 Apr 2025 10:48:50 +0200 Subject: [PATCH 10/58] changed the runner to choose verifiers based on the amount of given solvers --- verifiers/verifier_runner.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/verifiers/verifier_runner.py b/verifiers/verifier_runner.py index 248350b3..7fefa18d 100644 --- a/verifiers/verifier_runner.py +++ b/verifiers/verifier_runner.py @@ -1,14 +1,16 @@ import glob import math +import random import warnings from os.path import join from verifiers import * from fuzz_test_utils import Fuzz_Test_ErrorTypes -def get_all_verifiers() -> list: - # TODO: kiezen adhv aantal gegeven solvers - # return [Solution_Verifier,Optimization_Verifier,Model_Count_Verifier,Metamorphic_Verifier,Equivalance_Verifier] - return [Solver_Vote_Count_Verifier, Solver_Vote_Sat_Verifier] +def get_all_verifiers(single_solver) -> list: + if single_solver: + return [Solution_Verifier,Optimization_Verifier,Model_Count_Verifier,Metamorphic_Verifier,Equivalance_Verifier] + else: + return [Solver_Vote_Count_Verifier, Solver_Vote_Sat_Verifier] def run_verifiers(current_amount_of_tests, current_amount_of_error, lock, solver: list[str], mutations_per_model: int, folders: list, max_error_treshold: int, output_dir: str, time_limit: float) -> None: """ @@ -37,7 +39,12 @@ def run_verifiers(current_amount_of_tests, current_amount_of_error, lock, solver execution_time = 0 try: while time.time() < time_limit and current_amount_of_error.value < max_error_treshold: - random_verifier = random.choice(get_all_verifiers())(**verifier_kwargs) + if isinstance(solver, str): + random_verifier = random.choice(get_all_verifiers(single_solver=True))(**verifier_kwargs) + elif isinstance(solver, list): + random_verifier = random.choice(get_all_verifiers(single_solver=False))(**verifier_kwargs) + else: + raise Exception(f"The given solvers are not in the correct format. Should be either str or list, but is {type(solver)}.") fmodels = [] for folder in folders: fmodels.extend(glob.glob(join(folder,random_verifier.getType(), "*"))) From 24184c1f8109c3b739898772c3c1b19bb9897b12 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Fri, 11 Apr 2025 10:54:39 +0200 Subject: [PATCH 11/58] type_aware_operator_replacement now also checks whether the expression it wants to changes comes from an operator that requires a specific return type. Also a bunch of refactorings in almost every function I wrote. --- fuzz_test_utils/mutators.py | 340 +++++++++++++++++++++++++----------- 1 file changed, 236 insertions(+), 104 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 97a2ddd3..7ee89c16 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -909,7 +909,7 @@ def aritmetic_comparison_morph(const): raise MetamorphicError(aritmetic_comparison_morph, cons, e) -def type_aware_operator_replacement(constraints): +def type_aware_operator_replacement(constraints: list): """ Replaces a random operator of a random constraint from a list of given constraints. IMPORTANT: This can change satisfiability of the constraint! Only to be used with verifiers that allow this! @@ -928,33 +928,34 @@ def type_aware_operator_replacement(constraints): - [Bool, Bool] -> Bool : and or -> - [Bool, ...] -> Bool : and or - [Bool] -> Bool : not - - Int, Int -> Int : sum sub mul div mod pow + - Int, Int -> Int : sum sub mul div mod pow (IMPORTANT: div mod gives problems with domains containing 0, pow with domains containing negative numbers) - [Int, ...] -> Int : sum - [Int] -> Int : - - [Int, ...] [Int, ...] (arrays of same len) -> Int : wsum """ try: final_cons = copy.deepcopy(constraints) - cons_set = set(constraints) - # pick a random constraint and calculate their mutable expressions until there is at least 1 - candidates = list(cons_set) # or final_cons + # pick a random constraint and calculate whether they have a mutable expression until they do + candidates = list(set(constraints)) random.shuffle(candidates) for con in candidates: - exprs = get_all_mutable_op_exprs(con) + exprs = get_all_mutable_op_exprs(con) # e.g. for (x + y) // z > 0. The tree goes >(0, //(+(x,y), z)) + # which means either >, // or + can get mutated if exprs: break - else: + else: # In case there isn't any mutable expression in any constraint return final_cons - # remove the constraint from the constraints + # Remove the constraint from the constraints final_cons.remove(con) # Choose an expression to change expr = random.choice(exprs) + # Mutate this expression (= change the operator) mutate_op_expression(expr, con) - # add the changed constraint back + # Add the changed constraint back final_cons.append(con) return final_cons @@ -962,7 +963,8 @@ def type_aware_operator_replacement(constraints): return Exception(e) -def mutate_op_expression(expr, con): +# TODO: use this to make a "strengthen_expression" and "weaken_expression" +def mutate_op_expression(expr: Expression, con: Expression): """ Mutates the constraint containing the expression by mutating said expression. Only to be called when the expression is known to be in the constraint. @@ -972,7 +974,7 @@ def mutate_op_expression(expr, con): - con: the constraint containing the expression ~ No return. Mutates the constraint! """ - # types that can be converted into each-other + # Types that can be converted into each-other comparisons = {'==', '!=', '<', '<=', '>', '>='} int_ops = {'sum', 'sub', 'mul', 'pow', 'mod', 'div'} @@ -992,25 +994,25 @@ def mutate_op_expression(expr, con): raise ValueError(f"Unknown operator type: {expr.name}. (You should not be able to get here)") new_operator = random.choice(list(possible_replacements)) - expr.name = new_operator + expr.name = new_operator # Mutate expression by changing its name return - # recursively search in arguments + # Recursively search for the expression in arguments if hasattr(con, "args"): for arg in con.args: mutate_op_expression(expr, arg) return -def get_all_mutable_op_exprs(con): +def get_all_mutable_op_exprs(con: Expression): """ Returns a list of all expressions inside the given constraint of which the - operator can be mutated into another one. This can be extended with other operators. + operator can be mutated into another one. - ~ Paremeters: + ~ Parameters: - con: a single constraint, possibly containing multiple expressions ~ Return: - - mutable_exprs: all expressions in the constraint that can be mutated (safely) + - mutable_exprs: all expressions in the constraint that can be mutated """ comparisons = {'==', '!=', '<', '<=', '>', '>='} int_ops = {'sum', 'sub', 'mul', 'div', 'mod', 'pow'} @@ -1029,16 +1031,20 @@ def get_all_mutable_op_exprs(con): return mutable_exprs -# Helper function to get all expressions WITH an operator in a given constraint -def get_all_op_exprs(con): +def get_all_op_exprs(con: Expression): + """ + Helper function to get all expressions WITH an operator in a given constraint + """ if type(con) in {Comparison, Operator}: return sum((get_all_op_exprs(arg) for arg in con.args), []) + [con] # All subexpressions + current expression else: return [] -# Helper function to get all expressions WITHOUT an operator in a given constraint -def get_all_non_op_exprs(con): +def get_all_non_op_exprs(con: Expression): + """ + Helper function to get all expressions WITHOUT an operator in a given constraint + """ if hasattr(con, 'args') and not isinstance(con, NDVarArray) and con.name != 'boolval': return sum((get_all_non_op_exprs(arg) for arg in con.args), []) elif isinstance(con, list): @@ -1050,28 +1056,41 @@ def get_all_non_op_exprs(con): return [con] -# Helper function to get all epxressions in a given constraint (Might be unnecessary but let's use this for now) -def get_all_exprs(con): +def get_all_exprs(con: Expression): + """ + Helper function to get all expressions in a given constraint (Might be unnecessary but let's use this for now) + """ return get_all_op_exprs(con)[::-1] + get_all_non_op_exprs(con) -def get_all_exprs_mult(cons): +def get_all_exprs_mult(cons: list): + """ + Helper function to get all expressions in a given list of constraints (e.g. to get all possible arguments for an + expression replacement) + """ all_exprs = [] for con in cons: all_exprs += get_all_exprs(con) return all_exprs -def satisfies_args(func: Function, ints: int, bools: int, values: int, vars: int, has_bool_return: bool): +def satisfies_args(func: Function, ints: int, bools: int, constants: int, vars: int, has_bool_return: bool): """ - returns whether the given function `func` can work with the given amount of integers `ints` - and booleans `bools` and the given return type `has_bool_return` + Returns whether a new function of the given type `func` can be created with the given amount of arguments. + ~ Parameters: + - `ints`: The amount of integers in the possible arguments (including constants and variables) + - bools`: The amount of booleans in the possible arguments (including variables) + - `constants`: The amount of constants in the possible arguments (only numbers) + - `vars`: The amount of variables in the possible arguments (intvars and boolvars) + - `has_bool_return`: Indicates whether the return type should be boolean (True) or int (False) + ~ Returns: + - True if it is possible to create a new function of given type with the given arguments, False otherwise """ match func.type: case 'op' | 'comp': match func.name: case 'wsum': - return values >= 1 and ints + bools >= func.min_args and has_bool_return == func.bool_return + return constants >= 1 and ints + bools >= func.min_args and has_bool_return == func.bool_return case _: return ((func.int_args == -1 or func.int_args <= ints + bools) and # enough int args (func.bool_args == -1 or func.bool_args <= bools) and # enough bool args @@ -1083,45 +1102,58 @@ def satisfies_args(func: Function, ints: int, bools: int, values: int, vars: int case 'Abs' | 'NValue' | 'Count': return ints + bools >= func.min_args and not has_bool_return case 'Minimum' | 'Maximum' | 'Element': # We make these have only ints, so it always has the same return type - return values >= func.min_args and not has_bool_return + return constants >= func.min_args and not has_bool_return case 'Among' | 'NValueExcept': - return values >= 1 and ints >= func.min_args - 1 and not has_bool_return + return constants >= 1 and ints >= func.min_args - 1 and not has_bool_return case 'gcon': match func.name: - case 'Circuit' | 'Inverse' | 'GlobalCardinalityCount': - return values >= func.min_args and has_bool_return + case 'GlobalCardinalityCount': + return constants >= func.min_args and has_bool_return case 'IfThenElse' | 'Xor': return bools >= func.min_args and has_bool_return case 'Table' | 'NegativeTable': return vars >= 1 and ints + bools >= func.min_args - 1 and has_bool_return case 'LexLess' | 'LexLessEq' | 'LexChainLess' | 'LexChainLessEq': return vars >= func.min_args and has_bool_return + case 'Circuit' | 'Inverse': + return has_bool_return # No arguments required. We fill in the arrays with logical numbers case _: return ints + bools >= func.min_args and has_bool_return - -def get_new_operator(func: Function, ints, bools, vals, variables): +def get_new_operator(func: Function, ints: list, bools: list, constants: list, variables: list): + """ + Creates a new function of the given type with the arguments given + ~ Parameters: + - `ints`: The integers in the possible arguments (including constants and variables) + - `bools`: The booleans in the possible arguments (including variables) + - `constants`: The constants in the possible arguments (only numbers) + - `variables`: The variables in the possible arguments (intvars and boolvars) + ~ Returns: + - A new function with arguments from the given lists (or other arguments based on the Function `func`) + """ comb = ints + bools match func.type: case 'op' | 'comp': # Separate logic for wsum if func.name == 'wsum': # (-1, 0, False, 2, max_args, 2) - amnt_args = random.randint(func.min_args // 2, min(len(vals), func.max_args)//2) + amnt_args = random.randint(func.min_args // 2, min(len(constants), func.max_args) // 2) # First take constants - constants = random.sample(vals, amnt_args) + constants = random.sample(constants, amnt_args) # Then the other expressions others = random.sample(comb, amnt_args) return Operator(func.name, [constants, others]) # Logic for all other operators and comparisons is the same if func.int_args == -1: - amnt_args = random.randint(func.min_args, min(len(comb), func.max_args)) # Take at least min_args and at most max_args arguments + amnt_args = random.randint(func.min_args, min(len(comb), + func.max_args)) # Take at least min_args and at most max_args arguments args = random.sample(comb, amnt_args) elif func.int_args > 0: args = random.sample(comb, func.int_args) if func.bool_args == -1: - amnt_args = random.randint(func.min_args, min(len(bools), func.max_args)) # Take at least min_args and at most max_args arguments + amnt_args = random.randint(func.min_args, min(len(bools), + func.max_args)) # Take at least min_args and at most max_args arguments args = random.sample(bools, amnt_args) elif func.bool_args > 0: args = random.sample(bools, func.bool_args) @@ -1134,11 +1166,11 @@ def get_new_operator(func: Function, ints, bools, vals, variables): case 'Abs': args = random.choice(comb), case 'Minimum' | 'Maximum': - amnt_args = random.randint(func.min_args, min(len(vals), func.max_args)) - args = random.sample(vals, amnt_args), + amnt_args = random.randint(func.min_args, min(len(constants), func.max_args)) + args = random.sample(constants, amnt_args), case 'Element': - amnt_args = random.randint(func.min_args, min(len(vals), func.max_args)) - first_arg = random.sample(vals, amnt_args) + amnt_args = random.randint(func.min_args, min(len(constants), func.max_args)) + first_arg = random.sample(constants, amnt_args) idx = random.randint(0, amnt_args - 1) args = first_arg, idx case 'NValue': @@ -1150,15 +1182,15 @@ def get_new_operator(func: Function, ints, bools, vals, variables): last_arg = random.choice(comb) args = first_arg, last_arg case 'Among': - amnt_fst_arg = random.randint(func.min_args//2, min(len(comb), func.max_args)//2) - amnt_snd_arg = random.randint(func.min_args//2, min(len(vals), func.max_args)//2) + amnt_fst_arg = random.randint(func.min_args // 2, min(len(comb), func.max_args) // 2) + amnt_snd_arg = random.randint(func.min_args // 2, min(len(constants), func.max_args) // 2) first_arg = random.sample(comb, amnt_fst_arg) - second_arg = random.sample(vals, amnt_snd_arg) + second_arg = random.sample(constants, amnt_snd_arg) args = first_arg, second_arg case 'NValueExcept': - amnt_fst_arg = random.randint(func.min_args//2, min(len(comb), func.max_args)//2) + amnt_fst_arg = random.randint(func.min_args // 2, min(len(comb), func.max_args) // 2) first_arg = random.sample(comb, amnt_fst_arg) - second_arg = random.choice(vals) + second_arg = random.choice(constants) args = first_arg, second_arg return func.func(*args) case 'gcon': @@ -1167,25 +1199,25 @@ def get_new_operator(func: Function, ints, bools, vals, variables): amnt_args = random.randint(func.min_args, min(len(comb), func.max_args)) args = random.sample(comb, amnt_args) case 'AllDifferentExceptN' | 'AllEqualExceptN': - amnt_fst_args = random.randint(func.min_args//2, min(len(comb), func.max_args)//2) - amnt_snd_args = random.randint(func.min_args//2, min(len(comb), func.max_args - amnt_fst_args)) + amnt_fst_args = random.randint(func.min_args // 2, min(len(comb), func.max_args) // 2) + amnt_snd_args = random.randint(func.min_args // 2, min(len(comb), func.max_args - amnt_fst_args)) args = random.sample(comb, amnt_fst_args), random.sample(comb, amnt_snd_args) case 'LexLess' | 'LexLessEq': - half_amnt_args = random.randint(func.min_args//2, min(len(variables), func.max_args)//2) + half_amnt_args = random.randint(func.min_args // 2, min(len(variables), func.max_args) // 2) args = random.sample(variables, half_amnt_args), random.sample(variables, half_amnt_args) case 'LexChainLess' | 'LexChainLessEq': - amnt_args = random.randint(func.min_args, min(len(variables), func.max_args)//2) + amnt_args = random.randint(func.min_args // 2, min(len(variables), func.max_args) // 2) all_args = random.sample(variables, amnt_args) - divisors = [i for i in range(1, amnt_args) if amnt_args % i == 0] + divisors = [i for i in range(1, amnt_args + 1) if amnt_args % i == 0] fst_dimension = random.choice(divisors) snd_dimension = int(amnt_args / fst_dimension) args = [all_args[i * fst_dimension:(i + 1) * fst_dimension] for i in range(snd_dimension)], case 'Circuit': - amnt_args = random.randint(func.min_args, min(len(vals), func.max_args)) - args = random.sample(vals, amnt_args), + amnt_args = random.randint(func.min_args, func.max_args) + args = random.sample(range(amnt_args), amnt_args), case 'Inverse': - amnt_args = random.randint(func.min_args//2, min(len(vals), func.max_args)//2) - args = random.sample(range(1, amnt_args+1), amnt_args), random.sample(range(1, amnt_args+1), amnt_args) + amnt_args = random.randint(func.min_args // 2, func.max_args // 2) + args = random.sample(range(amnt_args), amnt_args), random.sample(range(amnt_args), amnt_args) case 'IfThenElse': amnt_args = random.randint(func.min_args, min(len(bools), func.max_args)) args = random.sample(bools, amnt_args) @@ -1193,40 +1225,58 @@ def get_new_operator(func: Function, ints, bools, vals, variables): amnt_args = random.randint(func.min_args, min(len(bools), func.max_args)) args = random.sample(bools, amnt_args), case 'Table' | 'NegativeTable': - amnt_fst_arg = random.randint(1, min(len(variables), func.max_args//4)) - amnt_snd_args = random.randint(1, min(len(comb), func.max_args-amnt_fst_arg) // amnt_fst_arg) * amnt_fst_arg + amnt_fst_arg = random.randint(1, min(len(variables), func.max_args // 4)) + amnt_snd_args = random.randint(1, min(len(comb), + func.max_args - amnt_fst_arg) // amnt_fst_arg) * amnt_fst_arg fst_args = random.sample(variables, amnt_fst_arg) snd_args = random.sample(comb, amnt_snd_args) - snd_args_transformed = [snd_args[i * amnt_fst_arg:(i + 1) * amnt_fst_arg] for i in range(int(amnt_snd_args/amnt_fst_arg))] + snd_args_transformed = [snd_args[i * amnt_fst_arg:(i + 1) * amnt_fst_arg] for i in + range(int(amnt_snd_args / amnt_fst_arg))] args = fst_args, snd_args_transformed case 'InDomain': amnt_args = random.randint(func.min_args, min(len(comb), func.max_args)) all_args = random.sample(comb, amnt_args) args = all_args[0], all_args[1:] case 'NoOverlap': - amnt_args = random.randint(func.min_args, min(len(comb), func.max_args//3)) - args = random.sample(comb, amnt_args), random.sample(comb, amnt_args), random.sample(comb, amnt_args) + amnt_args = random.randint(func.min_args, min(len(comb), func.max_args // 3)) + args = random.sample(comb, amnt_args), random.sample(comb, amnt_args), random.sample(comb, + amnt_args) case 'GlobalCardinalityCount': - amnt_fst_args = random.randint(1, min(len(vals), func.max_args - 2)) - amnt_snd_args = random.randint(1, min(len(vals), (func.max_args - amnt_fst_args)//2)) + amnt_fst_args = random.randint(1, min(len(constants), func.max_args - 2)) + amnt_snd_args = random.randint(1, min(len(constants), (func.max_args - amnt_fst_args) // 2)) counts = [random.randint(0, amnt_fst_args) for _ in range(amnt_snd_args)] - args = random.sample(vals, amnt_fst_args), counts, random.sample(vals, amnt_snd_args) + args = random.sample(constants, amnt_fst_args), counts, random.sample(constants, amnt_snd_args) case _: args = [] return func.func(*args) -def get_operator(args, has_bool_return): + +def get_operator(args: list, ret_type: str | bool): """ - Returns a new expression that needs fewer arguments of each type than available in `args`. - It has a boolean return type if `has_bool_return` is True, otherwise it has an int return type. + Randomly generates a new Expression that can be created with the given arguments `args` and return type `ret_type`. + ~ Parameters: + - `args`: all arguments that should be in the newly generated function. + - `ret_type`: the return type that the new function should have (usually generated by `get_return_type`). + There are four options: + -> 'constant' for numbers only + -> 'variable' for variables only + -> True for boolean (including variables) + -> False for int (including numbers and variables) + ~ Returns: + - A new Expression with arguments from the given list and with given return type """ ints = [e for e in args if not (is_boolexpr(e) or isinstance(e, list))] bools = [e for e in args if is_boolexpr(e)] - values = [e for e in args if not hasattr(e, 'value')] # Is this the only way to extract constants only? (e.g. for wsum) + constants = [e for e in args if + not hasattr(e, 'value')] # Is this the only way to extract constants only? (e.g. for wsum) + if ret_type == 'constant': + return random.choice(constants) # Some expressions can't be replaced by functions variables = get_variables(args) + if ret_type == 'variable': + return random.choice(variables) # Some expressions can't be replaced by functions ints_cnt = len(ints) bools_cnt = len(bools) - vals_cnt = len(values) + constants_cnt = len(constants) vars_cnt = len(variables) max_args = 12 @@ -1274,8 +1324,8 @@ def get_operator(args, has_bool_return): Element: ('gfun', -1, 0, None, 2, max_args), # [...], idx | Can return a boolean but this is not known beforehand (min 2, max /, /) # Denk best enkel de array vullen en de idx gewoon tussen 0 en len-1 pakken Count: ('gfun', -1, 0, False, 2, max_args), # [...], expr | (min 2, max /, /) - Among: ('gfun', -1, 0, False, 2, max_args), # [...], [...] | Second array can only have values, no expressions (not even BoolVal()) (min 2, max /, /) - NValueExcept: ('gfun', -1, 0, False, 2, max_args) # [...], val | Second argument can only have values, no expressions (not even BoolVal()) (min 2, max /, /) + Among: ('gfun', -1, 0, False, 2, max_args), # [...], [...] | Second array can only have constants, no expressions (not even BoolVal()) (min 2, max /, /) + NValueExcept: ('gfun', -1, 0, False, 2, max_args) # [...], val | Second argument can only have constants, no expressions (not even BoolVal()) (min 2, max /, /) }.items() } # Global constraints @@ -1284,9 +1334,9 @@ def get_operator(args, has_bool_return): name: Function(name.__name__, name, *attrs) for name, attrs in { AllDifferent: ('gcon', -1, 0, True, 2, max_args), # [...] | (min 2, max /, /) - AllDifferentExceptN: ('gcon', -1, 0, True, 2, max_args), # [...], [...] | Second arg can also be a single non-list value (min 2, max /, /) + AllDifferentExceptN: ('gcon', -1, 0, True, 2, max_args), # [...], [...] | Second arg can also be a single non-list constant (min 2, max /, /) AllEqual: ('gcon', -1, 0, True, 2, max_args), # [...] | (min 2, max /, /) - AllEqualExceptN: ('gcon', -1, 0, True, 2, max_args), # [...], [...] | Second arg can also be a single non-list value (min 2, max /, /) + AllEqualExceptN: ('gcon', -1, 0, True, 2, max_args), # [...], [...] | Second arg can also be a single non-list constant (min 2, max /, /) Circuit: ('gcon', -1, 0, True, 2, max_args), # [...] | Can only have ints, NO BOOLS! (min 2, max /, /) Inverse: ('gcon', -1, 0, True, 2, max_args, 2), # [...], [...] | Can only have ints, NO BOOLS! (min 2, max /, 2n) Table: ('gcon', -1, 0, True, 2, max_args), # [...], [[...],[...],...] | First argument only variables, Second argument should have a multiple amnt of args as the first one (min 2, max /, n + mn?) @@ -1311,55 +1361,64 @@ def get_operator(args, has_bool_return): } all_ops = ops | comps | global_fns | global_cons # print(f"ints_cnt={ints_cnt},bools_cnt={bools_cnt},has_bool_return={has_bool_return}") - after = {k: v for k, v in all_ops.items() if satisfies_args(v, ints_cnt, bools_cnt, vals_cnt, vars_cnt, has_bool_return)} + assert isinstance(ret_type, bool), f"Getting here means the return type should be a boolean. It is {ret_type}." + after = {k: v for k, v in all_ops.items() if + satisfies_args(v, ints_cnt, bools_cnt, constants_cnt, vars_cnt, ret_type)} if after: func = random.choice(list(after.values())) else: return None - return get_new_operator(func, ints, bools, values, variables) + return get_new_operator(func, ints, bools, constants, variables) + -def find_all_occurrences(con, target_node): +def find_all_occurrences(con: Expression, target_expr: Expression): """ - Recursively finds all occurrences of `target_node` in the expression `con`. - Returns a list of paths (as tuples) to each occurrence. + Recursively finds all occurrences of `target_expr` in the expression `con`. + ~ Parameters: + - `con`: constraint in which we search + - `target_expr`: the expression we're searching the occurrences for + ~ Returns: + - `occurrences`: a list of paths (as tuples) to each occurrence. """ occurrences = [] - if con is target_node: + if con is target_expr: occurrences.append(()) # Current node is the target - if hasattr(con, 'args') and con.name != 'boolval': + if hasattr(con, 'args') and not isinstance(con, NDVarArray) and con.name != 'boolval': for i, arg in enumerate(con.args): - for path in find_all_occurrences(arg, target_node): + for path in find_all_occurrences(arg, target_expr): occurrences.append((i,) + path) # Add index to the path elif isinstance(con, list): for i, arg in enumerate(con): - for path in find_all_occurrences(arg, target_node): + for path in find_all_occurrences(arg, target_expr): occurrences.append((i,) + path) return occurrences -def replace_at_path(con, path, new_expr): +def replace_at_path(con: Expression, path: tuple, new_expr: Expression): """ - Replace the subexpression at the given `path` in the expression tree `con` with `new_expr`. - - Parameters: - ~ con: The constraint in which we will replace an expression - ~ path: The path to the expression we will mutate (e.g. y has path (0, 1) in (x > y) -> p) - ~ new_expr: The new expression which will be at the specified `path` - Returns: - ~ con: The mutated constraint + Replace the Expression at the given `path` in the expression tree `con` with `new_expr`. + + ~ Parameters: + - `con`: The constraint in which we will replace an expression + - `path`: The path to the expression we will mutate (e.g. y has path (0, 1) in (x > y) -> p) + - `new_expr`: The new expression which will be at the specified `path` + ~ Returns: + - `con`: The mutated constraint """ - if not path: # END OF PATH + if not path: # Replace main expression return new_expr new_expr = copy.deepcopy(new_expr) # Important to avoid infinite recursion! parent = con + # Traverse the parents until we have the final parent for idx in path[:-1]: - if hasattr(parent, 'args') and parent.name != 'boolval': + if hasattr(parent, 'args') and not isinstance(parent, NDVarArray) and parent.name != 'boolval': parent = parent.args[idx] elif isinstance(parent, list): parent = parent[idx] + # Change the arguments of the parent - if hasattr(parent, 'args') and parent.name != 'boolval': + if hasattr(parent, 'args') and not isinstance(parent, NDVarArray) and parent.name != 'boolval': args = list(parent.args) args[path[-1]] = new_expr parent.update_args(args) @@ -1368,13 +1427,82 @@ def replace_at_path(con, path, new_expr): return con -def mutate_con(con, old_expr, new_expr): - paths = find_all_occurrences(con, old_expr) - rand_path = random.choice(paths) - return replace_at_path(con, rand_path, new_expr) +def expr_at_path(con: Expression, path: tuple, expr: Expression): + """ + Helper function for checking whether the given expression is on the given path in the given constraint. + Upon encountering `None` in a path, it will return True. + """ + if not path and con is expr: + return True + # Traverse the parents until we have the final parent + for idx in path: + if idx is not None: + if hasattr(con, 'args') and not isinstance(con, NDVarArray) and con.name != 'boolval': + con = con.args[idx] + elif isinstance(con, list): + con = con[idx] + else: + return len(find_all_occurrences(con, expr)) > 0 + return con is expr -def type_aware_expression_replacement(constraints): +def get_return_type(expr: Expression, con: Expression): + """ + Function to get the return type of expressions that are allowed to replace `expr` in a given constraint `con`. + There are four options: + - boolean (any type of boolean including 'Expression's which are of type boolean (e.g. BoolVal(True), x == 5, p, ...)) + - int (any type of integer including 'Expression's which are of type int (e.g. 1, 1 + 2, x, ...)) + - constant (integers that are constants. No 'Expression's allowed. (e.g. 1, 2, ...; NOT: 1 + 2, x, ...)) + - variable (variables that aren't 'Expression's. (e.g. x, y, p, q; NOT: x == y, x > y, ...)) + ~ Parameters: + - `expr`: the expression which we want to mutate + - `con`: the constraint in which the expression is found + ~ Returns: + - `path`: the path at which the expression would be mutated + - `ret_type`: the type that the expression is allowed to be mutated by, given either as a string or a boolean. + """ + # Define functions of which the arguments are restricted to be constants and the remaining path length the argument would be in + constant_restricted_functions = {Minimum: (1, (None,)), + Maximum: (1, (None,)), + Element: (2, (None,)), + Among: (2, (1, None)), + NValueExcept: (1, (1,)), + Circuit: (1, (None,)), + Inverse: (2, (None,)), + GlobalCardinalityCount: (2, (None,)), + 'wsum': (2, (0, None))} + variable_restricted_functions = {Table: (2, (0, None)), + NegativeTable: (2, (0, None)), + LexLess: (2, (None,)), + LexLessEq: (2, (None,)), + LexChainLess: (2, (None,)), + LexChainLessEq: (2, (None,))} + + paths = find_all_occurrences(con, expr) + path = random.choice(paths) + path_len = len(path) + for i, idx in enumerate(path): + if type(con) in constant_restricted_functions: + remaining_path_len, remaining_path = constant_restricted_functions[type(con)] + if path_len - i == remaining_path_len and expr_at_path(con, remaining_path, expr): + return path, 'constant' + elif isinstance(con, Operator) and con.name == 'wsum': + remaining_path_len, remaining_path = constant_restricted_functions['wsum'] + if path_len - i == remaining_path_len and expr_at_path(con, remaining_path, expr): + return path, 'constant' + elif type(con) in variable_restricted_functions: + remaining_path_len, remaining_path = variable_restricted_functions[type(con)] + if path_len - i == remaining_path_len and expr_at_path(con, remaining_path, expr): + return path, 'variable' + if hasattr(con, 'args') and not isinstance(con, NDVarArray) and con.name != 'boolval': + con = con.args[idx] + elif isinstance(con, list): + con = con[idx] + # We should only get here if the argument isn't in one of the functions above + return path, is_boolexpr(expr) + + +def type_aware_expression_replacement(constraints: list): """ Replaces a random expression of a random constraint from a list of given constraints. It replaces this expression by an operator with the same return type that takes as arguments the other expressions from all constraints. @@ -1383,9 +1511,9 @@ def type_aware_expression_replacement(constraints): function to swap out the constraints instead of adding them. ~ Parameters: - - constraints: a list of all the constraints to possibly be mutated + - `constraints`: a list of all the constraints to possibly be mutated ~ Return: - - final_cons: a list of the same constraints where one constraint has a mutated expression + - `final_cons`: a list of the same constraints where one constraint has a mutated expression """ final_cons = copy.deepcopy(constraints) # print(f"All constraints at the moment: {final_cons}") @@ -1393,17 +1521,17 @@ def type_aware_expression_replacement(constraints): rand_con = random.choice(final_cons) all_con_exprs = get_all_exprs(rand_con) expr = random.choice(all_con_exprs) - has_bool_return = is_boolexpr(expr) + path, ret_type = get_return_type(expr, rand_con) # Also gives us the taken path of the expression in the constraint # print(f"Changing constarint: {rand_con}") # print(f"Old expression: {expr}") # 2. Tel het aantal resterende params van elk type (van alle constraints of enkel in de constraint zelf?) all_exprs = get_all_exprs_mult(final_cons) # 3. Zoek een operator die <= aantal params nodig heeft met zelfde return type - new_expr = get_operator(all_exprs, has_bool_return) + new_expr = get_operator(all_exprs, ret_type) # 4. Vervang expression (+ vervang constraint) # print(f"New expression: {new_expr}") if new_expr: - new_con = mutate_con(rand_con, old_expr=expr, new_expr=new_expr) + new_con = replace_at_path(rand_con, path, new_expr=new_expr) # 5. Return the new constraints # final_cons.remove(rand_con) DOES NOT WORK because it uses == instead of 'is' index = None @@ -1416,13 +1544,17 @@ def type_aware_expression_replacement(constraints): final_cons.append(new_con) return final_cons + class MetamorphicError(Exception): pass + ''' returns a list of aritmetic expressions (as lists of indexes to traverse the expression tree) that occur in the input expression. One (random) candidate is taken from each level of the expression if there exists one ''' + + def pickaritmetic(con, log=[], candidates=[]): if hasattr(con, 'name'): if con.name == 'wsum': From a624d3f765ad76d0ee8753e59e65e82ef82dce3c Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Sun, 13 Apr 2025 23:04:51 +0200 Subject: [PATCH 12/58] added 20% chance to perform generation type mutation --- verifiers/solver_voting_count_verifier.py | 53 ++++++++++++++++++++-- verifiers/solver_voting_sat_verifier.py | 54 +++++++++++++++++++++-- 2 files changed, 101 insertions(+), 6 deletions(-) diff --git a/verifiers/solver_voting_count_verifier.py b/verifiers/solver_voting_count_verifier.py index 19188175..4a44a1df 100644 --- a/verifiers/solver_voting_count_verifier.py +++ b/verifiers/solver_voting_count_verifier.py @@ -35,9 +35,8 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti semanticFusionCountingwsum, semanticFusionCounting, semanticFusionCountingMinus, - semanticFusionCountingwsum, - type_aware_operator_replacement, - type_aware_expression_replacement] + semanticFusionCountingwsum] + self.gen_mutators = [type_aware_operator_replacement, type_aware_expression_replacement] self.mutators = [] self.original_model = None @@ -65,6 +64,54 @@ def initialize_run(self, is_rerun=False) -> None: self.mutators = [copy.deepcopy( self.cons)] # keep track of list of og_cons alternated with mutators that transformed it into the next list of og_cons. + def generate_mutations(self) -> None: + """ + Will generate random mutations based on mutations_per_model for the model + """ + for i in range(self.mutations_per_model): + # choose a metamorphic mutation, don't choose any from exclude_dict + + if random.random() < 0.8: + m = random.choice(self.mm_mutators) + else: + m = random.choice(self.gen_mutators) # 20% chance to choose gen-type mutator + + self.mutators += [self.seed] + # an error can occur in the transformations, so even before the solve call. + # log function and arguments in that case + self.mutators += [m] + try: + if m in {type_aware_operator_replacement, type_aware_expression_replacement}: + self.cons = m(self.cons) # apply an operator change and REPLACE constraints + else: + self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints + self.mutators += [copy.deepcopy(self.cons)] + except MetamorphicError as exc: + # add to exclude_dict, to avoid running into the same error + if self.model_file in self.exclude_dict: + self.exclude_dict[self.model_file] += [m] + else: + self.exclude_dict[self.model_file] = [m] + function, argument, e = exc.args + if isinstance(e, CPMpyException): + # expected behavior if we throw a cpmpy exception, do not log + return None + elif function == semanticFusion: + return None + # don't log semanticfusion crash + + print('I', end='', flush=True) + return dict(type=Fuzz_Test_ErrorTypes.internalfunctioncrash, + originalmodel_file=self.model_file, + exception=e, + function=function, + argument=argument, + stacktrace=traceback.format_exc(), + mutators=self.mutators, + constraints=self.cons, + originalmodel=self.original_model + ) + return None def verify_model(self, is_rerun=False) -> dict: try: model = cp.Model(self.cons) diff --git a/verifiers/solver_voting_sat_verifier.py b/verifiers/solver_voting_sat_verifier.py index 26b89992..f7f09fef 100644 --- a/verifiers/solver_voting_sat_verifier.py +++ b/verifiers/solver_voting_sat_verifier.py @@ -35,9 +35,8 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti semanticFusionCountingwsum, semanticFusionCounting, semanticFusionCountingMinus, - semanticFusionCountingwsum, - type_aware_operator_replacement, - type_aware_expression_replacement] + semanticFusionCountingwsum] + self.gen_mutators = [type_aware_operator_replacement, type_aware_expression_replacement] self.mutators = [] self.original_model = None @@ -56,6 +55,55 @@ def initialize_run(self, is_rerun=False) -> None: self.mutators = [copy.deepcopy( self.cons)] # keep track of list of og_cons alternated with mutators that transformed it into the next list of og_cons. + def generate_mutations(self) -> None: + """ + Will generate random mutations based on mutations_per_model for the model + """ + for i in range(self.mutations_per_model): + # choose a metamorphic mutation, don't choose any from exclude_dict + + if random.random() < 0.8: + m = random.choice(self.mm_mutators) + else: + m = random.choice(self.gen_mutators) # 20% chance to choose gen-type mutator + + self.mutators += [self.seed] + # an error can occur in the transformations, so even before the solve call. + # log function and arguments in that case + self.mutators += [m] + try: + if m in {type_aware_operator_replacement, type_aware_expression_replacement}: + self.cons = m(self.cons) # apply an operator change and REPLACE constraints + else: + self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints + self.mutators += [copy.deepcopy(self.cons)] + except MetamorphicError as exc: + # add to exclude_dict, to avoid running into the same error + if self.model_file in self.exclude_dict: + self.exclude_dict[self.model_file] += [m] + else: + self.exclude_dict[self.model_file] = [m] + function, argument, e = exc.args + if isinstance(e, CPMpyException): + # expected behavior if we throw a cpmpy exception, do not log + return None + elif function == semanticFusion: + return None + # don't log semanticfusion crash + + print('I', end='', flush=True) + return dict(type=Fuzz_Test_ErrorTypes.internalfunctioncrash, + originalmodel_file=self.model_file, + exception=e, + function=function, + argument=argument, + stacktrace=traceback.format_exc(), + mutators=self.mutators, + constraints=self.cons, + originalmodel=self.original_model + ) + return None + def verify_model(self, is_rerun=False) -> dict: try: model = cp.Model(self.cons) From 1c44e8e962455cab72d3084b9a65f120b935c4c8 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Sun, 13 Apr 2025 23:05:26 +0200 Subject: [PATCH 13/58] some bug fixes when generating new functions --- fuzz_test_utils/mutators.py | 51 +++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 7ee89c16..0e24d50f 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1048,13 +1048,19 @@ def get_all_non_op_exprs(con: Expression): if hasattr(con, 'args') and not isinstance(con, NDVarArray) and con.name != 'boolval': return sum((get_all_non_op_exprs(arg) for arg in con.args), []) elif isinstance(con, list): - if all([is_num(e) for e in con]): # wsum constants - return [] - else: - return [e for e in con] + return sum((get_all_non_op_exprs(e) for e in con), []) else: return [con] +# import cpmpy as cp +# m = cp.Model() +# p = cp.boolvar() +# a = cp.boolvar() +# m += cp.NegativeTable([p],[[a], [(~(a)) >= (p)], [p], [a], [(~p).implies(sum([-1*a, -1*p]) <= -2)]]) +# con = m.constraints[0] +# print(type(con.args[1])) +# print(con) +# m.solve() def get_all_exprs(con: Expression): """ @@ -1091,6 +1097,8 @@ def satisfies_args(func: Function, ints: int, bools: int, constants: int, vars: match func.name: case 'wsum': return constants >= 1 and ints + bools >= func.min_args and has_bool_return == func.bool_return + case 'pow': + return constants >= 1 and ints + bools >= 1 and has_bool_return == func.bool_return case _: return ((func.int_args == -1 or func.int_args <= ints + bools) and # enough int args (func.bool_args == -1 or func.bool_args <= bools) and # enough bool args @@ -1112,7 +1120,7 @@ def satisfies_args(func: Function, ints: int, bools: int, constants: int, vars: case 'IfThenElse' | 'Xor': return bools >= func.min_args and has_bool_return case 'Table' | 'NegativeTable': - return vars >= 1 and ints + bools >= func.min_args - 1 and has_bool_return + return vars >= 1 and constants >= max(1, func.min_args - 1) and has_bool_return case 'LexLess' | 'LexLessEq' | 'LexChainLess' | 'LexChainLessEq': return vars >= func.min_args and has_bool_return case 'Circuit' | 'Inverse': @@ -1135,7 +1143,7 @@ def get_new_operator(func: Function, ints: list, bools: list, constants: list, v comb = ints + bools match func.type: case 'op' | 'comp': - # Separate logic for wsum + # Separate logic for wsum and pow if func.name == 'wsum': # (-1, 0, False, 2, max_args, 2) amnt_args = random.randint(func.min_args // 2, min(len(constants), func.max_args) // 2) # First take constants @@ -1143,7 +1151,8 @@ def get_new_operator(func: Function, ints: list, bools: list, constants: list, v # Then the other expressions others = random.sample(comb, amnt_args) return Operator(func.name, [constants, others]) - + if func.name == 'pow': + args = random.choice(comb), random.choice(constants) # Logic for all other operators and comparisons is the same if func.int_args == -1: amnt_args = random.randint(func.min_args, min(len(comb), @@ -1171,7 +1180,8 @@ def get_new_operator(func: Function, ints: list, bools: list, constants: list, v case 'Element': amnt_args = random.randint(func.min_args, min(len(constants), func.max_args)) first_arg = random.sample(constants, amnt_args) - idx = random.randint(0, amnt_args - 1) + # idx = random.randint(0, amnt_args - 1) + idx = random.choice(ints) args = first_arg, idx case 'NValue': amnt_args = random.randint(func.min_args, min(len(comb), func.max_args)) @@ -1225,11 +1235,11 @@ def get_new_operator(func: Function, ints: list, bools: list, constants: list, v amnt_args = random.randint(func.min_args, min(len(bools), func.max_args)) args = random.sample(bools, amnt_args), case 'Table' | 'NegativeTable': - amnt_fst_arg = random.randint(1, min(len(variables), func.max_args // 4)) - amnt_snd_args = random.randint(1, min(len(comb), + amnt_fst_arg = random.randint(1, min(len(variables), len(constants), func.max_args // 4)) + amnt_snd_args = random.randint(1, min(len(constants), func.max_args - amnt_fst_arg) // amnt_fst_arg) * amnt_fst_arg fst_args = random.sample(variables, amnt_fst_arg) - snd_args = random.sample(comb, amnt_snd_args) + snd_args = random.sample(constants, amnt_snd_args) snd_args_transformed = [snd_args[i * amnt_fst_arg:(i + 1) * amnt_fst_arg] for i in range(int(amnt_snd_args / amnt_fst_arg))] args = fst_args, snd_args_transformed @@ -1267,12 +1277,15 @@ def get_operator(args: list, ret_type: str | bool): """ ints = [e for e in args if not (is_boolexpr(e) or isinstance(e, list))] bools = [e for e in args if is_boolexpr(e)] - constants = [e for e in args if - not hasattr(e, 'value')] # Is this the only way to extract constants only? (e.g. for wsum) - if ret_type == 'constant': - return random.choice(constants) # Some expressions can't be replaced by functions + constants = [e for e in args if isinstance(e, int)] variables = get_variables(args) - if ret_type == 'variable': + if ret_type == 'constant': + if constants: + return random.choice(constants) # Some expressions can't be replaced by functions + intvars = [e for e in variables if not (is_boolexpr(e) or isinstance(e, list))] + if intvars: + return random.choice(intvars) + if ret_type == 'variable' and variables: return random.choice(variables) # Some expressions can't be replaced by functions ints_cnt = len(ints) bools_cnt = len(bools) @@ -1470,6 +1483,10 @@ def get_return_type(expr: Expression, con: Expression): Circuit: (1, (None,)), Inverse: (2, (None,)), GlobalCardinalityCount: (2, (None,)), + Table: (3, (1, None)), + NegativeTable: (3, (1, None)), + Count: (1, (1, None)), + 'pow': (1, (1, None)), 'wsum': (2, (0, None))} variable_restricted_functions = {Table: (2, (0, None)), NegativeTable: (2, (0, None)), @@ -1515,11 +1532,13 @@ def type_aware_expression_replacement(constraints: list): ~ Return: - `final_cons`: a list of the same constraints where one constraint has a mutated expression """ + # print("="*100) final_cons = copy.deepcopy(constraints) # print(f"All constraints at the moment: {final_cons}") # 1. Neem een (random) expression van een (random) constraint en de return type rand_con = random.choice(final_cons) all_con_exprs = get_all_exprs(rand_con) + # print(f"all_con_exprs: {all_con_exprs}") expr = random.choice(all_con_exprs) path, ret_type = get_return_type(expr, rand_con) # Also gives us the taken path of the expression in the constraint # print(f"Changing constarint: {rand_con}") From 3da3a5f7f7bbfb7e9baab9dfe9aa3cc1c54844ba Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Wed, 16 Apr 2025 09:33:29 +0200 Subject: [PATCH 14/58] bugfixes in generation of new expressions --- fuzz_test_utils/mutators.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 0e24d50f..96d4923e 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1153,6 +1153,7 @@ def get_new_operator(func: Function, ints: list, bools: list, constants: list, v return Operator(func.name, [constants, others]) if func.name == 'pow': args = random.choice(comb), random.choice(constants) + return Operator(func.name, args) # Logic for all other operators and comparisons is the same if func.int_args == -1: amnt_args = random.randint(func.min_args, min(len(comb), @@ -1193,7 +1194,7 @@ def get_new_operator(func: Function, ints: list, bools: list, constants: list, v args = first_arg, last_arg case 'Among': amnt_fst_arg = random.randint(func.min_args // 2, min(len(comb), func.max_args) // 2) - amnt_snd_arg = random.randint(func.min_args // 2, min(len(constants), func.max_args) // 2) + amnt_snd_arg = random.randint(func.min_args // 2, min(len(constants), func.max_args // 2)) first_arg = random.sample(comb, amnt_fst_arg) second_arg = random.sample(constants, amnt_snd_arg) args = first_arg, second_arg @@ -1216,12 +1217,11 @@ def get_new_operator(func: Function, ints: list, bools: list, constants: list, v half_amnt_args = random.randint(func.min_args // 2, min(len(variables), func.max_args) // 2) args = random.sample(variables, half_amnt_args), random.sample(variables, half_amnt_args) case 'LexChainLess' | 'LexChainLessEq': - amnt_args = random.randint(func.min_args // 2, min(len(variables), func.max_args) // 2) - all_args = random.sample(variables, amnt_args) - divisors = [i for i in range(1, amnt_args + 1) if amnt_args % i == 0] - fst_dimension = random.choice(divisors) - snd_dimension = int(amnt_args / fst_dimension) - args = [all_args[i * fst_dimension:(i + 1) * fst_dimension] for i in range(snd_dimension)], + amnt_fst_args = random.randint(1, min(len(variables), func.max_args // 4)) + fst_args = random.sample(variables, amnt_fst_args) + amnt_snd_args = random.randint(1, func.max_args // 4) # the amnt of arrays of len of the first one + snd_args = [random.sample(variables, amnt_fst_args) for _ in range(amnt_snd_args)] + args = [fst_args] + snd_args, case 'Circuit': amnt_args = random.randint(func.min_args, func.max_args) args = random.sample(range(amnt_args), amnt_args), @@ -1275,7 +1275,7 @@ def get_operator(args: list, ret_type: str | bool): ~ Returns: - A new Expression with arguments from the given list and with given return type """ - ints = [e for e in args if not (is_boolexpr(e) or isinstance(e, list))] + ints = [e for e in args if not (is_boolexpr(e) or isinstance(e, list) or isinstance(e, NDVarArray) or isinstance(e, tuple))] bools = [e for e in args if is_boolexpr(e)] constants = [e for e in args if isinstance(e, int)] variables = get_variables(args) @@ -1503,8 +1503,8 @@ def get_return_type(expr: Expression, con: Expression): remaining_path_len, remaining_path = constant_restricted_functions[type(con)] if path_len - i == remaining_path_len and expr_at_path(con, remaining_path, expr): return path, 'constant' - elif isinstance(con, Operator) and con.name == 'wsum': - remaining_path_len, remaining_path = constant_restricted_functions['wsum'] + elif isinstance(con, Operator) and (con.name == 'wsum' or con.name == 'pow'): + remaining_path_len, remaining_path = constant_restricted_functions[con.name] if path_len - i == remaining_path_len and expr_at_path(con, remaining_path, expr): return path, 'constant' elif type(con) in variable_restricted_functions: From 6227201c95f2d711702b18e55d13cd54b10f1ec1 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Wed, 16 Apr 2025 09:34:31 +0200 Subject: [PATCH 15/58] Addition of fuctions to classify bugs while testing (hardcoded) --- fuzz_test_utils/output_writer.py | 50 ++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/fuzz_test_utils/output_writer.py b/fuzz_test_utils/output_writer.py index 98399c8e..9c7d91a5 100644 --- a/fuzz_test_utils/output_writer.py +++ b/fuzz_test_utils/output_writer.py @@ -3,6 +3,9 @@ from os.path import join from datetime import datetime +from fuzz_test_utils import type_aware_operator_replacement, type_aware_expression_replacement + + def create_error_output_text(error_data: dict) -> str: """ This helper function will create a more readable text from the error_data dict @@ -24,6 +27,53 @@ def create_error_output_text(error_data: dict) -> str: # return a more readable/user friendly error description ready to write to a file return f"An error occured while running a test\n\nUsed solver: {error_data['solver']}\n{verifier_text}\nWith {error_data['mutations_per_model']} mutations per model\nWith seed: {error_data['seed']}\nThe test failed in {execution_time_text}\n\nError Details:\n{error_text}" +def match_conditions(exc_str): + return [ + (lambda s: "'bool' object is not iterable" in s, "01_bool_obj_not_iterable/"), + (lambda s: "slice indices must be integers or None or have an __index__ method" in s, "02_slice_indices_must_be_ints/"), + (lambda s: all(x in s for x in ["Expecting value: line", "column ", "(char "]), "03_line_x_column_x/"), + (lambda s: any(x in s for x in ["lhs cannot be an integer at this point!", "not supported: model.get_or_make_boolean_index(boolval(False))"]), "04_lhs_no_int/"), + (lambda s: "'bool' object has no attribute 'implies'" in s, "05_bool_obj_no_implies/"), + (lambda s: " has no constraints" in s, "06_no_constraints_TO_FIX/"), + (lambda s: any(x in s for x in ["An int_mod must have a strictly positive modulo argument", "The domain of the divisor cannot contain 0", "Modulo with a divisor domain containing 0 is not supported.", "Power operator: For integer values, exponent must be non-negative:"]), "07_div0_pow-neg/"), + (lambda s: "object of type '_BoolVarImpl' has no len()" in s, "08_object_type_boolvarimpl_no_len/"), + (lambda s: "'int' object has no attribute 'lb'" in s, "09_int_obj_no_attr_lb/"), + (lambda s: all(x in s for x in ["Cannot convert", "to Choco variable"]), "10_cannot_convert_to_choco_var/"), + (lambda s: any(x in s for x in ["Translation of gurobi status 11 to CPMpy status not implemented", "KeyboardInterrupt", "cannot access local variable 'proc' where it is not associated with a value"]), "11_keyboard_interrupt/"), + (lambda s: "Cannot modify read-only attribute 'args', use 'update_args()'" in s, "12_cant_modify_args/"), + (lambda s: "Gurobi only supports division by constants, but got " in s, "13_gurobi_only_div_cst/"), + (lambda s: any(x in s for x in ["Could not resolve host: ", "Maximum number of failing server authorization attempts reached"]), "_temp/"), + (lambda s: "'int' object has no attribute 'get_bounds'" in s, "14_int_objc_no_attr_getbounds/"), + (lambda s: all(x in s for x in ["Domain of ", " only contains 0"]), "15_dom_only_contains_0/"), + (lambda s: "'int' object has no attribute 'handle'" in s, "16_int_obj_no_attr_handle/"), + (lambda s: "not an linear expression: boolval(True)" in s, "17_not_lin_expr_boolval/"), + (lambda s: "empty range for randrange()" in s, "18_empty_randrange/"), + (lambda s: "Amount of solutions of the two solvers are not equal." in s, "19_amnt_sol_neq/"), + (lambda s: "in method 'int_array_set', argument 2 of type 'int'" in s, "20_argx_type_y/"), + (lambda s: "'bool' object has no attribute 'has_subexpr'" in s, "21_bool_obj_no_has_subexpr") + ] + +def get_output_dir(error_data): + output_dir = "test_output/" + error = error_data["error"] + exception = error["exception"] + exc_str = str(exception) + mutators = error['mutators'][2::3] + # Split into two different directories for errors including generation-type mutations + if any([fn in {type_aware_operator_replacement, type_aware_expression_replacement} for fn in mutators]): + output_dir += "GEN/" + else: + output_dir += "MUT/" + + # Check whether the exception matches some format + for condition, path in match_conditions(exc_str): + if condition(exc_str): + output_dir += path + break + else: + output_dir += "UNKNOWN/" + + return output_dir def write_error(error_data: dict, output_dir: str) -> None: """ From 57b4085ffa77520e1d0970611d3db5c4c01280c4 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Wed, 16 Apr 2025 09:35:59 +0200 Subject: [PATCH 16/58] Removed check before mutations to increase testing performance --- verifiers/solver_voting_count_verifier.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/verifiers/solver_voting_count_verifier.py b/verifiers/solver_voting_count_verifier.py index 4a44a1df..62cd164d 100644 --- a/verifiers/solver_voting_count_verifier.py +++ b/verifiers/solver_voting_count_verifier.py @@ -53,11 +53,6 @@ def initialize_run(self, is_rerun=False) -> None: assert len(self.solvers) == 2, f"2 solvers required, {len(self.solvers)} given." if 'gurobi' in [s.lower() for s in self.solvers]: self.sol_lim = 10000 # TODO: is hardcode best idea? - self.sol_count_1 = cp.Model(self.cons).solveAll(solver=self.solvers[0],solution_limit=self.sol_lim) - self.sol_count_2 = cp.Model(self.cons).solveAll(solver=self.solvers[1],solution_limit=self.sol_lim) - else: - self.sol_count_1 = cp.Model(self.cons).solveAll(solver=self.solvers[0]) - self.sol_count_2 = cp.Model(self.cons).solveAll(solver=self.solvers[1]) # assert self.sol_count_1 == self.sol_count_2, f"{self.solvers} don't agree on amount of solutions (before mutations): {self.sol_count_1} and {self.sol_count_2}" @@ -143,8 +138,7 @@ def verify_model(self, is_rerun=False) -> dict: originalmodel_file=self.model_file, exception=f"Amount of solutions of the two solvers are not equal." f" #Solutions of {solver_1}: {new_count_1}." - f" #Solutions of {solver_2}: {new_count_2}." - f" Before: {self.sol_count_1} and {self.sol_count_2}", + f" #Solutions of {solver_2}: {new_count_2}.", constraints=self.cons, mutators=self.mutators, model=model, From 4b7cf8995aee0c99201d4dc14b0597961a983c45 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Wed, 16 Apr 2025 09:36:46 +0200 Subject: [PATCH 17/58] Added logic to classify bugs while testing --- verifiers/verifier_runner.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/verifiers/verifier_runner.py b/verifiers/verifier_runner.py index 7fefa18d..1070ca5f 100644 --- a/verifiers/verifier_runner.py +++ b/verifiers/verifier_runner.py @@ -1,9 +1,11 @@ import glob import math +import os import random import warnings from os.path import join +from fuzz_test_utils.output_writer import get_output_dir from verifiers import * from fuzz_test_utils import Fuzz_Test_ErrorTypes def get_all_verifiers(single_solver) -> list: @@ -59,6 +61,8 @@ def run_verifiers(current_amount_of_tests, current_amount_of_error, lock, solver lock.acquire() try: error_data = {'verifier':random_verifier.getName(),'solver' : solver, 'mutations_per_model' : mutations_per_model, "seed": random_seed, "execution_time": execution_time, "error" :error} + output_dir = get_output_dir(error_data) if get_output_dir(error_data) else output_dir + os.makedirs(output_dir, exist_ok=True) # create if it doesn't already exist write_error(error_data,output_dir) current_amount_of_error.value +=1 finally: From 1fb4456947969d7ed5c53c104102beab0b710056 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Thu, 17 Apr 2025 11:10:11 +0200 Subject: [PATCH 18/58] Changed major bug in choosing model file where the choice was made using the same seed --- verifiers/verifier_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verifiers/verifier_runner.py b/verifiers/verifier_runner.py index 1070ca5f..9d3f0dac 100644 --- a/verifiers/verifier_runner.py +++ b/verifiers/verifier_runner.py @@ -51,7 +51,7 @@ def run_verifiers(current_amount_of_tests, current_amount_of_error, lock, solver for folder in folders: fmodels.extend(glob.glob(join(folder,random_verifier.getType(), "*"))) if len(fmodels) > 0: - fmodel = random.choice(fmodels) + fmodel = random.Random().choice(fmodels) # random.choice used the random.seed()! Same models were being tested! start_time = time.time() error = random_verifier.run(fmodel) From a4811a512fa3576313047f338e0f4c2418f0bae5 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Thu, 17 Apr 2025 11:57:01 +0200 Subject: [PATCH 19/58] Addition of new verifier and accompanying mutator --- fuzz_test_utils/mutators.py | 224 ++++++++++++-- verifiers/__init__.py | 4 + verifiers/strengthening_weakening_verifier.py | 277 ++++++++++++++++++ verifiers/verifier_runner.py | 4 +- 4 files changed, 490 insertions(+), 19 deletions(-) create mode 100644 verifiers/strengthening_weakening_verifier.py diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 96d4923e..3e9992f5 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -963,7 +963,6 @@ def type_aware_operator_replacement(constraints: list): return Exception(e) -# TODO: use this to make a "strengthen_expression" and "weaken_expression" def mutate_op_expression(expr: Expression, con: Expression): """ Mutates the constraint containing the expression by mutating said expression. @@ -1047,20 +1046,11 @@ def get_all_non_op_exprs(con: Expression): """ if hasattr(con, 'args') and not isinstance(con, NDVarArray) and con.name != 'boolval': return sum((get_all_non_op_exprs(arg) for arg in con.args), []) - elif isinstance(con, list): + elif isinstance(con, list) or isinstance(con, NDVarArray): return sum((get_all_non_op_exprs(e) for e in con), []) else: return [con] -# import cpmpy as cp -# m = cp.Model() -# p = cp.boolvar() -# a = cp.boolvar() -# m += cp.NegativeTable([p],[[a], [(~(a)) >= (p)], [p], [a], [(~p).implies(sum([-1*a, -1*p]) <= -2)]]) -# con = m.constraints[0] -# print(type(con.args[1])) -# print(con) -# m.solve() def get_all_exprs(con: Expression): """ @@ -1282,7 +1272,7 @@ def get_operator(args: list, ret_type: str | bool): if ret_type == 'constant': if constants: return random.choice(constants) # Some expressions can't be replaced by functions - intvars = [e for e in variables if not (is_boolexpr(e) or isinstance(e, list))] + intvars = [e for e in variables if not (is_boolexpr(e) or isinstance(e, list) or isinstance(e, NDVarArray))] if intvars: return random.choice(intvars) if ret_type == 'variable' and variables: @@ -1400,7 +1390,7 @@ def find_all_occurrences(con: Expression, target_expr: Expression): for i, arg in enumerate(con.args): for path in find_all_occurrences(arg, target_expr): occurrences.append((i,) + path) # Add index to the path - elif isinstance(con, list): + elif isinstance(con, list) or isinstance(con, NDVarArray): for i, arg in enumerate(con): for path in find_all_occurrences(arg, target_expr): occurrences.append((i,) + path) @@ -1427,7 +1417,7 @@ def replace_at_path(con: Expression, path: tuple, new_expr: Expression): for idx in path[:-1]: if hasattr(parent, 'args') and not isinstance(parent, NDVarArray) and parent.name != 'boolval': parent = parent.args[idx] - elif isinstance(parent, list): + elif isinstance(parent, list) or isinstance(parent, NDVarArray): parent = parent[idx] # Change the arguments of the parent @@ -1435,7 +1425,7 @@ def replace_at_path(con: Expression, path: tuple, new_expr: Expression): args = list(parent.args) args[path[-1]] = new_expr parent.update_args(args) - elif isinstance(parent, list): + elif isinstance(parent, list) or isinstance(parent, NDVarArray): parent[path[-1]] = new_expr return con @@ -1452,7 +1442,7 @@ def expr_at_path(con: Expression, path: tuple, expr: Expression): if idx is not None: if hasattr(con, 'args') and not isinstance(con, NDVarArray) and con.name != 'boolval': con = con.args[idx] - elif isinstance(con, list): + elif isinstance(con, list) or isinstance(con, NDVarArray): con = con[idx] else: return len(find_all_occurrences(con, expr)) > 0 @@ -1513,7 +1503,7 @@ def get_return_type(expr: Expression, con: Expression): return path, 'variable' if hasattr(con, 'args') and not isinstance(con, NDVarArray) and con.name != 'boolval': con = con.args[idx] - elif isinstance(con, list): + elif isinstance(con, list) or isinstance(con, NDVarArray): con = con[idx] # We should only get here if the argument isn't in one of the functions above return path, is_boolexpr(expr) @@ -1564,6 +1554,206 @@ def type_aware_expression_replacement(constraints: list): return final_cons +def has_positive_parity(expr: Expression, con: Expression, curr_path: tuple) -> tuple | None: + """ + Function to retrieve the parity of an expression `expr` in the given constraint `con`. This means it shows + whether the constraint weakens or strengthens when the expression does. + ~ Parameters: + - `expr`: the expression that would be strengthened/weakened. + - `con`: the constraint that would be strengthened/weakened. + ~ Returns: + - `pos_parity`: True if the constraint strengthens (weakens) upon the expression strengthening, + False if it doesn't, + None if it is unknown. + """ + # Basecase 1: `expr` cannot be strengthened or weakened + if hasattr(expr, 'name'): + # NOTE: these are the simplest operators to strengthen/weaken (by just changing the operator into another one). + # Other operators could be changed in another way too (e.g. add/remove elements in the second argument + # of the expression x in [1, 2, 3, 4]). This could be included later and then changed accordingly in + # `strengthening_weakening_mutator()`. + changeable_ops = {'and', 'or', '->', 'xor', '==', '!=', '<=', '<', '>=', '>'} + if expr.name not in changeable_ops: + return None + else: + return None + + # Basecase 2: + if expr is con: + return True, curr_path + + # Recursively check in the arguments and change result upon encountering "not" operators + if con.name == 'not': + curr_path += 0, + neg_res = has_positive_parity(expr, con.args[0], curr_path) + return not neg_res, curr_path if neg_res is not None else None + if con.name == 'and' or con.name == 'or': + l, r = con.args + subtrees = [(0, l), (1, r)] + random.shuffle(subtrees) + for path, subtree in subtrees: + if any(expr is e for e in get_all_exprs(subtree)): # check if expr in l or r + curr_path += path, + return has_positive_parity(expr, subtree, curr_path) + raise Exception(f"The given expression {expr} is not in either of the subtrees {l} or {r}.") + + # If the constraint is anything else, we don't know the parity + return None + + +def strengthen_expr(expr: Expression, path: tuple, con: Expression) -> Expression: + """ + Strengthen the given expression `expr` in the given constraint `con`. + ~ Parameters: + - `expr`: the expression that will be strengthened. + - `con`: the constraint that will be strengthened/weakened. + ~ Returns: + - `con`: the constraint after the mutation. + """ + match expr.name: # {'or', '->', '!=', '<=', '>='} + case 'or': # and, xor, !=, <, > + args = expr.args + if len(args) != 2: + return con + comps = ['!=', '<', '>'] + ops = ['and'] + others = ['xor'] + new_op = random.choice(comps + ops + others) + case '->': # and, ==, < + comps = ['==', '<'] + ops = ['and'] + new_op = random.choice(comps + ops) + args = expr.args + case '!=': # <, > + comps = ['<', '>'] + new_op = random.choice(comps) + args = expr.args + case '<=': # <, == + comps = ['<', '=='] + new_op = random.choice(comps) + args = expr.args + case '>=': # >, == + comps = ['>', '=='] + new_op = random.choice(comps) + args = expr.args + if new_op in comps: + expr = Comparison(new_op, *args) + con = replace_at_path(con, path, expr) + elif new_op in ops: + expr = Operator(new_op, args) + con = replace_at_path(con, path, expr) + elif new_op == 'xor': + expr = Xor(args) + con = replace_at_path(con, path, expr) + return con + + +def weaken_expr(expr: Expression, path: tuple, con: Expression) -> Expression: + """ + Weaken the given expression `expr` in the given constraint `con`. + ~ Parameters: + - `expr`: the expression that will be weakened. + - `con`: the constraint that will be strengthened/weakened. + ~ Returns: + - `con`: the constraint after the mutation. + """ + match expr.name: # {'and', 'xor', '==', '<', '>'} + case 'and': # or, ->, ==, <=, >= + args = expr.args + if len(args) != 2: + return con + comps = ['==', '<=', '>='] + ops = ['or', '->'] + new_op = random.choice(comps + ops) + case 'xor': # or + args = expr.args + if len(args) != 2: + return con + new_op = 'or' + case '==': # <=, >= + comps = ['<=', '>='] + new_op = random.choice(comps) + args = expr.args + case '<': # !=, <= + comps = ['!=', '<='] + new_op = random.choice(comps) + args = expr.args + case '>': # != >= + comps = ['!=', '>='] + new_op = random.choice(comps) + args = expr.args + if new_op in comps: + expr = Comparison(new_op, *args) + con = replace_at_path(con, path, expr) + elif new_op in ops: + expr = Operator(new_op, args) + con = replace_at_path(con, path, expr) + elif new_op == 'xor': + expr = Xor(args) + con = replace_at_path(con, path, expr) + return con + + +def is_changeable(strengthen: bool, expr: Expression, pos_parity: bool) -> bool: + if strengthen ^ pos_parity: # weakening + return expr.name not in {'or', '->', '!=', '<=', '>='} + else: # strengthening + return expr.name not in {'and', 'xor', '==', '>', '<'} + + +def strengthening_weakening_mutator(constraints: list, strengthen: bool = True) -> list | Exception: + """ + Strengthens or weakens a (random?) constraint from a list of given constraints by replacing an operator of this constraint. + IMPORTANT: This can change satisfiability of the constraint! Only to be used with verifiers that allow this! + This means it returns a list of ALL constraints and has to be handled accordingly in the 'generate_mutations' + function to swap out the constraints instead of adding them. + + ~ Parameters: + - `constraints`: a list of all the constraints to possibly be mutated + - `strengthen`: a boolean indicating whether the mutator should strengthen or weaken a constraint + ~ Return: + - `final_cons`: a list of the same constraints where one constraint has a mutated operator + """ + try: + print_str = "strength" if strengthen else "weak" + final_cons = copy.deepcopy(constraints) + # pick a random constraint and calculate whether they have a mutable expression until they do + candidates = list(set(constraints)) + random.shuffle(candidates) + for con in candidates: + exprs = [] + for e in get_all_exprs(con): + parity = has_positive_parity(e, con, curr_path=tuple()) + if parity is not None and is_changeable(strengthen, e, parity[0]): + exprs.append((e, parity)) + if exprs: + break + else: # In case there isn't any mutable (weakening/strengthening depending on `strengthen`) expression in any constraint + # print(f"Couldn't find a constraint to {print_str}en.") + return final_cons + + # Remove the constraint from the constraints + final_cons.remove(con) + + # Choose an expression to change + expr, (pos_parity, path) = random.choice(exprs) + + # print(f"{print_str}ening constraint {con} by changing expression {expr}.") + + # Mutate this expression + if strengthen ^ pos_parity: # weaken if parity is different from `strengthen` + con = weaken_expr(expr, path, con) + else: + con = strengthen_expr(expr, path, con) + + # Add the changed constraint back + final_cons.append(con) + return final_cons + + except Exception as e: + return Exception(e) + + class MetamorphicError(Exception): pass diff --git a/verifiers/__init__.py b/verifiers/__init__.py index 13220ba6..6f9ab637 100644 --- a/verifiers/__init__.py +++ b/verifiers/__init__.py @@ -15,6 +15,7 @@ from .optimization_verifier import Optimization_Verifier from .solver_voting_sat_verifier import Solver_Vote_Sat_Verifier from .solver_voting_count_verifier import Solver_Vote_Count_Verifier +from .strengthening_weakening_verifier import Strengthening_Weakening_Verifier from .verifier_runner import run_verifiers, get_all_verifiers @@ -40,6 +41,9 @@ def lookup_verifier(verfier_name: str) -> Verifier: elif verfier_name == "solver_vote_count_verifier": return Solver_Vote_Count_Verifier + elif verfier_name == "strengthening_weakening_verifier": + return Strengthening_Weakening_Verifier + else: raise ValueError(f"Error verifier with name {verfier_name} does not exist") return None \ No newline at end of file diff --git a/verifiers/strengthening_weakening_verifier.py b/verifiers/strengthening_weakening_verifier.py new file mode 100644 index 00000000..58f55191 --- /dev/null +++ b/verifiers/strengthening_weakening_verifier.py @@ -0,0 +1,277 @@ +from verifiers import * + +class Strengthening_Weakening_Verifier(Verifier): + """ + The Solver Count Verifier will verify if the satisfiability is the same for all solvers after running multiple mutations + """ + + def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, time_limit: float, seed: int): + self.name = "strengthening_weakening_verifier" + self.type = 'sat' + + self.solvers = solver + + self.mutations_per_model = mutations_per_model + self.exclude_dict = exclude_dict + self.time_limit = time_limit + self.seed = seed + self.mm_mutators = [xor_morph, and_morph, or_morph, implies_morph, not_morph, + linearize_constraint_morph, + flatten_morph, + only_numexpr_equality_morph, + normalized_numexpr_morph, + reify_rewrite_morph, + only_bv_reifies_morph, + only_positive_bv_morph, + flat2cnf_morph, + toplevel_list_morph, + decompose_in_tree_morph, + push_down_negation_morph, + simplify_boolean_morph, + canonical_comparison_morph, + aritmetic_comparison_morph, + semanticFusionCounting, + semanticFusionCountingMinus, + semanticFusionCountingwsum, + semanticFusionCounting, + semanticFusionCountingMinus, + semanticFusionCountingwsum] + self.gen_mutators = [type_aware_operator_replacement, type_aware_expression_replacement] + self.mutators = [] + self.original_model = None + + def initialize_run(self, is_rerun=False) -> None: + if self.original_model == None: + with open(self.model_file, 'rb') as fpcl: + self.original_model = pickle.loads(fpcl.read()) + self.cons = self.original_model.constraints + assert (len(self.cons) > 0), f"{self.model_file} has no constraints" + self.cons = toplevel_list(self.cons) + if is_rerun: + print([(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)]) + + assert len(self.solvers) == 2, f"2 solvers required, {len(self.solvers)} given." + if 'gurobi' in [s.lower() for s in self.solvers]: + self.sol_lim = 10000 # TODO: is hardcode best idea? + + # assert self.sol_count_1 == self.sol_count_2, f"{self.solvers} don't agree on amount of solutions (before mutations): {self.sol_count_1} and {self.sol_count_2}" + + self.mutators = [copy.deepcopy( + self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. + + def generate_mutations(self) -> None | dict: + """ + Will generate random mutations based on mutations_per_model for the model + """ + for i in range(self.mutations_per_model): + + # choose a mutator. 33% of the time, this will be a strengthening/weakening mutation. + rand = random.random() + if rand <= 0.33: + m = strengthening_weakening_mutator + elif rand <= 0.8633: # ~~ remaining 80% of 0.67 (8/15) + m = random.choice(self.mm_mutators) + else: + m = random.choice(self.gen_mutators) # ~~ remaining 20% to choose gen-type mutator (2/15) + + self.mutators += [self.seed] + # an error can occur in the transformations, so even before the solve call. + # log function and arguments in that case + self.mutators += [m] + try: + if m in self.gen_mutators: + self.cons = m(self.cons) # apply a generative (non-metamorphic) mutation and REPLACE constraints + elif m == strengthening_weakening_mutator: + model = cp.Model(self.cons) + solver_1 = self.solvers[0] + solver_2 = self.solvers[1] + if hasattr(self, 'sol_lim'): + count_1 = model.solveAll(solver=solver_1, solution_limit=self.sol_lim) + else: + count_1 = model.solveAll(solver=solver_1) + if count_1 > 1: + self.cons = m(self.cons, strengthen=True) + elif count_1 < 1: + self.cons = m(self.cons, strengthen=False) + elif random.random() < 0.8: # If only 1 solution remains, we just go on normally instead + m = random.choice(self.mm_mutators) + self.cons += m(self.cons) + else: + m = random.choice(self.gen_mutators) + self.cons = m(self.cons) + else: + self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints + self.mutators += [copy.deepcopy(self.cons)] + + except MetamorphicError as exc: + # add to exclude_dict, to avoid running into the same error + if self.model_file in self.exclude_dict: + self.exclude_dict[self.model_file] += [m] + else: + self.exclude_dict[self.model_file] = [m] + function, argument, e = exc.args + if isinstance(e, CPMpyException): + # expected behavior if we throw a cpmpy exception, do not log + return None + elif function == semanticFusion: + return None + # don't log semanticfusion crash + + print('I', end='', flush=True) + return dict(type=Fuzz_Test_ErrorTypes.internalfunctioncrash, + originalmodel_file=self.model_file, + exception=e, + function=function, + argument=argument, + stacktrace=traceback.format_exc(), + mutators=self.mutators, + constraints=self.cons, + originalmodel=self.original_model + ) + return None + + + def verify_model(self, is_rerun=False) -> None | dict: + try: + model = cp.Model(self.cons) + if is_rerun: + print([(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)]) + time_limit = max(1, min(200, + self.time_limit - time.time())) # set the max time limit to the given time limit or to 1 if the self.time_limit-time.time() would be smaller then 1 + + solver_1 = self.solvers[0] + solver_2 = self.solvers[1] + if hasattr(self, 'sol_lim'): + new_count_1 = model.solveAll(solver=solver_1, solution_limit=self.sol_lim) + new_count_2 = model.solveAll(solver=solver_2, solution_limit=self.sol_lim) + else: + new_count_1 = model.solveAll(solver=solver_1) + new_count_2 = model.solveAll(solver=solver_2) + + if model.status().runtime > time_limit - 10: + # timeout, skip + print('T', end='', flush=True) + return None + elif new_count_1 == new_count_2: + # has to be same + print('.', end='', flush=True) + return None + else: + print('X', end='', flush=True) + return dict(type=Fuzz_Test_ErrorTypes.failed_model, + originalmodel_file=self.model_file, + exception=f"Amount of solutions of the two solvers are not equal." + f" #Solutions of {solver_1}: {new_count_1}." + f" #Solutions of {solver_2}: {new_count_2}.", + constraints=self.cons, + mutators=self.mutators, + model=model, + originalmodel=self.original_model + ) + + except Exception as e: + if isinstance(e, (CPMpyException, NotImplementedError)): + # expected error message, ignore + return None + print('E', end='', flush=True) + return dict(type=Fuzz_Test_ErrorTypes.internalcrash, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + mutators=self.mutators, + model=model, + originalmodel=self.original_model + ) + # if you got here, the model failed... + return dict(type=Fuzz_Test_ErrorTypes.failed_model, + originalmodel_file=self.model_file, + constraints=self.cons, + mutators=self.mutators, + model=newModel, + originalmodel=self.original_model + ) + + def run(self, model_file: str) -> dict: + """ + This function will run a single test on the given model + """ + try: + random.seed(self.seed) + self.model_file = model_file + self.initialize_run() + gen_mutations_error = self.generate_mutations() + + # check if no error occurred while generation the mutations + if gen_mutations_error == None: + return self.verify_model() + else: + return gen_mutations_error + except AssertionError as e: + print("A", end='', flush=True) + error_type = Fuzz_Test_ErrorTypes.crashed_model + if "is not sat" in str(e): + error_type = Fuzz_Test_ErrorTypes.unsat_model + elif "has no constraints" in str(e): + error_type = Fuzz_Test_ErrorTypes.no_constraints_model + return dict(type=error_type, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + originalmodel=self.original_model + ) + + except Exception as e: + print('C', end='', flush=True) + return dict(type=Fuzz_Test_ErrorTypes.crashed_model, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + mutators=self.mutators, + originalmodel=self.original_model + ) + + def rerun(self, error: dict) -> dict: + """ + This function will rerun a previous failed test + """ + try: + random.seed(self.seed) + self.model_file = error["originalmodel_file"] + self.original_model = error["originalmodel"] + self.exclude_dict = {} + self.initialize_run(is_rerun=True) + gen_mutations_error = self.generate_mutations() + + # check if no error occured while generation the mutations + if gen_mutations_error == None: + return self.verify_model(is_rerun=True) + else: + return gen_mutations_error + + except AssertionError as e: + print("A", end='', flush=True) + type = Fuzz_Test_ErrorTypes.crashed_model + if "is not sat" in str(e): + type = Fuzz_Test_ErrorTypes.unsat_model + elif "has no constraints" in str(e): + type = Fuzz_Test_ErrorTypes.no_constraints_model + return dict(type=type, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + originalmodel=self.original_model + ) + + except Exception as e: + print('C', end='', flush=True) + return dict(type=Fuzz_Test_ErrorTypes.crashed_model, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + originalmodel=self.original_model + ) \ No newline at end of file diff --git a/verifiers/verifier_runner.py b/verifiers/verifier_runner.py index 1070ca5f..b55e5345 100644 --- a/verifiers/verifier_runner.py +++ b/verifiers/verifier_runner.py @@ -12,7 +12,7 @@ def get_all_verifiers(single_solver) -> list: if single_solver: return [Solution_Verifier,Optimization_Verifier,Model_Count_Verifier,Metamorphic_Verifier,Equivalance_Verifier] else: - return [Solver_Vote_Count_Verifier, Solver_Vote_Sat_Verifier] + return [Solver_Vote_Count_Verifier, Solver_Vote_Sat_Verifier, Strengthening_Weakening_Verifier] def run_verifiers(current_amount_of_tests, current_amount_of_error, lock, solver: list[str], mutations_per_model: int, folders: list, max_error_treshold: int, output_dir: str, time_limit: float) -> None: """ @@ -51,7 +51,7 @@ def run_verifiers(current_amount_of_tests, current_amount_of_error, lock, solver for folder in folders: fmodels.extend(glob.glob(join(folder,random_verifier.getType(), "*"))) if len(fmodels) > 0: - fmodel = random.choice(fmodels) + fmodel = random.Random().choice(fmodels) # random.choice used the random.seed()! Same models were being tested! start_time = time.time() error = random_verifier.run(fmodel) From c6a7fb7137b47c0c0223d2883664dd57e13d10e2 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Thu, 17 Apr 2025 11:57:47 +0200 Subject: [PATCH 20/58] Some refactoring of the old verifiers --- verifiers/solver_voting_count_verifier.py | 10 +++++----- verifiers/solver_voting_sat_verifier.py | 10 +++++----- verifiers/verifier.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/verifiers/solver_voting_count_verifier.py b/verifiers/solver_voting_count_verifier.py index 62cd164d..56970012 100644 --- a/verifiers/solver_voting_count_verifier.py +++ b/verifiers/solver_voting_count_verifier.py @@ -57,9 +57,9 @@ def initialize_run(self, is_rerun=False) -> None: # assert self.sol_count_1 == self.sol_count_2, f"{self.solvers} don't agree on amount of solutions (before mutations): {self.sol_count_1} and {self.sol_count_2}" self.mutators = [copy.deepcopy( - self.cons)] # keep track of list of og_cons alternated with mutators that transformed it into the next list of og_cons. + self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. - def generate_mutations(self) -> None: + def generate_mutations(self) -> None | dict: """ Will generate random mutations based on mutations_per_model for the model """ @@ -76,8 +76,8 @@ def generate_mutations(self) -> None: # log function and arguments in that case self.mutators += [m] try: - if m in {type_aware_operator_replacement, type_aware_expression_replacement}: - self.cons = m(self.cons) # apply an operator change and REPLACE constraints + if m in self.gen_mutators: + self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints else: self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints self.mutators += [copy.deepcopy(self.cons)] @@ -162,7 +162,7 @@ def verify_model(self, is_rerun=False) -> dict: # if you got here, the model failed... return dict(type=Fuzz_Test_ErrorTypes.failed_model, originalmodel_file=self.model_file, - constraints=self.og_cons, + constraints=self.cons, mutators=self.mutators, model=newModel, originalmodel=self.original_model diff --git a/verifiers/solver_voting_sat_verifier.py b/verifiers/solver_voting_sat_verifier.py index f7f09fef..3e305e4a 100644 --- a/verifiers/solver_voting_sat_verifier.py +++ b/verifiers/solver_voting_sat_verifier.py @@ -53,15 +53,15 @@ def initialize_run(self, is_rerun=False) -> None: # No other preparation necessary self.mutators = [copy.deepcopy( - self.cons)] # keep track of list of og_cons alternated with mutators that transformed it into the next list of og_cons. + self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. def generate_mutations(self) -> None: """ Will generate random mutations based on mutations_per_model for the model """ for i in range(self.mutations_per_model): - # choose a metamorphic mutation, don't choose any from exclude_dict + # choose a mutation if random.random() < 0.8: m = random.choice(self.mm_mutators) else: @@ -72,8 +72,8 @@ def generate_mutations(self) -> None: # log function and arguments in that case self.mutators += [m] try: - if m in {type_aware_operator_replacement, type_aware_expression_replacement}: - self.cons = m(self.cons) # apply an operator change and REPLACE constraints + if m in self.gen_mutators: + self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints else: self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints self.mutators += [copy.deepcopy(self.cons)] @@ -157,7 +157,7 @@ def verify_model(self, is_rerun=False) -> dict: # if you got here, the model failed... return dict(type=Fuzz_Test_ErrorTypes.failed_model, originalmodel_file=self.model_file, - constraints=self.og_cons, + constraints=self.cons, mutators=self.mutators, model=newModel, originalmodel=self.original_model diff --git a/verifiers/verifier.py b/verifiers/verifier.py index bf64d82d..64619077 100644 --- a/verifiers/verifier.py +++ b/verifiers/verifier.py @@ -57,7 +57,7 @@ def generate_mutations(self) -> None: # log function and arguments in that case self.mutators += [m] try: - if m in {type_aware_operator_replacement, type_aware_expression_replacement}: + if m in {type_aware_operator_replacement, type_aware_expression_replacement, strengthening_weakening_mutator}: self.cons = m(self.cons) # apply an operator change and REPLACE constraints else: self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints From b3c29a05344bb371df3aba90f7d149a8f3d5deec Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Thu, 17 Apr 2025 15:40:28 +0200 Subject: [PATCH 21/58] added some exception raising, added some bug classification, fixed parity calculation. (multiple files) --- fuzz_test_utils/mutators.py | 78 ++++++++++++++++---------------- fuzz_test_utils/output_writer.py | 4 +- 2 files changed, 43 insertions(+), 39 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 3e9992f5..a5f19d6f 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -960,7 +960,7 @@ def type_aware_operator_replacement(constraints: list): return final_cons except Exception as e: - return Exception(e) + raise Exception(e) def mutate_op_expression(expr: Expression, con: Expression): @@ -1135,7 +1135,7 @@ def get_new_operator(func: Function, ints: list, bools: list, constants: list, v case 'op' | 'comp': # Separate logic for wsum and pow if func.name == 'wsum': # (-1, 0, False, 2, max_args, 2) - amnt_args = random.randint(func.min_args // 2, min(len(constants), func.max_args) // 2) + amnt_args = random.randint(func.min_args // 2, min(len(constants), func.max_args // 2)) # First take constants constants = random.sample(constants, amnt_args) # Then the other expressions @@ -1522,36 +1522,39 @@ def type_aware_expression_replacement(constraints: list): ~ Return: - `final_cons`: a list of the same constraints where one constraint has a mutated expression """ - # print("="*100) - final_cons = copy.deepcopy(constraints) - # print(f"All constraints at the moment: {final_cons}") - # 1. Neem een (random) expression van een (random) constraint en de return type - rand_con = random.choice(final_cons) - all_con_exprs = get_all_exprs(rand_con) - # print(f"all_con_exprs: {all_con_exprs}") - expr = random.choice(all_con_exprs) - path, ret_type = get_return_type(expr, rand_con) # Also gives us the taken path of the expression in the constraint - # print(f"Changing constarint: {rand_con}") - # print(f"Old expression: {expr}") - # 2. Tel het aantal resterende params van elk type (van alle constraints of enkel in de constraint zelf?) - all_exprs = get_all_exprs_mult(final_cons) - # 3. Zoek een operator die <= aantal params nodig heeft met zelfde return type - new_expr = get_operator(all_exprs, ret_type) - # 4. Vervang expression (+ vervang constraint) - # print(f"New expression: {new_expr}") - if new_expr: - new_con = replace_at_path(rand_con, path, new_expr=new_expr) - # 5. Return the new constraints - # final_cons.remove(rand_con) DOES NOT WORK because it uses == instead of 'is' - index = None - for i, constraint in enumerate(final_cons): - if constraint is rand_con: - index = i - break - if index is not None: - del final_cons[index] - final_cons.append(new_con) - return final_cons + try: + # print("="*100) + final_cons = copy.deepcopy(constraints) + # print(f"All constraints at the moment: {final_cons}") + # 1. Neem een (random) expression van een (random) constraint en de return type + rand_con = random.choice(final_cons) + all_con_exprs = get_all_exprs(rand_con) + # print(f"all_con_exprs: {all_con_exprs}") + expr = random.choice(all_con_exprs) + path, ret_type = get_return_type(expr, rand_con) # Also gives us the taken path of the expression in the constraint + # print(f"Changing constarint: {rand_con}") + # print(f"Old expression: {expr}") + # 2. Tel het aantal resterende params van elk type (van alle constraints of enkel in de constraint zelf?) + all_exprs = get_all_exprs_mult(final_cons) + # 3. Zoek een operator die <= aantal params nodig heeft met zelfde return type + new_expr = get_operator(all_exprs, ret_type) + # 4. Vervang expression (+ vervang constraint) + # print(f"New expression: {new_expr}") + if new_expr: + new_con = replace_at_path(rand_con, path, new_expr=new_expr) + # 5. Return the new constraints + # final_cons.remove(rand_con) DOES NOT WORK because it uses == instead of 'is' + index = None + for i, constraint in enumerate(final_cons): + if constraint is rand_con: + index = i + break + if index is not None: + del final_cons[index] + final_cons.append(new_con) + return final_cons + except Exception as e: + raise Exception(e) def has_positive_parity(expr: Expression, con: Expression, curr_path: tuple) -> tuple | None: @@ -1588,14 +1591,13 @@ def has_positive_parity(expr: Expression, con: Expression, curr_path: tuple) -> neg_res = has_positive_parity(expr, con.args[0], curr_path) return not neg_res, curr_path if neg_res is not None else None if con.name == 'and' or con.name == 'or': - l, r = con.args - subtrees = [(0, l), (1, r)] - random.shuffle(subtrees) + subtrees = list(enumerate(con.args)) + random.shuffle(subtrees) # to randomize argument selection for path, subtree in subtrees: - if any(expr is e for e in get_all_exprs(subtree)): # check if expr in l or r + if any(expr is e for e in get_all_exprs(subtree)): curr_path += path, return has_positive_parity(expr, subtree, curr_path) - raise Exception(f"The given expression {expr} is not in either of the subtrees {l} or {r}.") + raise Exception(f"The given expression {expr} is not in either of the subtrees {subtrees}.") # If the constraint is anything else, we don't know the parity return None @@ -1751,7 +1753,7 @@ def strengthening_weakening_mutator(constraints: list, strengthen: bool = True) return final_cons except Exception as e: - return Exception(e) + raise Exception(e) class MetamorphicError(Exception): diff --git a/fuzz_test_utils/output_writer.py b/fuzz_test_utils/output_writer.py index 9c7d91a5..76d4f36b 100644 --- a/fuzz_test_utils/output_writer.py +++ b/fuzz_test_utils/output_writer.py @@ -50,7 +50,9 @@ def match_conditions(exc_str): (lambda s: "empty range for randrange()" in s, "18_empty_randrange/"), (lambda s: "Amount of solutions of the two solvers are not equal." in s, "19_amnt_sol_neq/"), (lambda s: "in method 'int_array_set', argument 2 of type 'int'" in s, "20_argx_type_y/"), - (lambda s: "'bool' object has no attribute 'has_subexpr'" in s, "21_bool_obj_no_has_subexpr") + (lambda s: "'bool' object has no attribute 'has_subexpr'" in s, "21_bool_obj_no_has_subexpr"), + (lambda s: len(s) == 0, "22_empty_exception"), + (lambda s: "Not a known var " in s, "23_wsum_second_arg_vars") ] def get_output_dir(error_data): From 7b60431cd9afe25b77696b4d6bdfb831194d7ca6 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Fri, 18 Apr 2025 16:48:00 +0200 Subject: [PATCH 22/58] Change to InDomain to make first arg have only variables and second arg only constants --- fuzz_test_utils/mutators.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 3e9992f5..2320d2a4 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1109,7 +1109,7 @@ def satisfies_args(func: Function, ints: int, bools: int, constants: int, vars: return constants >= func.min_args and has_bool_return case 'IfThenElse' | 'Xor': return bools >= func.min_args and has_bool_return - case 'Table' | 'NegativeTable': + case 'Table' | 'NegativeTable' | 'InDomain': return vars >= 1 and constants >= max(1, func.min_args - 1) and has_bool_return case 'LexLess' | 'LexLessEq' | 'LexChainLess' | 'LexChainLessEq': return vars >= func.min_args and has_bool_return @@ -1135,7 +1135,7 @@ def get_new_operator(func: Function, ints: list, bools: list, constants: list, v case 'op' | 'comp': # Separate logic for wsum and pow if func.name == 'wsum': # (-1, 0, False, 2, max_args, 2) - amnt_args = random.randint(func.min_args // 2, min(len(constants), func.max_args) // 2) + amnt_args = random.randint(func.min_args // 2, min(len(constants), func.max_args // 2)) # First take constants constants = random.sample(constants, amnt_args) # Then the other expressions @@ -1183,13 +1183,13 @@ def get_new_operator(func: Function, ints: list, bools: list, constants: list, v last_arg = random.choice(comb) args = first_arg, last_arg case 'Among': - amnt_fst_arg = random.randint(func.min_args // 2, min(len(comb), func.max_args) // 2) + amnt_fst_arg = random.randint(func.min_args // 2, min(len(comb), func.max_args // 2)) amnt_snd_arg = random.randint(func.min_args // 2, min(len(constants), func.max_args // 2)) first_arg = random.sample(comb, amnt_fst_arg) second_arg = random.sample(constants, amnt_snd_arg) args = first_arg, second_arg case 'NValueExcept': - amnt_fst_arg = random.randint(func.min_args // 2, min(len(comb), func.max_args) // 2) + amnt_fst_arg = random.randint(func.min_args // 2, min(len(comb), func.max_args // 2)) first_arg = random.sample(comb, amnt_fst_arg) second_arg = random.choice(constants) args = first_arg, second_arg @@ -1200,11 +1200,11 @@ def get_new_operator(func: Function, ints: list, bools: list, constants: list, v amnt_args = random.randint(func.min_args, min(len(comb), func.max_args)) args = random.sample(comb, amnt_args) case 'AllDifferentExceptN' | 'AllEqualExceptN': - amnt_fst_args = random.randint(func.min_args // 2, min(len(comb), func.max_args) // 2) + amnt_fst_args = random.randint(func.min_args // 2, min(len(comb), func.max_args // 2)) amnt_snd_args = random.randint(func.min_args // 2, min(len(comb), func.max_args - amnt_fst_args)) args = random.sample(comb, amnt_fst_args), random.sample(comb, amnt_snd_args) case 'LexLess' | 'LexLessEq': - half_amnt_args = random.randint(func.min_args // 2, min(len(variables), func.max_args) // 2) + half_amnt_args = random.randint(func.min_args // 2, min(len(variables), func.max_args // 2)) args = random.sample(variables, half_amnt_args), random.sample(variables, half_amnt_args) case 'LexChainLess' | 'LexChainLessEq': amnt_fst_args = random.randint(1, min(len(variables), func.max_args // 4)) @@ -1234,9 +1234,10 @@ def get_new_operator(func: Function, ints: list, bools: list, constants: list, v range(int(amnt_snd_args / amnt_fst_arg))] args = fst_args, snd_args_transformed case 'InDomain': - amnt_args = random.randint(func.min_args, min(len(comb), func.max_args)) - all_args = random.sample(comb, amnt_args) - args = all_args[0], all_args[1:] + fst_arg = random.choice(variables) + amnt_snd_args = random.randint(func.min_args - 1, min(len(constants), func.max_args - 1)) + snd_args = random.sample(constants, amnt_snd_args) + args = fst_arg, snd_args case 'NoOverlap': amnt_args = random.randint(func.min_args, min(len(comb), func.max_args // 3)) args = random.sample(comb, amnt_args), random.sample(comb, amnt_args), random.sample(comb, From 801e85899f0c5a88361c4ee2715a4532a033c328 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Fri, 18 Apr 2025 16:50:22 +0200 Subject: [PATCH 23/58] Removed is_rerun argument --- verifiers/verifier.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/verifiers/verifier.py b/verifiers/verifier.py index 64619077..7191f9b2 100644 --- a/verifiers/verifier.py +++ b/verifiers/verifier.py @@ -158,12 +158,12 @@ def rerun(self,error: dict) -> dict: self.model_file = error["originalmodel_file"] self.original_model = error["originalmodel"] self.exclude_dict = {} - self.initialize_run(is_rerun=True) + self.initialize_run() gen_mutations_error = self.generate_mutations() # check if no error occured while generation the mutations if gen_mutations_error == None: - return self.verify_model(is_rerun=True) + return self.verify_model() else: return gen_mutations_error # self.og_cons = error["constraints"] From fe104058df6f5a4f0f814d04562696ab6cb5c839 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Fri, 18 Apr 2025 16:57:23 +0200 Subject: [PATCH 24/58] added variables to output dict and removed is_rerun variable to print these variables --- verifiers/solver_voting_count_verifier.py | 20 +++++++++----- verifiers/solver_voting_sat_verifier.py | 18 ++++++++----- verifiers/strengthening_weakening_verifier.py | 26 +++++++++++++------ 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/verifiers/solver_voting_count_verifier.py b/verifiers/solver_voting_count_verifier.py index 56970012..bd42b6ff 100644 --- a/verifiers/solver_voting_count_verifier.py +++ b/verifiers/solver_voting_count_verifier.py @@ -1,3 +1,4 @@ +from result_attributes import get_nr_mm_mutations, get_nr_mutations, get_nr_gen_mutations from verifiers import * class Solver_Vote_Count_Verifier(Verifier): @@ -40,15 +41,13 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti self.mutators = [] self.original_model = None - def initialize_run(self, is_rerun=False) -> None: + def initialize_run(self) -> None: if self.original_model == None: with open(self.model_file, 'rb') as fpcl: self.original_model = pickle.loads(fpcl.read()) self.cons = self.original_model.constraints assert (len(self.cons) > 0), f"{self.model_file} has no constraints" self.cons = toplevel_list(self.cons) - if is_rerun: - print([(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)]) assert len(self.solvers) == 2, f"2 solvers required, {len(self.solvers)} given." if 'gurobi' in [s.lower() for s in self.solvers]: @@ -64,7 +63,6 @@ def generate_mutations(self) -> None | dict: Will generate random mutations based on mutations_per_model for the model """ for i in range(self.mutations_per_model): - # choose a metamorphic mutation, don't choose any from exclude_dict if random.random() < 0.8: m = random.choice(self.mm_mutators) @@ -104,15 +102,17 @@ def generate_mutations(self) -> None | dict: stacktrace=traceback.format_exc(), mutators=self.mutators, constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) return None def verify_model(self, is_rerun=False) -> dict: try: model = cp.Model(self.cons) - if is_rerun: - print([(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)]) - time_limit = max(1, min(200, + + one_run_time_lim = 30 # TODO: change back to 200 after results? + time_limit = max(1, min(one_run_time_lim, self.time_limit - time.time())) # set the max time limit to the given time limit or to 1 if the self.time_limit-time.time() would be smaller then 1 solver_1 = self.solvers[0] @@ -142,6 +142,8 @@ def verify_model(self, is_rerun=False) -> dict: constraints=self.cons, mutators=self.mutators, model=model, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) @@ -157,6 +159,8 @@ def verify_model(self, is_rerun=False) -> dict: constraints=self.cons, mutators=self.mutators, model=model, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) # if you got here, the model failed... @@ -165,5 +169,7 @@ def verify_model(self, is_rerun=False) -> dict: constraints=self.cons, mutators=self.mutators, model=newModel, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) diff --git a/verifiers/solver_voting_sat_verifier.py b/verifiers/solver_voting_sat_verifier.py index 3e305e4a..21a4959f 100644 --- a/verifiers/solver_voting_sat_verifier.py +++ b/verifiers/solver_voting_sat_verifier.py @@ -40,15 +40,13 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti self.mutators = [] self.original_model = None - def initialize_run(self, is_rerun=False) -> None: + def initialize_run(self) -> None: if self.original_model == None: with open(self.model_file, 'rb') as fpcl: self.original_model = pickle.loads(fpcl.read()) self.cons = self.original_model.constraints assert (len(self.cons) > 0), f"{self.model_file} has no constraints" self.cons = toplevel_list(self.cons) - if is_rerun: - print([(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)]) # No other preparation necessary @@ -100,11 +98,14 @@ def generate_mutations(self) -> None: stacktrace=traceback.format_exc(), mutators=self.mutators, constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) return None - def verify_model(self, is_rerun=False) -> dict: + + def check_for_bug(self) -> None | dict: try: model = cp.Model(self.cons) time_limit = max(1, min(200, @@ -132,11 +133,12 @@ def verify_model(self, is_rerun=False) -> dict: return dict(type=Fuzz_Test_ErrorTypes.failed_model, originalmodel_file=self.model_file, exception=f"Results of the two solvers are not equal." - f" Result of {solver_1}: {solver_1_print}." - f" Result of {solver_2}: {solver_2_print}.", + f" Result of {self.solvers[0]}: {solver_1_print}." + f" Result of {self.solvers[1]}: {solver_2_print}.", constraints=self.cons, mutators=self.mutators, model=model, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)], originalmodel=self.original_model ) @@ -152,6 +154,8 @@ def verify_model(self, is_rerun=False) -> dict: constraints=self.cons, mutators=self.mutators, model=model, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) # if you got here, the model failed... @@ -160,5 +164,7 @@ def verify_model(self, is_rerun=False) -> dict: constraints=self.cons, mutators=self.mutators, model=newModel, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) diff --git a/verifiers/strengthening_weakening_verifier.py b/verifiers/strengthening_weakening_verifier.py index 58f55191..f7422ac7 100644 --- a/verifiers/strengthening_weakening_verifier.py +++ b/verifiers/strengthening_weakening_verifier.py @@ -40,15 +40,13 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti self.mutators = [] self.original_model = None - def initialize_run(self, is_rerun=False) -> None: + def initialize_run(self) -> None: if self.original_model == None: with open(self.model_file, 'rb') as fpcl: self.original_model = pickle.loads(fpcl.read()) self.cons = self.original_model.constraints assert (len(self.cons) > 0), f"{self.model_file} has no constraints" self.cons = toplevel_list(self.cons) - if is_rerun: - print([(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)]) assert len(self.solvers) == 2, f"2 solvers required, {len(self.solvers)} given." if 'gurobi' in [s.lower() for s in self.solvers]: @@ -126,16 +124,15 @@ def generate_mutations(self) -> None | dict: stacktrace=traceback.format_exc(), mutators=self.mutators, constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)], originalmodel=self.original_model ) return None - def verify_model(self, is_rerun=False) -> None | dict: + def verify_model(self) -> None | dict: try: model = cp.Model(self.cons) - if is_rerun: - print([(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)]) time_limit = max(1, min(200, self.time_limit - time.time())) # set the max time limit to the given time limit or to 1 if the self.time_limit-time.time() would be smaller then 1 @@ -166,6 +163,7 @@ def verify_model(self, is_rerun=False) -> None | dict: constraints=self.cons, mutators=self.mutators, model=model, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)], originalmodel=self.original_model ) @@ -181,6 +179,8 @@ def verify_model(self, is_rerun=False) -> None | dict: constraints=self.cons, mutators=self.mutators, model=model, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) # if you got here, the model failed... @@ -189,6 +189,8 @@ def verify_model(self, is_rerun=False) -> None | dict: constraints=self.cons, mutators=self.mutators, model=newModel, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) @@ -219,6 +221,8 @@ def run(self, model_file: str) -> dict: exception=e, stacktrace=traceback.format_exc(), constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) @@ -230,6 +234,8 @@ def run(self, model_file: str) -> dict: stacktrace=traceback.format_exc(), constraints=self.cons, mutators=self.mutators, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) @@ -242,12 +248,12 @@ def rerun(self, error: dict) -> dict: self.model_file = error["originalmodel_file"] self.original_model = error["originalmodel"] self.exclude_dict = {} - self.initialize_run(is_rerun=True) + self.initialize_run() gen_mutations_error = self.generate_mutations() # check if no error occured while generation the mutations if gen_mutations_error == None: - return self.verify_model(is_rerun=True) + return self.verify_model() else: return gen_mutations_error @@ -263,6 +269,8 @@ def rerun(self, error: dict) -> dict: exception=e, stacktrace=traceback.format_exc(), constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) @@ -273,5 +281,7 @@ def rerun(self, error: dict) -> dict: exception=e, stacktrace=traceback.format_exc(), constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) \ No newline at end of file From 747c00542b5b874696b7408f8d9477d86f0c4e7e Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Fri, 18 Apr 2025 16:58:18 +0200 Subject: [PATCH 25/58] some minor additions to classification of bugs + removal of GEN/MUT split. Better split should be coming soon. --- fuzz_test_utils/output_writer.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/fuzz_test_utils/output_writer.py b/fuzz_test_utils/output_writer.py index 9c7d91a5..07765d52 100644 --- a/fuzz_test_utils/output_writer.py +++ b/fuzz_test_utils/output_writer.py @@ -29,7 +29,7 @@ def create_error_output_text(error_data: dict) -> str: def match_conditions(exc_str): return [ - (lambda s: "'bool' object is not iterable" in s, "01_bool_obj_not_iterable/"), + (lambda s: any(x in s for x in ["'bool' object is not iterable", "'_BoolVarImpl' object is not iterable"]), "01_bool_obj_not_iterable/"), (lambda s: "slice indices must be integers or None or have an __index__ method" in s, "02_slice_indices_must_be_ints/"), (lambda s: all(x in s for x in ["Expecting value: line", "column ", "(char "]), "03_line_x_column_x/"), (lambda s: any(x in s for x in ["lhs cannot be an integer at this point!", "not supported: model.get_or_make_boolean_index(boolval(False))"]), "04_lhs_no_int/"), @@ -50,20 +50,15 @@ def match_conditions(exc_str): (lambda s: "empty range for randrange()" in s, "18_empty_randrange/"), (lambda s: "Amount of solutions of the two solvers are not equal." in s, "19_amnt_sol_neq/"), (lambda s: "in method 'int_array_set', argument 2 of type 'int'" in s, "20_argx_type_y/"), - (lambda s: "'bool' object has no attribute 'has_subexpr'" in s, "21_bool_obj_no_has_subexpr") + (lambda s: "'bool' object has no attribute 'has_subexpr'" in s, "21_bool_obj_no_has_subexpr/"), + (lambda s: "'int' object has no attribute 'is_bool'" in s, "22_int_obj_no_is_bool/") ] def get_output_dir(error_data): - output_dir = "test_output/" + output_dir = "quicktest/" error = error_data["error"] exception = error["exception"] exc_str = str(exception) - mutators = error['mutators'][2::3] - # Split into two different directories for errors including generation-type mutations - if any([fn in {type_aware_operator_replacement, type_aware_expression_replacement} for fn in mutators]): - output_dir += "GEN/" - else: - output_dir += "MUT/" # Check whether the exception matches some format for condition, path in match_conditions(exc_str): From 7ef203752f072bc656b10cbcc433951c0f6bd4ed Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Sat, 19 Apr 2025 12:05:23 +0200 Subject: [PATCH 26/58] added variables (along with their bounds) to the error dictionaries --- verifiers/verifier.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/verifiers/verifier.py b/verifiers/verifier.py index 7191f9b2..348403c6 100644 --- a/verifiers/verifier.py +++ b/verifiers/verifier.py @@ -85,6 +85,8 @@ def generate_mutations(self) -> None: stacktrace=traceback.format_exc(), mutators=self.mutators, constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) return None @@ -133,6 +135,8 @@ def run(self, model_file: str) -> dict: exception=e, stacktrace=traceback.format_exc(), constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) @@ -143,6 +147,8 @@ def run(self, model_file: str) -> dict: exception=e, stacktrace=traceback.format_exc(), constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], mutators=self.mutators, originalmodel=self.original_model ) @@ -181,6 +187,8 @@ def rerun(self,error: dict) -> dict: exception=e, stacktrace=traceback.format_exc(), constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) @@ -191,6 +199,8 @@ def rerun(self,error: dict) -> dict: exception=e, stacktrace=traceback.format_exc(), constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], originalmodel=self.original_model ) From a4aef0f7a0a0af3b3a2f2bed115ba7c3c2649c6d Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Wed, 23 Apr 2025 12:45:47 +0200 Subject: [PATCH 27/58] fixed bug where expression wasn't found when in NDVarArray --- fuzz_test_utils/mutators.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 2320d2a4..b6eff33e 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1385,8 +1385,10 @@ def find_all_occurrences(con: Expression, target_expr: Expression): - `occurrences`: a list of paths (as tuples) to each occurrence. """ occurrences = [] - if con is target_expr: - occurrences.append(()) # Current node is the target + # np.int32 didn't match with `is` + if (isinstance(con, np.int32) and isinstance(target_expr, np.int32) and con == target_expr) or \ + con is target_expr: + occurrences.append(()) if hasattr(con, 'args') and not isinstance(con, NDVarArray) and con.name != 'boolval': for i, arg in enumerate(con.args): for path in find_all_occurrences(arg, target_expr): From 76c6913f5959dacbab14a159d50b9d0722f6881e Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Wed, 23 Apr 2025 12:54:24 +0200 Subject: [PATCH 28/58] added function writing errors to csv + addition of some new error classification --- fuzz_test_utils/output_writer.py | 175 ++++++++++++++++++++++++++++--- 1 file changed, 163 insertions(+), 12 deletions(-) diff --git a/fuzz_test_utils/output_writer.py b/fuzz_test_utils/output_writer.py index 07765d52..9b7ec4b9 100644 --- a/fuzz_test_utils/output_writer.py +++ b/fuzz_test_utils/output_writer.py @@ -1,9 +1,11 @@ import math import pickle +import csv +import glob +import os from os.path import join from datetime import datetime - -from fuzz_test_utils import type_aware_operator_replacement, type_aware_expression_replacement +from fuzz_test_utils.mutators import type_aware_operator_replacement, type_aware_expression_replacement, strengthening_weakening_mutator def create_error_output_text(error_data: dict) -> str: @@ -35,7 +37,7 @@ def match_conditions(exc_str): (lambda s: any(x in s for x in ["lhs cannot be an integer at this point!", "not supported: model.get_or_make_boolean_index(boolval(False))"]), "04_lhs_no_int/"), (lambda s: "'bool' object has no attribute 'implies'" in s, "05_bool_obj_no_implies/"), (lambda s: " has no constraints" in s, "06_no_constraints_TO_FIX/"), - (lambda s: any(x in s for x in ["An int_mod must have a strictly positive modulo argument", "The domain of the divisor cannot contain 0", "Modulo with a divisor domain containing 0 is not supported.", "Power operator: For integer values, exponent must be non-negative:"]), "07_div0_pow-neg/"), + (lambda s: any(x in s for x in ["or-tools does not accept a 'modulo' operation where '0' is in the domain of the divisor", "An int_mod must have a strictly positive modulo argument", "The domain of the divisor cannot contain 0", "Modulo with a divisor domain containing 0 is not supported.", "Power operator: For integer values, exponent must be non-negative:"]), "07_div0_pow-neg/"), (lambda s: "object of type '_BoolVarImpl' has no len()" in s, "08_object_type_boolvarimpl_no_len/"), (lambda s: "'int' object has no attribute 'lb'" in s, "09_int_obj_no_attr_lb/"), (lambda s: all(x in s for x in ["Cannot convert", "to Choco variable"]), "10_cannot_convert_to_choco_var/"), @@ -46,16 +48,18 @@ def match_conditions(exc_str): (lambda s: "'int' object has no attribute 'get_bounds'" in s, "14_int_objc_no_attr_getbounds/"), (lambda s: all(x in s for x in ["Domain of ", " only contains 0"]), "15_dom_only_contains_0/"), (lambda s: "'int' object has no attribute 'handle'" in s, "16_int_obj_no_attr_handle/"), - (lambda s: "not an linear expression: boolval(True)" in s, "17_not_lin_expr_boolval/"), + (lambda s: "not an linear expression: " in s, "17_not_lin_expr_bool/"), (lambda s: "empty range for randrange()" in s, "18_empty_randrange/"), - (lambda s: "Amount of solutions of the two solvers are not equal." in s, "19_amnt_sol_neq/"), + (lambda s: "Results of the solvers are not equal. Solver results:" in s, "19_amnt_sol_neq/"), (lambda s: "in method 'int_array_set', argument 2 of type 'int'" in s, "20_argx_type_y/"), (lambda s: "'bool' object has no attribute 'has_subexpr'" in s, "21_bool_obj_no_has_subexpr/"), - (lambda s: "'int' object has no attribute 'is_bool'" in s, "22_int_obj_no_is_bool/") + (lambda s: "'int' object has no attribute 'is_bool'" in s, "22_int_obj_no_is_bool/"), + (lambda s: "not supported: model.get_or_make_boolean_index(" in s, "23_not_supported_get_or_make_boolean_index/"), + (lambda s: "'BoolVal' object has no attribute 'get_integer_var_value_map'" in s, "24_not_known_var/") ] -def get_output_dir(error_data): - output_dir = "quicktest/" +def get_logging_dir(error_data, logging_dir): + logging_dir += "/" error = error_data["error"] exception = error["exception"] exc_str = str(exception) @@ -63,12 +67,12 @@ def get_output_dir(error_data): # Check whether the exception matches some format for condition, path in match_conditions(exc_str): if condition(exc_str): - output_dir += path + logging_dir += path break else: - output_dir += "UNKNOWN/" + logging_dir += "UNKNOWN/" - return output_dir + return logging_dir def write_error(error_data: dict, output_dir: str) -> None: """ @@ -85,4 +89,151 @@ def write_error(error_data: dict, output_dir: str) -> None: pickle.dump(error_data, file=ff) with open(join(output_dir, f"{error_data['error']['type'].name}_{date_text}.txt"), "w") as ff: - ff.write(create_error_output_text(error_data)) \ No newline at end of file + ff.write(create_error_output_text(error_data)) + + +# Below are all help functions for writing the csv file +def get_bug_class(error_data): + exc_str = str(error_data['error']['exception']) + # Check whether the exception matches some format + for condition, classification in match_conditions(exc_str): + if condition(exc_str): + bug_class = classification[3:-1] + break + else: + bug_class = "UNKNOWN" + return bug_class + + +def get_verifier(error_data): + return error_data['verifier'] + + +def get_solvers(error_data): + return error_data['solver'] + + +def get_seed(error_data): + return error_data['error']['seed'] + + +def get_time_taken(error_data): + return error_data['execution_time'] + + +def get_bug_type(error_data): + return error_data['error']['type'] + + +def get_exception(error_data): + return error_data['error']['exception'] + + +def get_original_cons(error_data): + return error_data['error']['originalmodel'].constraints + + +def get_current_cons(error_data): + return error_data['error']['constraints'] + + +def get_mutations(error_data): + if 'mutators' in error_data['error']: + return error_data['error']['mutators'][2::3] + else: + return [] + + +def get_nr_mm_mutations(error_data): + return len([m for m in get_mutations(error_data) if + m not in {type_aware_expression_replacement, type_aware_operator_replacement, + strengthening_weakening_mutator}]) + + +def get_nr_gen_mutations(error_data): + return len([m for m in get_mutations(error_data) if + m in {type_aware_expression_replacement, type_aware_operator_replacement, + strengthening_weakening_mutator}]) + + +def get_nr_mutations(error_data): + return len(get_mutations(error_data)) + + +def get_bugged_solver(error_data): + error = error_data['error'] + all_solvers = ['minizinc', 'ortools', 'gurobi', 'choco', 'z3'] + if 'stacktrace' in error: + stacktrace = error['stacktrace'] + for solver in all_solvers: + if solver in str(stacktrace): + return solver + elif 'exception' in error: # In case of a failed model (no crash) + exc = error['exception'] + possibly_bugged = [s for s in all_solvers if s in str(exc)] + if possibly_bugged: + return possibly_bugged + return 'UNKNOWN' + + +def get_objective(error_data): + return error_data['error']['originalmodel'].objective + + +def get_variables(error_data): + return error_data['error']['variables'] + + +def get_nr_solve_checks(error_data): + return error_data['error']['nr_solve_checks'] + + +def get_cause(error_data): + return error_data['error']['caused_by'] + + +def get_nr_timed_out_solve_calls(error_data): + return error_data['error']['nr_timed_out'] + + +def write_csv(error_data: dict, output_path) -> None: + columns_and_functions = [ + ("bug_class", get_bug_class), + ("verifier", get_verifier), + ("solvers", get_solvers), + ("seed", get_seed), + ("time_taken", get_time_taken), + ("bug_type", get_bug_type), + ("exception", get_exception), + ("original_constraints", get_original_cons), + ("current_constraints", get_current_cons), + ("total_nr_mutations", get_nr_mutations), + ("nr_mm_mutations", get_nr_mm_mutations), + ("nr_gen_mutations", get_nr_gen_mutations), + ("bugged_solver", get_bugged_solver), + ("objective", get_objective), + ("variables", get_variables), + ("nr_solve_checks", get_nr_solve_checks), + ("bug_cause", get_cause), + ("nr_timed_out_solve_calls", get_nr_timed_out_solve_calls) + ] + + file_exists = os.path.isfile(output_path) + + with open(output_path, mode='a', newline='', encoding='utf-8') as csvfile: + writer = csv.writer(csvfile) + + if not file_exists: + # Write header + headers = [col for col, _ in columns_and_functions] + writer.writerow(headers) + + # Write the error data row + row = [] + for _, func in columns_and_functions: + try: + value = func(error_data) + except Exception as e: + value = f"ERROR: {e}" + row.append(value) + writer.writerow(row) From e103a5579574240455658c5c29dfba1e0d401e4d Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Wed, 23 Apr 2025 12:55:26 +0200 Subject: [PATCH 29/58] Minor change to account for change in seed implementation for some verifiers --- verifiers/verifier.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/verifiers/verifier.py b/verifiers/verifier.py index 348403c6..0a3d5678 100644 --- a/verifiers/verifier.py +++ b/verifiers/verifier.py @@ -41,7 +41,7 @@ def __init__(self, name: str, type: str, solver: str, mutations_per_model: int, self.original_model = None - def generate_mutations(self) -> None: + def generate_mutations(self) -> dict | None: """ Will generate random mutations based on mutations_per_model for the model """ @@ -160,7 +160,11 @@ def rerun(self,error: dict) -> dict: This function will rerun a previous failed test """ try: - random.seed(self.seed) + if 'seed' in error: + run_seed = error['seed'] + random.seed(run_seed) + else: + random.seed(self.seed) self.model_file = error["originalmodel_file"] self.original_model = error["originalmodel"] self.exclude_dict = {} From d5b06b25c7e6400d290a589c7eea59ec159aab20 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 29 Apr 2025 11:34:51 +0200 Subject: [PATCH 30/58] changes to make final results possible --- fuzz_test_utils/output_writer.py | 3 +- verifiers/solver_voting_count_verifier.py | 304 +++++++++++++-- verifiers/solver_voting_sat_verifier.py | 300 ++++++++++++-- verifiers/strengthening_weakening_verifier.py | 366 +++++++++++++++--- verifiers/verifier_runner.py | 13 +- 5 files changed, 833 insertions(+), 153 deletions(-) diff --git a/fuzz_test_utils/output_writer.py b/fuzz_test_utils/output_writer.py index 9b7ec4b9..450b3d33 100644 --- a/fuzz_test_utils/output_writer.py +++ b/fuzz_test_utils/output_writer.py @@ -55,7 +55,7 @@ def match_conditions(exc_str): (lambda s: "'bool' object has no attribute 'has_subexpr'" in s, "21_bool_obj_no_has_subexpr/"), (lambda s: "'int' object has no attribute 'is_bool'" in s, "22_int_obj_no_is_bool/"), (lambda s: "not supported: model.get_or_make_boolean_index(" in s, "23_not_supported_get_or_make_boolean_index/"), - (lambda s: "'BoolVal' object has no attribute 'get_integer_var_value_map'" in s, "24_not_known_var/") + (lambda s: any(x in s for x in ["'BoolVal' object has no attribute 'get_integer_var_value_map'", "Not a known var "]), "24_not_known_var/") ] def get_logging_dir(error_data, logging_dir): @@ -210,6 +210,7 @@ def write_csv(error_data: dict, output_path) -> None: ("total_nr_mutations", get_nr_mutations), ("nr_mm_mutations", get_nr_mm_mutations), ("nr_gen_mutations", get_nr_gen_mutations), + ("mutations", get_mutations), ("bugged_solver", get_bugged_solver), ("objective", get_objective), ("variables", get_variables), diff --git a/verifiers/solver_voting_count_verifier.py b/verifiers/solver_voting_count_verifier.py index bd42b6ff..9b7f164e 100644 --- a/verifiers/solver_voting_count_verifier.py +++ b/verifiers/solver_voting_count_verifier.py @@ -1,4 +1,3 @@ -from result_attributes import get_nr_mm_mutations, get_nr_mutations, get_nr_gen_mutations from verifiers import * class Solver_Vote_Count_Verifier(Verifier): @@ -15,7 +14,7 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti self.mutations_per_model = mutations_per_model self.exclude_dict = exclude_dict self.time_limit = time_limit - self.seed = seed + self.seed = random.Random().random() self.mm_mutators = [xor_morph, and_morph, or_morph, implies_morph, not_morph, linearize_constraint_morph, flatten_morph, @@ -40,6 +39,10 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti self.gen_mutators = [type_aware_operator_replacement, type_aware_expression_replacement] self.mutators = [] self.original_model = None + self.nr_solve_checks = 0 + self.bug_cause = 'STARTMODEL' + self.nr_timed_out = 0 + self.last_mut = None def initialize_run(self) -> None: if self.original_model == None: @@ -63,11 +66,14 @@ def generate_mutations(self) -> None | dict: Will generate random mutations based on mutations_per_model for the model """ for i in range(self.mutations_per_model): - - if random.random() < 0.8: - m = random.choice(self.mm_mutators) - else: - m = random.choice(self.gen_mutators) # 20% chance to choose gen-type mutator + # choose a mutation (not in exclude_dict) + valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( + self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( + set(self.mm_mutators).union(set(self.gen_mutators))) + if random.random() <= 0.8: # 80% chance to choose metamorphic mutation + m = random.choice([mm_mut for mm_mut in self.mm_mutators if mm_mut in valid_mutators]) + else: # 20% chance to choose generation-based mutation + m = random.choice([gen_mut for gen_mut in self.gen_mutators if gen_mut in valid_mutators]) self.mutators += [self.seed] # an error can occur in the transformations, so even before the solve call. @@ -75,9 +81,13 @@ def generate_mutations(self) -> None | dict: self.mutators += [m] try: if m in self.gen_mutators: + self.bug_cause = 'during GEN' self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints + self.bug_cause = 'GEN' else: + self.bug_cause = 'during MM' self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints + self.bug_cause = 'MM' self.mutators += [copy.deepcopy(self.cons)] except MetamorphicError as exc: # add to exclude_dict, to avoid running into the same error @@ -94,7 +104,8 @@ def generate_mutations(self) -> None | dict: # don't log semanticfusion crash print('I', end='', flush=True) - return dict(type=Fuzz_Test_ErrorTypes.internalfunctioncrash, + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.internalfunctioncrash, originalmodel_file=self.model_file, exception=e, function=function, @@ -104,55 +115,81 @@ def generate_mutations(self) -> None | dict: constraints=self.cons, variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)], - originalmodel=self.original_model + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out ) return None - def verify_model(self, is_rerun=False) -> dict: + def verify_model(self, is_bug_check=False) -> None | dict: try: model = cp.Model(self.cons) - one_run_time_lim = 30 # TODO: change back to 200 after results? - time_limit = max(1, min(one_run_time_lim, + # if is_bug_check: + # max_search_time = 20 + # else: + # max_search_time = 40 + max_search_time = 40 + + time_limit = max(1, min(max_search_time, # TODO: change `max_search_time` back to 200 self.time_limit - time.time())) # set the max time limit to the given time limit or to 1 if the self.time_limit-time.time() would be smaller then 1 - solver_1 = self.solvers[0] - solver_2 = self.solvers[1] - if hasattr(self, 'sol_lim'): - new_count_1 = model.solveAll(solver=solver_1, solution_limit=self.sol_lim) - new_count_2 = model.solveAll(solver=solver_2, solution_limit=self.sol_lim) - else: - new_count_1 = model.solveAll(solver=solver_1) - new_count_2 = model.solveAll(solver=solver_2) + # Get the actual solver results and their execution times. + # We do it this way because a solver might crash, meaning the other solver doesn't get a turn. + solvers_results = [] + solvers_times = [] + for s in self.solvers: + self.nr_solve_checks += 1 + if hasattr(self, 'sol_lim'): + solvers_results.append(model.solveAll(solver=s, solution_limit=self.sol_lim, time_limit=time_limit)) + solvers_times.append(model.status().runtime) + else: + solvers_results.append(model.solveAll(solver=s, time_limit=time_limit)) + solvers_times.append(model.status().runtime) - if model.status().runtime > time_limit - 10: + nr_timed_out_solvers = sum([t > time_limit * 0.8 for t in solvers_times]) + if nr_timed_out_solvers > 0: # timeout, skip - print('T', end='', flush=True) + self.nr_timed_out += nr_timed_out_solvers + if not is_bug_check: + print('T', end='', flush=True) + else: + self.bug_cause = 'UNKNOWN' return None - elif new_count_1 == new_count_2: + elif all(s1 == s2 for i, s1 in enumerate(solvers_results) for j, s2 in enumerate(solvers_results) if i < j): # has to be same - print('.', end='', flush=True) + if not is_bug_check: + print('.', end='', flush=True) return None else: - print('X', end='', flush=True) - return dict(type=Fuzz_Test_ErrorTypes.failed_model, + solver_results_str = ", ".join( + f"{solver}: {result}" for solver, result in zip(self.solvers, solvers_results)) + if is_bug_check: + print('X', end='', flush=True) + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.failed_model, originalmodel_file=self.model_file, - exception=f"Amount of solutions of the two solvers are not equal." - f" #Solutions of {solver_1}: {new_count_1}." - f" #Solutions of {solver_2}: {new_count_2}.", + exception=f"Results of the solvers are not equal. Solver results: {solver_results_str}.", constraints=self.cons, mutators=self.mutators, model=model, variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)], - originalmodel=self.original_model + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out ) except Exception as e: if isinstance(e, (CPMpyException, NotImplementedError)): # expected error message, ignore return None - print('E', end='', flush=True) - return dict(type=Fuzz_Test_ErrorTypes.internalcrash, + if is_bug_check: + print('E', end='', flush=True) + + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.internalcrash, originalmodel_file=self.model_file, exception=e, stacktrace=traceback.format_exc(), @@ -161,15 +198,196 @@ def verify_model(self, is_rerun=False) -> dict: model=model, variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)], - originalmodel=self.original_model + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out ) - # if you got here, the model failed... - return dict(type=Fuzz_Test_ErrorTypes.failed_model, - originalmodel_file=self.model_file, - constraints=self.cons, - mutators=self.mutators, - model=newModel, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model - ) + + def run(self, model_file: str) -> dict | None: + """ + This function will run a single tests on the given model + """ + try: + random.seed(self.seed) + self.model_file = model_file + self.initialize_run() + gen_mutations_error = self.generate_mutations() + + # check if no error occured while generation the mutations + if gen_mutations_error == None: + # FOLLOWING 5 LINES CHANGED! + verify_model_error = self.verify_model() + if verify_model_error == None: + return None + else: + return self.find_error_rerun(verify_model_error) + else: + return gen_mutations_error # This error requires no rerun + except AssertionError as e: + print("A", end='', flush=True) + error_type = Fuzz_Test_ErrorTypes.crashed_model + if "is not sat" in str(e): + error_type = Fuzz_Test_ErrorTypes.unsat_model + elif "has no constraints" in str(e): + error_type = Fuzz_Test_ErrorTypes.no_constraints_model + return dict(seed=self.seed, + type=error_type, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + except Exception as e: + print('C', end='', flush=True) + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.crashed_model, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + mutators=self.mutators, + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + def find_error_rerun(self, error_dict) -> dict: + try: + random.seed(self.seed) + error_type = error_dict['type'] + self.initialize_run() # initialize empty (self.)model, cons, mutators + + # This should always be the case + if error_type in [Fuzz_Test_ErrorTypes.internalcrash, Fuzz_Test_ErrorTypes.failed_model]: # Error type 'E', often during model.solve() or solveAll or type 'X' + return self.bug_search_run_and_verify_model() + + except AssertionError as e: + print("A", end='', flush=True) + type = Fuzz_Test_ErrorTypes.crashed_model + if "is not sat" in str(e): + type = Fuzz_Test_ErrorTypes.unsat_model + elif "has no constraints" in str(e): + type = Fuzz_Test_ErrorTypes.no_constraints_model + return dict(seed=self.seed, + type=type, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + except Exception as e: + print('C', end='', flush=True) + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.crashed_model, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + def bug_search_run_and_verify_model(self) -> dict: + for _ in range(self.mutations_per_model): + last_bug_cause = self.bug_cause + + # Generate the type of mutation that will happen + valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( + self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( + set(self.mm_mutators).union(set(self.gen_mutators))) + if random.random() < 0.8: # 80% chance to choose metamorphic mutation + m = random.choice([mm_mut for mm_mut in self.mm_mutators if mm_mut in valid_mutators]) + new_mut_type = 'MM' + else: # 20% chance to choose generation-based mutation + m = random.choice([gen_mut for gen_mut in self.gen_mutators if gen_mut in valid_mutators]) + new_mut_type = 'GEN' + + # Check whether verify_model returns an error before the new mutation, because the cause is then at the old mutation + if new_mut_type != last_bug_cause: + verify_model_error = self.verify_model(is_bug_check=True) + if verify_model_error is not None: + return verify_model_error + + # Then, apply the new mutation (which shouldn't give an error itself) + gen_mut_error = self.apply_single_mutation(m) + assert gen_mut_error is None, "There should be no errors related to the application of mutations here." + + # Finally, check the model at the end. This SHOULD give an error + verify_model_error = self.verify_model(is_bug_check=True) + if verify_model_error is not None: + return verify_model_error + else: + print('_', end='', flush=True) + + def apply_single_mutation(self, m) -> dict | None: + """ + Will generate one random mutation and apply it to the model + """ + self.mutators += [self.seed] + # an error can occur in the transformations, so even before the solve call. + # log function and arguments in that case + self.mutators += [m] + try: + if m in self.gen_mutators: + self.bug_cause = f'during GEN, after {self.bug_cause}' + self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints + self.bug_cause = 'GEN' + else: + self.bug_cause = f'during MM, after {self.bug_cause}' + self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints + self.bug_cause = 'MM' + self.mutators += [copy.deepcopy(self.cons)] + except MetamorphicError as exc: + # add to exclude_dict, to avoid running into the same error + if self.model_file in self.exclude_dict: + self.exclude_dict[self.model_file] += [m] + else: + self.exclude_dict[self.model_file] = [m] + function, argument, e = exc.args + if isinstance(e, CPMpyException): + # expected behavior if we throw a cpmpy exception, do not log + return None + elif function == semanticFusion: + return None + # don't log semanticfusion crash + + print('I', end='', flush=True) + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.internalfunctioncrash, + originalmodel_file=self.model_file, + exception=e, + function=function, + argument=argument, + stacktrace=traceback.format_exc(), + mutators=self.mutators, + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + return None \ No newline at end of file diff --git a/verifiers/solver_voting_sat_verifier.py b/verifiers/solver_voting_sat_verifier.py index 21a4959f..5d58bce0 100644 --- a/verifiers/solver_voting_sat_verifier.py +++ b/verifiers/solver_voting_sat_verifier.py @@ -14,7 +14,7 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti self.mutations_per_model = mutations_per_model self.exclude_dict = exclude_dict self.time_limit = time_limit - self.seed = seed + self.seed = random.Random().random() self.mm_mutators = [xor_morph, and_morph, or_morph, implies_morph, not_morph, linearize_constraint_morph, flatten_morph, @@ -39,6 +39,10 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti self.gen_mutators = [type_aware_operator_replacement, type_aware_expression_replacement] self.mutators = [] self.original_model = None + self.nr_solve_checks = 0 + self.bug_cause = 'STARTMODEL' + self.nr_timed_out = 0 + self.last_mut = None def initialize_run(self) -> None: if self.original_model == None: @@ -53,17 +57,18 @@ def initialize_run(self) -> None: self.mutators = [copy.deepcopy( self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. - def generate_mutations(self) -> None: + def generate_mutations(self) -> dict | None: """ Will generate random mutations based on mutations_per_model for the model """ for i in range(self.mutations_per_model): - - # choose a mutation - if random.random() < 0.8: - m = random.choice(self.mm_mutators) - else: - m = random.choice(self.gen_mutators) # 20% chance to choose gen-type mutator + # choose a mutation (not in exclude_dict) + valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( + self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list(set(self.mm_mutators).union(set(self.gen_mutators))) + if random.random() <= 0.8: # 80% chance to choose metamorphic mutation + m = random.choice([mm_mut for mm_mut in self.mm_mutators if mm_mut in valid_mutators]) + else: # 20% chance to choose generation-based mutation + m = random.choice([gen_mut for gen_mut in self.gen_mutators if gen_mut in valid_mutators]) self.mutators += [self.seed] # an error can occur in the transformations, so even before the solve call. @@ -71,9 +76,13 @@ def generate_mutations(self) -> None: self.mutators += [m] try: if m in self.gen_mutators: + self.bug_cause = 'during GEN' self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints + self.bug_cause = 'GEN' else: + self.bug_cause = 'during MM' self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints + self.bug_cause = 'MM' self.mutators += [copy.deepcopy(self.cons)] except MetamorphicError as exc: # add to exclude_dict, to avoid running into the same error @@ -90,7 +99,8 @@ def generate_mutations(self) -> None: # don't log semanticfusion crash print('I', end='', flush=True) - return dict(type=Fuzz_Test_ErrorTypes.internalfunctioncrash, + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.internalfunctioncrash, originalmodel_file=self.model_file, exception=e, function=function, @@ -100,54 +110,77 @@ def generate_mutations(self) -> None: constraints=self.cons, variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)], - originalmodel=self.original_model + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out ) return None - def check_for_bug(self) -> None | dict: + def verify_model(self, is_bug_check=False) -> None | dict: try: model = cp.Model(self.cons) - time_limit = max(1, min(200, + + if is_bug_check: + max_search_time = 10 + else: + max_search_time = 20 + + time_limit = max(1, min(max_search_time, # TODO: change `max_search_time` back to 200 self.time_limit - time.time())) # set the max time limit to the given time limit or to 1 if the self.time_limit-time.time() would be smaller then 1 - # choosing the solvers - solver_1 = self.solvers[0] - solver_2 = self.solvers[1] - solver_1_is_sat = model.solve(solver=solver_1, time_limit=time_limit) - solver_2_is_sat = model.solve(solver=solver_2, time_limit=time_limit) - # for prettier exception printing - solver_1_print = "sat" if solver_1_is_sat else "unsat" - solver_2_print = "sat" if solver_2_is_sat else "unsat" + # Get the actual solver results and their execution times. + # We do it this way because a solver might crash, meaning the other solver doesn't get a turn. + solvers_results = [] + solvers_times = [] + for s in self.solvers: + self.nr_solve_checks += 1 + solvers_results.append(model.solve(solver=s, time_limit=time_limit)) + solvers_times.append(model.status().runtime) - if model.status().runtime > time_limit - 10: + nr_timed_out_solvers = sum([t > time_limit * 0.8 for t in solvers_times]) + if nr_timed_out_solvers > 0: # timeout, skip - print('T', end='', flush=True) + self.nr_timed_out += nr_timed_out_solvers + if not is_bug_check: + print('T', end='', flush=True) + else: + self.bug_cause = 'UNKNOWN' return None - elif solver_1_is_sat == solver_2_is_sat: + elif all(s1 == s2 for i, s1 in enumerate(solvers_results) for j, s2 in enumerate(solvers_results) if i < j): # has to be same - print('.', end='', flush=True) + if not is_bug_check: + print('.', end='', flush=True) return None else: - print('X', end='', flush=True) - return dict(type=Fuzz_Test_ErrorTypes.failed_model, + solver_results_str = ", ".join( + f"{solver}: {result}" for solver, result in zip(self.solvers, solvers_results)) + if is_bug_check: + print('X', end='', flush=True) + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.failed_model, originalmodel_file=self.model_file, - exception=f"Results of the two solvers are not equal." - f" Result of {self.solvers[0]}: {solver_1_print}." - f" Result of {self.solvers[1]}: {solver_2_print}.", + exception=f"Results of the solvers are not equal. Solver results: {solver_results_str}.", constraints=self.cons, mutators=self.mutators, model=model, variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)], - originalmodel=self.original_model + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out ) except Exception as e: if isinstance(e, (CPMpyException, NotImplementedError)): # expected error message, ignore return None - print('E', end='', flush=True) - return dict(type=Fuzz_Test_ErrorTypes.internalcrash, + if is_bug_check: + print('E', end='', flush=True) + + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.internalcrash, originalmodel_file=self.model_file, exception=e, stacktrace=traceback.format_exc(), @@ -156,15 +189,196 @@ def check_for_bug(self) -> None | dict: model=model, variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)], - originalmodel=self.original_model + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + def run(self, model_file: str) -> dict | None: + """ + This function will run a single tests on the given model + """ + try: + random.seed(self.seed) + self.model_file = model_file + self.initialize_run() + gen_mutations_error = self.generate_mutations() + + # check if no error occured while generation the mutations + if gen_mutations_error == None: + # FOLLOWING 5 LINES CHANGED! + verify_model_error = self.verify_model() + if verify_model_error == None: + return None + else: + return self.find_error_rerun(verify_model_error) + else: + return gen_mutations_error # This error requires no rerun + except AssertionError as e: + print("A", end='', flush=True) + error_type = Fuzz_Test_ErrorTypes.crashed_model + if "is not sat" in str(e): + error_type = Fuzz_Test_ErrorTypes.unsat_model + elif "has no constraints" in str(e): + error_type = Fuzz_Test_ErrorTypes.no_constraints_model + return dict(seed=self.seed, + type=error_type, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out ) - # if you got here, the model failed... - return dict(type=Fuzz_Test_ErrorTypes.failed_model, - originalmodel_file=self.model_file, - constraints=self.cons, - mutators=self.mutators, - model=newModel, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model - ) + + except Exception as e: + print('C', end='', flush=True) + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.crashed_model, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + mutators=self.mutators, + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + def find_error_rerun(self, error_dict) -> dict: + try: + random.seed(self.seed) + error_type = error_dict['type'] + self.initialize_run() # initialize empty (self.)model, cons, mutators + + # This should always be the case + if error_type in [Fuzz_Test_ErrorTypes.internalcrash, Fuzz_Test_ErrorTypes.failed_model]: # Error type 'E', often during model.solve() or solveAll or type 'X' + return self.bug_search_run_and_verify_model() + + except AssertionError as e: + print("A", end='', flush=True) + type = Fuzz_Test_ErrorTypes.crashed_model + if "is not sat" in str(e): + type = Fuzz_Test_ErrorTypes.unsat_model + elif "has no constraints" in str(e): + type = Fuzz_Test_ErrorTypes.no_constraints_model + return dict(seed=self.seed, + type=type, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + except Exception as e: + print('C', end='', flush=True) + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.crashed_model, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + def bug_search_run_and_verify_model(self) -> dict: + for _ in range(self.mutations_per_model): + last_bug_cause = self.bug_cause + + # Generate the type of mutation that will happen + valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( + self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( + set(self.mm_mutators).union(set(self.gen_mutators))) + if random.random() < 0.8: # 80% chance to choose metamorphic mutation + m = random.choice([mm_mut for mm_mut in self.mm_mutators if mm_mut in valid_mutators]) + new_mut_type = 'MM' + else: # 20% chance to choose generation-based mutation + m = random.choice([gen_mut for gen_mut in self.gen_mutators if gen_mut in valid_mutators]) + new_mut_type = 'GEN' + + # Check whether verify_model returns an error before the new mutation, because the cause is then at the old mutation + if new_mut_type != last_bug_cause: + verify_model_error = self.verify_model(is_bug_check=True) + if verify_model_error is not None: + return verify_model_error + + # Then, apply the new mutation (which shouldn't give an error itself) + gen_mut_error = self.apply_single_mutation(m) + assert gen_mut_error is None, "There should be no errors related to the application of mutations here." + + # Finally, check the model at the end. This SHOULD give an error + verify_model_error = self.verify_model(is_bug_check=True) + if verify_model_error is not None: + return verify_model_error + else: + print('_', end='', flush=True) + + def apply_single_mutation(self, m) -> dict | None: + """ + Will generate one random mutation and apply it to the model + """ + self.mutators += [self.seed] + # an error can occur in the transformations, so even before the solve call. + # log function and arguments in that case + self.mutators += [m] + try: + if m in self.gen_mutators: + self.bug_cause = f'during GEN, after {self.bug_cause}' + self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints + self.bug_cause = 'GEN' + else: + self.bug_cause = f'during MM, after {self.bug_cause}' + self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints + self.bug_cause = 'MM' + self.mutators += [copy.deepcopy(self.cons)] + except MetamorphicError as exc: + # add to exclude_dict, to avoid running into the same error + if self.model_file in self.exclude_dict: + self.exclude_dict[self.model_file] += [m] + else: + self.exclude_dict[self.model_file] = [m] + function, argument, e = exc.args + if isinstance(e, CPMpyException): + # expected behavior if we throw a cpmpy exception, do not log + return None + elif function == semanticFusion: + return None + # don't log semanticfusion crash + + print('I', end='', flush=True) + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.internalfunctioncrash, + originalmodel_file=self.model_file, + exception=e, + function=function, + argument=argument, + stacktrace=traceback.format_exc(), + mutators=self.mutators, + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + return None diff --git a/verifiers/strengthening_weakening_verifier.py b/verifiers/strengthening_weakening_verifier.py index f7422ac7..0fe9ae70 100644 --- a/verifiers/strengthening_weakening_verifier.py +++ b/verifiers/strengthening_weakening_verifier.py @@ -1,3 +1,5 @@ +import random + from verifiers import * class Strengthening_Weakening_Verifier(Verifier): @@ -14,7 +16,7 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti self.mutations_per_model = mutations_per_model self.exclude_dict = exclude_dict self.time_limit = time_limit - self.seed = seed + self.seed = random.Random().random() self.mm_mutators = [xor_morph, and_morph, or_morph, implies_morph, not_morph, linearize_constraint_morph, flatten_morph, @@ -39,6 +41,10 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti self.gen_mutators = [type_aware_operator_replacement, type_aware_expression_replacement] self.mutators = [] self.original_model = None + self.nr_solve_checks = 0 + self.bug_cause = 'STARTMODEL' + self.nr_timed_out = 0 + self.last_mut = None def initialize_run(self) -> None: if self.original_model == None: @@ -64,13 +70,17 @@ def generate_mutations(self) -> None | dict: for i in range(self.mutations_per_model): # choose a mutator. 33% of the time, this will be a strengthening/weakening mutation. + # choose a mutation (not in exclude_dict) + valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)).union({strengthening_weakening_mutator}) - set( + self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( + set(self.mm_mutators).union(set(self.gen_mutators)).union({strengthening_weakening_mutator})) rand = random.random() if rand <= 0.33: - m = strengthening_weakening_mutator + m = strengthening_weakening_mutator if strengthening_weakening_mutator in valid_mutators else random.choice(self.mm_mutators) elif rand <= 0.8633: # ~~ remaining 80% of 0.67 (8/15) - m = random.choice(self.mm_mutators) + m = random.choice([mm_mut for mm_mut in self.mm_mutators if mm_mut in valid_mutators]) else: - m = random.choice(self.gen_mutators) # ~~ remaining 20% to choose gen-type mutator (2/15) + m = random.choice([gen_mut for gen_mut in self.gen_mutators if gen_mut in valid_mutators]) self.mutators += [self.seed] # an error can occur in the transformations, so even before the solve call. @@ -78,27 +88,44 @@ def generate_mutations(self) -> None | dict: self.mutators += [m] try: if m in self.gen_mutators: + self.bug_cause = 'during GEN' self.cons = m(self.cons) # apply a generative (non-metamorphic) mutation and REPLACE constraints + self.bug_cause = 'GEN' elif m == strengthening_weakening_mutator: model = cp.Model(self.cons) - solver_1 = self.solvers[0] - solver_2 = self.solvers[1] + # s = random.choice(self.solvers) if 'ortools' not in self.solvers else 'ortools' + s = 'ortools' # TODO: CHANGE + # print("I add to nrsolvechecks") + self.nr_solve_checks += 1 if hasattr(self, 'sol_lim'): - count_1 = model.solveAll(solver=solver_1, solution_limit=self.sol_lim) + count = model.solveAll(solver=s, solution_limit=self.sol_lim, time_limit=5) # should find at least 1 solution in 5s else: - count_1 = model.solveAll(solver=solver_1) - if count_1 > 1: + count = model.solveAll(solver=s, time_limit=5) + if count > 1: + self.bug_cause = 'during STR' self.cons = m(self.cons, strengthen=True) - elif count_1 < 1: + self.bug_cause = 'STR' + elif count < 1: + self.bug_cause = 'during WKN' self.cons = m(self.cons, strengthen=False) + self.bug_cause = 'WKN' elif random.random() < 0.8: # If only 1 solution remains, we just go on normally instead m = random.choice(self.mm_mutators) + self.bug_cause = 'during MM' self.cons += m(self.cons) + self.bug_cause = 'MM' else: m = random.choice(self.gen_mutators) + self.bug_cause = 'during GEN' self.cons = m(self.cons) + self.bug_cause = 'GEN' else: - self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints + self.bug_cause = 'during MM' + self.cons += m(self.cons) + self.bug_cause = 'MM' + if not m == self.mutators[-1]: + self.mutators[-1] = m + # print(f"Mutator in iteration {i} is {self.mutators[-1]}.") self.mutators += [copy.deepcopy(self.cons)] except MetamorphicError as exc: @@ -116,7 +143,8 @@ def generate_mutations(self) -> None | dict: # don't log semanticfusion crash print('I', end='', flush=True) - return dict(type=Fuzz_Test_ErrorTypes.internalfunctioncrash, + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.internalfunctioncrash, originalmodel_file=self.model_file, exception=e, function=function, @@ -125,54 +153,83 @@ def generate_mutations(self) -> None | dict: mutators=self.mutators, constraints=self.cons, variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)], - originalmodel=self.original_model + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out ) return None - - def verify_model(self) -> None | dict: + def verify_model(self, is_bug_check=False) -> None | dict: try: model = cp.Model(self.cons) - time_limit = max(1, min(200, + + # if is_bug_check: + # max_search_time = 20 + # else: + # max_search_time = 40 + max_search_time = 40 + + time_limit = max(1, min(max_search_time, # TODO: change `max_search_time` back to 200 self.time_limit - time.time())) # set the max time limit to the given time limit or to 1 if the self.time_limit-time.time() would be smaller then 1 - solver_1 = self.solvers[0] - solver_2 = self.solvers[1] - if hasattr(self, 'sol_lim'): - new_count_1 = model.solveAll(solver=solver_1, solution_limit=self.sol_lim) - new_count_2 = model.solveAll(solver=solver_2, solution_limit=self.sol_lim) - else: - new_count_1 = model.solveAll(solver=solver_1) - new_count_2 = model.solveAll(solver=solver_2) + # Get the actual solver results and their execution times. + # We do it this way because a solver might crash, meaning the other solver doesn't get a turn. + solvers_results = [] + solvers_times = [] + for s in self.solvers: + # print("I add to nrsolvechecks") + self.nr_solve_checks += 1 + if hasattr(self, 'sol_lim'): + solvers_results.append(model.solveAll(solver=s, solution_limit=self.sol_lim, time_limit=time_limit)) + solvers_times.append(model.status().runtime) + else: + solvers_results.append(model.solveAll(solver=s, time_limit=time_limit)) + solvers_times.append(model.status().runtime) - if model.status().runtime > time_limit - 10: + nr_timed_out_solvers = sum([t > time_limit * 0.8 for t in solvers_times]) + if nr_timed_out_solvers > 0: # timeout, skip - print('T', end='', flush=True) + self.nr_timed_out += nr_timed_out_solvers + if not is_bug_check: + print('T', end='', flush=True) + else: + self.bug_cause = 'UNKNOWN' return None - elif new_count_1 == new_count_2: + elif all(s1 == s2 for i, s1 in enumerate(solvers_results) for j, s2 in enumerate(solvers_results) if i < j): # has to be same - print('.', end='', flush=True) + if not is_bug_check: + print('.', end='', flush=True) return None else: - print('X', end='', flush=True) - return dict(type=Fuzz_Test_ErrorTypes.failed_model, + solver_results_str = ", ".join( + f"{solver}: {result}" for solver, result in zip(self.solvers, solvers_results)) + if is_bug_check: + print('X', end='', flush=True) + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.failed_model, originalmodel_file=self.model_file, - exception=f"Amount of solutions of the two solvers are not equal." - f" #Solutions of {solver_1}: {new_count_1}." - f" #Solutions of {solver_2}: {new_count_2}.", + exception=f"Results of the solvers are not equal. Solver results: {solver_results_str}.", constraints=self.cons, mutators=self.mutators, model=model, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)], - originalmodel=self.original_model + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out ) except Exception as e: if isinstance(e, (CPMpyException, NotImplementedError)): # expected error message, ignore return None - print('E', end='', flush=True) - return dict(type=Fuzz_Test_ErrorTypes.internalcrash, + if is_bug_check: + print('E', end='', flush=True) + + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.internalcrash, originalmodel_file=self.model_file, exception=e, stacktrace=traceback.format_exc(), @@ -181,22 +238,16 @@ def verify_model(self) -> None | dict: model=model, variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)], - originalmodel=self.original_model + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out ) - # if you got here, the model failed... - return dict(type=Fuzz_Test_ErrorTypes.failed_model, - originalmodel_file=self.model_file, - constraints=self.cons, - mutators=self.mutators, - model=newModel, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model - ) - - def run(self, model_file: str) -> dict: + + + def run(self, model_file: str) -> dict | None: """ - This function will run a single test on the given model + This function will run a single tests on the given model """ try: random.seed(self.seed) @@ -204,11 +255,16 @@ def run(self, model_file: str) -> dict: self.initialize_run() gen_mutations_error = self.generate_mutations() - # check if no error occurred while generation the mutations + # check if no error occured while generation the mutations if gen_mutations_error == None: - return self.verify_model() + # FOLLOWING 5 LINES CHANGED! + verify_model_error = self.verify_model() + if verify_model_error == None: + return None + else: + return self.find_error_rerun(verify_model_error) else: - return gen_mutations_error + return gen_mutations_error # This error requires no rerun except AssertionError as e: print("A", end='', flush=True) error_type = Fuzz_Test_ErrorTypes.crashed_model @@ -216,35 +272,216 @@ def run(self, model_file: str) -> dict: error_type = Fuzz_Test_ErrorTypes.unsat_model elif "has no constraints" in str(e): error_type = Fuzz_Test_ErrorTypes.no_constraints_model - return dict(type=error_type, + return dict(seed=self.seed, + type=error_type, originalmodel_file=self.model_file, exception=e, stacktrace=traceback.format_exc(), constraints=self.cons, variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)], - originalmodel=self.original_model + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out ) except Exception as e: print('C', end='', flush=True) - return dict(type=Fuzz_Test_ErrorTypes.crashed_model, + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.crashed_model, originalmodel_file=self.model_file, exception=e, stacktrace=traceback.format_exc(), constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], mutators=self.mutators, + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + def find_error_rerun(self, error_dict) -> dict: + try: + # print("WE ARE NOW RUNNING A FIND_ERROR_RERUN!") + random.seed(self.seed) + error_type = error_dict['type'] + self.initialize_run() # initialize empty (self.)model, cons, mutators + + # This should always be the case + if error_type in [Fuzz_Test_ErrorTypes.internalcrash, Fuzz_Test_ErrorTypes.failed_model]: # Error type 'E', often during model.solve() or solveAll or type 'X' + return self.bug_search_run_and_verify_model() + + except AssertionError as e: + print("A", end='', flush=True) + type = Fuzz_Test_ErrorTypes.crashed_model + if "is not sat" in str(e): + type = Fuzz_Test_ErrorTypes.unsat_model + elif "has no constraints" in str(e): + type = Fuzz_Test_ErrorTypes.no_constraints_model + return dict(seed=self.seed, + type=type, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in get_variables(self.cons)], - originalmodel=self.original_model + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out ) + except Exception as e: + print('C', end='', flush=True) + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.crashed_model, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + def bug_search_run_and_verify_model(self) -> dict: + for _ in range(self.mutations_per_model): + last_bug_cause = self.bug_cause + + # choose a mutator. 33% of the time, this will be a strengthening/weakening mutation. + # choose a mutation (not in exclude_dict) + valid_mutators = list( + set(self.mm_mutators).union(set(self.gen_mutators)).union({strengthening_weakening_mutator}) - set( + self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( + set(self.mm_mutators).union(set(self.gen_mutators)).union({strengthening_weakening_mutator})) + rand = random.random() + if rand <= 0.33: + m = strengthening_weakening_mutator if strengthening_weakening_mutator in valid_mutators else random.choice(self.mm_mutators) + new_mut_type = 'STRWK' + elif rand <= 0.8633: # ~~ remaining 80% of 0.67 (8/15) + m = random.choice([mm_mut for mm_mut in self.mm_mutators if mm_mut in valid_mutators]) + new_mut_type = 'MM' + else: + m = random.choice([gen_mut for gen_mut in self.gen_mutators if gen_mut in valid_mutators]) + new_mut_type = 'GEN' + + # Check whether verify_model returns an error before the new mutation, because the cause is then at the old mutation + if new_mut_type != last_bug_cause: + verify_model_error = self.verify_model(is_bug_check=True) + if verify_model_error is not None: + return verify_model_error + + # Then, apply the new mutation (which shouldn't give an error itself) + gen_mut_error = self.apply_single_mutation(m) + assert gen_mut_error is None, "There should be no errors related to the application of mutations here." + + # Finally, check the model at the end. This SHOULD give an error + verify_model_error = self.verify_model(is_bug_check=True) + if verify_model_error is not None: + return verify_model_error + else: + print('_', end='', flush=True) + + def apply_single_mutation(self, m) -> dict | None: + """ + Will generate one random mutation and apply it to the model + """ + self.mutators += [self.seed] + # an error can occur in the transformations, so even before the solve call. + # log function and arguments in that case + self.mutators += [m] + try: + if m in self.gen_mutators: + self.bug_cause = 'during GEN' + self.cons = m(self.cons) # apply a generative (non-metamorphic) mutation and REPLACE constraints + self.bug_cause = 'GEN' + elif m == strengthening_weakening_mutator: + model = cp.Model(self.cons) + # s = random.choice(self.solvers) if 'ortools' not in self.solvers else 'ortools' + s = 'ortools' # TODO: CHANGE + # print("I add to nrsolvechecks") + self.nr_solve_checks += 1 + if hasattr(self, 'sol_lim'): + count = model.solveAll(solver=s, solution_limit=self.sol_lim, + time_limit=5) # should find at least 1 solution in 5s + else: + count = model.solveAll(solver=s, time_limit=5) + if count > 1: + self.bug_cause = 'during STR' + self.cons = m(self.cons, strengthen=True) + self.bug_cause = 'STR' + elif count < 1: + self.bug_cause = 'during WKN' + self.cons = m(self.cons, strengthen=False) + self.bug_cause = 'WKN' + elif random.random() < 0.8: # If only 1 solution remains, we just go on normally instead + m = random.choice(self.mm_mutators) + self.bug_cause = 'during MM' + self.cons += m(self.cons) + self.bug_cause = 'MM' + else: + m = random.choice(self.gen_mutators) + self.bug_cause = 'during GEN' + self.cons = m(self.cons) + self.bug_cause = 'GEN' + else: + self.bug_cause = 'during MM' + self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints + self.bug_cause = 'MM' + if not m == self.mutators[-1]: + self.mutators[-1] = m + # print(f"Mutator in bug_find is {self.mutators[-1]}.") + self.mutators += [copy.deepcopy(self.cons)] + except MetamorphicError as exc: + # add to exclude_dict, to avoid running into the same error + if self.model_file in self.exclude_dict: + self.exclude_dict[self.model_file] += [m] + else: + self.exclude_dict[self.model_file] = [m] + function, argument, e = exc.args + if isinstance(e, CPMpyException): + # expected behavior if we throw a cpmpy exception, do not log + return None + elif function == semanticFusion: + return None + # don't log semanticfusion crash + + print('I', end='', flush=True) + return dict(seed=self.seed, + type=Fuzz_Test_ErrorTypes.internalfunctioncrash, + originalmodel_file=self.model_file, + exception=e, + function=function, + argument=argument, + stacktrace=traceback.format_exc(), + mutators=self.mutators, + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + return None + def rerun(self, error: dict) -> dict: """ This function will rerun a previous failed test """ try: - random.seed(self.seed) + if 'seed' in error: + run_seed = error['seed'] + random.seed(run_seed) + else: + random.seed(self.seed) self.model_file = error["originalmodel_file"] self.original_model = error["originalmodel"] self.exclude_dict = {} @@ -253,9 +490,16 @@ def rerun(self, error: dict) -> dict: # check if no error occured while generation the mutations if gen_mutations_error == None: - return self.verify_model() + # FOLLOWING 5 LINES CHANGED! + verify_model_error = self.verify_model() + if verify_model_error == None: + return None + else: + return self.find_error_rerun(verify_model_error) else: - return gen_mutations_error + return gen_mutations_error # This error requires no rerun + # self.og_cons = error["constraints"] + # return self.verify_model() except AssertionError as e: print("A", end='', flush=True) diff --git a/verifiers/verifier_runner.py b/verifiers/verifier_runner.py index b55e5345..3736366b 100644 --- a/verifiers/verifier_runner.py +++ b/verifiers/verifier_runner.py @@ -5,7 +5,7 @@ import warnings from os.path import join -from fuzz_test_utils.output_writer import get_output_dir +from fuzz_test_utils.output_writer import get_logging_dir, write_csv from verifiers import * from fuzz_test_utils import Fuzz_Test_ErrorTypes def get_all_verifiers(single_solver) -> list: @@ -60,11 +60,14 @@ def run_verifiers(current_amount_of_tests, current_amount_of_error, lock, solver if error is not None: lock.acquire() try: + if 'seed' in error: # Give every run its own seed, otherwise same mutations happen + random_seed = error['seed'] error_data = {'verifier':random_verifier.getName(),'solver' : solver, 'mutations_per_model' : mutations_per_model, "seed": random_seed, "execution_time": execution_time, "error" :error} - output_dir = get_output_dir(error_data) if get_output_dir(error_data) else output_dir - os.makedirs(output_dir, exist_ok=True) # create if it doesn't already exist - write_error(error_data,output_dir) - current_amount_of_error.value +=1 + logging_dir = get_logging_dir(error_data, output_dir) if get_logging_dir(error_data, output_dir) else output_dir + os.makedirs(logging_dir, exist_ok=True) # create if it doesn't already exist + write_error(error_data, logging_dir) + write_csv(error_data, 'csv_results.csv') + current_amount_of_error.value += 1 finally: lock.release() lock.acquire() From 2af99c7cc0174d94a9689f644a019f400dbe81ea Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 29 Apr 2025 11:35:52 +0200 Subject: [PATCH 31/58] bugfix with table constraints --- fuzz_test_utils/mutators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index b6eff33e..52ff5b7f 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1500,7 +1500,7 @@ def get_return_type(expr: Expression, con: Expression): remaining_path_len, remaining_path = constant_restricted_functions[con.name] if path_len - i == remaining_path_len and expr_at_path(con, remaining_path, expr): return path, 'constant' - elif type(con) in variable_restricted_functions: + if type(con) in variable_restricted_functions: remaining_path_len, remaining_path = variable_restricted_functions[type(con)] if path_len - i == remaining_path_len and expr_at_path(con, remaining_path, expr): return path, 'variable' From 5620574abb0501db4ca6ef7535421c5856eba352 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 29 Apr 2025 14:58:39 +0200 Subject: [PATCH 32/58] Addition of small new mutation type: replacing domain of a variable to a single value --- fuzz_test_utils/mutators.py | 20 +++++++++ verifiers/strengthening_weakening_verifier.py | 43 +++++++++++-------- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 52ff5b7f..609e1370 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1757,6 +1757,26 @@ def strengthening_weakening_mutator(constraints: list, strengthen: bool = True) return Exception(e) +def change_domain_mutator(constraints: list): + try: + # Take random variable + variables = get_variables(constraints) + rand_var = random.choice(variables) + + # Get its value by solving the model + Model(constraints).solve() + + # Replace its domain by its value + rand_var.lb = rand_var.value() + rand_var.ub = rand_var.value() + + # Return the given constraints to be compatible with how the other non-metamorphic mutators are called + return constraints + + except Exception as e: + raise Exception(e) + + class MetamorphicError(Exception): pass diff --git a/verifiers/strengthening_weakening_verifier.py b/verifiers/strengthening_weakening_verifier.py index 0fe9ae70..6586b0b1 100644 --- a/verifiers/strengthening_weakening_verifier.py +++ b/verifiers/strengthening_weakening_verifier.py @@ -39,6 +39,7 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti semanticFusionCountingMinus, semanticFusionCountingwsum] self.gen_mutators = [type_aware_operator_replacement, type_aware_expression_replacement] + self.str_wkn_mutators = [strengthening_weakening_mutator, change_domain_mutator] self.mutators = [] self.original_model = None self.nr_solve_checks = 0 @@ -71,16 +72,22 @@ def generate_mutations(self) -> None | dict: # choose a mutator. 33% of the time, this will be a strengthening/weakening mutation. # choose a mutation (not in exclude_dict) - valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)).union({strengthening_weakening_mutator}) - set( + valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)).union(set(self.str_wkn_mutators)) - set( self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( - set(self.mm_mutators).union(set(self.gen_mutators)).union({strengthening_weakening_mutator})) + set(self.mm_mutators).union(set(self.gen_mutators)).union(set(self.str_wkn_mutators))) rand = random.random() if rand <= 0.33: - m = strengthening_weakening_mutator if strengthening_weakening_mutator in valid_mutators else random.choice(self.mm_mutators) + mutator_list = self.str_wkn_mutators elif rand <= 0.8633: # ~~ remaining 80% of 0.67 (8/15) - m = random.choice([mm_mut for mm_mut in self.mm_mutators if mm_mut in valid_mutators]) + mutator_list = self.mm_mutators + else: + mutator_list = self.gen_mutators + + valid = [m for m in mutator_list if m in valid_mutators] + if valid: + m = random.choice(valid) else: - m = random.choice([gen_mut for gen_mut in self.gen_mutators if gen_mut in valid_mutators]) + continue # No valid mutator? => go to next mutation self.mutators += [self.seed] # an error can occur in the transformations, so even before the solve call. @@ -91,22 +98,18 @@ def generate_mutations(self) -> None | dict: self.bug_cause = 'during GEN' self.cons = m(self.cons) # apply a generative (non-metamorphic) mutation and REPLACE constraints self.bug_cause = 'GEN' - elif m == strengthening_weakening_mutator: + elif m in self.str_wkn_mutators: model = cp.Model(self.cons) - # s = random.choice(self.solvers) if 'ortools' not in self.solvers else 'ortools' - s = 'ortools' # TODO: CHANGE - # print("I add to nrsolvechecks") + s = random.choice(self.solvers) self.nr_solve_checks += 1 - if hasattr(self, 'sol_lim'): - count = model.solveAll(solver=s, solution_limit=self.sol_lim, time_limit=5) # should find at least 1 solution in 5s - else: - count = model.solveAll(solver=s, time_limit=5) + count = model.solveAll(solver=s, solution_limit=2, time_limit=5) # should find at least 1 solution in 5s if count > 1: self.bug_cause = 'during STR' - self.cons = m(self.cons, strengthen=True) + self.cons = m(self.cons) self.bug_cause = 'STR' elif count < 1: self.bug_cause = 'during WKN' + m = strengthening_weakening_mutator self.cons = m(self.cons, strengthen=False) self.bug_cause = 'WKN' elif random.random() < 0.8: # If only 1 solution remains, we just go on normally instead @@ -363,15 +366,21 @@ def bug_search_run_and_verify_model(self) -> dict: set(self.mm_mutators).union(set(self.gen_mutators)).union({strengthening_weakening_mutator})) rand = random.random() if rand <= 0.33: - m = strengthening_weakening_mutator if strengthening_weakening_mutator in valid_mutators else random.choice(self.mm_mutators) + mutator_list = self.str_wkn_mutators new_mut_type = 'STRWK' elif rand <= 0.8633: # ~~ remaining 80% of 0.67 (8/15) - m = random.choice([mm_mut for mm_mut in self.mm_mutators if mm_mut in valid_mutators]) + mutator_list = self.mm_mutators new_mut_type = 'MM' else: - m = random.choice([gen_mut for gen_mut in self.gen_mutators if gen_mut in valid_mutators]) + mutator_list = self.gen_mutators new_mut_type = 'GEN' + valid = [m for m in mutator_list if m in valid_mutators] + if valid: + m = random.choice(valid) + else: + continue # No valid mutator? => go to next mutation + # Check whether verify_model returns an error before the new mutation, because the cause is then at the old mutation if new_mut_type != last_bug_cause: verify_model_error = self.verify_model(is_bug_check=True) From cf197366c84019b9d6a906f242214c954a2e1d22 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 29 Apr 2025 15:02:43 +0200 Subject: [PATCH 33/58] small bug fix in case there are no valid mutators (unlikely) --- verifiers/solver_voting_count_verifier.py | 22 +++++++++++++++++----- verifiers/solver_voting_sat_verifier.py | 22 +++++++++++++++++----- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/verifiers/solver_voting_count_verifier.py b/verifiers/solver_voting_count_verifier.py index 9b7f164e..f559f852 100644 --- a/verifiers/solver_voting_count_verifier.py +++ b/verifiers/solver_voting_count_verifier.py @@ -71,9 +71,15 @@ def generate_mutations(self) -> None | dict: self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( set(self.mm_mutators).union(set(self.gen_mutators))) if random.random() <= 0.8: # 80% chance to choose metamorphic mutation - m = random.choice([mm_mut for mm_mut in self.mm_mutators if mm_mut in valid_mutators]) + mutator_list = self.mm_mutators else: # 20% chance to choose generation-based mutation - m = random.choice([gen_mut for gen_mut in self.gen_mutators if gen_mut in valid_mutators]) + mutator_list = self.gen_mutators + + valid = [m for m in mutator_list if m in valid_mutators] + if valid: + m = random.choice(valid) + else: + continue self.mutators += [self.seed] # an error can occur in the transformations, so even before the solve call. @@ -317,13 +323,19 @@ def bug_search_run_and_verify_model(self) -> dict: valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( set(self.mm_mutators).union(set(self.gen_mutators))) - if random.random() < 0.8: # 80% chance to choose metamorphic mutation - m = random.choice([mm_mut for mm_mut in self.mm_mutators if mm_mut in valid_mutators]) + if random.random() <= 0.8: # 80% chance to choose metamorphic mutation + mutator_list = self.mm_mutators new_mut_type = 'MM' else: # 20% chance to choose generation-based mutation - m = random.choice([gen_mut for gen_mut in self.gen_mutators if gen_mut in valid_mutators]) + mutator_list = self.gen_mutators new_mut_type = 'GEN' + valid = [m for m in mutator_list if m in valid_mutators] + if valid: + m = random.choice(valid) + else: + continue + # Check whether verify_model returns an error before the new mutation, because the cause is then at the old mutation if new_mut_type != last_bug_cause: verify_model_error = self.verify_model(is_bug_check=True) diff --git a/verifiers/solver_voting_sat_verifier.py b/verifiers/solver_voting_sat_verifier.py index 5d58bce0..e8b4e63f 100644 --- a/verifiers/solver_voting_sat_verifier.py +++ b/verifiers/solver_voting_sat_verifier.py @@ -66,9 +66,15 @@ def generate_mutations(self) -> dict | None: valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list(set(self.mm_mutators).union(set(self.gen_mutators))) if random.random() <= 0.8: # 80% chance to choose metamorphic mutation - m = random.choice([mm_mut for mm_mut in self.mm_mutators if mm_mut in valid_mutators]) + mutator_list = self.mm_mutators else: # 20% chance to choose generation-based mutation - m = random.choice([gen_mut for gen_mut in self.gen_mutators if gen_mut in valid_mutators]) + mutator_list = self.gen_mutators + + valid = [m for m in mutator_list if m in valid_mutators] + if valid: + m = random.choice(valid) + else: + continue self.mutators += [self.seed] # an error can occur in the transformations, so even before the solve call. @@ -308,13 +314,19 @@ def bug_search_run_and_verify_model(self) -> dict: valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( set(self.mm_mutators).union(set(self.gen_mutators))) - if random.random() < 0.8: # 80% chance to choose metamorphic mutation - m = random.choice([mm_mut for mm_mut in self.mm_mutators if mm_mut in valid_mutators]) + if random.random() <= 0.8: # 80% chance to choose metamorphic mutation + mutator_list = self.mm_mutators new_mut_type = 'MM' else: # 20% chance to choose generation-based mutation - m = random.choice([gen_mut for gen_mut in self.gen_mutators if gen_mut in valid_mutators]) + mutator_list = self.gen_mutators new_mut_type = 'GEN' + valid = [m for m in mutator_list if m in valid_mutators] + if valid: + m = random.choice(valid) + else: + continue + # Check whether verify_model returns an error before the new mutation, because the cause is then at the old mutation if new_mut_type != last_bug_cause: verify_model_error = self.verify_model(is_bug_check=True) From 58ebc1f8e7cd0504f9f06f46fe2ff0bff3105831 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Sat, 3 May 2025 15:03:17 +0200 Subject: [PATCH 34/58] fixed bug where parity check always checked two arguments exactly --- fuzz_test_utils/mutators.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 609e1370..51d5d2d5 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1591,14 +1591,14 @@ def has_positive_parity(expr: Expression, con: Expression, curr_path: tuple) -> neg_res = has_positive_parity(expr, con.args[0], curr_path) return not neg_res, curr_path if neg_res is not None else None if con.name == 'and' or con.name == 'or': - l, r = con.args - subtrees = [(0, l), (1, r)] + args = con.args + subtrees = list(enumerate(args)) random.shuffle(subtrees) for path, subtree in subtrees: - if any(expr is e for e in get_all_exprs(subtree)): # check if expr in l or r + if any(expr is e for e in get_all_exprs(subtree)): # check if expr is in the subtree curr_path += path, return has_positive_parity(expr, subtree, curr_path) - raise Exception(f"The given expression {expr} is not in either of the subtrees {l} or {r}.") + raise Exception(f"The given expression {expr} is not in any of the arguments: {args}.") # If the constraint is anything else, we don't know the parity return None @@ -1754,7 +1754,7 @@ def strengthening_weakening_mutator(constraints: list, strengthen: bool = True) return final_cons except Exception as e: - return Exception(e) + raise Exception(e) def change_domain_mutator(constraints: list): From 4147ef2178930311c735f75aa22c94a117fd8ac5 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Sat, 3 May 2025 15:05:37 +0200 Subject: [PATCH 35/58] Added rerun to find where the bug occurred when bug happened during generation of mutations --- verifiers/solver_voting_count_verifier.py | 18 +++++++---- verifiers/solver_voting_sat_verifier.py | 18 +++++++---- verifiers/strengthening_weakening_verifier.py | 32 +++++++++++-------- 3 files changed, 41 insertions(+), 27 deletions(-) diff --git a/verifiers/solver_voting_count_verifier.py b/verifiers/solver_voting_count_verifier.py index f559f852..2475e081 100644 --- a/verifiers/solver_voting_count_verifier.py +++ b/verifiers/solver_voting_count_verifier.py @@ -109,7 +109,6 @@ def generate_mutations(self) -> None | dict: return None # don't log semanticfusion crash - print('I', end='', flush=True) return dict(seed=self.seed, type=Fuzz_Test_ErrorTypes.internalfunctioncrash, originalmodel_file=self.model_file, @@ -156,11 +155,10 @@ def verify_model(self, is_bug_check=False) -> None | dict: nr_timed_out_solvers = sum([t > time_limit * 0.8 for t in solvers_times]) if nr_timed_out_solvers > 0: # timeout, skip + self.bug_cause = 'UNKNOWN' self.nr_timed_out += nr_timed_out_solvers if not is_bug_check: print('T', end='', flush=True) - else: - self.bug_cause = 'UNKNOWN' return None elif all(s1 == s2 for i, s1 in enumerate(solvers_results) for j, s2 in enumerate(solvers_results) if i < j): # has to be same @@ -229,7 +227,7 @@ def run(self, model_file: str) -> dict | None: else: return self.find_error_rerun(verify_model_error) else: - return gen_mutations_error # This error requires no rerun + return self.find_error_rerun(gen_mutations_error) except AssertionError as e: print("A", end='', flush=True) error_type = Fuzz_Test_ErrorTypes.crashed_model @@ -277,6 +275,9 @@ def find_error_rerun(self, error_dict) -> dict: # This should always be the case if error_type in [Fuzz_Test_ErrorTypes.internalcrash, Fuzz_Test_ErrorTypes.failed_model]: # Error type 'E', often during model.solve() or solveAll or type 'X' return self.bug_search_run_and_verify_model() + elif error_type == Fuzz_Test_ErrorTypes.internalfunctioncrash: + mutations = error_dict['mutators'][2::3] + return self.bug_search_run_and_verify_model(nr_mutations=len(mutations)) except AssertionError as e: print("A", end='', flush=True) @@ -315,7 +316,9 @@ def find_error_rerun(self, error_dict) -> dict: nr_timed_out=self.nr_timed_out ) - def bug_search_run_and_verify_model(self) -> dict: + def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: + if nr_mutations is not None: + self.mutations_per_model = nr_mutations for _ in range(self.mutations_per_model): last_bug_cause = self.bug_cause @@ -342,9 +345,10 @@ def bug_search_run_and_verify_model(self) -> dict: if verify_model_error is not None: return verify_model_error - # Then, apply the new mutation (which shouldn't give an error itself) + # Then, apply the new mutation and check whether it gives an error gen_mut_error = self.apply_single_mutation(m) - assert gen_mut_error is None, "There should be no errors related to the application of mutations here." + if gen_mut_error is not None: + return gen_mut_error # Finally, check the model at the end. This SHOULD give an error verify_model_error = self.verify_model(is_bug_check=True) diff --git a/verifiers/solver_voting_sat_verifier.py b/verifiers/solver_voting_sat_verifier.py index e8b4e63f..e7698350 100644 --- a/verifiers/solver_voting_sat_verifier.py +++ b/verifiers/solver_voting_sat_verifier.py @@ -104,7 +104,6 @@ def generate_mutations(self) -> dict | None: return None # don't log semanticfusion crash - print('I', end='', flush=True) return dict(seed=self.seed, type=Fuzz_Test_ErrorTypes.internalfunctioncrash, originalmodel_file=self.model_file, @@ -148,11 +147,10 @@ def verify_model(self, is_bug_check=False) -> None | dict: nr_timed_out_solvers = sum([t > time_limit * 0.8 for t in solvers_times]) if nr_timed_out_solvers > 0: # timeout, skip + self.bug_cause = 'UNKNOWN' self.nr_timed_out += nr_timed_out_solvers if not is_bug_check: print('T', end='', flush=True) - else: - self.bug_cause = 'UNKNOWN' return None elif all(s1 == s2 for i, s1 in enumerate(solvers_results) for j, s2 in enumerate(solvers_results) if i < j): # has to be same @@ -220,7 +218,7 @@ def run(self, model_file: str) -> dict | None: else: return self.find_error_rerun(verify_model_error) else: - return gen_mutations_error # This error requires no rerun + return self.find_error_rerun(gen_mutations_error) except AssertionError as e: print("A", end='', flush=True) error_type = Fuzz_Test_ErrorTypes.crashed_model @@ -268,6 +266,9 @@ def find_error_rerun(self, error_dict) -> dict: # This should always be the case if error_type in [Fuzz_Test_ErrorTypes.internalcrash, Fuzz_Test_ErrorTypes.failed_model]: # Error type 'E', often during model.solve() or solveAll or type 'X' return self.bug_search_run_and_verify_model() + elif error_type == Fuzz_Test_ErrorTypes.internalfunctioncrash: + mutations = error_dict['mutators'][2::3] + return self.bug_search_run_and_verify_model(nr_mutations=len(mutations)) except AssertionError as e: print("A", end='', flush=True) @@ -306,7 +307,9 @@ def find_error_rerun(self, error_dict) -> dict: nr_timed_out=self.nr_timed_out ) - def bug_search_run_and_verify_model(self) -> dict: + def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: + if nr_mutations is not None: + self.mutations_per_model = nr_mutations for _ in range(self.mutations_per_model): last_bug_cause = self.bug_cause @@ -333,9 +336,10 @@ def bug_search_run_and_verify_model(self) -> dict: if verify_model_error is not None: return verify_model_error - # Then, apply the new mutation (which shouldn't give an error itself) + # Then, apply the new mutation and check whether it gives an error gen_mut_error = self.apply_single_mutation(m) - assert gen_mut_error is None, "There should be no errors related to the application of mutations here." + if gen_mut_error is not None: + return gen_mut_error # Finally, check the model at the end. This SHOULD give an error verify_model_error = self.verify_model(is_bug_check=True) diff --git a/verifiers/strengthening_weakening_verifier.py b/verifiers/strengthening_weakening_verifier.py index 6586b0b1..9fecacfc 100644 --- a/verifiers/strengthening_weakening_verifier.py +++ b/verifiers/strengthening_weakening_verifier.py @@ -76,9 +76,9 @@ def generate_mutations(self) -> None | dict: self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( set(self.mm_mutators).union(set(self.gen_mutators)).union(set(self.str_wkn_mutators))) rand = random.random() - if rand <= 0.33: + if rand <= 1/3: mutator_list = self.str_wkn_mutators - elif rand <= 0.8633: # ~~ remaining 80% of 0.67 (8/15) + elif rand <= 1/3 + 0.8 * 2/3: # ~~ remaining 80% mutator_list = self.mm_mutators else: mutator_list = self.gen_mutators @@ -104,7 +104,8 @@ def generate_mutations(self) -> None | dict: self.nr_solve_checks += 1 count = model.solveAll(solver=s, solution_limit=2, time_limit=5) # should find at least 1 solution in 5s if count > 1: - self.bug_cause = 'during STR' + if m == strengthening_weakening_mutator: + self.bug_cause = 'during STR' self.cons = m(self.cons) self.bug_cause = 'STR' elif count < 1: @@ -145,7 +146,6 @@ def generate_mutations(self) -> None | dict: return None # don't log semanticfusion crash - print('I', end='', flush=True) return dict(seed=self.seed, type=Fuzz_Test_ErrorTypes.internalfunctioncrash, originalmodel_file=self.model_file, @@ -193,11 +193,10 @@ def verify_model(self, is_bug_check=False) -> None | dict: nr_timed_out_solvers = sum([t > time_limit * 0.8 for t in solvers_times]) if nr_timed_out_solvers > 0: # timeout, skip + self.bug_cause = 'UNKNOWN' self.nr_timed_out += nr_timed_out_solvers if not is_bug_check: print('T', end='', flush=True) - else: - self.bug_cause = 'UNKNOWN' return None elif all(s1 == s2 for i, s1 in enumerate(solvers_results) for j, s2 in enumerate(solvers_results) if i < j): # has to be same @@ -267,7 +266,7 @@ def run(self, model_file: str) -> dict | None: else: return self.find_error_rerun(verify_model_error) else: - return gen_mutations_error # This error requires no rerun + return self.find_error_rerun(gen_mutations_error) except AssertionError as e: print("A", end='', flush=True) error_type = Fuzz_Test_ErrorTypes.crashed_model @@ -316,6 +315,9 @@ def find_error_rerun(self, error_dict) -> dict: # This should always be the case if error_type in [Fuzz_Test_ErrorTypes.internalcrash, Fuzz_Test_ErrorTypes.failed_model]: # Error type 'E', often during model.solve() or solveAll or type 'X' return self.bug_search_run_and_verify_model() + elif error_type == Fuzz_Test_ErrorTypes.internalfunctioncrash: + mutations = error_dict['mutators'][2::3] + return self.bug_search_run_and_verify_model(nr_mutations=len(mutations)) except AssertionError as e: print("A", end='', flush=True) @@ -354,7 +356,9 @@ def find_error_rerun(self, error_dict) -> dict: nr_timed_out=self.nr_timed_out ) - def bug_search_run_and_verify_model(self) -> dict: + def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: + if nr_mutations is not None: + self.mutations_per_model = nr_mutations for _ in range(self.mutations_per_model): last_bug_cause = self.bug_cause @@ -365,10 +369,10 @@ def bug_search_run_and_verify_model(self) -> dict: self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( set(self.mm_mutators).union(set(self.gen_mutators)).union({strengthening_weakening_mutator})) rand = random.random() - if rand <= 0.33: + if rand <= 1/3: mutator_list = self.str_wkn_mutators new_mut_type = 'STRWK' - elif rand <= 0.8633: # ~~ remaining 80% of 0.67 (8/15) + elif rand <= 1/3 + 0.8 * 2/3: # ~~ remaining 80% mutator_list = self.mm_mutators new_mut_type = 'MM' else: @@ -387,9 +391,10 @@ def bug_search_run_and_verify_model(self) -> dict: if verify_model_error is not None: return verify_model_error - # Then, apply the new mutation (which shouldn't give an error itself) + # Then, apply the new mutation and check whether it gives an error gen_mut_error = self.apply_single_mutation(m) - assert gen_mut_error is None, "There should be no errors related to the application of mutations here." + if gen_mut_error is not None: + return gen_mut_error # Finally, check the model at the end. This SHOULD give an error verify_model_error = self.verify_model(is_bug_check=True) @@ -423,7 +428,8 @@ def apply_single_mutation(self, m) -> dict | None: else: count = model.solveAll(solver=s, time_limit=5) if count > 1: - self.bug_cause = 'during STR' + if m == strengthening_weakening_mutator: # solve call happening otherwise + self.bug_cause = 'during STR' self.cons = m(self.cons, strengthen=True) self.bug_cause = 'STR' elif count < 1: From 56389b42e2b54cd0fb0f12d55cc7ec9c2115bbba Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Mon, 5 May 2025 15:25:24 +0200 Subject: [PATCH 36/58] Small adjustment to mutator probabilities --- verifiers/strengthening_weakening_verifier.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/verifiers/strengthening_weakening_verifier.py b/verifiers/strengthening_weakening_verifier.py index 9fecacfc..6637a1d7 100644 --- a/verifiers/strengthening_weakening_verifier.py +++ b/verifiers/strengthening_weakening_verifier.py @@ -78,7 +78,7 @@ def generate_mutations(self) -> None | dict: rand = random.random() if rand <= 1/3: mutator_list = self.str_wkn_mutators - elif rand <= 1/3 + 0.8 * 2/3: # ~~ remaining 80% + elif rand - 1/3 <= 2/3 * 0.8: # ~~ remaining 80% mutator_list = self.mm_mutators else: mutator_list = self.gen_mutators @@ -113,7 +113,7 @@ def generate_mutations(self) -> None | dict: m = strengthening_weakening_mutator self.cons = m(self.cons, strengthen=False) self.bug_cause = 'WKN' - elif random.random() < 0.8: # If only 1 solution remains, we just go on normally instead + elif random.random() <= 0.8: # If only 1 solution remains, we just go on normally instead m = random.choice(self.mm_mutators) self.bug_cause = 'during MM' self.cons += m(self.cons) @@ -372,7 +372,7 @@ def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: if rand <= 1/3: mutator_list = self.str_wkn_mutators new_mut_type = 'STRWK' - elif rand <= 1/3 + 0.8 * 2/3: # ~~ remaining 80% + elif rand - 1/3 <= 2/3 * 0.8: # ~~ remaining 80% mutator_list = self.mm_mutators new_mut_type = 'MM' else: @@ -436,7 +436,7 @@ def apply_single_mutation(self, m) -> dict | None: self.bug_cause = 'during WKN' self.cons = m(self.cons, strengthen=False) self.bug_cause = 'WKN' - elif random.random() < 0.8: # If only 1 solution remains, we just go on normally instead + elif random.random() <= 0.8: # If only 1 solution remains, we just go on normally instead m = random.choice(self.mm_mutators) self.bug_cause = 'during MM' self.cons += m(self.cons) From 0947f5459aa0bdf7053850c402de349273e6e72e Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Mon, 5 May 2025 18:39:47 +0200 Subject: [PATCH 37/58] Added some global constraints to the weakening and strengthening mutators --- fuzz_test_utils/mutators.py | 119 ++++++++++++++++++++++++++++-------- 1 file changed, 93 insertions(+), 26 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index a39f01f4..0ade088a 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -936,7 +936,11 @@ def type_aware_operator_replacement(constraints: list): try: final_cons = copy.deepcopy(constraints) # pick a random constraint and calculate whether they have a mutable expression until they do - candidates = list(set(constraints)) + candidates = [] + for item in constraints: + if not any(item is cand for cand in candidates): + candidates.append(item) + # candidates = list(set(constraints)) # does not work with NDVarArray random.shuffle(candidates) for con in candidates: exprs = get_all_mutable_op_exprs(con) # e.g. for (x + y) // z > 0. The tree goes >(0, //(+(x,y), z)) @@ -1045,7 +1049,7 @@ def get_all_non_op_exprs(con: Expression): Helper function to get all expressions WITHOUT an operator in a given constraint """ if hasattr(con, 'args') and not isinstance(con, NDVarArray) and con.name != 'boolval': - return sum((get_all_non_op_exprs(arg) for arg in con.args), []) + return sum((get_all_non_op_exprs(arg) for arg in con.args), [con]) elif isinstance(con, list) or isinstance(con, NDVarArray): return sum((get_all_non_op_exprs(e) for e in con), []) else: @@ -1573,13 +1577,17 @@ def has_positive_parity(expr: Expression, con: Expression, curr_path: tuple) -> None if it is unknown. """ # Basecase 1: `expr` cannot be strengthened or weakened + changeable_ops = {'and', 'or', '->', 'xor', '==', '!=', '<=', '<', '>=', '>'} + changeable_globals = {AllDifferent, AllDifferentExceptN, AllEqual, AllEqualExceptN, + Table, NegativeTable, IncreasingStrict, DecreasingStrict, + LexLess, LexChainLess, Increasing, Decreasing, LexLessEq, + LexChainLessEq, InDomain} if hasattr(expr, 'name'): # NOTE: these are the simplest operators to strengthen/weaken (by just changing the operator into another one). # Other operators could be changed in another way too (e.g. add/remove elements in the second argument # of the expression x in [1, 2, 3, 4]). This could be included later and then changed accordingly in # `strengthening_weakening_mutator()`. - changeable_ops = {'and', 'or', '->', 'xor', '==', '!=', '<=', '<', '>=', '>'} - if expr.name not in changeable_ops: + if not (expr.name in changeable_ops or type(expr) in changeable_globals): return None else: return None @@ -1642,15 +1650,37 @@ def strengthen_expr(expr: Expression, path: tuple, con: Expression) -> Expressio comps = ['>', '=='] new_op = random.choice(comps) args = expr.args - if new_op in comps: - expr = Comparison(new_op, *args) - con = replace_at_path(con, path, expr) - elif new_op in ops: - expr = Operator(new_op, args) - con = replace_at_path(con, path, expr) - elif new_op == 'xor': - expr = Xor(args) - con = replace_at_path(con, path, expr) + + match expr: + case Increasing(): + expr.name = 'IncreasingStrict' + case Decreasing(): + expr.name = 'DecreasingStrict' + case LexLessEq(): + expr.name = 'LexLess' + case LexChainLessEq(): + expr.name = 'LexChainLess' + case NegativeTable(): + fst_args, snd_args = expr.args + random_idx = random.randrange(len(fst_args)) + new_fst_args = fst_args[:random_idx] + fst_args[random_idx + 1:] + new_snd_args = [arg[:random_idx] + arg[random_idx + 1:] for arg in snd_args] + expr.update_args((new_fst_args, new_snd_args)) + case InDomain(): + fst_args, snd_args = expr.args + random_idx = random.randrange(len(snd_args)) + new_snd_args = snd_args[:random_idx] + snd_args[random_idx + 1:] + expr.update_args((fst_args, new_snd_args)) + + if 'new_op' in locals(): # Rewrite this function later + if new_op in comps: + expr = Comparison(new_op, *args) + elif new_op in ops: + expr = Operator(new_op, args) + elif new_op == 'xor': + expr = Xor(args) + + con = replace_at_path(con, path, expr) return con @@ -1688,23 +1718,55 @@ def weaken_expr(expr: Expression, path: tuple, con: Expression) -> Expression: comps = ['!=', '>='] new_op = random.choice(comps) args = expr.args - if new_op in comps: - expr = Comparison(new_op, *args) - con = replace_at_path(con, path, expr) - elif new_op in ops: - expr = Operator(new_op, args) - con = replace_at_path(con, path, expr) - elif new_op == 'xor': - expr = Xor(args) - con = replace_at_path(con, path, expr) + + match expr: + case IncreasingStrict(): + expr.name = 'Increasing' + case DecreasingStrict(): + expr.name = 'Decreasing' + case LexLess(): + expr.name = 'LexLessEq' + case LexChainLess(): + expr.name = 'LexChainLessEq' + case Table(): + fst_args, snd_args = expr.args + random_idx = random.randrange(len(fst_args)) + new_fst_args = fst_args[:random_idx] + fst_args[random_idx + 1:] + new_snd_args = [arg[:random_idx] + arg[random_idx + 1:] for arg in snd_args] + expr.update_args((new_fst_args, new_snd_args)) + case AllDifferent() | AllEqual(): + old_args = expr.args + random_idx = random.randrange(len(old_args)) + new_args = old_args[:random_idx] + old_args[random_idx + 1:] + expr.update_args(new_args) + case AllDifferentExceptN() | AllEqualExceptN(): + fst_args, snd_args = expr.args + random_idx = random.randrange(len(fst_args)) + new_fst_args = fst_args[:random_idx] + fst_args[random_idx + 1:] + expr.update_args((new_fst_args, snd_args)) + + if 'new_op' in locals(): # Rewrite this function later + if new_op in comps: + expr = Comparison(new_op, *args) + elif new_op in ops: + expr = Operator(new_op, args) + elif new_op == 'xor': + expr = Xor(args) + + con = replace_at_path(con, path, expr) return con def is_changeable(strengthen: bool, expr: Expression, pos_parity: bool) -> bool: if strengthen ^ pos_parity: # weakening - return expr.name not in {'or', '->', '!=', '<=', '>='} + is_changeable_op = expr.name in {'and', 'xor', '==', '>', '<'} + is_changeable_global = type(expr) in {AllDifferent, AllDifferentExceptN, AllEqual, AllEqualExceptN, + Table, NegativeTable, IncreasingStrict, DecreasingStrict, + LexLess, LexChainLess} else: # strengthening - return expr.name not in {'and', 'xor', '==', '>', '<'} + is_changeable_op = expr.name in {'or', '->', '!=', '<=', '>='} + is_changeable_global = type(expr) in {Increasing, Decreasing, LexLessEq, LexChainLessEq, NegativeTable, InDomain} + return is_changeable_op or is_changeable_global def strengthening_weakening_mutator(constraints: list, strengthen: bool = True) -> list | Exception: @@ -1721,10 +1783,15 @@ def strengthening_weakening_mutator(constraints: list, strengthen: bool = True) - `final_cons`: a list of the same constraints where one constraint has a mutated operator """ try: - print_str = "strength" if strengthen else "weak" + # print_str = "strength" if strengthen else "weak" final_cons = copy.deepcopy(constraints) # pick a random constraint and calculate whether they have a mutable expression until they do - candidates = list(set(constraints)) + + candidates = [] + for item in constraints: + if not any(item is cand for cand in candidates): + candidates.append(item) + # candidates = list(set(constraints)) # does not work with NDVarArray random.shuffle(candidates) for con in candidates: exprs = [] From 6a732ed78d6aac3a03dcad6c75942dfca49b864a Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Mon, 5 May 2025 19:05:58 +0200 Subject: [PATCH 38/58] Some small refactorings --- fuzz_test_utils/mutators.py | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 0ade088a..0ab75c98 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1530,25 +1530,24 @@ def type_aware_expression_replacement(constraints: list): - `final_cons`: a list of the same constraints where one constraint has a mutated expression """ try: - # print("="*100) final_cons = copy.deepcopy(constraints) - # print(f"All constraints at the moment: {final_cons}") + # 1. Neem een (random) expression van een (random) constraint en de return type rand_con = random.choice(final_cons) all_con_exprs = get_all_exprs(rand_con) - # print(f"all_con_exprs: {all_con_exprs}") expr = random.choice(all_con_exprs) path, ret_type = get_return_type(expr, rand_con) # Also gives us the taken path of the expression in the constraint - # print(f"Changing constarint: {rand_con}") - # print(f"Old expression: {expr}") - # 2. Tel het aantal resterende params van elk type (van alle constraints of enkel in de constraint zelf?) + + # 2. Tel het aantal resterende params van elk type all_exprs = get_all_exprs_mult(final_cons) + # 3. Zoek een operator die <= aantal params nodig heeft met zelfde return type new_expr = get_operator(all_exprs, ret_type) + # 4. Vervang expression (+ vervang constraint) - # print(f"New expression: {new_expr}") if new_expr: new_con = replace_at_path(rand_con, path, new_expr=new_expr) + # 5. Return the new constraints # final_cons.remove(rand_con) DOES NOT WORK because it uses == instead of 'is' index = None @@ -1577,16 +1576,16 @@ def has_positive_parity(expr: Expression, con: Expression, curr_path: tuple) -> None if it is unknown. """ # Basecase 1: `expr` cannot be strengthened or weakened - changeable_ops = {'and', 'or', '->', 'xor', '==', '!=', '<=', '<', '>=', '>'} - changeable_globals = {AllDifferent, AllDifferentExceptN, AllEqual, AllEqualExceptN, - Table, NegativeTable, IncreasingStrict, DecreasingStrict, - LexLess, LexChainLess, Increasing, Decreasing, LexLessEq, - LexChainLessEq, InDomain} if hasattr(expr, 'name'): # NOTE: these are the simplest operators to strengthen/weaken (by just changing the operator into another one). # Other operators could be changed in another way too (e.g. add/remove elements in the second argument # of the expression x in [1, 2, 3, 4]). This could be included later and then changed accordingly in # `strengthening_weakening_mutator()`. + changeable_ops = {'and', 'or', '->', 'xor', '==', '!=', '<=', '<', '>=', '>'} + changeable_globals = {AllDifferent, AllDifferentExceptN, AllEqual, AllEqualExceptN, + Table, NegativeTable, IncreasingStrict, DecreasingStrict, + LexLess, LexChainLess, Increasing, Decreasing, LexLessEq, + LexChainLessEq, InDomain} if not (expr.name in changeable_ops or type(expr) in changeable_globals): return None else: @@ -1672,7 +1671,7 @@ def strengthen_expr(expr: Expression, path: tuple, con: Expression) -> Expressio new_snd_args = snd_args[:random_idx] + snd_args[random_idx + 1:] expr.update_args((fst_args, new_snd_args)) - if 'new_op' in locals(): # Rewrite this function later + if 'new_op' in locals(): # Rewrite this later if new_op in comps: expr = Comparison(new_op, *args) elif new_op in ops: @@ -1745,7 +1744,7 @@ def weaken_expr(expr: Expression, path: tuple, con: Expression) -> Expression: new_fst_args = fst_args[:random_idx] + fst_args[random_idx + 1:] expr.update_args((new_fst_args, snd_args)) - if 'new_op' in locals(): # Rewrite this function later + if 'new_op' in locals(): # Rewrite this later if new_op in comps: expr = Comparison(new_op, *args) elif new_op in ops: @@ -1758,6 +1757,7 @@ def weaken_expr(expr: Expression, path: tuple, con: Expression) -> Expression: def is_changeable(strengthen: bool, expr: Expression, pos_parity: bool) -> bool: + # Sets to be extended when more weakening and strengthening options get added if strengthen ^ pos_parity: # weakening is_changeable_op = expr.name in {'and', 'xor', '==', '>', '<'} is_changeable_global = type(expr) in {AllDifferent, AllDifferentExceptN, AllEqual, AllEqualExceptN, @@ -1783,10 +1783,9 @@ def strengthening_weakening_mutator(constraints: list, strengthen: bool = True) - `final_cons`: a list of the same constraints where one constraint has a mutated operator """ try: - # print_str = "strength" if strengthen else "weak" final_cons = copy.deepcopy(constraints) - # pick a random constraint and calculate whether they have a mutable expression until they do + # pick a random constraint and calculate whether they have a mutable expression until they do candidates = [] for item in constraints: if not any(item is cand for cand in candidates): @@ -1802,7 +1801,6 @@ def strengthening_weakening_mutator(constraints: list, strengthen: bool = True) if exprs: break else: # In case there isn't any mutable (weakening/strengthening depending on `strengthen`) expression in any constraint - # print(f"Couldn't find a constraint to {print_str}en.") return final_cons # Remove the constraint from the constraints @@ -1811,8 +1809,6 @@ def strengthening_weakening_mutator(constraints: list, strengthen: bool = True) # Choose an expression to change expr, (pos_parity, path) = random.choice(exprs) - # print(f"{print_str}ening constraint {con} by changing expression {expr}.") - # Mutate this expression if strengthen ^ pos_parity: # weaken if parity is different from `strengthen` con = weaken_expr(expr, path, con) From 384702ef7e31cd4d1f6c05d90223fac25da29cab Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Sun, 18 May 2025 10:51:32 +0200 Subject: [PATCH 39/58] Added mm_mutation probability as parameter --- fuzz_test.py | 3 +- fuzz_test_rerunner.py | 7 ++--- verifiers/solver_voting_count_verifier.py | 28 ++++++++++++------- verifiers/solver_voting_sat_verifier.py | 23 +++++++++++---- verifiers/strengthening_weakening_verifier.py | 28 ++++++++++++------- 5 files changed, 58 insertions(+), 31 deletions(-) diff --git a/fuzz_test.py b/fuzz_test.py index 0b65c76b..d4eedfbc 100644 --- a/fuzz_test.py +++ b/fuzz_test.py @@ -34,6 +34,7 @@ def check_positive(value): parser.add_argument("--max-minutes", help = "The maximum time (in minutes) the tests should run (by default the tests will run forever). The tests will quit sooner if max-bugs was set and reached or an keyboardinterrupt occured", required=False, default=math.inf ,type=check_positive) parser.add_argument("-mpm","--mutations-per-model", help = "The amount of mutations that will be executed on every model", required=False, default=5 ,type=check_positive) parser.add_argument("-p","--amount-of-processes", help = "The amount of processes that will be used to run the tests", required=False, default=cpu_count()-1 ,type=check_positive) # the -1 is for the main process + parser.add_argument("--mm-prob", help="The probability that a metamorphic mutation will be chosen in case of a verifier that allows other mutations", required=False, default=1, type=float) args = parser.parse_args() models = [] max_failed_tests = args.max_failed_tests @@ -65,7 +66,7 @@ def check_positive(value): # creating processes to run all the tests processes = [] - process_args = (current_amount_of_tests, current_amount_of_error, lock, args.solver, args.mutations_per_model ,models ,max_failed_tests,args.output_dir, max_time) + process_args = (current_amount_of_tests, current_amount_of_error, lock, args.solver, args.mutations_per_model ,models ,max_failed_tests,args.output_dir, max_time, args.mm_prob) for x in range(args.amount_of_processes): processes.append(Process(target=run_verifiers,args=process_args)) diff --git a/fuzz_test_rerunner.py b/fuzz_test_rerunner.py index 17b4e4bd..4bc80f79 100644 --- a/fuzz_test_rerunner.py +++ b/fuzz_test_rerunner.py @@ -33,11 +33,10 @@ def rerun_file(failed_model_file,output_dir ): error_data = pickle.loads(fpcl.read()) random.seed(error_data["seed"]) if error_data["error"]["type"].name != "fuzz_test_crash": # if it is a fuzz_test crash error we skip it - if type(error_data["solver"]) == str: # old solver voting implementation (e.g. "minizinc,z3") - solver = error_data["solver"].split(',') - else: - solver = error_data["solver"] + solver = error_data["solver"] verifier_kwargs = {'solver': solver, "mutations_per_model": error_data["mutations_per_model"], "exclude_dict": {}, "time_limit": time.time()*3600, "seed": error_data["seed"]} + if 'mm_prob' in error_data["error"]: + verifier_kwargs['mm_prob'] = error_data["error"]["mm_prob"] error = lookup_verifier(error_data["verifier"])(**verifier_kwargs).rerun(error_data["error"]) error_data["error"] = error diff --git a/verifiers/solver_voting_count_verifier.py b/verifiers/solver_voting_count_verifier.py index 2475e081..06cd4c62 100644 --- a/verifiers/solver_voting_count_verifier.py +++ b/verifiers/solver_voting_count_verifier.py @@ -2,10 +2,10 @@ class Solver_Vote_Count_Verifier(Verifier): """ - The Solver Count Verifier will verify if the satisfiability is the same for all solvers after running multiple mutations + The Solver Count Verifier will verify if the amount of solutions is the same for all solvers after running multiple mutations """ - def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, time_limit: float, seed: int): + def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, time_limit: float, seed: int, mm_prob: float): self.name = "solver_vote_count_verifier" self.type = 'sat' @@ -43,6 +43,7 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti self.bug_cause = 'STARTMODEL' self.nr_timed_out = 0 self.last_mut = None + self.mm_prob = mm_prob def initialize_run(self) -> None: if self.original_model == None: @@ -51,12 +52,11 @@ def initialize_run(self) -> None: self.cons = self.original_model.constraints assert (len(self.cons) > 0), f"{self.model_file} has no constraints" self.cons = toplevel_list(self.cons) - - assert len(self.solvers) == 2, f"2 solvers required, {len(self.solvers)} given." - if 'gurobi' in [s.lower() for s in self.solvers]: + assert len(self.solvers) > 1, f"More than 1 solver required, given solvers: {self.solvers}." + if 'gurobi' in [s.lower() for s in self.solvers]: # Because gurobi can't run solveAll without solution_limit self.sol_lim = 10000 # TODO: is hardcode best idea? - # assert self.sol_count_1 == self.sol_count_2, f"{self.solvers} don't agree on amount of solutions (before mutations): {self.sol_count_1} and {self.sol_count_2}" + # Optional: Check before applying the mutations. This should never fail... self.mutators = [copy.deepcopy( self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. @@ -70,9 +70,9 @@ def generate_mutations(self) -> None | dict: valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( set(self.mm_mutators).union(set(self.gen_mutators))) - if random.random() <= 0.8: # 80% chance to choose metamorphic mutation + if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation mutator_list = self.mm_mutators - else: # 20% chance to choose generation-based mutation + else: # 1-mm_prob to choose generation-based mutation mutator_list = self.gen_mutators valid = [m for m in mutator_list if m in valid_mutators] @@ -110,6 +110,7 @@ def generate_mutations(self) -> None | dict: # don't log semanticfusion crash return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.internalfunctioncrash, originalmodel_file=self.model_file, exception=e, @@ -171,6 +172,7 @@ def verify_model(self, is_bug_check=False) -> None | dict: if is_bug_check: print('X', end='', flush=True) return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.failed_model, originalmodel_file=self.model_file, exception=f"Results of the solvers are not equal. Solver results: {solver_results_str}.", @@ -193,6 +195,7 @@ def verify_model(self, is_bug_check=False) -> None | dict: print('E', end='', flush=True) return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.internalcrash, originalmodel_file=self.model_file, exception=e, @@ -236,6 +239,7 @@ def run(self, model_file: str) -> dict | None: elif "has no constraints" in str(e): error_type = Fuzz_Test_ErrorTypes.no_constraints_model return dict(seed=self.seed, + mm_prob=self.mm_prob, type=error_type, originalmodel_file=self.model_file, exception=e, @@ -252,6 +256,7 @@ def run(self, model_file: str) -> dict | None: except Exception as e: print('C', end='', flush=True) return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.crashed_model, originalmodel_file=self.model_file, exception=e, @@ -287,6 +292,7 @@ def find_error_rerun(self, error_dict) -> dict: elif "has no constraints" in str(e): type = Fuzz_Test_ErrorTypes.no_constraints_model return dict(seed=self.seed, + mm_prob=self.mm_prob, type=type, originalmodel_file=self.model_file, exception=e, @@ -303,6 +309,7 @@ def find_error_rerun(self, error_dict) -> dict: except Exception as e: print('C', end='', flush=True) return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.crashed_model, originalmodel_file=self.model_file, exception=e, @@ -326,10 +333,10 @@ def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( set(self.mm_mutators).union(set(self.gen_mutators))) - if random.random() <= 0.8: # 80% chance to choose metamorphic mutation + if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation mutator_list = self.mm_mutators new_mut_type = 'MM' - else: # 20% chance to choose generation-based mutation + else: # 1-mm_prob to choose generation-based mutation mutator_list = self.gen_mutators new_mut_type = 'GEN' @@ -391,6 +398,7 @@ def apply_single_mutation(self, m) -> dict | None: print('I', end='', flush=True) return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.internalfunctioncrash, originalmodel_file=self.model_file, exception=e, diff --git a/verifiers/solver_voting_sat_verifier.py b/verifiers/solver_voting_sat_verifier.py index e7698350..1df3c7e0 100644 --- a/verifiers/solver_voting_sat_verifier.py +++ b/verifiers/solver_voting_sat_verifier.py @@ -2,10 +2,10 @@ class Solver_Vote_Sat_Verifier(Verifier): """ - The Solver Count Verifier will verify if the satisfiability is the same for all solvers after running multiple mutations + The Solver Satisfiability Verifier will verify if the satisfiability is the same for all solvers after running multiple mutations """ - def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, time_limit: float, seed: int): + def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, time_limit: float, seed: int, mm_prob: float): self.name = "solver_vote_sat_verifier" self.type = 'sat' @@ -43,6 +43,7 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti self.bug_cause = 'STARTMODEL' self.nr_timed_out = 0 self.last_mut = None + self.mm_prob = mm_prob def initialize_run(self) -> None: if self.original_model == None: @@ -51,8 +52,10 @@ def initialize_run(self) -> None: self.cons = self.original_model.constraints assert (len(self.cons) > 0), f"{self.model_file} has no constraints" self.cons = toplevel_list(self.cons) + assert len(self.solvers) > 1, f"More than 1 solver required, given solvers: {self.solvers}." # No other preparation necessary + # Optional: Check before applying the mutations. This should never fail... self.mutators = [copy.deepcopy( self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. @@ -65,9 +68,9 @@ def generate_mutations(self) -> dict | None: # choose a mutation (not in exclude_dict) valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list(set(self.mm_mutators).union(set(self.gen_mutators))) - if random.random() <= 0.8: # 80% chance to choose metamorphic mutation + if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation mutator_list = self.mm_mutators - else: # 20% chance to choose generation-based mutation + else: # 1-mm_prob to choose generation-based mutation mutator_list = self.gen_mutators valid = [m for m in mutator_list if m in valid_mutators] @@ -105,6 +108,7 @@ def generate_mutations(self) -> dict | None: # don't log semanticfusion crash return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.internalfunctioncrash, originalmodel_file=self.model_file, exception=e, @@ -163,6 +167,7 @@ def verify_model(self, is_bug_check=False) -> None | dict: if is_bug_check: print('X', end='', flush=True) return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.failed_model, originalmodel_file=self.model_file, exception=f"Results of the solvers are not equal. Solver results: {solver_results_str}.", @@ -184,6 +189,7 @@ def verify_model(self, is_bug_check=False) -> None | dict: print('E', end='', flush=True) return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.internalcrash, originalmodel_file=self.model_file, exception=e, @@ -227,6 +233,7 @@ def run(self, model_file: str) -> dict | None: elif "has no constraints" in str(e): error_type = Fuzz_Test_ErrorTypes.no_constraints_model return dict(seed=self.seed, + mm_prob=self.mm_prob, type=error_type, originalmodel_file=self.model_file, exception=e, @@ -243,6 +250,7 @@ def run(self, model_file: str) -> dict | None: except Exception as e: print('C', end='', flush=True) return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.crashed_model, originalmodel_file=self.model_file, exception=e, @@ -278,6 +286,7 @@ def find_error_rerun(self, error_dict) -> dict: elif "has no constraints" in str(e): type = Fuzz_Test_ErrorTypes.no_constraints_model return dict(seed=self.seed, + mm_prob=self.mm_prob, type=type, originalmodel_file=self.model_file, exception=e, @@ -294,6 +303,7 @@ def find_error_rerun(self, error_dict) -> dict: except Exception as e: print('C', end='', flush=True) return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.crashed_model, originalmodel_file=self.model_file, exception=e, @@ -317,10 +327,10 @@ def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( set(self.mm_mutators).union(set(self.gen_mutators))) - if random.random() <= 0.8: # 80% chance to choose metamorphic mutation + if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation mutator_list = self.mm_mutators new_mut_type = 'MM' - else: # 20% chance to choose generation-based mutation + else: # 1-mm_prob to choose generation-based mutation mutator_list = self.gen_mutators new_mut_type = 'GEN' @@ -382,6 +392,7 @@ def apply_single_mutation(self, m) -> dict | None: print('I', end='', flush=True) return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.internalfunctioncrash, originalmodel_file=self.model_file, exception=e, diff --git a/verifiers/strengthening_weakening_verifier.py b/verifiers/strengthening_weakening_verifier.py index 6637a1d7..db3a8a48 100644 --- a/verifiers/strengthening_weakening_verifier.py +++ b/verifiers/strengthening_weakening_verifier.py @@ -4,10 +4,10 @@ class Strengthening_Weakening_Verifier(Verifier): """ - The Solver Count Verifier will verify if the satisfiability is the same for all solvers after running multiple mutations + The Strengthening Weakening Verifier will verify if the amount of solutions is the same for all solvers after running multiple mutations """ - def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, time_limit: float, seed: int): + def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, time_limit: float, seed: int, mm_prob: float): self.name = "strengthening_weakening_verifier" self.type = 'sat' @@ -46,6 +46,7 @@ def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, ti self.bug_cause = 'STARTMODEL' self.nr_timed_out = 0 self.last_mut = None + self.mm_prob = mm_prob def initialize_run(self) -> None: if self.original_model == None: @@ -54,12 +55,11 @@ def initialize_run(self) -> None: self.cons = self.original_model.constraints assert (len(self.cons) > 0), f"{self.model_file} has no constraints" self.cons = toplevel_list(self.cons) - - assert len(self.solvers) == 2, f"2 solvers required, {len(self.solvers)} given." - if 'gurobi' in [s.lower() for s in self.solvers]: + assert len(self.solvers) > 1, f"More than 1 solver required, given solvers: {self.solvers}." + if 'gurobi' in [s.lower() for s in self.solvers]: # Because gurobi can't run solveAll without solution_limit self.sol_lim = 10000 # TODO: is hardcode best idea? - # assert self.sol_count_1 == self.sol_count_2, f"{self.solvers} don't agree on amount of solutions (before mutations): {self.sol_count_1} and {self.sol_count_2}" + # Optional: Check before applying the mutations. This should never fail... self.mutators = [copy.deepcopy( self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. @@ -78,7 +78,7 @@ def generate_mutations(self) -> None | dict: rand = random.random() if rand <= 1/3: mutator_list = self.str_wkn_mutators - elif rand - 1/3 <= 2/3 * 0.8: # ~~ remaining 80% + elif rand - 1/3 <= 2/3 * self.mm_prob: # ~~ remaining mm_prob mutator_list = self.mm_mutators else: mutator_list = self.gen_mutators @@ -113,7 +113,7 @@ def generate_mutations(self) -> None | dict: m = strengthening_weakening_mutator self.cons = m(self.cons, strengthen=False) self.bug_cause = 'WKN' - elif random.random() <= 0.8: # If only 1 solution remains, we just go on normally instead + elif random.random() <= self.mm_prob: # If only 1 solution remains, we just go on normally instead m = random.choice(self.mm_mutators) self.bug_cause = 'during MM' self.cons += m(self.cons) @@ -147,6 +147,7 @@ def generate_mutations(self) -> None | dict: # don't log semanticfusion crash return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.internalfunctioncrash, originalmodel_file=self.model_file, exception=e, @@ -209,6 +210,7 @@ def verify_model(self, is_bug_check=False) -> None | dict: if is_bug_check: print('X', end='', flush=True) return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.failed_model, originalmodel_file=self.model_file, exception=f"Results of the solvers are not equal. Solver results: {solver_results_str}.", @@ -231,6 +233,7 @@ def verify_model(self, is_bug_check=False) -> None | dict: print('E', end='', flush=True) return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.internalcrash, originalmodel_file=self.model_file, exception=e, @@ -275,6 +278,7 @@ def run(self, model_file: str) -> dict | None: elif "has no constraints" in str(e): error_type = Fuzz_Test_ErrorTypes.no_constraints_model return dict(seed=self.seed, + mm_prob=self.mm_prob, type=error_type, originalmodel_file=self.model_file, exception=e, @@ -291,6 +295,7 @@ def run(self, model_file: str) -> dict | None: except Exception as e: print('C', end='', flush=True) return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.crashed_model, originalmodel_file=self.model_file, exception=e, @@ -327,6 +332,7 @@ def find_error_rerun(self, error_dict) -> dict: elif "has no constraints" in str(e): type = Fuzz_Test_ErrorTypes.no_constraints_model return dict(seed=self.seed, + mm_prob=self.mm_prob, type=type, originalmodel_file=self.model_file, exception=e, @@ -343,6 +349,7 @@ def find_error_rerun(self, error_dict) -> dict: except Exception as e: print('C', end='', flush=True) return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.crashed_model, originalmodel_file=self.model_file, exception=e, @@ -372,7 +379,7 @@ def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: if rand <= 1/3: mutator_list = self.str_wkn_mutators new_mut_type = 'STRWK' - elif rand - 1/3 <= 2/3 * 0.8: # ~~ remaining 80% + elif rand - 1/3 <= 2/3 * self.mm_prob: # ~~ remaining mm_prob mutator_list = self.mm_mutators new_mut_type = 'MM' else: @@ -436,7 +443,7 @@ def apply_single_mutation(self, m) -> dict | None: self.bug_cause = 'during WKN' self.cons = m(self.cons, strengthen=False) self.bug_cause = 'WKN' - elif random.random() <= 0.8: # If only 1 solution remains, we just go on normally instead + elif random.random() <= self.mm_prob: # If only 1 solution remains, we just go on normally instead m = random.choice(self.mm_mutators) self.bug_cause = 'during MM' self.cons += m(self.cons) @@ -470,6 +477,7 @@ def apply_single_mutation(self, m) -> dict | None: print('I', end='', flush=True) return dict(seed=self.seed, + mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.internalfunctioncrash, originalmodel_file=self.model_file, exception=e, From 032833b87034bf9c3fdb5ea1b5501d04382e2421 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Sun, 18 May 2025 10:52:28 +0200 Subject: [PATCH 40/58] Added new equivalence verifier for solver voting --- verifiers/__init__.py | 4 + verifiers/solver_voting_eq_verifier.py | 419 +++++++++++++++++++++++++ 2 files changed, 423 insertions(+) create mode 100644 verifiers/solver_voting_eq_verifier.py diff --git a/verifiers/__init__.py b/verifiers/__init__.py index 6f9ab637..1d097967 100644 --- a/verifiers/__init__.py +++ b/verifiers/__init__.py @@ -16,6 +16,7 @@ from .solver_voting_sat_verifier import Solver_Vote_Sat_Verifier from .solver_voting_count_verifier import Solver_Vote_Count_Verifier from .strengthening_weakening_verifier import Strengthening_Weakening_Verifier +from .solver_voting_eq_verifier import Solver_Vote_Eq_Verifier from .verifier_runner import run_verifiers, get_all_verifiers @@ -44,6 +45,9 @@ def lookup_verifier(verfier_name: str) -> Verifier: elif verfier_name == "strengthening_weakening_verifier": return Strengthening_Weakening_Verifier + elif verfier_name == "solver_vote_eq_verifier": + return Solver_Vote_Eq_Verifier + else: raise ValueError(f"Error verifier with name {verfier_name} does not exist") return None \ No newline at end of file diff --git a/verifiers/solver_voting_eq_verifier.py b/verifiers/solver_voting_eq_verifier.py new file mode 100644 index 00000000..6c0b8aad --- /dev/null +++ b/verifiers/solver_voting_eq_verifier.py @@ -0,0 +1,419 @@ +from verifiers import * + +class Solver_Vote_Eq_Verifier(Verifier): + """ + The Solver Equivalence Verifier will verify if the solutions are the same for all solvers after running multiple mutations + """ + + def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, time_limit: float, seed: int, mm_prob: float): + self.name = "solver_vote_eq_verifier" + self.type = 'sat' + + self.solvers = solver + + self.mutations_per_model = mutations_per_model + self.exclude_dict = exclude_dict + self.time_limit = time_limit + self.seed = random.Random().random() + self.mm_mutators = [xor_morph, and_morph, or_morph, implies_morph, not_morph, + linearize_constraint_morph, + flatten_morph, + only_numexpr_equality_morph, + normalized_numexpr_morph, + reify_rewrite_morph, + only_bv_reifies_morph, + only_positive_bv_morph, + flat2cnf_morph, + toplevel_list_morph, + decompose_in_tree_morph, + push_down_negation_morph, + simplify_boolean_morph, + canonical_comparison_morph, + aritmetic_comparison_morph, + semanticFusionCounting, + semanticFusionCountingMinus, + semanticFusionCountingwsum, + semanticFusionCounting, + semanticFusionCountingMinus, + semanticFusionCountingwsum] + self.gen_mutators = [type_aware_operator_replacement, type_aware_expression_replacement] + self.mutators = [] + self.original_model = None + self.nr_solve_checks = 0 + self.bug_cause = 'STARTMODEL' + self.nr_timed_out = 0 + self.last_mut = None + self.mm_prob = mm_prob + + def initialize_run(self) -> None: + if self.original_model == None: + with open(self.model_file, 'rb') as fpcl: + self.original_model = pickle.loads(fpcl.read()) + self.cons = self.original_model.constraints + assert (len(self.cons) > 0), f"{self.model_file} has no constraints" + self.cons = toplevel_list(self.cons) + assert len(self.solvers) > 1, f"More than 1 solver required, given solvers: {self.solvers}." + if 'gurobi' in [s.lower() for s in self.solvers]: # Because gurobi can't run solveAll without solution_limit + self.sol_lim = 10000 # TODO: is hardcode best idea? + + self.original_vars = get_variables(self.cons) # New auxiliary variables will be added so we need to do this here + + self.mutators = [copy.deepcopy( + self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. + + def generate_mutations(self) -> None | dict: + """ + Will generate random mutations based on mutations_per_model for the model + """ + for i in range(self.mutations_per_model): + # choose a mutation (not in exclude_dict) + valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( + self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( + set(self.mm_mutators).union(set(self.gen_mutators))) + if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation + mutator_list = self.mm_mutators + else: # 1-mm_prob to choose generation-based mutation + mutator_list = self.gen_mutators + + valid = [m for m in mutator_list if m in valid_mutators] + if valid: + m = random.choice(valid) + else: + continue + + self.mutators += [self.seed] + # an error can occur in the transformations, so even before the solve call. + # log function and arguments in that case + self.mutators += [m] + try: + if m in self.gen_mutators: + self.bug_cause = 'during GEN' + self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints + self.bug_cause = 'GEN' + else: + self.bug_cause = 'during MM' + self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints + self.bug_cause = 'MM' + self.mutators += [copy.deepcopy(self.cons)] + except MetamorphicError as exc: + # add to exclude_dict, to avoid running into the same error + if self.model_file in self.exclude_dict: + self.exclude_dict[self.model_file] += [m] + else: + self.exclude_dict[self.model_file] = [m] + function, argument, e = exc.args + if isinstance(e, CPMpyException): + # expected behavior if we throw a cpmpy exception, do not log + return None + elif function == semanticFusion: + return None + # don't log semanticfusion crash + + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=Fuzz_Test_ErrorTypes.internalfunctioncrash, + originalmodel_file=self.model_file, + exception=e, + function=function, + argument=argument, + stacktrace=traceback.format_exc(), + mutators=self.mutators, + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + return None + def verify_model(self, is_bug_check=False) -> None | dict: + try: + model = cp.Model(self.cons) + + # if is_bug_check: + # max_search_time = 20 + # else: + # max_search_time = 40 + max_search_time = 40 + + time_limit = max(1, min(max_search_time, # TODO: change `max_search_time` back to 200 + self.time_limit - time.time())) # set the max time limit to the given time limit or to 1 if the self.time_limit-time.time() would be smaller then 1 + + # Get the actual solver results and their execution times. + # We do it this way because a solver might crash, meaning the other solver doesn't get a turn. + all_sols = [set() for i, s in enumerate(self.solvers)] + solvers_times = [] + for i, s in enumerate(self.solvers): + self.nr_solve_checks += 1 + if hasattr(self, 'sol_lim'): + model.solveAll(solver=s, time_limit=time_limit, display=lambda: all_sols[i].add( + tuple([v.value() for v in self.original_vars])), solution_limit=self.sol_lim) + solvers_times.append(model.status().runtime) + else: + model.solveAll(solver=s, time_limit=time_limit, display=lambda: all_sols[i].add( + tuple([v.value() for v in self.original_vars]))) + solvers_times.append(model.status().runtime) + + nr_timed_out_solvers = sum([t > time_limit * 0.8 for t in solvers_times]) + if nr_timed_out_solvers > 0: + # timeout, skip + self.bug_cause = 'UNKNOWN' + self.nr_timed_out += nr_timed_out_solvers + if not is_bug_check: + print('T', end='', flush=True) + return None + elif all(len(s1.symmetric_difference(s2)) == 0 for i, s1 in enumerate(all_sols) for j, s2 in enumerate(all_sols) if i < j): + # has to be same + if not is_bug_check: + print('.', end='', flush=True) + return None + else: + solver_results_str = ", ".join( + f"{solver}: {result}" for solver, result in zip(self.solvers, all_sols)) + if is_bug_check: + print('X', end='', flush=True) + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=Fuzz_Test_ErrorTypes.failed_model, + originalmodel_file=self.model_file, + exception=f"Results of the solvers are not equal. Solver results: {solver_results_str}.", + constraints=self.cons, + mutators=self.mutators, + model=model, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + except Exception as e: + if isinstance(e, (CPMpyException, NotImplementedError)): + # expected error message, ignore + return None + if is_bug_check: + print('E', end='', flush=True) + + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=Fuzz_Test_ErrorTypes.internalcrash, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + mutators=self.mutators, + model=model, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + def run(self, model_file: str) -> dict | None: + """ + This function will run a single tests on the given model + """ + try: + random.seed(self.seed) + self.model_file = model_file + self.initialize_run() + gen_mutations_error = self.generate_mutations() + + # check if no error occured while generation the mutations + if gen_mutations_error == None: + # FOLLOWING 5 LINES CHANGED! + verify_model_error = self.verify_model() + if verify_model_error == None: + return None + else: + return self.find_error_rerun(verify_model_error) + else: + return self.find_error_rerun(gen_mutations_error) + except AssertionError as e: + print("A", end='', flush=True) + error_type = Fuzz_Test_ErrorTypes.crashed_model + if "is not sat" in str(e): + error_type = Fuzz_Test_ErrorTypes.unsat_model + elif "has no constraints" in str(e): + error_type = Fuzz_Test_ErrorTypes.no_constraints_model + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=error_type, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + except Exception as e: + print('C', end='', flush=True) + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=Fuzz_Test_ErrorTypes.crashed_model, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + mutators=self.mutators, + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + def find_error_rerun(self, error_dict) -> dict: + try: + random.seed(self.seed) + error_type = error_dict['type'] + self.initialize_run() # initialize empty (self.)model, cons, mutators + + # This should always be the case + if error_type in [Fuzz_Test_ErrorTypes.internalcrash, Fuzz_Test_ErrorTypes.failed_model]: # Error type 'E', often during model.solve() or solveAll or type 'X' + return self.bug_search_run_and_verify_model() + elif error_type == Fuzz_Test_ErrorTypes.internalfunctioncrash: + mutations = error_dict['mutators'][2::3] + return self.bug_search_run_and_verify_model(nr_mutations=len(mutations)) + + except AssertionError as e: + print("A", end='', flush=True) + type = Fuzz_Test_ErrorTypes.crashed_model + if "is not sat" in str(e): + type = Fuzz_Test_ErrorTypes.unsat_model + elif "has no constraints" in str(e): + type = Fuzz_Test_ErrorTypes.no_constraints_model + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=type, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + except Exception as e: + print('C', end='', flush=True) + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=Fuzz_Test_ErrorTypes.crashed_model, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: + if nr_mutations is not None: + self.mutations_per_model = nr_mutations + for _ in range(self.mutations_per_model): + last_bug_cause = self.bug_cause + + # Generate the type of mutation that will happen + valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( + self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( + set(self.mm_mutators).union(set(self.gen_mutators))) + if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation + mutator_list = self.mm_mutators + new_mut_type = 'MM' + else: # 1-mm_prob to choose generation-based mutation + mutator_list = self.gen_mutators + new_mut_type = 'GEN' + + valid = [m for m in mutator_list if m in valid_mutators] + if valid: + m = random.choice(valid) + else: + continue + + # Check whether verify_model returns an error before the new mutation, because the cause is then at the old mutation + if new_mut_type != last_bug_cause: + verify_model_error = self.verify_model(is_bug_check=True) + if verify_model_error is not None: + return verify_model_error + + # Then, apply the new mutation and check whether it gives an error + gen_mut_error = self.apply_single_mutation(m) + if gen_mut_error is not None: + return gen_mut_error + + # Finally, check the model at the end. This SHOULD give an error + verify_model_error = self.verify_model(is_bug_check=True) + if verify_model_error is not None: + return verify_model_error + else: + print('_', end='', flush=True) + + def apply_single_mutation(self, m) -> dict | None: + """ + Will generate one random mutation and apply it to the model + """ + self.mutators += [self.seed] + # an error can occur in the transformations, so even before the solve call. + # log function and arguments in that case + self.mutators += [m] + try: + if m in self.gen_mutators: + self.bug_cause = f'during GEN, after {self.bug_cause}' + self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints + self.bug_cause = 'GEN' + else: + self.bug_cause = f'during MM, after {self.bug_cause}' + self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints + self.bug_cause = 'MM' + self.mutators += [copy.deepcopy(self.cons)] + except MetamorphicError as exc: + # add to exclude_dict, to avoid running into the same error + if self.model_file in self.exclude_dict: + self.exclude_dict[self.model_file] += [m] + else: + self.exclude_dict[self.model_file] = [m] + function, argument, e = exc.args + if isinstance(e, CPMpyException): + # expected behavior if we throw a cpmpy exception, do not log + return None + elif function == semanticFusion: + return None + # don't log semanticfusion crash + + print('I', end='', flush=True) + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=Fuzz_Test_ErrorTypes.internalfunctioncrash, + originalmodel_file=self.model_file, + exception=e, + function=function, + argument=argument, + stacktrace=traceback.format_exc(), + mutators=self.mutators, + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + return None \ No newline at end of file From 33d883d3d8cf3457acf8f2c3efaee8de3970a73e Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Sun, 18 May 2025 11:11:34 +0200 Subject: [PATCH 41/58] Added new verifier that checks single solution --- verifiers/__init__.py | 4 + verifiers/solver_voting_sol_verifier.py | 416 ++++++++++++++++++++++++ 2 files changed, 420 insertions(+) create mode 100644 verifiers/solver_voting_sol_verifier.py diff --git a/verifiers/__init__.py b/verifiers/__init__.py index 1d097967..b8dd4412 100644 --- a/verifiers/__init__.py +++ b/verifiers/__init__.py @@ -17,6 +17,7 @@ from .solver_voting_count_verifier import Solver_Vote_Count_Verifier from .strengthening_weakening_verifier import Strengthening_Weakening_Verifier from .solver_voting_eq_verifier import Solver_Vote_Eq_Verifier +from .solver_voting_sol_verifier import Solver_Vote_Sol_Verifier from .verifier_runner import run_verifiers, get_all_verifiers @@ -48,6 +49,9 @@ def lookup_verifier(verfier_name: str) -> Verifier: elif verfier_name == "solver_vote_eq_verifier": return Solver_Vote_Eq_Verifier + elif verfier_name == "solver_vote_sol_verifier": + return Solver_Vote_Sol_Verifier + else: raise ValueError(f"Error verifier with name {verfier_name} does not exist") return None \ No newline at end of file diff --git a/verifiers/solver_voting_sol_verifier.py b/verifiers/solver_voting_sol_verifier.py new file mode 100644 index 00000000..ffc93811 --- /dev/null +++ b/verifiers/solver_voting_sol_verifier.py @@ -0,0 +1,416 @@ +from verifiers import * + +class Solver_Vote_Sol_Verifier(Verifier): + """ + The Solver Sol Verifier will verify if a single solution is also in all other solvers after running multiple mutations + """ + + def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, time_limit: float, seed: int, mm_prob: float): + self.name = "solver_vote_sol_verifier" + self.type = 'sat' + + self.solvers = solver + + self.mutations_per_model = mutations_per_model + self.exclude_dict = exclude_dict + self.time_limit = time_limit + self.seed = random.Random().random() + self.mm_mutators = [xor_morph, and_morph, or_morph, implies_morph, not_morph, + linearize_constraint_morph, + flatten_morph, + only_numexpr_equality_morph, + normalized_numexpr_morph, + reify_rewrite_morph, + only_bv_reifies_morph, + only_positive_bv_morph, + flat2cnf_morph, + toplevel_list_morph, + decompose_in_tree_morph, + push_down_negation_morph, + simplify_boolean_morph, + canonical_comparison_morph, + aritmetic_comparison_morph, + semanticFusionCounting, + semanticFusionCountingMinus, + semanticFusionCountingwsum, + semanticFusionCounting, + semanticFusionCountingMinus, + semanticFusionCountingwsum] + self.gen_mutators = [type_aware_operator_replacement, type_aware_expression_replacement] + self.mutators = [] + self.original_model = None + self.nr_solve_checks = 0 + self.bug_cause = 'STARTMODEL' + self.nr_timed_out = 0 + self.last_mut = None + self.mm_prob = mm_prob + + def initialize_run(self) -> None: + if self.original_model == None: + with open(self.model_file, 'rb') as fpcl: + self.original_model = pickle.loads(fpcl.read()) + self.cons = self.original_model.constraints + assert (len(self.cons) > 0), f"{self.model_file} has no constraints" + self.cons = toplevel_list(self.cons) + assert len(self.solvers) > 1, f"More than 1 solver required, given solvers: {self.solvers}." + + self.og_vars = get_variables(self.cons) + + self.mutators = [copy.deepcopy( + self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. + + def generate_mutations(self) -> None | dict: + """ + Will generate random mutations based on mutations_per_model for the model + """ + for i in range(self.mutations_per_model): + # choose a mutation (not in exclude_dict) + valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( + self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( + set(self.mm_mutators).union(set(self.gen_mutators))) + if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation + mutator_list = self.mm_mutators + else: # 1-mm_prob to choose generation-based mutation + mutator_list = self.gen_mutators + + valid = [m for m in mutator_list if m in valid_mutators] + if valid: + m = random.choice(valid) + else: + continue + + self.mutators += [self.seed] + # an error can occur in the transformations, so even before the solve call. + # log function and arguments in that case + self.mutators += [m] + try: + if m in self.gen_mutators: + self.bug_cause = 'during GEN' + self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints + self.bug_cause = 'GEN' + else: + self.bug_cause = 'during MM' + self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints + self.bug_cause = 'MM' + self.mutators += [copy.deepcopy(self.cons)] + except MetamorphicError as exc: + # add to exclude_dict, to avoid running into the same error + if self.model_file in self.exclude_dict: + self.exclude_dict[self.model_file] += [m] + else: + self.exclude_dict[self.model_file] = [m] + function, argument, e = exc.args + if isinstance(e, CPMpyException): + # expected behavior if we throw a cpmpy exception, do not log + return None + elif function == semanticFusion: + return None + # don't log semanticfusion crash + + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=Fuzz_Test_ErrorTypes.internalfunctioncrash, + originalmodel_file=self.model_file, + exception=e, + function=function, + argument=argument, + stacktrace=traceback.format_exc(), + mutators=self.mutators, + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + return None + def verify_model(self, is_bug_check=False) -> None | dict: + try: + model = cp.Model(self.cons) + + # if is_bug_check: + # max_search_time = 20 + # else: + # max_search_time = 40 + max_search_time = 40 + + time_limit = max(1, min(max_search_time, # TODO: change `max_search_time` back to 200 + self.time_limit - time.time())) # set the max time limit to the given time limit or to 1 if the self.time_limit-time.time() would be smaller then 1 + + # Get the actual solver results and their execution times. + # We do it this way because a solver might crash, meaning the other solver doesn't get a turn. + rand_solver = random.choice(self.solvers) + model.solve(solver=rand_solver, time_limit=time_limit) + sol = [var == var.value() for var in self.og_vars if var.value() is not None] + model += sol + + solvers_results = [] + solvers_times = [] + for s in self.solvers: + self.nr_solve_checks += 1 + solvers_results.append(model.solve(solver=s, time_limit=time_limit)) + solvers_times.append(model.status().runtime) + + nr_timed_out_solvers = sum([t > time_limit * 0.8 for t in solvers_times]) + if nr_timed_out_solvers > 0: + # timeout, skip + self.bug_cause = 'UNKNOWN' + self.nr_timed_out += nr_timed_out_solvers + if not is_bug_check: + print('T', end='', flush=True) + return None + elif all(s1 == s2 for i, s1 in enumerate(solvers_results) for j, s2 in enumerate(solvers_results) if i < j): + # has to be same + if not is_bug_check: + print('.', end='', flush=True) + return None + else: + solver_results_str = ", ".join( + f"{solver}: {result}" for solver, result in zip(self.solvers, solvers_results)) + if is_bug_check: + print('X', end='', flush=True) + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=Fuzz_Test_ErrorTypes.failed_model, + originalmodel_file=self.model_file, + exception=f"Results of the solvers are not equal. Solver results: {solver_results_str}.", + constraints=self.cons, + mutators=self.mutators, + model=model, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + except Exception as e: + if isinstance(e, (CPMpyException, NotImplementedError)): + # expected error message, ignore + return None + if is_bug_check: + print('E', end='', flush=True) + + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=Fuzz_Test_ErrorTypes.internalcrash, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + mutators=self.mutators, + model=model, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + def run(self, model_file: str) -> dict | None: + """ + This function will run a single tests on the given model + """ + try: + random.seed(self.seed) + self.model_file = model_file + self.initialize_run() + gen_mutations_error = self.generate_mutations() + + # check if no error occured while generation the mutations + if gen_mutations_error == None: + # FOLLOWING 5 LINES CHANGED! + verify_model_error = self.verify_model() + if verify_model_error == None: + return None + else: + return self.find_error_rerun(verify_model_error) + else: + return self.find_error_rerun(gen_mutations_error) + except AssertionError as e: + print("A", end='', flush=True) + error_type = Fuzz_Test_ErrorTypes.crashed_model + if "is not sat" in str(e): + error_type = Fuzz_Test_ErrorTypes.unsat_model + elif "has no constraints" in str(e): + error_type = Fuzz_Test_ErrorTypes.no_constraints_model + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=error_type, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + except Exception as e: + print('C', end='', flush=True) + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=Fuzz_Test_ErrorTypes.crashed_model, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + mutators=self.mutators, + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + def find_error_rerun(self, error_dict) -> dict: + try: + random.seed(self.seed) + error_type = error_dict['type'] + self.initialize_run() # initialize empty (self.)model, cons, mutators + + # This should always be the case + if error_type in [Fuzz_Test_ErrorTypes.internalcrash, Fuzz_Test_ErrorTypes.failed_model]: # Error type 'E', often during model.solve() or solveAll or type 'X' + return self.bug_search_run_and_verify_model() + elif error_type == Fuzz_Test_ErrorTypes.internalfunctioncrash: + mutations = error_dict['mutators'][2::3] + return self.bug_search_run_and_verify_model(nr_mutations=len(mutations)) + + except AssertionError as e: + print("A", end='', flush=True) + type = Fuzz_Test_ErrorTypes.crashed_model + if "is not sat" in str(e): + type = Fuzz_Test_ErrorTypes.unsat_model + elif "has no constraints" in str(e): + type = Fuzz_Test_ErrorTypes.no_constraints_model + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=type, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + except Exception as e: + print('C', end='', flush=True) + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=Fuzz_Test_ErrorTypes.crashed_model, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: + if nr_mutations is not None: + self.mutations_per_model = nr_mutations + for _ in range(self.mutations_per_model): + last_bug_cause = self.bug_cause + + # Generate the type of mutation that will happen + valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( + self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( + set(self.mm_mutators).union(set(self.gen_mutators))) + if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation + mutator_list = self.mm_mutators + new_mut_type = 'MM' + else: # 1-mm_prob to choose generation-based mutation + mutator_list = self.gen_mutators + new_mut_type = 'GEN' + + valid = [m for m in mutator_list if m in valid_mutators] + if valid: + m = random.choice(valid) + else: + continue + + # Check whether verify_model returns an error before the new mutation, because the cause is then at the old mutation + if new_mut_type != last_bug_cause: + verify_model_error = self.verify_model(is_bug_check=True) + if verify_model_error is not None: + return verify_model_error + + # Then, apply the new mutation and check whether it gives an error + gen_mut_error = self.apply_single_mutation(m) + if gen_mut_error is not None: + return gen_mut_error + + # Finally, check the model at the end. This SHOULD give an error + verify_model_error = self.verify_model(is_bug_check=True) + if verify_model_error is not None: + return verify_model_error + else: + print('_', end='', flush=True) + + def apply_single_mutation(self, m) -> dict | None: + """ + Will generate one random mutation and apply it to the model + """ + self.mutators += [self.seed] + # an error can occur in the transformations, so even before the solve call. + # log function and arguments in that case + self.mutators += [m] + try: + if m in self.gen_mutators: + self.bug_cause = f'during GEN, after {self.bug_cause}' + self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints + self.bug_cause = 'GEN' + else: + self.bug_cause = f'during MM, after {self.bug_cause}' + self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints + self.bug_cause = 'MM' + self.mutators += [copy.deepcopy(self.cons)] + except MetamorphicError as exc: + # add to exclude_dict, to avoid running into the same error + if self.model_file in self.exclude_dict: + self.exclude_dict[self.model_file] += [m] + else: + self.exclude_dict[self.model_file] = [m] + function, argument, e = exc.args + if isinstance(e, CPMpyException): + # expected behavior if we throw a cpmpy exception, do not log + return None + elif function == semanticFusion: + return None + # don't log semanticfusion crash + + print('I', end='', flush=True) + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=Fuzz_Test_ErrorTypes.internalfunctioncrash, + originalmodel_file=self.model_file, + exception=e, + function=function, + argument=argument, + stacktrace=traceback.format_exc(), + mutators=self.mutators, + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + return None \ No newline at end of file From 7ef2dc222ae540221e5c8086fc02995b3b34334f Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Sun, 18 May 2025 11:13:52 +0200 Subject: [PATCH 42/58] refactorings and removal of the 'multiple' argument in Function class --- fuzz_test_utils/mutators.py | 80 +++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 43 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 0ab75c98..3764f14b 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1,7 +1,6 @@ import copy import random -import cpmpy import numpy as np from cpmpy.expressions.globalfunctions import GlobalFunction, Abs, Minimum, Maximum, Element, Count, Among, NValue, \ NValueExcept @@ -31,8 +30,7 @@ class Function: def __init__(self, name, func, type_: str, int_args: int, bool_args: int, bool_return: bool | None, min_args: int = None, - max_args: int = None, - multiple: int = 1): + max_args: int = None): """ type = string that describes the type of function it is int_args = the amount of args of type int it requires @@ -40,7 +38,6 @@ def __init__(self, name, func, type_: str, int_args: int, bool_args: int, bool_return = a boolean representing whether it returns a boolean (False means int return type, None means it can be either) min_args = the minimum amount of args the function takes max_args = the maximum amount of args the function takes - multiple = the arguments have to be a multiple of this int """ self.name = name self.func = func @@ -50,16 +47,13 @@ def __init__(self, name, func, type_: str, int_args: int, bool_args: int, self.bool_return = bool_return self.min_args = min_args self.max_args = max_args - self.multiple = multiple def __repr__(self): return (f"Operation({self.name}, {self.type}, {self.int_args}, {self.bool_args}, " - f"{self.bool_return}, min_args={self.min_args}, max_args={self.max_args}, multiple={self.multiple})") + f"{self.bool_return}, min_args={self.min_args}, max_args={self.max_args})") '''TRUTH TABLE BASED MORPHS''' - - def not_morph(cons): con = random.choice(cons) ncon = ~con @@ -1058,7 +1052,7 @@ def get_all_non_op_exprs(con: Expression): def get_all_exprs(con: Expression): """ - Helper function to get all expressions in a given constraint (Might be unnecessary but let's use this for now) + Helper function to get all expressions in a given constraint """ return get_all_op_exprs(con)[::-1] + get_all_non_op_exprs(con) @@ -1123,7 +1117,7 @@ def satisfies_args(func: Function, ints: int, bools: int, constants: int, vars: return ints + bools >= func.min_args and has_bool_return -def get_new_operator(func: Function, ints: list, bools: list, constants: list, variables: list): +def generate_new_operator(func: Function, ints: list, bools: list, constants: list, variables: list): """ Creates a new function of the given type with the arguments given ~ Parameters: @@ -1286,11 +1280,11 @@ def get_operator(args: list, ret_type: str | bool): bools_cnt = len(bools) constants_cnt = len(constants) vars_cnt = len(variables) - max_args = 12 + max_args = 12 # TODO: parametriseer? # Operators: ops = { - # name: (type, int_args, bool_args, bool_return, min_args, max_args, multiple) .._args -1 = n-ary, min 2 + # name: (type, int_args, bool_args, bool_return, min_args, max_args) .._args -1 = n-ary, min 2 name: Function(name, name, *attrs) for name, attrs in { 'and': ('op', 0, -1, True, 2, max_args), @@ -1298,7 +1292,7 @@ def get_operator(args: list, ret_type: str | bool): '->': ('op', 0, 2, True, 2, 2), 'not': ('op', 0, 1, True, 1, 1), 'sum': ('op', -1, 0, False, 2, max_args), - 'wsum': ('op', -1, 0, False, 2, max_args, 2), + 'wsum': ('op', -1, 0, False, 2, max_args), 'sub': ('op', 2, 0, False, 2, 2), 'mul': ('op', 2, 0, False, 2, 2), 'div': ('op', 2, 0, False, 2, 2), @@ -1326,14 +1320,14 @@ def get_operator(args: list, ret_type: str | bool): name: Function(name.__name__, name, *attrs) for name, attrs in { Abs: ('gfun', 1, 0, False, 1, 1), # expr | (min 1, max 1, /) - Minimum: ('gfun', -1, 0, None, 2, max_args), # [...] | Can return a boolean but this is not known beforehand (min 2, max /, /) - Maximum: ('gfun', -1, 0, None, 2, max_args), # [...] | Can return a boolean but this is not known beforehand (min 2, max /, /) - NValue: ('gfun', -1, 0, False, 2, max_args), # [...] | (min 2, max /, /) - Element: ('gfun', -1, 0, None, 2, max_args), # [...], idx | Can return a boolean but this is not known beforehand (min 2, max /, /) + Minimum: ('gfun', -1, 0, None, 2, max_args), # [...] | Can return a boolean but this is not known beforehand (min 2, max /) + Maximum: ('gfun', -1, 0, None, 2, max_args), # [...] | Can return a boolean but this is not known beforehand (min 2, max /) + NValue: ('gfun', -1, 0, False, 2, max_args), # [...] | (min 2, max /) + Element: ('gfun', -1, 0, None, 2, max_args), # [...], idx | Can return a boolean but this is not known beforehand (min 2, max /) # Denk best enkel de array vullen en de idx gewoon tussen 0 en len-1 pakken - Count: ('gfun', -1, 0, False, 2, max_args), # [...], expr | (min 2, max /, /) - Among: ('gfun', -1, 0, False, 2, max_args), # [...], [...] | Second array can only have constants, no expressions (not even BoolVal()) (min 2, max /, /) - NValueExcept: ('gfun', -1, 0, False, 2, max_args) # [...], val | Second argument can only have constants, no expressions (not even BoolVal()) (min 2, max /, /) + Count: ('gfun', -1, 0, False, 2, max_args), # [...], expr | (min 2, max /) + Among: ('gfun', -1, 0, False, 2, max_args), # [...], [...] | Second array can only have constants, no expressions (not even BoolVal()) (min 2, max /) + NValueExcept: ('gfun', -1, 0, False, 2, max_args) # [...], val | Second argument can only have constants, no expressions (not even BoolVal()) (min 2, max /) }.items() } # Global constraints @@ -1341,30 +1335,30 @@ def get_operator(args: list, ret_type: str | bool): # name: (type, int_args, bool_args, bool_return, min_args, max_args, multiple) .._args -1 = n-ary, min 2 name: Function(name.__name__, name, *attrs) for name, attrs in { - AllDifferent: ('gcon', -1, 0, True, 2, max_args), # [...] | (min 2, max /, /) - AllDifferentExceptN: ('gcon', -1, 0, True, 2, max_args), # [...], [...] | Second arg can also be a single non-list constant (min 2, max /, /) - AllEqual: ('gcon', -1, 0, True, 2, max_args), # [...] | (min 2, max /, /) - AllEqualExceptN: ('gcon', -1, 0, True, 2, max_args), # [...], [...] | Second arg can also be a single non-list constant (min 2, max /, /) - Circuit: ('gcon', -1, 0, True, 2, max_args), # [...] | Can only have ints, NO BOOLS! (min 2, max /, /) - Inverse: ('gcon', -1, 0, True, 2, max_args, 2), # [...], [...] | Can only have ints, NO BOOLS! (min 2, max /, 2n) - Table: ('gcon', -1, 0, True, 2, max_args), # [...], [[...],[...],...] | First argument only variables, Second argument should have a multiple amnt of args as the first one (min 2, max /, n + mn?) - NegativeTable: ('gcon', -1, 0, True, 2, max_args), # [...], [[...],[...],...] | First argument only variables, Second argument should have a multiple amnt of args as the first one (min 2, max /, n + mn?) - IfThenElse: ('gcon', 0, 3, True, 3, 3), # arg1, arg2, arg3 (min 3, max 3, /) - InDomain: ('gcon', -1, 0, True, 2, max_args), # val, [...] | (min 2, max /, /) - Xor: ('gcon', 0, -1, True, 1, max_args), # [...] | (min 1, max /, /) + AllDifferent: ('gcon', -1, 0, True, 2, max_args), # [...] | (min 2, max /) + AllDifferentExceptN: ('gcon', -1, 0, True, 2, max_args), # [...], [...] | Second arg can also be a single non-list constant (min 2, max /) + AllEqual: ('gcon', -1, 0, True, 2, max_args), # [...] | (min 2, max /) + AllEqualExceptN: ('gcon', -1, 0, True, 2, max_args), # [...], [...] | Second arg can also be a single non-list constant (min 2, max /) + Circuit: ('gcon', -1, 0, True, 2, max_args), # [...] | Can only have ints, NO BOOLS! (min 2, max /) + Inverse: ('gcon', -1, 0, True, 2, max_args), # [...], [...] | Can only have ints, NO BOOLS! (min 2, max /) + Table: ('gcon', -1, 0, True, 2, max_args), # [...], [[...],[...],...] | First argument only variables, Second argument should have a multiple amnt of args as the first one (min 2, max /) + NegativeTable: ('gcon', -1, 0, True, 2, max_args), # [...], [[...],[...],...] | First argument only variables, Second argument should have a multiple amnt of args as the first one (min 2, max /) + IfThenElse: ('gcon', 0, 3, True, 3, 3), # arg1, arg2, arg3 (min 3, max 3) + InDomain: ('gcon', -1, 0, True, 2, max_args), # val, [...] | (min 2, max /) + Xor: ('gcon', 0, -1, True, 1, max_args), # [...] | (min 1, max /) # Cumulative: (-1, 0, True), # st, dur, end, demand, cap (Ingewikkelde constraint) # Precedence: (?, ?, True), # (Ingewikkelde constraint) - NoOverlap: ('gcon', -1, 0, True, 3, max_args, 3), # [...], [...], [...] | Three lists all have same length (min 3, max /, 3n) + NoOverlap: ('gcon', -1, 0, True, 3, max_args), # [...], [...], [...] | Three lists all have same length (min 3, max /) GlobalCardinalityCount: ('gcon', -1, 0, True, 2, max_args), # [...], [...], [...] | The first and last list have to be the same length and - # they all have to be ints, NO BOOLS (min 2, max /, /) + # they all have to be ints, NO BOOLS (min 2, max /) Increasing: ('gcon', -1, 0, True, 2, max_args), # [...] (min 2, max /, /) Decreasing: ('gcon', -1, 0, True, 2, max_args), # [...] (min 2, max /, /) IncreasingStrict: ('gcon', -1, 0, True, 2, max_args), # [...] (min 2, max /, /) DecreasingStrict: ('gcon', -1, 0, True, 2, max_args), # [...] (min 2, max /, /) - LexLess: ('gcon', -1, 0, True, 2, max_args, 2), # [...], [...] | Lists have same length (min 2, max /, 2n), ONLY VARS - LexLessEq: ('gcon', -1, 0, True, 2, max_args, 2), # [...], [...] | Lists have same length (min 2, max /, 2n), ONLY VARS - LexChainLess: ('gcon', -1, 0, True, 2, max_args), # [...][...] | Rows have same length (min 2, max /, mn), ONLY VARS - LexChainLessEq: ('gcon', -1, 0, True, 2, max_args), # [...][...] | Rows have same length (min 2, max /, mn), ONLY VARS + LexLess: ('gcon', -1, 0, True, 2, max_args), # [...], [...] | Lists have same length (min 2, max /), ONLY VARS + LexLessEq: ('gcon', -1, 0, True, 2, max_args), # [...], [...] | Lists have same length (min 2, max /), ONLY VARS + LexChainLess: ('gcon', -1, 0, True, 2, max_args), # [...][...] | Rows have same length (min 2, max /), ONLY VARS + LexChainLessEq: ('gcon', -1, 0, True, 2, max_args), # [...][...] | Rows have same length (min 2, max /), ONLY VARS }.items() } all_ops = ops | comps | global_fns | global_cons @@ -1376,7 +1370,7 @@ def get_operator(args: list, ret_type: str | bool): func = random.choice(list(after.values())) else: return None - return get_new_operator(func, ints, bools, constants, variables) + return generate_new_operator(func, ints, bools, constants, variables) def find_all_occurrences(con: Expression, target_expr: Expression): @@ -1537,7 +1531,7 @@ def type_aware_expression_replacement(constraints: list): all_con_exprs = get_all_exprs(rand_con) expr = random.choice(all_con_exprs) path, ret_type = get_return_type(expr, rand_con) # Also gives us the taken path of the expression in the constraint - + # (Might be more than one occurrence so we should take the right one) # 2. Tel het aantal resterende params van elk type all_exprs = get_all_exprs_mult(final_cons) @@ -1577,10 +1571,8 @@ def has_positive_parity(expr: Expression, con: Expression, curr_path: tuple) -> """ # Basecase 1: `expr` cannot be strengthened or weakened if hasattr(expr, 'name'): - # NOTE: these are the simplest operators to strengthen/weaken (by just changing the operator into another one). - # Other operators could be changed in another way too (e.g. add/remove elements in the second argument - # of the expression x in [1, 2, 3, 4]). This could be included later and then changed accordingly in - # `strengthening_weakening_mutator()`. + # NOTE: these are not necessarily the only expressions that can be strengthened/weakened. + # (some double work is being done in function `is_changeable` so to do?) changeable_ops = {'and', 'or', '->', 'xor', '==', '!=', '<=', '<', '>=', '>'} changeable_globals = {AllDifferent, AllDifferentExceptN, AllEqual, AllEqualExceptN, Table, NegativeTable, IncreasingStrict, DecreasingStrict, @@ -1623,6 +1615,7 @@ def strengthen_expr(expr: Expression, path: tuple, con: Expression) -> Expressio ~ Returns: - `con`: the constraint after the mutation. """ + # TODO: 'and' & 'or' strengthenable by adding/removing args match expr.name: # {'or', '->', '!=', '<=', '>='} case 'or': # and, xor, !=, <, > args = expr.args @@ -1692,6 +1685,7 @@ def weaken_expr(expr: Expression, path: tuple, con: Expression) -> Expression: ~ Returns: - `con`: the constraint after the mutation. """ + # TODO: 'and' & 'or' weakenable by removing/adding args match expr.name: # {'and', 'xor', '==', '<', '>'} case 'and': # or, ->, ==, <=, >= args = expr.args From b093dbc0469829d5192255cb41f1d5ec2b4ff352 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Sun, 18 May 2025 12:02:41 +0200 Subject: [PATCH 43/58] Added a new superclass for all the solver voting verifiers, because all of them used the same functions. --- verifiers/solver_voting_count_verifier.py | 274 +-------------- verifiers/solver_voting_eq_verifier.py | 274 +-------------- verifiers/solver_voting_sat_verifier.py | 276 +-------------- verifiers/solver_voting_sol_verifier.py | 275 +-------------- verifiers/solver_voting_verifier.py | 395 ++++++++++++++++++++++ 5 files changed, 407 insertions(+), 1087 deletions(-) create mode 100644 verifiers/solver_voting_verifier.py diff --git a/verifiers/solver_voting_count_verifier.py b/verifiers/solver_voting_count_verifier.py index 06cd4c62..2a40d9e8 100644 --- a/verifiers/solver_voting_count_verifier.py +++ b/verifiers/solver_voting_count_verifier.py @@ -1,6 +1,8 @@ from verifiers import * +from verifiers.solver_voting_verifier import Solver_Voting_Verifier -class Solver_Vote_Count_Verifier(Verifier): + +class Solver_Vote_Count_Verifier(Solver_Voting_Verifier): """ The Solver Count Verifier will verify if the amount of solutions is the same for all solvers after running multiple mutations """ @@ -61,72 +63,7 @@ def initialize_run(self) -> None: self.mutators = [copy.deepcopy( self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. - def generate_mutations(self) -> None | dict: - """ - Will generate random mutations based on mutations_per_model for the model - """ - for i in range(self.mutations_per_model): - # choose a mutation (not in exclude_dict) - valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( - self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( - set(self.mm_mutators).union(set(self.gen_mutators))) - if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation - mutator_list = self.mm_mutators - else: # 1-mm_prob to choose generation-based mutation - mutator_list = self.gen_mutators - - valid = [m for m in mutator_list if m in valid_mutators] - if valid: - m = random.choice(valid) - else: - continue - self.mutators += [self.seed] - # an error can occur in the transformations, so even before the solve call. - # log function and arguments in that case - self.mutators += [m] - try: - if m in self.gen_mutators: - self.bug_cause = 'during GEN' - self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints - self.bug_cause = 'GEN' - else: - self.bug_cause = 'during MM' - self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints - self.bug_cause = 'MM' - self.mutators += [copy.deepcopy(self.cons)] - except MetamorphicError as exc: - # add to exclude_dict, to avoid running into the same error - if self.model_file in self.exclude_dict: - self.exclude_dict[self.model_file] += [m] - else: - self.exclude_dict[self.model_file] = [m] - function, argument, e = exc.args - if isinstance(e, CPMpyException): - # expected behavior if we throw a cpmpy exception, do not log - return None - elif function == semanticFusion: - return None - # don't log semanticfusion crash - - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=Fuzz_Test_ErrorTypes.internalfunctioncrash, - originalmodel_file=self.model_file, - exception=e, - function=function, - argument=argument, - stacktrace=traceback.format_exc(), - mutators=self.mutators, - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - return None def verify_model(self, is_bug_check=False) -> None | dict: try: model = cp.Model(self.cons) @@ -210,208 +147,3 @@ def verify_model(self, is_bug_check=False) -> None | dict: caused_by=self.bug_cause, nr_timed_out=self.nr_timed_out ) - - def run(self, model_file: str) -> dict | None: - """ - This function will run a single tests on the given model - """ - try: - random.seed(self.seed) - self.model_file = model_file - self.initialize_run() - gen_mutations_error = self.generate_mutations() - - # check if no error occured while generation the mutations - if gen_mutations_error == None: - # FOLLOWING 5 LINES CHANGED! - verify_model_error = self.verify_model() - if verify_model_error == None: - return None - else: - return self.find_error_rerun(verify_model_error) - else: - return self.find_error_rerun(gen_mutations_error) - except AssertionError as e: - print("A", end='', flush=True) - error_type = Fuzz_Test_ErrorTypes.crashed_model - if "is not sat" in str(e): - error_type = Fuzz_Test_ErrorTypes.unsat_model - elif "has no constraints" in str(e): - error_type = Fuzz_Test_ErrorTypes.no_constraints_model - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=error_type, - originalmodel_file=self.model_file, - exception=e, - stacktrace=traceback.format_exc(), - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - - except Exception as e: - print('C', end='', flush=True) - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=Fuzz_Test_ErrorTypes.crashed_model, - originalmodel_file=self.model_file, - exception=e, - stacktrace=traceback.format_exc(), - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - mutators=self.mutators, - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - - def find_error_rerun(self, error_dict) -> dict: - try: - random.seed(self.seed) - error_type = error_dict['type'] - self.initialize_run() # initialize empty (self.)model, cons, mutators - - # This should always be the case - if error_type in [Fuzz_Test_ErrorTypes.internalcrash, Fuzz_Test_ErrorTypes.failed_model]: # Error type 'E', often during model.solve() or solveAll or type 'X' - return self.bug_search_run_and_verify_model() - elif error_type == Fuzz_Test_ErrorTypes.internalfunctioncrash: - mutations = error_dict['mutators'][2::3] - return self.bug_search_run_and_verify_model(nr_mutations=len(mutations)) - - except AssertionError as e: - print("A", end='', flush=True) - type = Fuzz_Test_ErrorTypes.crashed_model - if "is not sat" in str(e): - type = Fuzz_Test_ErrorTypes.unsat_model - elif "has no constraints" in str(e): - type = Fuzz_Test_ErrorTypes.no_constraints_model - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=type, - originalmodel_file=self.model_file, - exception=e, - stacktrace=traceback.format_exc(), - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - - except Exception as e: - print('C', end='', flush=True) - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=Fuzz_Test_ErrorTypes.crashed_model, - originalmodel_file=self.model_file, - exception=e, - stacktrace=traceback.format_exc(), - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - - def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: - if nr_mutations is not None: - self.mutations_per_model = nr_mutations - for _ in range(self.mutations_per_model): - last_bug_cause = self.bug_cause - - # Generate the type of mutation that will happen - valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( - self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( - set(self.mm_mutators).union(set(self.gen_mutators))) - if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation - mutator_list = self.mm_mutators - new_mut_type = 'MM' - else: # 1-mm_prob to choose generation-based mutation - mutator_list = self.gen_mutators - new_mut_type = 'GEN' - - valid = [m for m in mutator_list if m in valid_mutators] - if valid: - m = random.choice(valid) - else: - continue - - # Check whether verify_model returns an error before the new mutation, because the cause is then at the old mutation - if new_mut_type != last_bug_cause: - verify_model_error = self.verify_model(is_bug_check=True) - if verify_model_error is not None: - return verify_model_error - - # Then, apply the new mutation and check whether it gives an error - gen_mut_error = self.apply_single_mutation(m) - if gen_mut_error is not None: - return gen_mut_error - - # Finally, check the model at the end. This SHOULD give an error - verify_model_error = self.verify_model(is_bug_check=True) - if verify_model_error is not None: - return verify_model_error - else: - print('_', end='', flush=True) - - def apply_single_mutation(self, m) -> dict | None: - """ - Will generate one random mutation and apply it to the model - """ - self.mutators += [self.seed] - # an error can occur in the transformations, so even before the solve call. - # log function and arguments in that case - self.mutators += [m] - try: - if m in self.gen_mutators: - self.bug_cause = f'during GEN, after {self.bug_cause}' - self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints - self.bug_cause = 'GEN' - else: - self.bug_cause = f'during MM, after {self.bug_cause}' - self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints - self.bug_cause = 'MM' - self.mutators += [copy.deepcopy(self.cons)] - except MetamorphicError as exc: - # add to exclude_dict, to avoid running into the same error - if self.model_file in self.exclude_dict: - self.exclude_dict[self.model_file] += [m] - else: - self.exclude_dict[self.model_file] = [m] - function, argument, e = exc.args - if isinstance(e, CPMpyException): - # expected behavior if we throw a cpmpy exception, do not log - return None - elif function == semanticFusion: - return None - # don't log semanticfusion crash - - print('I', end='', flush=True) - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=Fuzz_Test_ErrorTypes.internalfunctioncrash, - originalmodel_file=self.model_file, - exception=e, - function=function, - argument=argument, - stacktrace=traceback.format_exc(), - mutators=self.mutators, - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - return None \ No newline at end of file diff --git a/verifiers/solver_voting_eq_verifier.py b/verifiers/solver_voting_eq_verifier.py index 6c0b8aad..a8edd4d7 100644 --- a/verifiers/solver_voting_eq_verifier.py +++ b/verifiers/solver_voting_eq_verifier.py @@ -1,6 +1,8 @@ from verifiers import * +from verifiers.solver_voting_verifier import Solver_Voting_Verifier -class Solver_Vote_Eq_Verifier(Verifier): + +class Solver_Vote_Eq_Verifier(Solver_Voting_Verifier): """ The Solver Equivalence Verifier will verify if the solutions are the same for all solvers after running multiple mutations """ @@ -61,72 +63,7 @@ def initialize_run(self) -> None: self.mutators = [copy.deepcopy( self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. - def generate_mutations(self) -> None | dict: - """ - Will generate random mutations based on mutations_per_model for the model - """ - for i in range(self.mutations_per_model): - # choose a mutation (not in exclude_dict) - valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( - self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( - set(self.mm_mutators).union(set(self.gen_mutators))) - if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation - mutator_list = self.mm_mutators - else: # 1-mm_prob to choose generation-based mutation - mutator_list = self.gen_mutators - - valid = [m for m in mutator_list if m in valid_mutators] - if valid: - m = random.choice(valid) - else: - continue - self.mutators += [self.seed] - # an error can occur in the transformations, so even before the solve call. - # log function and arguments in that case - self.mutators += [m] - try: - if m in self.gen_mutators: - self.bug_cause = 'during GEN' - self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints - self.bug_cause = 'GEN' - else: - self.bug_cause = 'during MM' - self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints - self.bug_cause = 'MM' - self.mutators += [copy.deepcopy(self.cons)] - except MetamorphicError as exc: - # add to exclude_dict, to avoid running into the same error - if self.model_file in self.exclude_dict: - self.exclude_dict[self.model_file] += [m] - else: - self.exclude_dict[self.model_file] = [m] - function, argument, e = exc.args - if isinstance(e, CPMpyException): - # expected behavior if we throw a cpmpy exception, do not log - return None - elif function == semanticFusion: - return None - # don't log semanticfusion crash - - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=Fuzz_Test_ErrorTypes.internalfunctioncrash, - originalmodel_file=self.model_file, - exception=e, - function=function, - argument=argument, - stacktrace=traceback.format_exc(), - mutators=self.mutators, - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - return None def verify_model(self, is_bug_check=False) -> None | dict: try: model = cp.Model(self.cons) @@ -212,208 +149,3 @@ def verify_model(self, is_bug_check=False) -> None | dict: caused_by=self.bug_cause, nr_timed_out=self.nr_timed_out ) - - def run(self, model_file: str) -> dict | None: - """ - This function will run a single tests on the given model - """ - try: - random.seed(self.seed) - self.model_file = model_file - self.initialize_run() - gen_mutations_error = self.generate_mutations() - - # check if no error occured while generation the mutations - if gen_mutations_error == None: - # FOLLOWING 5 LINES CHANGED! - verify_model_error = self.verify_model() - if verify_model_error == None: - return None - else: - return self.find_error_rerun(verify_model_error) - else: - return self.find_error_rerun(gen_mutations_error) - except AssertionError as e: - print("A", end='', flush=True) - error_type = Fuzz_Test_ErrorTypes.crashed_model - if "is not sat" in str(e): - error_type = Fuzz_Test_ErrorTypes.unsat_model - elif "has no constraints" in str(e): - error_type = Fuzz_Test_ErrorTypes.no_constraints_model - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=error_type, - originalmodel_file=self.model_file, - exception=e, - stacktrace=traceback.format_exc(), - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - - except Exception as e: - print('C', end='', flush=True) - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=Fuzz_Test_ErrorTypes.crashed_model, - originalmodel_file=self.model_file, - exception=e, - stacktrace=traceback.format_exc(), - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - mutators=self.mutators, - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - - def find_error_rerun(self, error_dict) -> dict: - try: - random.seed(self.seed) - error_type = error_dict['type'] - self.initialize_run() # initialize empty (self.)model, cons, mutators - - # This should always be the case - if error_type in [Fuzz_Test_ErrorTypes.internalcrash, Fuzz_Test_ErrorTypes.failed_model]: # Error type 'E', often during model.solve() or solveAll or type 'X' - return self.bug_search_run_and_verify_model() - elif error_type == Fuzz_Test_ErrorTypes.internalfunctioncrash: - mutations = error_dict['mutators'][2::3] - return self.bug_search_run_and_verify_model(nr_mutations=len(mutations)) - - except AssertionError as e: - print("A", end='', flush=True) - type = Fuzz_Test_ErrorTypes.crashed_model - if "is not sat" in str(e): - type = Fuzz_Test_ErrorTypes.unsat_model - elif "has no constraints" in str(e): - type = Fuzz_Test_ErrorTypes.no_constraints_model - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=type, - originalmodel_file=self.model_file, - exception=e, - stacktrace=traceback.format_exc(), - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - - except Exception as e: - print('C', end='', flush=True) - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=Fuzz_Test_ErrorTypes.crashed_model, - originalmodel_file=self.model_file, - exception=e, - stacktrace=traceback.format_exc(), - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - - def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: - if nr_mutations is not None: - self.mutations_per_model = nr_mutations - for _ in range(self.mutations_per_model): - last_bug_cause = self.bug_cause - - # Generate the type of mutation that will happen - valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( - self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( - set(self.mm_mutators).union(set(self.gen_mutators))) - if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation - mutator_list = self.mm_mutators - new_mut_type = 'MM' - else: # 1-mm_prob to choose generation-based mutation - mutator_list = self.gen_mutators - new_mut_type = 'GEN' - - valid = [m for m in mutator_list if m in valid_mutators] - if valid: - m = random.choice(valid) - else: - continue - - # Check whether verify_model returns an error before the new mutation, because the cause is then at the old mutation - if new_mut_type != last_bug_cause: - verify_model_error = self.verify_model(is_bug_check=True) - if verify_model_error is not None: - return verify_model_error - - # Then, apply the new mutation and check whether it gives an error - gen_mut_error = self.apply_single_mutation(m) - if gen_mut_error is not None: - return gen_mut_error - - # Finally, check the model at the end. This SHOULD give an error - verify_model_error = self.verify_model(is_bug_check=True) - if verify_model_error is not None: - return verify_model_error - else: - print('_', end='', flush=True) - - def apply_single_mutation(self, m) -> dict | None: - """ - Will generate one random mutation and apply it to the model - """ - self.mutators += [self.seed] - # an error can occur in the transformations, so even before the solve call. - # log function and arguments in that case - self.mutators += [m] - try: - if m in self.gen_mutators: - self.bug_cause = f'during GEN, after {self.bug_cause}' - self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints - self.bug_cause = 'GEN' - else: - self.bug_cause = f'during MM, after {self.bug_cause}' - self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints - self.bug_cause = 'MM' - self.mutators += [copy.deepcopy(self.cons)] - except MetamorphicError as exc: - # add to exclude_dict, to avoid running into the same error - if self.model_file in self.exclude_dict: - self.exclude_dict[self.model_file] += [m] - else: - self.exclude_dict[self.model_file] = [m] - function, argument, e = exc.args - if isinstance(e, CPMpyException): - # expected behavior if we throw a cpmpy exception, do not log - return None - elif function == semanticFusion: - return None - # don't log semanticfusion crash - - print('I', end='', flush=True) - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=Fuzz_Test_ErrorTypes.internalfunctioncrash, - originalmodel_file=self.model_file, - exception=e, - function=function, - argument=argument, - stacktrace=traceback.format_exc(), - mutators=self.mutators, - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - return None \ No newline at end of file diff --git a/verifiers/solver_voting_sat_verifier.py b/verifiers/solver_voting_sat_verifier.py index 1df3c7e0..c92f2831 100644 --- a/verifiers/solver_voting_sat_verifier.py +++ b/verifiers/solver_voting_sat_verifier.py @@ -1,6 +1,8 @@ from verifiers import * +from verifiers.solver_voting_verifier import Solver_Voting_Verifier -class Solver_Vote_Sat_Verifier(Verifier): + +class Solver_Vote_Sat_Verifier(Solver_Voting_Verifier): """ The Solver Satisfiability Verifier will verify if the satisfiability is the same for all solvers after running multiple mutations """ @@ -60,73 +62,6 @@ def initialize_run(self) -> None: self.mutators = [copy.deepcopy( self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. - def generate_mutations(self) -> dict | None: - """ - Will generate random mutations based on mutations_per_model for the model - """ - for i in range(self.mutations_per_model): - # choose a mutation (not in exclude_dict) - valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( - self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list(set(self.mm_mutators).union(set(self.gen_mutators))) - if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation - mutator_list = self.mm_mutators - else: # 1-mm_prob to choose generation-based mutation - mutator_list = self.gen_mutators - - valid = [m for m in mutator_list if m in valid_mutators] - if valid: - m = random.choice(valid) - else: - continue - - self.mutators += [self.seed] - # an error can occur in the transformations, so even before the solve call. - # log function and arguments in that case - self.mutators += [m] - try: - if m in self.gen_mutators: - self.bug_cause = 'during GEN' - self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints - self.bug_cause = 'GEN' - else: - self.bug_cause = 'during MM' - self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints - self.bug_cause = 'MM' - self.mutators += [copy.deepcopy(self.cons)] - except MetamorphicError as exc: - # add to exclude_dict, to avoid running into the same error - if self.model_file in self.exclude_dict: - self.exclude_dict[self.model_file] += [m] - else: - self.exclude_dict[self.model_file] = [m] - function, argument, e = exc.args - if isinstance(e, CPMpyException): - # expected behavior if we throw a cpmpy exception, do not log - return None - elif function == semanticFusion: - return None - # don't log semanticfusion crash - - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=Fuzz_Test_ErrorTypes.internalfunctioncrash, - originalmodel_file=self.model_file, - exception=e, - function=function, - argument=argument, - stacktrace=traceback.format_exc(), - mutators=self.mutators, - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - return None - - def verify_model(self, is_bug_check=False) -> None | dict: try: model = cp.Model(self.cons) @@ -204,208 +139,3 @@ def verify_model(self, is_bug_check=False) -> None | dict: caused_by=self.bug_cause, nr_timed_out=self.nr_timed_out ) - - def run(self, model_file: str) -> dict | None: - """ - This function will run a single tests on the given model - """ - try: - random.seed(self.seed) - self.model_file = model_file - self.initialize_run() - gen_mutations_error = self.generate_mutations() - - # check if no error occured while generation the mutations - if gen_mutations_error == None: - # FOLLOWING 5 LINES CHANGED! - verify_model_error = self.verify_model() - if verify_model_error == None: - return None - else: - return self.find_error_rerun(verify_model_error) - else: - return self.find_error_rerun(gen_mutations_error) - except AssertionError as e: - print("A", end='', flush=True) - error_type = Fuzz_Test_ErrorTypes.crashed_model - if "is not sat" in str(e): - error_type = Fuzz_Test_ErrorTypes.unsat_model - elif "has no constraints" in str(e): - error_type = Fuzz_Test_ErrorTypes.no_constraints_model - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=error_type, - originalmodel_file=self.model_file, - exception=e, - stacktrace=traceback.format_exc(), - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - - except Exception as e: - print('C', end='', flush=True) - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=Fuzz_Test_ErrorTypes.crashed_model, - originalmodel_file=self.model_file, - exception=e, - stacktrace=traceback.format_exc(), - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - mutators=self.mutators, - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - - def find_error_rerun(self, error_dict) -> dict: - try: - random.seed(self.seed) - error_type = error_dict['type'] - self.initialize_run() # initialize empty (self.)model, cons, mutators - - # This should always be the case - if error_type in [Fuzz_Test_ErrorTypes.internalcrash, Fuzz_Test_ErrorTypes.failed_model]: # Error type 'E', often during model.solve() or solveAll or type 'X' - return self.bug_search_run_and_verify_model() - elif error_type == Fuzz_Test_ErrorTypes.internalfunctioncrash: - mutations = error_dict['mutators'][2::3] - return self.bug_search_run_and_verify_model(nr_mutations=len(mutations)) - - except AssertionError as e: - print("A", end='', flush=True) - type = Fuzz_Test_ErrorTypes.crashed_model - if "is not sat" in str(e): - type = Fuzz_Test_ErrorTypes.unsat_model - elif "has no constraints" in str(e): - type = Fuzz_Test_ErrorTypes.no_constraints_model - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=type, - originalmodel_file=self.model_file, - exception=e, - stacktrace=traceback.format_exc(), - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - - except Exception as e: - print('C', end='', flush=True) - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=Fuzz_Test_ErrorTypes.crashed_model, - originalmodel_file=self.model_file, - exception=e, - stacktrace=traceback.format_exc(), - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - - def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: - if nr_mutations is not None: - self.mutations_per_model = nr_mutations - for _ in range(self.mutations_per_model): - last_bug_cause = self.bug_cause - - # Generate the type of mutation that will happen - valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( - self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( - set(self.mm_mutators).union(set(self.gen_mutators))) - if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation - mutator_list = self.mm_mutators - new_mut_type = 'MM' - else: # 1-mm_prob to choose generation-based mutation - mutator_list = self.gen_mutators - new_mut_type = 'GEN' - - valid = [m for m in mutator_list if m in valid_mutators] - if valid: - m = random.choice(valid) - else: - continue - - # Check whether verify_model returns an error before the new mutation, because the cause is then at the old mutation - if new_mut_type != last_bug_cause: - verify_model_error = self.verify_model(is_bug_check=True) - if verify_model_error is not None: - return verify_model_error - - # Then, apply the new mutation and check whether it gives an error - gen_mut_error = self.apply_single_mutation(m) - if gen_mut_error is not None: - return gen_mut_error - - # Finally, check the model at the end. This SHOULD give an error - verify_model_error = self.verify_model(is_bug_check=True) - if verify_model_error is not None: - return verify_model_error - else: - print('_', end='', flush=True) - - def apply_single_mutation(self, m) -> dict | None: - """ - Will generate one random mutation and apply it to the model - """ - self.mutators += [self.seed] - # an error can occur in the transformations, so even before the solve call. - # log function and arguments in that case - self.mutators += [m] - try: - if m in self.gen_mutators: - self.bug_cause = f'during GEN, after {self.bug_cause}' - self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints - self.bug_cause = 'GEN' - else: - self.bug_cause = f'during MM, after {self.bug_cause}' - self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints - self.bug_cause = 'MM' - self.mutators += [copy.deepcopy(self.cons)] - except MetamorphicError as exc: - # add to exclude_dict, to avoid running into the same error - if self.model_file in self.exclude_dict: - self.exclude_dict[self.model_file] += [m] - else: - self.exclude_dict[self.model_file] = [m] - function, argument, e = exc.args - if isinstance(e, CPMpyException): - # expected behavior if we throw a cpmpy exception, do not log - return None - elif function == semanticFusion: - return None - # don't log semanticfusion crash - - print('I', end='', flush=True) - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=Fuzz_Test_ErrorTypes.internalfunctioncrash, - originalmodel_file=self.model_file, - exception=e, - function=function, - argument=argument, - stacktrace=traceback.format_exc(), - mutators=self.mutators, - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - return None diff --git a/verifiers/solver_voting_sol_verifier.py b/verifiers/solver_voting_sol_verifier.py index ffc93811..4eb245cf 100644 --- a/verifiers/solver_voting_sol_verifier.py +++ b/verifiers/solver_voting_sol_verifier.py @@ -1,6 +1,8 @@ from verifiers import * +from verifiers.solver_voting_verifier import Solver_Voting_Verifier -class Solver_Vote_Sol_Verifier(Verifier): + +class Solver_Vote_Sol_Verifier(Solver_Voting_Verifier): """ The Solver Sol Verifier will verify if a single solution is also in all other solvers after running multiple mutations """ @@ -59,72 +61,6 @@ def initialize_run(self) -> None: self.mutators = [copy.deepcopy( self.cons)] # keep track of list of cons alternated with mutators that transformed it into the next list of cons. - def generate_mutations(self) -> None | dict: - """ - Will generate random mutations based on mutations_per_model for the model - """ - for i in range(self.mutations_per_model): - # choose a mutation (not in exclude_dict) - valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( - self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( - set(self.mm_mutators).union(set(self.gen_mutators))) - if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation - mutator_list = self.mm_mutators - else: # 1-mm_prob to choose generation-based mutation - mutator_list = self.gen_mutators - - valid = [m for m in mutator_list if m in valid_mutators] - if valid: - m = random.choice(valid) - else: - continue - - self.mutators += [self.seed] - # an error can occur in the transformations, so even before the solve call. - # log function and arguments in that case - self.mutators += [m] - try: - if m in self.gen_mutators: - self.bug_cause = 'during GEN' - self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints - self.bug_cause = 'GEN' - else: - self.bug_cause = 'during MM' - self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints - self.bug_cause = 'MM' - self.mutators += [copy.deepcopy(self.cons)] - except MetamorphicError as exc: - # add to exclude_dict, to avoid running into the same error - if self.model_file in self.exclude_dict: - self.exclude_dict[self.model_file] += [m] - else: - self.exclude_dict[self.model_file] = [m] - function, argument, e = exc.args - if isinstance(e, CPMpyException): - # expected behavior if we throw a cpmpy exception, do not log - return None - elif function == semanticFusion: - return None - # don't log semanticfusion crash - - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=Fuzz_Test_ErrorTypes.internalfunctioncrash, - originalmodel_file=self.model_file, - exception=e, - function=function, - argument=argument, - stacktrace=traceback.format_exc(), - mutators=self.mutators, - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - return None def verify_model(self, is_bug_check=False) -> None | dict: try: model = cp.Model(self.cons) @@ -209,208 +145,3 @@ def verify_model(self, is_bug_check=False) -> None | dict: caused_by=self.bug_cause, nr_timed_out=self.nr_timed_out ) - - def run(self, model_file: str) -> dict | None: - """ - This function will run a single tests on the given model - """ - try: - random.seed(self.seed) - self.model_file = model_file - self.initialize_run() - gen_mutations_error = self.generate_mutations() - - # check if no error occured while generation the mutations - if gen_mutations_error == None: - # FOLLOWING 5 LINES CHANGED! - verify_model_error = self.verify_model() - if verify_model_error == None: - return None - else: - return self.find_error_rerun(verify_model_error) - else: - return self.find_error_rerun(gen_mutations_error) - except AssertionError as e: - print("A", end='', flush=True) - error_type = Fuzz_Test_ErrorTypes.crashed_model - if "is not sat" in str(e): - error_type = Fuzz_Test_ErrorTypes.unsat_model - elif "has no constraints" in str(e): - error_type = Fuzz_Test_ErrorTypes.no_constraints_model - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=error_type, - originalmodel_file=self.model_file, - exception=e, - stacktrace=traceback.format_exc(), - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - - except Exception as e: - print('C', end='', flush=True) - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=Fuzz_Test_ErrorTypes.crashed_model, - originalmodel_file=self.model_file, - exception=e, - stacktrace=traceback.format_exc(), - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - mutators=self.mutators, - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - - def find_error_rerun(self, error_dict) -> dict: - try: - random.seed(self.seed) - error_type = error_dict['type'] - self.initialize_run() # initialize empty (self.)model, cons, mutators - - # This should always be the case - if error_type in [Fuzz_Test_ErrorTypes.internalcrash, Fuzz_Test_ErrorTypes.failed_model]: # Error type 'E', often during model.solve() or solveAll or type 'X' - return self.bug_search_run_and_verify_model() - elif error_type == Fuzz_Test_ErrorTypes.internalfunctioncrash: - mutations = error_dict['mutators'][2::3] - return self.bug_search_run_and_verify_model(nr_mutations=len(mutations)) - - except AssertionError as e: - print("A", end='', flush=True) - type = Fuzz_Test_ErrorTypes.crashed_model - if "is not sat" in str(e): - type = Fuzz_Test_ErrorTypes.unsat_model - elif "has no constraints" in str(e): - type = Fuzz_Test_ErrorTypes.no_constraints_model - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=type, - originalmodel_file=self.model_file, - exception=e, - stacktrace=traceback.format_exc(), - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - - except Exception as e: - print('C', end='', flush=True) - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=Fuzz_Test_ErrorTypes.crashed_model, - originalmodel_file=self.model_file, - exception=e, - stacktrace=traceback.format_exc(), - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - - def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: - if nr_mutations is not None: - self.mutations_per_model = nr_mutations - for _ in range(self.mutations_per_model): - last_bug_cause = self.bug_cause - - # Generate the type of mutation that will happen - valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( - self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( - set(self.mm_mutators).union(set(self.gen_mutators))) - if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation - mutator_list = self.mm_mutators - new_mut_type = 'MM' - else: # 1-mm_prob to choose generation-based mutation - mutator_list = self.gen_mutators - new_mut_type = 'GEN' - - valid = [m for m in mutator_list if m in valid_mutators] - if valid: - m = random.choice(valid) - else: - continue - - # Check whether verify_model returns an error before the new mutation, because the cause is then at the old mutation - if new_mut_type != last_bug_cause: - verify_model_error = self.verify_model(is_bug_check=True) - if verify_model_error is not None: - return verify_model_error - - # Then, apply the new mutation and check whether it gives an error - gen_mut_error = self.apply_single_mutation(m) - if gen_mut_error is not None: - return gen_mut_error - - # Finally, check the model at the end. This SHOULD give an error - verify_model_error = self.verify_model(is_bug_check=True) - if verify_model_error is not None: - return verify_model_error - else: - print('_', end='', flush=True) - - def apply_single_mutation(self, m) -> dict | None: - """ - Will generate one random mutation and apply it to the model - """ - self.mutators += [self.seed] - # an error can occur in the transformations, so even before the solve call. - # log function and arguments in that case - self.mutators += [m] - try: - if m in self.gen_mutators: - self.bug_cause = f'during GEN, after {self.bug_cause}' - self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints - self.bug_cause = 'GEN' - else: - self.bug_cause = f'during MM, after {self.bug_cause}' - self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints - self.bug_cause = 'MM' - self.mutators += [copy.deepcopy(self.cons)] - except MetamorphicError as exc: - # add to exclude_dict, to avoid running into the same error - if self.model_file in self.exclude_dict: - self.exclude_dict[self.model_file] += [m] - else: - self.exclude_dict[self.model_file] = [m] - function, argument, e = exc.args - if isinstance(e, CPMpyException): - # expected behavior if we throw a cpmpy exception, do not log - return None - elif function == semanticFusion: - return None - # don't log semanticfusion crash - - print('I', end='', flush=True) - return dict(seed=self.seed, - mm_prob=self.mm_prob, - type=Fuzz_Test_ErrorTypes.internalfunctioncrash, - originalmodel_file=self.model_file, - exception=e, - function=function, - argument=argument, - stacktrace=traceback.format_exc(), - mutators=self.mutators, - constraints=self.cons, - variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in - get_variables(self.cons)], - originalmodel=self.original_model, - nr_solve_checks=self.nr_solve_checks, - caused_by=self.bug_cause, - nr_timed_out=self.nr_timed_out - ) - return None \ No newline at end of file diff --git a/verifiers/solver_voting_verifier.py b/verifiers/solver_voting_verifier.py new file mode 100644 index 00000000..4575d4d1 --- /dev/null +++ b/verifiers/solver_voting_verifier.py @@ -0,0 +1,395 @@ +import random + +from verifiers import * + + +class Solver_Voting_Verifier(Verifier): + """ + The base class containing the base functions for each verifier. + """ + + def __init__(self, solver: str, mutations_per_model: int, exclude_dict: dict, time_limit: float, seed: int, mm_prob: float): + self.solvers = solver + + self.mutations_per_model = mutations_per_model + self.exclude_dict = exclude_dict + self.time_limit = time_limit + self.seed = random.Random().random() + self.mm_mutators = [xor_morph, and_morph, or_morph, implies_morph, not_morph, + linearize_constraint_morph, + flatten_morph, + only_numexpr_equality_morph, + normalized_numexpr_morph, + reify_rewrite_morph, + only_bv_reifies_morph, + only_positive_bv_morph, + flat2cnf_morph, + toplevel_list_morph, + decompose_in_tree_morph, + push_down_negation_morph, + simplify_boolean_morph, + canonical_comparison_morph, + aritmetic_comparison_morph, + semanticFusionCounting, + semanticFusionCountingMinus, + semanticFusionCountingwsum, + semanticFusionCounting, + semanticFusionCountingMinus, + semanticFusionCountingwsum] + self.gen_mutators = [type_aware_operator_replacement, type_aware_expression_replacement] + self.mutators = [] + self.original_model = None + self.nr_solve_checks = 0 + self.bug_cause = 'STARTMODEL' + self.nr_timed_out = 0 + self.last_mut = None + self.mm_prob = mm_prob + + def generate_mutations(self) -> None | dict: + """ + Will generate random mutations based on mutations_per_model for the model + """ + for i in range(self.mutations_per_model): + # choose a mutation (not in exclude_dict) + valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( + self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( + set(self.mm_mutators).union(set(self.gen_mutators))) + if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation + mutator_list = self.mm_mutators + else: # 1-mm_prob to choose generation-based mutation + mutator_list = self.gen_mutators + + valid = [m for m in mutator_list if m in valid_mutators] + if valid: + m = random.choice(valid) + else: + continue + + self.mutators += [self.seed] + # an error can occur in the transformations, so even before the solve call. + # log function and arguments in that case + self.mutators += [m] + try: + if m in self.gen_mutators: + self.bug_cause = 'during GEN' + self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints + self.bug_cause = 'GEN' + else: + self.bug_cause = 'during MM' + self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints + self.bug_cause = 'MM' + self.mutators += [copy.deepcopy(self.cons)] + except MetamorphicError as exc: + # add to exclude_dict, to avoid running into the same error + if self.model_file in self.exclude_dict: + self.exclude_dict[self.model_file] += [m] + else: + self.exclude_dict[self.model_file] = [m] + function, argument, e = exc.args + if isinstance(e, CPMpyException): + # expected behavior if we throw a cpmpy exception, do not log + return None + elif function == semanticFusion: + return None + # don't log semanticfusion crash + + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=Fuzz_Test_ErrorTypes.internalfunctioncrash, + originalmodel_file=self.model_file, + exception=e, + function=function, + argument=argument, + stacktrace=traceback.format_exc(), + mutators=self.mutators, + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + return None + + def initialize_run(self) -> None: + """ + Abstract function that gets executed before generating the mutation, + This function is ued for getting the right data from the model. + Each verifier needs to implement this function + """ + raise NotImplementedError(f"method 'initialize_run' is not implemented for class {type(self)}") + + def verify_model(self) -> dict: + """ + Abstract function that will solve the newly created model with the mutations. + It will check if the test succeeded or not. + Each verifier needs to implement this function + """ + raise NotImplementedError(f"method 'verify_model' is not implemented for class {type(self)}") + + def run(self, model_file: str) -> dict | None: + """ + This function will run a single tests on the given model + """ + try: + random.seed(self.seed) + self.model_file = model_file + self.initialize_run() + gen_mutations_error = self.generate_mutations() + + # check if no error occured while generation the mutations + if gen_mutations_error == None: + # FOLLOWING 5 LINES CHANGED! + verify_model_error = self.verify_model() + if verify_model_error == None: + return None + else: + return self.find_error_rerun(verify_model_error) + else: + return self.find_error_rerun(gen_mutations_error) + except AssertionError as e: + print("A", end='', flush=True) + error_type = Fuzz_Test_ErrorTypes.crashed_model + if "is not sat" in str(e): + error_type = Fuzz_Test_ErrorTypes.unsat_model + elif "has no constraints" in str(e): + error_type = Fuzz_Test_ErrorTypes.no_constraints_model + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=error_type, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + except Exception as e: + print('C', end='', flush=True) + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=Fuzz_Test_ErrorTypes.crashed_model, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + mutators=self.mutators, + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + def find_error_rerun(self, error_dict) -> dict: + try: + random.seed(self.seed) + error_type = error_dict['type'] + self.initialize_run() # initialize empty (self.)model, cons, mutators + + # This should always be the case + if error_type in [Fuzz_Test_ErrorTypes.internalcrash, Fuzz_Test_ErrorTypes.failed_model]: # Error type 'E', often during model.solve() or solveAll or type 'X' + return self.bug_search_run_and_verify_model() + elif error_type == Fuzz_Test_ErrorTypes.internalfunctioncrash: + mutations = error_dict['mutators'][2::3] + return self.bug_search_run_and_verify_model(nr_mutations=len(mutations)) + + except AssertionError as e: + print("A", end='', flush=True) + type = Fuzz_Test_ErrorTypes.crashed_model + if "is not sat" in str(e): + type = Fuzz_Test_ErrorTypes.unsat_model + elif "has no constraints" in str(e): + type = Fuzz_Test_ErrorTypes.no_constraints_model + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=type, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + except Exception as e: + print('C', end='', flush=True) + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=Fuzz_Test_ErrorTypes.crashed_model, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + + def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: + if nr_mutations is not None: + self.mutations_per_model = nr_mutations + for _ in range(self.mutations_per_model): + last_bug_cause = self.bug_cause + + # Generate the type of mutation that will happen + valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( + self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( + set(self.mm_mutators).union(set(self.gen_mutators))) + if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation + mutator_list = self.mm_mutators + new_mut_type = 'MM' + else: # 1-mm_prob to choose generation-based mutation + mutator_list = self.gen_mutators + new_mut_type = 'GEN' + + valid = [m for m in mutator_list if m in valid_mutators] + if valid: + m = random.choice(valid) + else: + continue + + # Check whether verify_model returns an error before the new mutation, because the cause is then at the old mutation + if new_mut_type != last_bug_cause: + verify_model_error = self.verify_model(is_bug_check=True) + if verify_model_error is not None: + return verify_model_error + + # Then, apply the new mutation and check whether it gives an error + gen_mut_error = self.apply_single_mutation(m) + if gen_mut_error is not None: + return gen_mut_error + + # Finally, check the model at the end. This SHOULD give an error + verify_model_error = self.verify_model(is_bug_check=True) + if verify_model_error is not None: + return verify_model_error + else: + print('_', end='', flush=True) + + def apply_single_mutation(self, m) -> dict | None: + """ + Will generate one random mutation and apply it to the model + """ + self.mutators += [self.seed] + # an error can occur in the transformations, so even before the solve call. + # log function and arguments in that case + self.mutators += [m] + try: + if m in self.gen_mutators: + self.bug_cause = f'during GEN, after {self.bug_cause}' + self.cons = m(self.cons) # apply a generative mutation and REPLACE constraints + self.bug_cause = 'GEN' + else: + self.bug_cause = f'during MM, after {self.bug_cause}' + self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints + self.bug_cause = 'MM' + self.mutators += [copy.deepcopy(self.cons)] + except MetamorphicError as exc: + # add to exclude_dict, to avoid running into the same error + if self.model_file in self.exclude_dict: + self.exclude_dict[self.model_file] += [m] + else: + self.exclude_dict[self.model_file] = [m] + function, argument, e = exc.args + if isinstance(e, CPMpyException): + # expected behavior if we throw a cpmpy exception, do not log + return None + elif function == semanticFusion: + return None + # don't log semanticfusion crash + + print('I', end='', flush=True) + return dict(seed=self.seed, + mm_prob=self.mm_prob, + type=Fuzz_Test_ErrorTypes.internalfunctioncrash, + originalmodel_file=self.model_file, + exception=e, + function=function, + argument=argument, + stacktrace=traceback.format_exc(), + mutators=self.mutators, + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model, + nr_solve_checks=self.nr_solve_checks, + caused_by=self.bug_cause, + nr_timed_out=self.nr_timed_out + ) + return None + + def rerun(self, error: dict) -> dict: + """ + This function will rerun a previous failed test + """ + try: + if 'seed' in error: + run_seed = error['seed'] + random.seed(run_seed) + else: + random.seed(self.seed) + self.model_file = error["originalmodel_file"] + self.original_model = error["originalmodel"] + self.exclude_dict = {} + self.initialize_run() + gen_mutations_error = self.generate_mutations() + + # check if no error occured while generation the mutations + if gen_mutations_error == None: + return self.verify_model() + else: + return gen_mutations_error + # self.og_cons = error["constraints"] + # return self.verify_model() + + except AssertionError as e: + print("A", end='', flush=True) + type = Fuzz_Test_ErrorTypes.crashed_model + if "is not sat" in str(e): + type = Fuzz_Test_ErrorTypes.unsat_model + elif "has no constraints" in str(e): + type = Fuzz_Test_ErrorTypes.no_constraints_model + return dict(type=type, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model + ) + + except Exception as e: + print('C', end='', flush=True) + return dict(type=Fuzz_Test_ErrorTypes.crashed_model, + originalmodel_file=self.model_file, + exception=e, + stacktrace=traceback.format_exc(), + constraints=self.cons, + variables=[(var, var.lb, var.ub) if not is_boolexpr(var) else (var, "bool") for var in + get_variables(self.cons)], + originalmodel=self.original_model + ) + + def getType(self) -> str: + """This function is used for getting the type of the problem the verifier verifies""" + return self.type + + def getName(self) -> str: + """This function is used for getting the name of the verifier""" + return self.name From 8a2a0d71ee77229c680ea774396cf21ba42d8e1e Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 20 May 2025 11:39:10 +0200 Subject: [PATCH 44/58] changed solve call before str/wkn to solve() instead of solveAll(). The mutator now strengthens upon sat and weakens upon unsat (before: strengthen upon #sol > 1, weaken upon #sol < 1). --- verifiers/strengthening_weakening_verifier.py | 25 +++---------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/verifiers/strengthening_weakening_verifier.py b/verifiers/strengthening_weakening_verifier.py index db3a8a48..458550e5 100644 --- a/verifiers/strengthening_weakening_verifier.py +++ b/verifiers/strengthening_weakening_verifier.py @@ -424,35 +424,18 @@ def apply_single_mutation(self, m) -> dict | None: self.cons = m(self.cons) # apply a generative (non-metamorphic) mutation and REPLACE constraints self.bug_cause = 'GEN' elif m == strengthening_weakening_mutator: - model = cp.Model(self.cons) - # s = random.choice(self.solvers) if 'ortools' not in self.solvers else 'ortools' - s = 'ortools' # TODO: CHANGE - # print("I add to nrsolvechecks") + s = random.choice(self.solvers) self.nr_solve_checks += 1 - if hasattr(self, 'sol_lim'): - count = model.solveAll(solver=s, solution_limit=self.sol_lim, - time_limit=5) # should find at least 1 solution in 5s - else: - count = model.solveAll(solver=s, time_limit=5) - if count > 1: + sat = cp.Model(self.cons).solve(solver=s) + if sat: if m == strengthening_weakening_mutator: # solve call happening otherwise self.bug_cause = 'during STR' self.cons = m(self.cons, strengthen=True) self.bug_cause = 'STR' - elif count < 1: + else: self.bug_cause = 'during WKN' self.cons = m(self.cons, strengthen=False) self.bug_cause = 'WKN' - elif random.random() <= self.mm_prob: # If only 1 solution remains, we just go on normally instead - m = random.choice(self.mm_mutators) - self.bug_cause = 'during MM' - self.cons += m(self.cons) - self.bug_cause = 'MM' - else: - m = random.choice(self.gen_mutators) - self.bug_cause = 'during GEN' - self.cons = m(self.cons) - self.bug_cause = 'GEN' else: self.bug_cause = 'during MM' self.cons += m(self.cons) # apply a metamorphic mutation and add to constraints From fef14a44e06b5e29440c6ea3a139262da3d39798 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 20 May 2025 11:57:58 +0200 Subject: [PATCH 45/58] Bugfix when looking at which mutator should be used --- verifiers/strengthening_weakening_verifier.py | 25 +++++-------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/verifiers/strengthening_weakening_verifier.py b/verifiers/strengthening_weakening_verifier.py index 458550e5..2b260e57 100644 --- a/verifiers/strengthening_weakening_verifier.py +++ b/verifiers/strengthening_weakening_verifier.py @@ -99,37 +99,24 @@ def generate_mutations(self) -> None | dict: self.cons = m(self.cons) # apply a generative (non-metamorphic) mutation and REPLACE constraints self.bug_cause = 'GEN' elif m in self.str_wkn_mutators: - model = cp.Model(self.cons) s = random.choice(self.solvers) self.nr_solve_checks += 1 - count = model.solveAll(solver=s, solution_limit=2, time_limit=5) # should find at least 1 solution in 5s - if count > 1: - if m == strengthening_weakening_mutator: + sat = cp.Model(self.cons).solve(solver=s) + if sat: + if m == strengthening_weakening_mutator: # solve call happening otherwise self.bug_cause = 'during STR' - self.cons = m(self.cons) + self.cons = m(self.cons, strengthen=True) self.bug_cause = 'STR' - elif count < 1: + else: self.bug_cause = 'during WKN' - m = strengthening_weakening_mutator self.cons = m(self.cons, strengthen=False) self.bug_cause = 'WKN' - elif random.random() <= self.mm_prob: # If only 1 solution remains, we just go on normally instead - m = random.choice(self.mm_mutators) - self.bug_cause = 'during MM' - self.cons += m(self.cons) - self.bug_cause = 'MM' - else: - m = random.choice(self.gen_mutators) - self.bug_cause = 'during GEN' - self.cons = m(self.cons) - self.bug_cause = 'GEN' else: self.bug_cause = 'during MM' self.cons += m(self.cons) self.bug_cause = 'MM' if not m == self.mutators[-1]: self.mutators[-1] = m - # print(f"Mutator in iteration {i} is {self.mutators[-1]}.") self.mutators += [copy.deepcopy(self.cons)] except MetamorphicError as exc: @@ -423,7 +410,7 @@ def apply_single_mutation(self, m) -> dict | None: self.bug_cause = 'during GEN' self.cons = m(self.cons) # apply a generative (non-metamorphic) mutation and REPLACE constraints self.bug_cause = 'GEN' - elif m == strengthening_weakening_mutator: + elif m in self.str_wkn_mutators: s = random.choice(self.solvers) self.nr_solve_checks += 1 sat = cp.Model(self.cons).solve(solver=s) From 883efe6701d82d04d6b37c2bd5b669e465ffb504 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 20 May 2025 14:06:31 +0200 Subject: [PATCH 46/58] changed domain mutator to take random subdomain or random larger domain instead of just one value. Also added some todo's for possible improvements --- fuzz_test_utils/mutators.py | 41 ++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 3764f14b..f518a342 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -971,6 +971,7 @@ def mutate_op_expression(expr: Expression, con: Expression): - con: the constraint containing the expression ~ No return. Mutates the constraint! """ + # TODO: other types of functions should also be added (e.g. global constraints/functions) # Types that can be converted into each-other comparisons = {'==', '!=', '<', '<=', '>', '>='} int_ops = {'sum', 'sub', 'mul', @@ -1011,6 +1012,7 @@ def get_all_mutable_op_exprs(con: Expression): ~ Return: - mutable_exprs: all expressions in the constraint that can be mutated """ + # TODO: other types of functions should also be added (e.g. global constraints/functions) comparisons = {'==', '!=', '<', '<=', '>', '>='} int_ops = {'sum', 'sub', 'mul', 'div', 'mod', 'pow'} logic_ops = {'and', 'or', '->'} @@ -1032,6 +1034,7 @@ def get_all_op_exprs(con: Expression): """ Helper function to get all expressions WITH an operator in a given constraint """ + # TODO: other types of functions should also be added (e.g. global constraints/functions) if type(con) in {Comparison, Operator}: return sum((get_all_op_exprs(arg) for arg in con.args), []) + [con] # All subexpressions + current expression else: @@ -1042,8 +1045,9 @@ def get_all_non_op_exprs(con: Expression): """ Helper function to get all expressions WITHOUT an operator in a given constraint """ + # TODO: other types of functions should also be added (e.g. global constraints/functions) if hasattr(con, 'args') and not isinstance(con, NDVarArray) and con.name != 'boolval': - return sum((get_all_non_op_exprs(arg) for arg in con.args), [con]) + return sum((get_all_non_op_exprs(arg) for arg in con.args), []) elif isinstance(con, list) or isinstance(con, NDVarArray): return sum((get_all_non_op_exprs(e) for e in con), []) else: @@ -1476,7 +1480,7 @@ def get_return_type(expr: Expression, con: Expression): GlobalCardinalityCount: (2, (None,)), Table: (3, (1, None)), NegativeTable: (3, (1, None)), - Count: (1, (1, None)), + # Count: (1, (1, None)), # TODO: I don't think this is necessary, so check this. 'pow': (1, (1, None)), 'wsum': (2, (0, None))} variable_restricted_functions = {Table: (2, (0, None)), @@ -1572,7 +1576,7 @@ def has_positive_parity(expr: Expression, con: Expression, curr_path: tuple) -> # Basecase 1: `expr` cannot be strengthened or weakened if hasattr(expr, 'name'): # NOTE: these are not necessarily the only expressions that can be strengthened/weakened. - # (some double work is being done in function `is_changeable` so to do?) + # (some double work is being done in function `is_changeable` so todo?) changeable_ops = {'and', 'or', '->', 'xor', '==', '!=', '<=', '<', '>=', '>'} changeable_globals = {AllDifferent, AllDifferentExceptN, AllEqual, AllEqualExceptN, Table, NegativeTable, IncreasingStrict, DecreasingStrict, @@ -1588,6 +1592,7 @@ def has_positive_parity(expr: Expression, con: Expression, curr_path: tuple) -> return True, curr_path # Recursively check in the arguments and change result upon encountering "not" operators + # TODO: extend with other operators. e.g. the left side of `->` also has negative parity if con.name == 'not': curr_path += 0, neg_res = has_positive_parity(expr, con.args[0], curr_path) @@ -1776,6 +1781,10 @@ def strengthening_weakening_mutator(constraints: list, strengthen: bool = True) ~ Return: - `final_cons`: a list of the same constraints where one constraint has a mutated operator """ + # TODO: right now: checks for possible expressions to strengthen/weaken and THEN calculates whether it should be + # strengthened or weakened. + # should be: calculate parity while searching for possible expressions and dismiss them if they can't be + # strengthened or weakened based on that. try: final_cons = copy.deepcopy(constraints) @@ -1817,18 +1826,26 @@ def strengthening_weakening_mutator(constraints: list, strengthen: bool = True) raise Exception(e) -def change_domain_mutator(constraints: list): +def change_domain_mutator(constraints: list, strengthen: bool): + # TODO? something else for boolean variables? try: # Take random variable variables = get_variables(constraints) - rand_var = random.choice(variables) - - # Get its value by solving the model - Model(constraints).solve() - - # Replace its domain by its value - rand_var.lb = rand_var.value() - rand_var.ub = rand_var.value() + if variables: # Improbable but it's possible there are no variables + rand_var = random.choice(variables) + + lb = rand_var.lb + ub = rand_var.ub + if strengthen: + rand_var.lb = random.randint(lb, ub - 1) + rand_var.ub = random.randint(rand_var.lb + 1, ub) + else: + expansion_param = 2 # How much bigger should the domain possibly be + avg = (ub + lb) / 2 + max_ub = int(avg + (ub - avg) * expansion_param) + min_lb = int(avg + (lb - avg) * expansion_param) + rand_var.lb = random.randint(min_lb, lb) + rand_var.ub = random.randint(ub, max_ub) # Return the given constraints to be compatible with how the other non-metamorphic mutators are called return constraints From 31aa18ff9e100daa5c79e44058c2ae0729313577 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 20 May 2025 14:24:09 +0200 Subject: [PATCH 47/58] bugfix to be compatible with the new domain changer --- verifiers/strengthening_weakening_verifier.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/verifiers/strengthening_weakening_verifier.py b/verifiers/strengthening_weakening_verifier.py index 2b260e57..505910a5 100644 --- a/verifiers/strengthening_weakening_verifier.py +++ b/verifiers/strengthening_weakening_verifier.py @@ -103,8 +103,7 @@ def generate_mutations(self) -> None | dict: self.nr_solve_checks += 1 sat = cp.Model(self.cons).solve(solver=s) if sat: - if m == strengthening_weakening_mutator: # solve call happening otherwise - self.bug_cause = 'during STR' + self.bug_cause = 'during STR' self.cons = m(self.cons, strengthen=True) self.bug_cause = 'STR' else: @@ -415,8 +414,7 @@ def apply_single_mutation(self, m) -> dict | None: self.nr_solve_checks += 1 sat = cp.Model(self.cons).solve(solver=s) if sat: - if m == strengthening_weakening_mutator: # solve call happening otherwise - self.bug_cause = 'during STR' + self.bug_cause = 'during STR' self.cons = m(self.cons, strengthen=True) self.bug_cause = 'STR' else: From 45fc23592d1934035835b83bcf8c1793e9e6e13c Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 20 May 2025 15:22:43 +0200 Subject: [PATCH 48/58] Small bugfix --- verifiers/solver_voting_verifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verifiers/solver_voting_verifier.py b/verifiers/solver_voting_verifier.py index 4575d4d1..dd86a995 100644 --- a/verifiers/solver_voting_verifier.py +++ b/verifiers/solver_voting_verifier.py @@ -120,7 +120,7 @@ def initialize_run(self) -> None: """ raise NotImplementedError(f"method 'initialize_run' is not implemented for class {type(self)}") - def verify_model(self) -> dict: + def verify_model(self, is_bug_check=False) -> dict: """ Abstract function that will solve the newly created model with the mutations. It will check if the test succeeded or not. From b9fcc439682890eada48e6d50032ff0ef3b81f98 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Wed, 21 May 2025 14:27:05 +0200 Subject: [PATCH 49/58] fixed bug when solution_limit was reached + fixed printing of error --- verifiers/solver_voting_eq_verifier.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/verifiers/solver_voting_eq_verifier.py b/verifiers/solver_voting_eq_verifier.py index a8edd4d7..d8958a70 100644 --- a/verifiers/solver_voting_eq_verifier.py +++ b/verifiers/solver_voting_eq_verifier.py @@ -84,7 +84,7 @@ def verify_model(self, is_bug_check=False) -> None | dict: for i, s in enumerate(self.solvers): self.nr_solve_checks += 1 if hasattr(self, 'sol_lim'): - model.solveAll(solver=s, time_limit=time_limit, display=lambda: all_sols[i].add( + res = model.solveAll(solver=s, time_limit=time_limit, display=lambda: all_sols[i].add( tuple([v.value() for v in self.original_vars])), solution_limit=self.sol_lim) solvers_times.append(model.status().runtime) else: @@ -100,21 +100,28 @@ def verify_model(self, is_bug_check=False) -> None | dict: if not is_bug_check: print('T', end='', flush=True) return None - elif all(len(s1.symmetric_difference(s2)) == 0 for i, s1 in enumerate(all_sols) for j, s2 in enumerate(all_sols) if i < j): + elif all(len(s1.symmetric_difference(s2)) == 0 for i, s1 in enumerate(all_sols) for j, s2 in enumerate(all_sols) if i < j) or res == self.sol_lim: # has to be same if not is_bug_check: print('.', end='', flush=True) return None else: - solver_results_str = ", ".join( - f"{solver}: {result}" for solver, result in zip(self.solvers, all_sols)) + solver_results_str = "" + from itertools import combinations + for (i, s1), (j, s2) in combinations(enumerate(all_sols), 2): + if len(solver_results_str) > 0: + solver_results_str += "\n" + solver_name_1 = self.solvers[i] + solver_name_2 = self.solvers[j] + diff = s1.symmetric_difference(s2) + solver_results_str += f"{solver_name_1} vs {solver_name_2}: {len(diff)} differences" if is_bug_check: print('X', end='', flush=True) return dict(seed=self.seed, mm_prob=self.mm_prob, type=Fuzz_Test_ErrorTypes.failed_model, originalmodel_file=self.model_file, - exception=f"Results of the solvers are not equal. Solver results: {solver_results_str}.", + exception=f"Results of the solvers are not equal. Solver results: {solver_results_str}", constraints=self.cons, mutators=self.mutators, model=model, From d3b7fa45220ad33f4b498e64f5910db80a6b972e Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Wed, 21 May 2025 14:27:50 +0200 Subject: [PATCH 50/58] fixed bug for domain changer where the chosen domain range was empty --- fuzz_test_utils/mutators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index f518a342..73bd0db6 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1837,8 +1837,8 @@ def change_domain_mutator(constraints: list, strengthen: bool): lb = rand_var.lb ub = rand_var.ub if strengthen: - rand_var.lb = random.randint(lb, ub - 1) - rand_var.ub = random.randint(rand_var.lb + 1, ub) + rand_var.lb = random.randint(lb, max(lb, ub - 1)) + rand_var.ub = random.randint(min(ub, rand_var.lb + 1), ub) else: expansion_param = 2 # How much bigger should the domain possibly be avg = (ub + lb) / 2 From f3b95870daa28f2e6406839583d3cc1655796fe7 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Wed, 21 May 2025 14:28:44 +0200 Subject: [PATCH 51/58] added last line of stacktrace to the output --- fuzz_test_utils/output_writer.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/fuzz_test_utils/output_writer.py b/fuzz_test_utils/output_writer.py index 450b3d33..0ae3e4ea 100644 --- a/fuzz_test_utils/output_writer.py +++ b/fuzz_test_utils/output_writer.py @@ -196,6 +196,21 @@ def get_nr_timed_out_solve_calls(error_data): return error_data['error']['nr_timed_out'] +def extract_last_stacktrace_lines(error_data) -> str: + stacktrace = error_data['error']['stacktrace'] + lines = stacktrace.strip().splitlines() + last_file_index = None + + for i, line in enumerate(lines): + if line.strip().startswith("File"): + last_file_index = i + + if last_file_index is not None: + return '\n'.join(lines[last_file_index:]).strip() + else: + return '' + + def write_csv(error_data: dict, output_path) -> None: columns_and_functions = [ ("bug_class", get_bug_class), @@ -205,6 +220,7 @@ def write_csv(error_data: dict, output_path) -> None: ("time_taken", get_time_taken), ("bug_type", get_bug_type), ("exception", get_exception), + ("last_stacktrace_line", extract_last_stacktrace_lines), ("original_constraints", get_original_cons), ("current_constraints", get_current_cons), ("total_nr_mutations", get_nr_mutations), From 8c6962a1db52d225225a913777affbe35e2ee8a2 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Wed, 21 May 2025 18:02:33 +0200 Subject: [PATCH 52/58] Added tuple support because apparently those are part of the startmodels sometimes --- fuzz_test_utils/mutators.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 73bd0db6..425b62f0 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1048,7 +1048,7 @@ def get_all_non_op_exprs(con: Expression): # TODO: other types of functions should also be added (e.g. global constraints/functions) if hasattr(con, 'args') and not isinstance(con, NDVarArray) and con.name != 'boolval': return sum((get_all_non_op_exprs(arg) for arg in con.args), []) - elif isinstance(con, list) or isinstance(con, NDVarArray): + elif isinstance(con, list) or isinstance(con, NDVarArray) or isinstance(con, tuple): return sum((get_all_non_op_exprs(e) for e in con), []) else: return [con] @@ -1275,7 +1275,7 @@ def get_operator(args: list, ret_type: str | bool): if ret_type == 'constant': if constants: return random.choice(constants) # Some expressions can't be replaced by functions - intvars = [e for e in variables if not (is_boolexpr(e) or isinstance(e, list) or isinstance(e, NDVarArray))] + intvars = [e for e in variables if not (is_boolexpr(e) or isinstance(e, list) or isinstance(e, NDVarArray) or isinstance(e, tuple))] if intvars: return random.choice(intvars) if ret_type == 'variable' and variables: @@ -1395,7 +1395,7 @@ def find_all_occurrences(con: Expression, target_expr: Expression): for i, arg in enumerate(con.args): for path in find_all_occurrences(arg, target_expr): occurrences.append((i,) + path) # Add index to the path - elif isinstance(con, list) or isinstance(con, NDVarArray): + elif isinstance(con, list) or isinstance(con, NDVarArray) or isinstance(con, tuple): for i, arg in enumerate(con): for path in find_all_occurrences(arg, target_expr): occurrences.append((i,) + path) @@ -1422,7 +1422,7 @@ def replace_at_path(con: Expression, path: tuple, new_expr: Expression): for idx in path[:-1]: if hasattr(parent, 'args') and not isinstance(parent, NDVarArray) and parent.name != 'boolval': parent = parent.args[idx] - elif isinstance(parent, list) or isinstance(parent, NDVarArray): + elif isinstance(parent, list) or isinstance(parent, NDVarArray) or isinstance(parent, tuple): parent = parent[idx] # Change the arguments of the parent @@ -1432,6 +1432,10 @@ def replace_at_path(con: Expression, path: tuple, new_expr: Expression): parent.update_args(args) elif isinstance(parent, list) or isinstance(parent, NDVarArray): parent[path[-1]] = new_expr + elif isinstance(parent, tuple): + parent = list(parent) + parent[path[-1]] = new_expr + parent = tuple(parent) return con @@ -1447,7 +1451,7 @@ def expr_at_path(con: Expression, path: tuple, expr: Expression): if idx is not None: if hasattr(con, 'args') and not isinstance(con, NDVarArray) and con.name != 'boolval': con = con.args[idx] - elif isinstance(con, list) or isinstance(con, NDVarArray): + elif isinstance(con, list) or isinstance(con, NDVarArray) or isinstance(con, tuple): con = con[idx] else: return len(find_all_occurrences(con, expr)) > 0 @@ -1508,7 +1512,7 @@ def get_return_type(expr: Expression, con: Expression): return path, 'variable' if hasattr(con, 'args') and not isinstance(con, NDVarArray) and con.name != 'boolval': con = con.args[idx] - elif isinstance(con, list) or isinstance(con, NDVarArray): + elif isinstance(con, list) or isinstance(con, NDVarArray) or isinstance(con, tuple): con = con[idx] # We should only get here if the argument isn't in one of the functions above return path, is_boolexpr(expr) From 7525b75f21b9e8d9ddb4c6f5a4d84e254614693b Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Wed, 21 May 2025 18:54:11 +0200 Subject: [PATCH 53/58] Element now cannot take a constant as index that is larger than the first argument --- fuzz_test_utils/mutators.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 425b62f0..27624799 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1174,7 +1174,8 @@ def generate_new_operator(func: Function, ints: list, bools: list, constants: li amnt_args = random.randint(func.min_args, min(len(constants), func.max_args)) first_arg = random.sample(constants, amnt_args) # idx = random.randint(0, amnt_args - 1) - idx = random.choice(ints) + constants_filtered = [e for e in constants if (isinstance(e, int) and e <= amnt_args - 1)] # Make sure you can't take an integer out of bounds + idx = random.choice(constants_filtered + variables) args = first_arg, idx case 'NValue': amnt_args = random.randint(func.min_args, min(len(comb), func.max_args)) From 39886af41a33dfd9e541034c1f3fe7443319dc5d Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Thu, 22 May 2025 17:37:34 +0200 Subject: [PATCH 54/58] bugfix when no solution limit --- verifiers/solver_voting_eq_verifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verifiers/solver_voting_eq_verifier.py b/verifiers/solver_voting_eq_verifier.py index d8958a70..6a0ee619 100644 --- a/verifiers/solver_voting_eq_verifier.py +++ b/verifiers/solver_voting_eq_verifier.py @@ -100,7 +100,7 @@ def verify_model(self, is_bug_check=False) -> None | dict: if not is_bug_check: print('T', end='', flush=True) return None - elif all(len(s1.symmetric_difference(s2)) == 0 for i, s1 in enumerate(all_sols) for j, s2 in enumerate(all_sols) if i < j) or res == self.sol_lim: + elif all(len(s1.symmetric_difference(s2)) == 0 for i, s1 in enumerate(all_sols) for j, s2 in enumerate(all_sols) if i < j) or (hasattr(self, 'sol_lim') and res == self.sol_lim): # has to be same if not is_bug_check: print('.', end='', flush=True) From fcf3d2cb370d1141e4eb546a8f7506b63e88cb8d Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Thu, 22 May 2025 17:38:16 +0200 Subject: [PATCH 55/58] some more classifications --- fuzz_test_utils/output_writer.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/fuzz_test_utils/output_writer.py b/fuzz_test_utils/output_writer.py index 0ae3e4ea..09f63b5d 100644 --- a/fuzz_test_utils/output_writer.py +++ b/fuzz_test_utils/output_writer.py @@ -39,7 +39,7 @@ def match_conditions(exc_str): (lambda s: " has no constraints" in s, "06_no_constraints_TO_FIX/"), (lambda s: any(x in s for x in ["or-tools does not accept a 'modulo' operation where '0' is in the domain of the divisor", "An int_mod must have a strictly positive modulo argument", "The domain of the divisor cannot contain 0", "Modulo with a divisor domain containing 0 is not supported.", "Power operator: For integer values, exponent must be non-negative:"]), "07_div0_pow-neg/"), (lambda s: "object of type '_BoolVarImpl' has no len()" in s, "08_object_type_boolvarimpl_no_len/"), - (lambda s: "'int' object has no attribute 'lb'" in s, "09_int_obj_no_attr_lb/"), + (lambda s: any(x in s for x in ["'int' object has no attribute 'lb'", "object has no attribute 'lb'"]), "09_int_obj_no_attr_lb/"), (lambda s: all(x in s for x in ["Cannot convert", "to Choco variable"]), "10_cannot_convert_to_choco_var/"), (lambda s: any(x in s for x in ["Translation of gurobi status 11 to CPMpy status not implemented", "KeyboardInterrupt", "cannot access local variable 'proc' where it is not associated with a value"]), "11_keyboard_interrupt/"), (lambda s: "Cannot modify read-only attribute 'args', use 'update_args()'" in s, "12_cant_modify_args/"), @@ -55,7 +55,18 @@ def match_conditions(exc_str): (lambda s: "'bool' object has no attribute 'has_subexpr'" in s, "21_bool_obj_no_has_subexpr/"), (lambda s: "'int' object has no attribute 'is_bool'" in s, "22_int_obj_no_is_bool/"), (lambda s: "not supported: model.get_or_make_boolean_index(" in s, "23_not_supported_get_or_make_boolean_index/"), - (lambda s: any(x in s for x in ["'BoolVal' object has no attribute 'get_integer_var_value_map'", "Not a known var "]), "24_not_known_var/") + (lambda s: any(x in s for x in ["'BoolVal' object has no attribute 'get_integer_var_value_map'", "Not a known var "]), "24_not_known_var/"), + (lambda s: "ut of memory" in s, "25_out_of_mem/"), + (lambda s: all(x in s for x in ["Unsupported boolexpr", "in reification"]), "26_unsupported_boolexpr/"), + (lambda s: "not a variable" in s, "27_not_a_variable/"), + (lambda s: "Invalid rhs argument for general constraint of indicator type" in s, "28_invalid_rhs_arg/"), + (lambda s: "MiniZinc stopped with a non-zero exit code, but did not output an error message." in s, "29_minizinc_nonzero_exitcode/"), + (lambda s: "The size of a performed interval must be >= 0 in constraint" in s, "30_interval_size_gt0/"), + (lambda s: "must be real number, not" in s, "31_must_be_real/"), + (lambda s: "n-ary operators require at least one argument" in s, "32_n-ary_ops_gt_1_arg/"), + (lambda s: "Bound requested for unknown expression" in s, "33_bound_request_unknown_expression/"), + (lambda s: "arrays cannot be elements of arrays" in s, "34_arrays_no_elem_of_array/"), + (lambda s: "not enough values to unpack" in s, "35_not_enough_values_unpack/"), ] def get_logging_dir(error_data, logging_dir): From 09a500e2d9c67c32c5f437bfc25ba048cb1f0ba7 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Thu, 22 May 2025 17:42:51 +0200 Subject: [PATCH 56/58] bugfix when looking where bug happened --- verifiers/strengthening_weakening_verifier.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/verifiers/strengthening_weakening_verifier.py b/verifiers/strengthening_weakening_verifier.py index 505910a5..24891da0 100644 --- a/verifiers/strengthening_weakening_verifier.py +++ b/verifiers/strengthening_weakening_verifier.py @@ -358,9 +358,9 @@ def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: # choose a mutator. 33% of the time, this will be a strengthening/weakening mutation. # choose a mutation (not in exclude_dict) valid_mutators = list( - set(self.mm_mutators).union(set(self.gen_mutators)).union({strengthening_weakening_mutator}) - set( + set(self.mm_mutators).union(set(self.gen_mutators)).union(self.str_wkn_mutators) - set( self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( - set(self.mm_mutators).union(set(self.gen_mutators)).union({strengthening_weakening_mutator})) + set(self.mm_mutators).union(set(self.gen_mutators)).union(self.str_wkn_mutators)) rand = random.random() if rand <= 1/3: mutator_list = self.str_wkn_mutators From 4a1a0e76b604b49314e2eb3d743a8ce7380894bc Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Sat, 24 May 2025 13:24:22 +0200 Subject: [PATCH 57/58] Removal of the exlude dict due to memory reasons for long runs (still in comments) --- verifiers/solver_voting_verifier.py | 38 ++++++++------- verifiers/strengthening_weakening_verifier.py | 48 ++++++++++--------- 2 files changed, 45 insertions(+), 41 deletions(-) diff --git a/verifiers/solver_voting_verifier.py b/verifiers/solver_voting_verifier.py index dd86a995..612f10dd 100644 --- a/verifiers/solver_voting_verifier.py +++ b/verifiers/solver_voting_verifier.py @@ -50,20 +50,21 @@ def generate_mutations(self) -> None | dict: Will generate random mutations based on mutations_per_model for the model """ for i in range(self.mutations_per_model): - # choose a mutation (not in exclude_dict) - valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( - self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( - set(self.mm_mutators).union(set(self.gen_mutators))) + # # choose a mutation (not in exclude_dict) + # valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( + # self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( + # set(self.mm_mutators).union(set(self.gen_mutators))) if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation mutator_list = self.mm_mutators else: # 1-mm_prob to choose generation-based mutation mutator_list = self.gen_mutators - valid = [m for m in mutator_list if m in valid_mutators] - if valid: - m = random.choice(valid) - else: - continue + # valid = [m for m in mutator_list if m in valid_mutators] + # if valid: + # m = random.choice(valid) + # else: + # continue + m = random.choice(mutator_list) self.mutators += [self.seed] # an error can occur in the transformations, so even before the solve call. @@ -246,10 +247,10 @@ def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: for _ in range(self.mutations_per_model): last_bug_cause = self.bug_cause - # Generate the type of mutation that will happen - valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( - self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( - set(self.mm_mutators).union(set(self.gen_mutators))) + # # Generate the type of mutation that will happen + # valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)) - set( + # self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( + # set(self.mm_mutators).union(set(self.gen_mutators))) if random.random() <= self.mm_prob: # mm_prob probability to choose metamorphic mutation mutator_list = self.mm_mutators new_mut_type = 'MM' @@ -257,11 +258,12 @@ def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: mutator_list = self.gen_mutators new_mut_type = 'GEN' - valid = [m for m in mutator_list if m in valid_mutators] - if valid: - m = random.choice(valid) - else: - continue + # valid = [m for m in mutator_list if m in valid_mutators] + # if valid: + # m = random.choice(valid) + # else: + # continue + m = random.choice(mutator_list) # Check whether verify_model returns an error before the new mutation, because the cause is then at the old mutation if new_mut_type != last_bug_cause: diff --git a/verifiers/strengthening_weakening_verifier.py b/verifiers/strengthening_weakening_verifier.py index 24891da0..f64fc290 100644 --- a/verifiers/strengthening_weakening_verifier.py +++ b/verifiers/strengthening_weakening_verifier.py @@ -72,22 +72,23 @@ def generate_mutations(self) -> None | dict: # choose a mutator. 33% of the time, this will be a strengthening/weakening mutation. # choose a mutation (not in exclude_dict) - valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)).union(set(self.str_wkn_mutators)) - set( - self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( - set(self.mm_mutators).union(set(self.gen_mutators)).union(set(self.str_wkn_mutators))) + # valid_mutators = list(set(self.mm_mutators).union(set(self.gen_mutators)).union(set(self.str_wkn_mutators)) - set( + # self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( + # set(self.mm_mutators).union(set(self.gen_mutators)).union(set(self.str_wkn_mutators))) rand = random.random() - if rand <= 1/3: + if rand <= 1/3 * (1 - self.mm_prob): mutator_list = self.str_wkn_mutators - elif rand - 1/3 <= 2/3 * self.mm_prob: # ~~ remaining mm_prob + elif rand - (1/3 * (1 - self.mm_prob)) <= 2/3 * self.mm_prob: # ~~ remaining mm_prob mutator_list = self.mm_mutators else: mutator_list = self.gen_mutators - valid = [m for m in mutator_list if m in valid_mutators] - if valid: - m = random.choice(valid) - else: - continue # No valid mutator? => go to next mutation + # valid = [m for m in mutator_list if m in valid_mutators] + # if valid: + # m = random.choice(valid) + # else: + # continue # No valid mutator? => go to next mutation + m = random.choice(mutator_list) self.mutators += [self.seed] # an error can occur in the transformations, so even before the solve call. @@ -355,28 +356,29 @@ def bug_search_run_and_verify_model(self, nr_mutations=None) -> dict: for _ in range(self.mutations_per_model): last_bug_cause = self.bug_cause - # choose a mutator. 33% of the time, this will be a strengthening/weakening mutation. - # choose a mutation (not in exclude_dict) - valid_mutators = list( - set(self.mm_mutators).union(set(self.gen_mutators)).union(self.str_wkn_mutators) - set( - self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( - set(self.mm_mutators).union(set(self.gen_mutators)).union(self.str_wkn_mutators)) + # # choose a mutator. 33% of the time, this will be a strengthening/weakening mutation. + # # choose a mutation (not in exclude_dict) + # valid_mutators = list( + # set(self.mm_mutators).union(set(self.gen_mutators)).union({strengthening_weakening_mutator}) - set( + # self.exclude_dict[self.model_file])) if self.model_file in self.exclude_dict else list( + # set(self.mm_mutators).union(set(self.gen_mutators)).union(self.str_wkn_mutators)) rand = random.random() - if rand <= 1/3: + if rand <= 1 / 3 * (1 - self.mm_prob): mutator_list = self.str_wkn_mutators new_mut_type = 'STRWK' - elif rand - 1/3 <= 2/3 * self.mm_prob: # ~~ remaining mm_prob + elif rand - (1 / 3 * (1 - self.mm_prob)) <= 2 / 3 * self.mm_prob: # ~~ remaining mm_prob mutator_list = self.mm_mutators new_mut_type = 'MM' else: mutator_list = self.gen_mutators new_mut_type = 'GEN' - valid = [m for m in mutator_list if m in valid_mutators] - if valid: - m = random.choice(valid) - else: - continue # No valid mutator? => go to next mutation + # valid = [m for m in mutator_list if m in valid_mutators] + # if valid: + # m = random.choice(valid) + # else: + # continue # No valid mutator? => go to next mutation + m = random.choice(mutator_list) # Check whether verify_model returns an error before the new mutation, because the cause is then at the old mutation if new_mut_type != last_bug_cause: From 3fbc608bc062c0f511224e817d4bd057e4283722 Mon Sep 17 00:00:00 2001 From: Josse Heylen Date: Tue, 28 Oct 2025 16:18:30 +0100 Subject: [PATCH 58/58] All changes together --- fuzz_test.py | 66 ++++++++++++++++++- fuzz_test_utils/mutators.py | 12 ++-- verifiers/model_counting_verifier.py | 2 +- verifiers/solver_voting_count_verifier.py | 2 +- verifiers/solver_voting_eq_verifier.py | 2 +- verifiers/strengthening_weakening_verifier.py | 2 +- verifiers/verifier_runner.py | 30 +++++++-- 7 files changed, 97 insertions(+), 19 deletions(-) diff --git a/fuzz_test.py b/fuzz_test.py index d4eedfbc..5baf87ca 100644 --- a/fuzz_test.py +++ b/fuzz_test.py @@ -6,6 +6,8 @@ import time from pathlib import Path from multiprocessing import Process,Lock, Manager, set_start_method,Pool, cpu_count +import csv +from datetime import datetime import cpmpy as cp @@ -66,9 +68,20 @@ def check_positive(value): # creating processes to run all the tests processes = [] - process_args = (current_amount_of_tests, current_amount_of_error, lock, args.solver, args.mutations_per_model ,models ,max_failed_tests,args.output_dir, max_time, args.mm_prob) + + # FOR EXPERIMENTS: + solvers = ['ortools', 'minizinc', 'choco', 'gurobi'] + verifiers = ["solver_vote_count_verifier", "solver_vote_eq_verifier", "solver_vote_sat_verifier", "solver_vote_sol_verifier", "strengthening_weakening_verifier"] + solver_counts = {s: manager.Value(f"{s}", 0) for s in solvers} + verifier_counts = {v: manager.Value(f"{v}", 0) for v in verifiers} + verifier_run_times = {v: manager.Value(f"{v}", 0) for v in verifiers} + from itertools import combinations + solver_combos = [list(c) for c in combinations(solvers, 2)] for x in range(args.amount_of_processes): + process_args = (current_amount_of_tests, current_amount_of_error, lock, solver_combos[x], + args.mutations_per_model, models, max_failed_tests, args.output_dir, max_time, args.mm_prob, + solver_counts, verifier_counts, verifier_run_times) # PAS TERUG AAN NA EXPERIMENTEN processes.append(Process(target=run_verifiers,args=process_args)) try: @@ -77,8 +90,14 @@ def check_positive(value): process.start() for process in processes: - process.join() - process.close() + process.join(timeout=args.max_minutes*60) # wait max double minutes + + # If any process is still alive after timeout, terminate it + for process in processes: + if process.is_alive(): + print(f"Forcefully terminating process {process.pid}", flush=True) + process.terminate() + process.join() # Clean up except KeyboardInterrupt as e: print("interrupting...",flush=True,end="\n") @@ -97,4 +116,45 @@ def check_positive(value): else: print("Succesfully executed " +str(current_amount_of_tests.value) + " tests, "+str(current_amount_of_error.value)+" tests failed",flush=True,end="\n") + run_name = args.output_dir + minutes_ran = args.max_minutes + mpm = args.mutations_per_model + mm_prob = args.mm_prob + amnt_tests = current_amount_of_tests.value + amnt_errors = current_amount_of_error.value + solver_count_vals = {s: c.value for s, c in solver_counts.items()} + verifier_count_vals = {v: c.value for v, c in verifier_counts.items()} + verifier_runtime_vals = {v: t.value for v, t in verifier_run_times.items()} + print(f"Run \"{run_name}\" ran for {minutes_ran} minutes and executed {amnt_tests} tests of which {amnt_errors} failed.") + print("Amount of times each solver ran for:", solver_count_vals) + print("Amount of times each verifier ran:", verifier_count_vals) + print("Time each verifier ran:", verifier_runtime_vals) + + # Prepare timestamp and filename + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + stats_filename = f"{run_name}_{timestamp}.csv" + + # Create the rows for the CSV + headers = ["run_name", "mutations_per_model", "mm_probability", "minutes_ran", "amnt_tests", "amnt_errors"] + row = [run_name, mpm, mm_prob, minutes_ran, amnt_tests, amnt_errors] + + # Optionally, flatten solver and verifier data into columns + for s, count in solver_count_vals.items(): + headers.append(f"solver_count_{s}") + row.append(count) + + for v, count in verifier_count_vals.items(): + headers.append(f"verifier_count_{v}") + row.append(count) + + for v, t in verifier_runtime_vals.items(): + headers.append(f"verifier_runtime_{v}") + row.append(t) + + # Save to CSV + with open(stats_filename, mode="w", newline="") as f: + writer = csv.writer(f) + writer.writerow(headers) + writer.writerow(row) + print(f"Saved run statistics to {stats_filename}") diff --git a/fuzz_test_utils/mutators.py b/fuzz_test_utils/mutators.py index 27624799..4182d4e9 100644 --- a/fuzz_test_utils/mutators.py +++ b/fuzz_test_utils/mutators.py @@ -1485,7 +1485,7 @@ def get_return_type(expr: Expression, con: Expression): GlobalCardinalityCount: (2, (None,)), Table: (3, (1, None)), NegativeTable: (3, (1, None)), - # Count: (1, (1, None)), # TODO: I don't think this is necessary, so check this. + # Count: (1, (1, None)), # TODO: Is this still necessary? 'pow': (1, (1, None)), 'wsum': (2, (0, None))} variable_restricted_functions = {Table: (2, (0, None)), @@ -1625,7 +1625,7 @@ def strengthen_expr(expr: Expression, path: tuple, con: Expression) -> Expressio ~ Returns: - `con`: the constraint after the mutation. """ - # TODO: 'and' & 'or' strengthenable by adding/removing args + # TODO: 'and' & 'or' strengthenable by adding/removing args + other functions too match expr.name: # {'or', '->', '!=', '<=', '>='} case 'or': # and, xor, !=, <, > args = expr.args @@ -1695,7 +1695,7 @@ def weaken_expr(expr: Expression, path: tuple, con: Expression) -> Expression: ~ Returns: - `con`: the constraint after the mutation. """ - # TODO: 'and' & 'or' weakenable by removing/adding args + # TODO: 'and' & 'or' weakenable by removing/adding args + other functions too match expr.name: # {'and', 'xor', '==', '<', '>'} case 'and': # or, ->, ==, <=, >= args = expr.args @@ -1834,9 +1834,9 @@ def strengthening_weakening_mutator(constraints: list, strengthen: bool = True) def change_domain_mutator(constraints: list, strengthen: bool): # TODO? something else for boolean variables? try: - # Take random variable - variables = get_variables(constraints) - if variables: # Improbable but it's possible there are no variables + # Take random integer variable + variables = [v for v in get_variables(constraints) if not is_boolexpr(v)] + if variables: # Don't change if no integer variables rand_var = random.choice(variables) lb = rand_var.lb diff --git a/verifiers/model_counting_verifier.py b/verifiers/model_counting_verifier.py index ad0aa58b..f226becc 100644 --- a/verifiers/model_counting_verifier.py +++ b/verifiers/model_counting_verifier.py @@ -35,7 +35,7 @@ def initialize_run(self) -> None: assert (len(self.cons)>0), f"{self.model_file} has no constraints" self.cons = toplevel_list(self.cons) if self.solver == 'gurobi': - self.sol_lim = 10000 # TODO: is hardcode best idea? + self.sol_lim = 10000 # Should this be hardcoded? self.sol_count = cp.Model(self.cons).solveAll(solver=self.solver,time_limit=max(1,min(250,self.time_limit-time.time())), solution_limit=self.sol_lim) else: self.sol_count = cp.Model(self.cons).solveAll(solver=self.solver,time_limit=max(1,min(250,self.time_limit-time.time()))) diff --git a/verifiers/solver_voting_count_verifier.py b/verifiers/solver_voting_count_verifier.py index 2a40d9e8..e4b69697 100644 --- a/verifiers/solver_voting_count_verifier.py +++ b/verifiers/solver_voting_count_verifier.py @@ -56,7 +56,7 @@ def initialize_run(self) -> None: self.cons = toplevel_list(self.cons) assert len(self.solvers) > 1, f"More than 1 solver required, given solvers: {self.solvers}." if 'gurobi' in [s.lower() for s in self.solvers]: # Because gurobi can't run solveAll without solution_limit - self.sol_lim = 10000 # TODO: is hardcode best idea? + self.sol_lim = 10000 # Should this be hardcoded? # Optional: Check before applying the mutations. This should never fail... diff --git a/verifiers/solver_voting_eq_verifier.py b/verifiers/solver_voting_eq_verifier.py index 6a0ee619..a9e3a3c3 100644 --- a/verifiers/solver_voting_eq_verifier.py +++ b/verifiers/solver_voting_eq_verifier.py @@ -56,7 +56,7 @@ def initialize_run(self) -> None: self.cons = toplevel_list(self.cons) assert len(self.solvers) > 1, f"More than 1 solver required, given solvers: {self.solvers}." if 'gurobi' in [s.lower() for s in self.solvers]: # Because gurobi can't run solveAll without solution_limit - self.sol_lim = 10000 # TODO: is hardcode best idea? + self.sol_lim = 10000 # Should this be hardcoded? self.original_vars = get_variables(self.cons) # New auxiliary variables will be added so we need to do this here diff --git a/verifiers/strengthening_weakening_verifier.py b/verifiers/strengthening_weakening_verifier.py index f64fc290..7a427175 100644 --- a/verifiers/strengthening_weakening_verifier.py +++ b/verifiers/strengthening_weakening_verifier.py @@ -57,7 +57,7 @@ def initialize_run(self) -> None: self.cons = toplevel_list(self.cons) assert len(self.solvers) > 1, f"More than 1 solver required, given solvers: {self.solvers}." if 'gurobi' in [s.lower() for s in self.solvers]: # Because gurobi can't run solveAll without solution_limit - self.sol_lim = 10000 # TODO: is hardcode best idea? + self.sol_lim = 10000 # Should this be hardcoded? # Optional: Check before applying the mutations. This should never fail... diff --git a/verifiers/verifier_runner.py b/verifiers/verifier_runner.py index 3736366b..ab5bae88 100644 --- a/verifiers/verifier_runner.py +++ b/verifiers/verifier_runner.py @@ -4,6 +4,7 @@ import random import warnings from os.path import join +import gc from fuzz_test_utils.output_writer import get_logging_dir, write_csv from verifiers import * @@ -12,9 +13,11 @@ def get_all_verifiers(single_solver) -> list: if single_solver: return [Solution_Verifier,Optimization_Verifier,Model_Count_Verifier,Metamorphic_Verifier,Equivalance_Verifier] else: - return [Solver_Vote_Count_Verifier, Solver_Vote_Sat_Verifier, Strengthening_Weakening_Verifier] + return [Solver_Vote_Count_Verifier, Solver_Vote_Sat_Verifier, Strengthening_Weakening_Verifier, + Solver_Vote_Eq_Verifier, Solver_Vote_Sol_Verifier] -def run_verifiers(current_amount_of_tests, current_amount_of_error, lock, solver: list[str], mutations_per_model: int, folders: list, max_error_treshold: int, output_dir: str, time_limit: float) -> None: +def run_verifiers(current_amount_of_tests, current_amount_of_error, lock, solver: list[str], mutations_per_model: int, folders: list, max_error_treshold: int, output_dir: str, time_limit: float, mm_prob: float, + solver_counts, verifier_counts, verifier_runtimes) -> None: """ This function will be used to run different verifiers @@ -36,6 +39,9 @@ def run_verifiers(current_amount_of_tests, current_amount_of_error, lock, solver random.seed(random_seed) solver = solver[0] if len(solver) == 1 else solver # Take the solver as a string if there is only one + if solver == 'pysat': + solver = random.Random().sample(['minizinc', 'ortools', 'gurobi', 'choco'], 2) # Andere solvers hebben problemen met hun time_limit + print(f"Running with solvers {solver}") verifier_kwargs = {"solver":solver, "mutations_per_model":mutations_per_model, "exclude_dict":exclude_dict,"time_limit": time_limit, "seed":random_seed} execution_time = 0 @@ -44,18 +50,25 @@ def run_verifiers(current_amount_of_tests, current_amount_of_error, lock, solver if isinstance(solver, str): random_verifier = random.choice(get_all_verifiers(single_solver=True))(**verifier_kwargs) elif isinstance(solver, list): + verifier_kwargs["mm_prob"] = mm_prob # add probability to choose mm_mut instead of gen_mut random_verifier = random.choice(get_all_verifiers(single_solver=False))(**verifier_kwargs) else: - raise Exception(f"The given solvers are not in the correct format. Should be either str or list, but is {type(solver)}.") + raise Exception(f"The given solvers are not in the correct format. Should be either a single solver (str) or a list of solvers ([str]), but is {type(solver)}.") fmodels = [] for folder in folders: fmodels.extend(glob.glob(join(folder,random_verifier.getType(), "*"))) if len(fmodels) > 0: fmodel = random.Random().choice(fmodels) # random.choice used the random.seed()! Same models were being tested! + for s in solver: + solver_counts[s].value += 1 + verifier_counts[random_verifier.name].value += 1 start_time = time.time() error = random_verifier.run(fmodel) - execution_time = math.floor(time.time() - start_time) + end_time = time.time() + execution_time = math.floor(end_time - start_time) + verifier_runtimes[random_verifier.name].value += end_time - start_time + # check if we got an error if error is not None: lock.acquire() @@ -66,15 +79,20 @@ def run_verifiers(current_amount_of_tests, current_amount_of_error, lock, solver logging_dir = get_logging_dir(error_data, output_dir) if get_logging_dir(error_data, output_dir) else output_dir os.makedirs(logging_dir, exist_ok=True) # create if it doesn't already exist write_error(error_data, logging_dir) - write_csv(error_data, 'csv_results.csv') + write_csv(error_data, output_dir+'.csv') current_amount_of_error.value += 1 finally: - lock.release() + lock.release() + # Memory fix? + del random_verifier + import gc + gc.collect() lock.acquire() try: current_amount_of_tests.value += 1 finally: lock.release() + print(f"Process {os.getpid()} with solvers {solver} exiting at {time.time()} > {time_limit}", flush=True) except Exception as e: print(traceback.format_exc(),flush=True)