From 1ed37cb8f4ecac736680db089984a6190e42b027 Mon Sep 17 00:00:00 2001 From: Afsoon Afzal Date: Wed, 11 Apr 2018 12:23:46 -0400 Subject: [PATCH 1/6] Skeleton of mutation based mission generator --- houston/generator/base.py | 3 +- houston/generator/mutation.py | 104 ++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 houston/generator/mutation.py diff --git a/houston/generator/base.py b/houston/generator/base.py index 856f278f..825ed524 100644 --- a/houston/generator/base.py +++ b/houston/generator/base.py @@ -244,8 +244,7 @@ def generate_and_run(self, seed, resource_limits, with_coverage=False): self.threads, stream, self.record_outcome, - with_coverage - ) + with_coverage) self.__resource_usage = ResourceUsage() self.__start_time = timeit.default_timer() self.tick() diff --git a/houston/generator/mutation.py b/houston/generator/mutation.py new file mode 100644 index 00000000..b6e94fff --- /dev/null +++ b/houston/generator/mutation.py @@ -0,0 +1,104 @@ +from houston.generator.base import MissionGenerator +from houston.mission import Mission + + +class MutationBasedMissionGenerator(MissionGenerator): + def __init__(self, + system, + initial_state, + env, + threads = 1, + action_generators = [], + max_num_actions = 10, + initial_mission = None): + super(RandomMissionGenerator, self).__init__(system, threads, action_generators, max_num_actions) + self.__initial_state = initial_state + self.__env = env + self.__initial_mission = initial_mission if initial_mission else self._generate_random_mission() + self.__in_progress_missions = {} + self.__most_fit_missions = [self.__initial_mission] #TODO this can be a dictionary instead of a list + + + @property + def initial_state(self): + """ + The initial state used by all missions produced by this generator. + """ + return self.__initial_state + + + @property + def initial_mission(self): + """ + Returns the initial mission to start the mutation from. + """ + return self.__initial_mission + + + @property + def env(self): + """ + The environment used by all missions produced by this generator. + """ + return self.__env + + + def _generate_action(self, schema): + generator = self.action_generator(schema) + if generator is None: + return schema.generate(self.rng) + return generator.generate_action_without_state(self.system, self.__env, self.rng) + + + def _generate_random_mission(self): + schemas = list(self.system.schemas.values()) + actions = [] + for _ in range(self.rng.randint(1, self.max_num_actions)): + schema = self.rng.choice(schemas) + actions.append(self._generate_action(schema)) + return Mission(self.__env, self.__initial_state, actions) + + + def _get_fitness(self, mission): + #TODO Get the fitness of this mission + outcome = self.outcomes[mission] + coverage = self.coverage[mission] + return 1.0 + + + def _mutate_mission(self, mission): + #TODO mutate a given mission + return mission + + + def generate_mission(self): + parent = self.rng.choice(self.__most_fit_missions) + mission = self._mutate_mission(parent) + self.__in_progress_missions[mission] = {'parent': parent} + return mission + + + def record_outcome(self, mission, outcome, coverage): + """ + Records the outcome of a given mission. The mission is logged to the + history, and its outcome is stored in the outcome dictionary. If the + mission failed, the mission is also added to the set of failed + missions. + """ + self.__history.append(mission) + self.__outcomes[mission] = outcome + self.__coverage[mission] = coverage + + if outcome.failed: + self.__failures.add(mission) + + if not mission in self.__in_progress_missions: + print("Something went wrong! mission is not in progress") + + fitness = self._get_fitness(mission) + parent = self.__in_progress_missions[mission]['parent'] + parent_fitness = self._get_fitness(parent) + if fitness >= parent_fitness: + self.__most_fit_missions.remove(parent) + self.__most_fit_missions.append(mission) + self.__in_progress_missions.pop(mission) From bb02f2969941689078ae365535528471230b1438 Mon Sep 17 00:00:00 2001 From: Afsoon Afzal Date: Wed, 11 Apr 2018 16:06:54 -0400 Subject: [PATCH 2/6] Adding a number of mutation operators --- houston/generator/mutation.py | 56 +++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/houston/generator/mutation.py b/houston/generator/mutation.py index b6e94fff..df4c6f1a 100644 --- a/houston/generator/mutation.py +++ b/houston/generator/mutation.py @@ -63,15 +63,56 @@ def _get_fitness(self, mission): #TODO Get the fitness of this mission outcome = self.outcomes[mission] coverage = self.coverage[mission] - return 1.0 + initial_coverage = self.coverage[self.__initial_mission] + + fitness = 1.0 + + if not outcome.passed: + fitness *= 5.0 + + similar_coverage = coverage.intersection(initial_coverage) + fitness += len(similar_coverage) + + return fitness + + + def _add_action_operator(self, mission): + actions = mission.actions + if len(actions) >= self.max_num_actions: + return None + schema = self.rng.choice(list(self.system.schemas.values())) + actions.insert(self.rng.randint(0, len(actions)-1), self._generate_action(schema)) + return Mission(self.__env, self.__initial_state, actions) + + + def _delete_action_operator(self, mission): + actions = mission.actions + if len(actions) <= 1: + return None + actions.pop(self.rng.randint(0, len(actions)-1) + return Mission(self.__env, self.__initial_state, actions) + + + def _edit_action_operator(self, mission): + #TODO implement editing an existing action + return None def _mutate_mission(self, mission): - #TODO mutate a given mission - return mission + mutation_operators = [self._add_action_operator, self._delete_action_operator, + self._edit_action_operator] + generated_mission = None + while not generated_mission: + operator = self.rng.choice(mutation_operators) + generated_mission = operator(mission) + return generated_mission def generate_mission(self): + if not self.__most_fit_missions: + self.__in_progress_missions[self.__initial_mission] = {'parent': None} + return self.__initial_mission + parent = self.rng.choice(self.__most_fit_missions) mission = self._mutate_mission(parent) self.__in_progress_missions[mission] = {'parent': parent} @@ -97,8 +138,17 @@ def record_outcome(self, mission, outcome, coverage): fitness = self._get_fitness(mission) parent = self.__in_progress_missions[mission]['parent'] + if not parent: + # initial mission + self.__most_fit_missions.append(mission) + self.__in_progress_missions.pop(mission) + return + parent_fitness = self._get_fitness(parent) if fitness >= parent_fitness: self.__most_fit_missions.remove(parent) self.__most_fit_missions.append(mission) self.__in_progress_missions.pop(mission) + + + From 240d9d373ec42f36b92a9317eaa8ec5d947e0082 Mon Sep 17 00:00:00 2001 From: Afsoon Afzal Date: Thu, 12 Apr 2018 14:26:44 -0400 Subject: [PATCH 3/6] Bug fixes and improvements --- example/hello_world.py | 30 +++++++++++++++++++++++------ houston/generator/__init__.py | 2 ++ houston/generator/mutation.py | 35 ++++++++++++++++++++++++---------- houston/generator/resources.py | 17 ++++++++++++++--- 4 files changed, 65 insertions(+), 19 deletions(-) diff --git a/example/hello_world.py b/example/hello_world.py index fb6733a8..c97597d7 100755 --- a/example/hello_world.py +++ b/example/hello_world.py @@ -2,7 +2,7 @@ import houston import bugzoo import json -from houston.generator.rand import RandomMissionGenerator +from houston.generator import * from houston.generator.resources import ResourceLimits from houston.mission import Mission from houston.runner import MissionRunnerPool @@ -94,6 +94,23 @@ def generate_and_run(sut, initial, environment, number_of_missions): print("DONE") +### Generate and run missions with mutation operator +def generate_and_run_mutation(sut, initial_state, environment, initial_mission, number_of_missions): + mission_generator = MutationBasedMissionGenerator(sut, initial, environment, initial_mission=initial_mission, action_generators=[CircleBasedGotoGenerator((-35.3632607, 149.1652351), 2.0)]) + resource_limits = ResourceLimits(number_of_missions*5, 1000, number_of_missions) + mission_generator.generate_and_run(100, resource_limits, with_coverage=True) + print("DONE") + with open("example/missions-mutation.json", "w") as f: + mission_descriptions = list(map(Mission.to_json, mission_generator.history)) + print(str(mission_descriptions)) + json.dump(mission_descriptions, f) + f.write("\n") + mission_descriptions = list(map(Mission.to_json, mission_generator.most_fit_missions)) + print(str(mission_descriptions)) + json.dump(mission_descriptions, f) + + + ### Generate and run missions with fault localization def generate_and_run_with_fl(sut, initial, environment, number_of_missions): mission_generator = RandomMissionGenerator(sut, initial, environment, max_num_actions=3, action_generators=[CircleBasedGotoGenerator((-35.3632607, 149.1652351), 2.0)]) @@ -108,19 +125,19 @@ def generate_and_run_with_fl(sut, initial, environment, number_of_missions): if __name__=="__main__": - #sut = houston.ardu.ArduRover('afrl:overflow') - sut = houston.ardu.ArduCopter('afrl:overflow') + sut = houston.ardu.ArduRover('afrl:overflow') + #sut = houston.ardu.ArduCopter('afrl:overflow') # mission description actions = [ houston.action.Action("arm", {'arm': True}), - houston.action.Action("takeoff", {'altitude': 3.0}), + #houston.action.Action("takeoff", {'altitude': 3.0}), houston.action.Action("goto", { 'latitude' : -35.361354, 'longitude': 149.165218, 'altitude' : 5.0 }), - houston.action.Action("setmode", {'mode': 'LAND'}), + #houston.action.Action("setmode", {'mode': 'LAND'}), houston.action.Action("arm", {'arm': False}) ] environment = houston.state.Environment({}) @@ -141,7 +158,8 @@ def generate_and_run_with_fl(sut, initial, environment, number_of_missions): #run_single_mission_with_coverage(sandbox, mission) #generate(sut, initial, environment, 100, 10) - run_all_missions(sut, "example/missions.json", False) + #run_all_missions(sut, "example/missions.json", False) + generate_and_run_mutation(sut, initial, environment, mission, 3) #generate_and_run_with_fl(sut, initial, environment, 5) #run_single_mission_with_coverage(sandbox, mission) diff --git a/houston/generator/__init__.py b/houston/generator/__init__.py index e69de29b..1e828ddc 100644 --- a/houston/generator/__init__.py +++ b/houston/generator/__init__.py @@ -0,0 +1,2 @@ +from houston.generator.rand import RandomMissionGenerator +from houston.generator.mutation import MutationBasedMissionGenerator diff --git a/houston/generator/mutation.py b/houston/generator/mutation.py index df4c6f1a..72e9f458 100644 --- a/houston/generator/mutation.py +++ b/houston/generator/mutation.py @@ -11,12 +11,12 @@ def __init__(self, action_generators = [], max_num_actions = 10, initial_mission = None): - super(RandomMissionGenerator, self).__init__(system, threads, action_generators, max_num_actions) + super(MutationBasedMissionGenerator, self).__init__(system, threads, action_generators, max_num_actions) self.__initial_state = initial_state self.__env = env self.__initial_mission = initial_mission if initial_mission else self._generate_random_mission() self.__in_progress_missions = {} - self.__most_fit_missions = [self.__initial_mission] #TODO this can be a dictionary instead of a list + self.__most_fit_missions = [] #TODO this can be a dictionary instead of a list @property @@ -43,6 +43,14 @@ def env(self): return self.__env + @property + def most_fit_missions(self): + """ + Return the most fit missions generated. + """ + return self.__most_fit_missions + + def _generate_action(self, schema): generator = self.action_generator(schema) if generator is None: @@ -61,6 +69,9 @@ def _generate_random_mission(self): def _get_fitness(self, mission): #TODO Get the fitness of this mission + if mission == self.__initial_mission: + return 1.0 + outcome = self.outcomes[mission] coverage = self.coverage[mission] initial_coverage = self.coverage[self.__initial_mission] @@ -89,7 +100,7 @@ def _delete_action_operator(self, mission): actions = mission.actions if len(actions) <= 1: return None - actions.pop(self.rng.randint(0, len(actions)-1) + actions.pop(self.rng.randint(0, len(actions)-1)) return Mission(self.__env, self.__initial_state, actions) @@ -104,6 +115,7 @@ def _mutate_mission(self, mission): generated_mission = None while not generated_mission: operator = self.rng.choice(mutation_operators) + print("Operator: {}".format(str(operator))) generated_mission = operator(mission) return generated_mission @@ -116,6 +128,7 @@ def generate_mission(self): parent = self.rng.choice(self.__most_fit_missions) mission = self._mutate_mission(parent) self.__in_progress_missions[mission] = {'parent': parent} + print("Mutated: {}\nfrom: {}".format(mission.to_json(), parent.to_json())) return mission @@ -126,12 +139,13 @@ def record_outcome(self, mission, outcome, coverage): mission failed, the mission is also added to the set of failed missions. """ - self.__history.append(mission) - self.__outcomes[mission] = outcome - self.__coverage[mission] = coverage + self.history.append(mission) + self.outcomes[mission] = outcome + self.coverage[mission] = coverage + if outcome.failed: - self.__failures.add(mission) + self.failures.add(mission) if not mission in self.__in_progress_missions: print("Something went wrong! mission is not in progress") @@ -140,14 +154,15 @@ def record_outcome(self, mission, outcome, coverage): parent = self.__in_progress_missions[mission]['parent'] if not parent: # initial mission - self.__most_fit_missions.append(mission) + self.most_fit_missions.append(mission) self.__in_progress_missions.pop(mission) return parent_fitness = self._get_fitness(parent) if fitness >= parent_fitness: - self.__most_fit_missions.remove(parent) - self.__most_fit_missions.append(mission) + if len(self.most_fit_missions) == self.resource_limits.num_missions_selected: + self.most_fit_missions.remove(parent) + self.most_fit_missions.append(mission) self.__in_progress_missions.pop(mission) diff --git a/houston/generator/resources.py b/houston/generator/resources.py index 547b6793..01b2c413 100644 --- a/houston/generator/resources.py +++ b/houston/generator/resources.py @@ -28,12 +28,14 @@ class ResourceLimits(object): @staticmethod def from_json(jsn): return ResourceLimits(num_missions = jsn['num_missions'], - running_time = jsn['running_time']) + running_time = jsn['running_time'], + num_missions_selected = jsn['num_missions_selected']) - def __init__(self, num_missions = None, running_time = None): + def __init__(self, num_missions = None, running_time = None, num_missions_selected = None): self.__num_missions = num_missions self.__running_time = running_time + self.__num_missions_selected = num_missions_selected def reached(self, usage): @@ -48,6 +50,14 @@ def reached(self, usage): return False + @property + def num_missions_selected(self): + """ + Number of missions that should be selected from all generated missions. + """ + return self.__num_missions_selected + + @property def num_missions(self): """ @@ -89,5 +99,6 @@ def to_json(self): """ return { 'num_missions': self.__num_missions, - 'running_time': self.__running_time + 'running_time': self.__running_time, + 'num_missions_selected': self.__num_missions_selected } From d01d20d75cbbe2f4b7da3fe00592a49a77838afd Mon Sep 17 00:00:00 2001 From: Afsoon Afzal Date: Thu, 12 Apr 2018 14:59:57 -0400 Subject: [PATCH 4/6] Updating fitness, accepting bad results 5% of the times --- houston/generator/mutation.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/houston/generator/mutation.py b/houston/generator/mutation.py index 72e9f458..04fcfc2b 100644 --- a/houston/generator/mutation.py +++ b/houston/generator/mutation.py @@ -68,9 +68,10 @@ def _generate_random_mission(self): def _get_fitness(self, mission): - #TODO Get the fitness of this mission - if mission == self.__initial_mission: - return 1.0 + """ + Returns a float number as the fitness. Higher is better. + """ + #TODO Come up with a reasonable fitness metric outcome = self.outcomes[mission] coverage = self.coverage[mission] @@ -79,10 +80,12 @@ def _get_fitness(self, mission): fitness = 1.0 if not outcome.passed: - fitness *= 5.0 + fitness *= 10.0 similar_coverage = coverage.intersection(initial_coverage) - fitness += len(similar_coverage) + fitness += (len(similar_coverage)/len(coverage))*15 + + fitness -= len(mission.actions)*3 return fitness @@ -159,8 +162,9 @@ def record_outcome(self, mission, outcome, coverage): return parent_fitness = self._get_fitness(parent) - if fitness >= parent_fitness: - if len(self.most_fit_missions) == self.resource_limits.num_missions_selected: + print("My fotness: {}, parent fitness: {}".format(fitness, parent_fitness)) + if fitness >= parent_fitness or self.rng.random() <= 0.05: + if len(self.most_fit_missions) >= self.resource_limits.num_missions_selected: self.most_fit_missions.remove(parent) self.most_fit_missions.append(mission) self.__in_progress_missions.pop(mission) From 7cb0d239a70fa487872b8c45ac8163524b56946b Mon Sep 17 00:00:00 2001 From: Afsoon Afzal Date: Fri, 13 Apr 2018 12:12:50 -0400 Subject: [PATCH 5/6] Initial implementation of edit operator --- houston/generator/mutation.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/houston/generator/mutation.py b/houston/generator/mutation.py index 04fcfc2b..a311cb80 100644 --- a/houston/generator/mutation.py +++ b/houston/generator/mutation.py @@ -108,8 +108,14 @@ def _delete_action_operator(self, mission): def _edit_action_operator(self, mission): - #TODO implement editing an existing action - return None + actions = mission.actions + index = self.rng.randint(0, len(actions)-1) + new_action = self._generate_action(self.system.schemas[actions[index].schema_name]) + if new_action.values == actions[index].values: + return None + actions[index] = new_action + + return Mission(self.__env, self.__initial_state, actions) def _mutate_mission(self, mission): @@ -162,7 +168,7 @@ def record_outcome(self, mission, outcome, coverage): return parent_fitness = self._get_fitness(parent) - print("My fotness: {}, parent fitness: {}".format(fitness, parent_fitness)) + print("My fitness: {}, parent fitness: {}".format(fitness, parent_fitness)) if fitness >= parent_fitness or self.rng.random() <= 0.05: if len(self.most_fit_missions) >= self.resource_limits.num_missions_selected: self.most_fit_missions.remove(parent) From ca75f49f0eb11bf053d5a136f4dee76a2caad3f9 Mon Sep 17 00:00:00 2001 From: Afsoon Afzal Date: Tue, 17 Apr 2018 15:55:41 -0400 Subject: [PATCH 6/6] Making initial mission mandatory --- example/hello_world.py | 2 +- houston/generator/mutation.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/example/hello_world.py b/example/hello_world.py index c97597d7..f975491a 100755 --- a/example/hello_world.py +++ b/example/hello_world.py @@ -96,7 +96,7 @@ def generate_and_run(sut, initial, environment, number_of_missions): ### Generate and run missions with mutation operator def generate_and_run_mutation(sut, initial_state, environment, initial_mission, number_of_missions): - mission_generator = MutationBasedMissionGenerator(sut, initial, environment, initial_mission=initial_mission, action_generators=[CircleBasedGotoGenerator((-35.3632607, 149.1652351), 2.0)]) + mission_generator = MutationBasedMissionGenerator(sut, initial, environment, initial_mission, action_generators=[CircleBasedGotoGenerator((-35.3632607, 149.1652351), 2.0)]) resource_limits = ResourceLimits(number_of_missions*5, 1000, number_of_missions) mission_generator.generate_and_run(100, resource_limits, with_coverage=True) print("DONE") diff --git a/houston/generator/mutation.py b/houston/generator/mutation.py index a311cb80..cdd137e7 100644 --- a/houston/generator/mutation.py +++ b/houston/generator/mutation.py @@ -7,14 +7,14 @@ def __init__(self, system, initial_state, env, + initial_mission, threads = 1, action_generators = [], - max_num_actions = 10, - initial_mission = None): + max_num_actions = 10): super(MutationBasedMissionGenerator, self).__init__(system, threads, action_generators, max_num_actions) self.__initial_state = initial_state self.__env = env - self.__initial_mission = initial_mission if initial_mission else self._generate_random_mission() + self.__initial_mission = initial_mission self.__in_progress_missions = {} self.__most_fit_missions = [] #TODO this can be a dictionary instead of a list