From 2a44e47b8d188b0b76faff4050fde79ce27d0693 Mon Sep 17 00:00:00 2001 From: ttchalakov Date: Mon, 20 Jun 2022 23:41:02 -0400 Subject: [PATCH 01/41] Added log barrier cost and old factory --- autompc/costs/barrier_cost.py | 137 ++++++++++++++++++++++++++++ autompc/ocp/barrier_cost_factory.py | 122 +++++++++++++++++++++++++ 2 files changed, 259 insertions(+) create mode 100644 autompc/costs/barrier_cost.py create mode 100644 autompc/ocp/barrier_cost_factory.py diff --git a/autompc/costs/barrier_cost.py b/autompc/costs/barrier_cost.py new file mode 100644 index 0000000..57a9016 --- /dev/null +++ b/autompc/costs/barrier_cost.py @@ -0,0 +1,137 @@ +# Created by Teodor Tchalakov, (ttcha2@illinois.edu) + +import numpy as np +import numpy.linalg as la + +from .cost import Cost + +class LogBarrierCost(Cost): + def __init__(self, system, boundedStates): + """ + Create barrier cost that approximates an inequality constraint. + Function does not exist outside the limit. + where : - b * ln ( a - x ) for upper limit + - b * ln ( a + x ) for lower limit + Parameters + ---------- + system : System + Robot system object. + boundedState : dict + Dictionary of { "observation/control name" : (limit, scale, upper)} + observation/control (x) : String + Observation/control name for which limit is specified. + limit (a) : double + limit value a that barrier is placed at. + scale (b) : double + Positive scalar to magnify the cost function. + scale: (0, inf) + upper : boolean + True if the limit is an upper limit. + False if the limit is a lower limit. + """ + super().__init__(system) + self.obsConfiguration = [] + self.ctrlsConfiguration = [] + + for variable in boundedStates.keys(): + config = boundedStates[variable] + # Check that scale is positive + if(config[1] < 0): + raise ValueError(f"{variable}'s log barrier must be positive, was {config[1]}") + elif(variable in system.observations): + self.obsConfiguration.append([variable, config]) + elif(variable in system.controls): + self.ctrlsConfiguration.append([variable, config]) + else: + raise ValueError(f"Variable {variable} is not in the given system") + + # Configs + self._is_quad = False + self._is_convex = True + self._is_diff = True + self._is_twice_diff = True + self._has_goal = False + + #Cost Function: + # - b * ln ( a - x ) upper limit + # - b * ln ( a + x ) lower limit + def eval_obs_cost(self, obs): + sum = 0 + for boundedObs in self.obsConfiguration: + variable, config = boundedObs + index = self.system.observations.index(variable) + limit, scale, upper = config + self._direction = -1 + if(upper): + direction = 1 + sum = sum + -scale * np.log(limit - (direction * obs[index])) + return sum + + #Jacobian: + # b / (a - x) upper limit + # -b / (a - x) lower limit + def eval_obs_cost_diff(self, obs): + jacobian = np.zeros(self.system.obs_dim) + for boundedObs in self.obsConfiguration: + variable, config = boundedObs + index = self.system.observations.index(variable) + limit, scale, upper = config + self._direction = -1 + if(upper): + direction = 1 + jacobian[boundedObs[0]] = direction * scale / (limit - obs[index]) + return self.eval_obs_cost(obs), jacobian + + #Hessian: + # b / (a - x)^2 upper limit + # b / (a - x)^2 lower limit + def eval_obs_cost_hess(self, obs): + hessian = np.zeros((self.system.obs_dim, self.system.obs_dim)) + for boundedObs in self.obsConfiguration: + variable, config = boundedObs + index = self.system.observations.index(variable) + limit, scale, _ = config + hessian[boundedObs[0]][boundedObs[0]] = scale / ((limit - obs[index])**2) + return *self.eval_obs_cost_diff, hessian + + def eval_ctrl_cost(self, ctrl): + sum = 0 + for boundedCtrl in self.ctrlsConfiguration: + variable, config = boundedCtrl + index = self.system.controls.index(variable) + limit, scale, upper = config + self._direction = -1 + if(upper): + direction = 1 + sum = sum + -scale * np.log(limit - (direction * ctrl[index])) + return sum + + def eval_ctrl_cost_diff(self, ctrl): + jacobian = np.zeros(self.system.ctrl_dim) + for boundedCtrl in self.ctrlsConfiguration: + variable, config = boundedCtrl + index = self.system.controls.index(variable) + limit, scale, upper = config + self._direction = -1 + if(upper): + direction = 1 + jacobian[boundedCtrl[0]] = direction * scale / (limit - ctrl[index]) + return self.eval_ctrl_cost(), jacobian + + def eval_ctrl_cost_hess(self, ctrl): + hessian = np.zeros((self.system.ctrl_dim, self.system.ctrl_dim)) + for boundedCtrl in self.ctrlsConfiguration: + variable, config = boundedCtrl + index = self.system.controls.index(variable) + limit, scale, _ = config + hessian[boundedCtrl[0]][boundedCtrl[0]] = scale / ((limit - ctrl[index])**2) + return *self.eval_ctrl_cost_diff(), hessian + + def eval_term_obs_cost(self, obs): + return 0 + + def eval_term_obs_cost_diff(self, obs): + return 0 + + def eval_term_obs_cost_hess(self, obs): + return 0 \ No newline at end of file diff --git a/autompc/ocp/barrier_cost_factory.py b/autompc/ocp/barrier_cost_factory.py new file mode 100644 index 0000000..10d66e2 --- /dev/null +++ b/autompc/ocp/barrier_cost_factory.py @@ -0,0 +1,122 @@ +# Created by Teodor Tchalakov (tcha2@illinois.edu), 2022-04-18 + +# Standard library includes +from collections import defaultdict + +# Internal library includes +from .cost_factory import CostFactory +from . import LogBarrierCost + +# External library includes +import numpy as np +import ConfigSpace as CS +import ConfigSpace.hyperparameters as CSH +import ConfigSpace.conditions as CSC + +def construct_default_bounds(): + return (1e-3, 1e4, 1.0, False) + +class LogBarrierCostFactory(CostFactory): + def __init__(self, system): + super().__init__(system) + + self._scale_bounds = defaultdict(construct_default_bounds) # Key: obsname, Value: (lower, upper, default, log_scale) + self._limits = [] # Key: obs/ctrlname, Value: (limit, upper) + self._scale_fixed = [] # Key: obs/ctrlname, Value: limit + + """ + boundedState : String + Name of observation or control in the system. + + limit : double + limit value that barrier is placed at. + + upper : boolean + True if the limit is an upper limit. + False if the limit is a lower limit. + """ + def set_limit(self, boundedState, limit, upper): + if(boundedState in self.system.observations or boundedState in self.system.controls): + self._limits[boundedState] = (limit, upper) + else: + raise ValueError(str(boundedState) + " is not in system") + + """ + boundedState : String + Name of observation or control in the system. + + limit : double + limit value that barrier is placed at. + + lower_bound : double + lower bound of configuration space + + upper_bound : double + upper bound of configuration space + + default : double + default value of configuration space + """ + def set_bounds(self, boundedState, lower_bound, upper_bound, default, log=False): + if(boundedState in self.system.observations or boundedState in self.system.controls): + if(boundedState in self._limits): + self._scale_bounds[boundedState] = (lower_bound, upper_bound, default, log) + else: + raise ValueError(str(boundedState) + " does not have a configured limit use set_limit") + else: + raise ValueError(str(boundedState) + " is not in system") + + def set_fixed_scale(self, boundedState, scale): + if(boundedState in self.system.observations or boundedState in self.system.controls): + if(boundedState in self._limits): + self._scale_fixed[boundedState] = scale + else: + raise ValueError(str(boundedState) + " does not have a configured limit use set_limit") + else: + raise ValueError(str(boundedState) + " is not in system") + + + def _get_hyperparameters(self, label, bounds_dict, fixed_dict): + hyperparameters = [] + for name in (self.system.controls + self.system.observations): + if name in fixed_dict or name not in self._limits: + continue + limit, upper = self._limits[name] + upper_string = "Upper" + if(not upper): + upper_string = "Lower" + lower_scale, upper_scale, default, log = bounds_dict[name] + hyper = CSH.UniformFloatHyperparameter("{}_{}_{}".format(name, upper_string, label), + lower=lower_scale, upper=upper_scale, default_value=default, log=log) + hyperparameters.append(hyper) + return hyperparameters + + def get_configuration_space(self): + cs = CS.ConfigurationSpace() + hypers = self._get_hyperparameters("LogBarrier", self._scale_bounds, self._scale_fixed) + cs.add_hyperparameters(hypers) + return cs + + def _get_boundedState(self, cfg, label, fixed_dict): + boundedStates = dict() + for name in (self.system.controls + self.system.observations): + if name in fixed_dict: + limit, upper = self._limits[name] + scale = self._scale_fixed[name] + boundedStates[name] = (limit, scale, upper) + elif name in self._limits: + limit, upper = self._limits[name] + upper_string = "Upper" + if(not upper): + upper_string = "Lower" + hyper_name = f"{name}_{upper_string}_{label}" + scale = cfg(hyper_name) + boundedStates[name] = (limit, scale, upper) + return boundedStates + + def __call__(self, cfg, task, trajs): + boundedStates = self._get_boundedState(cfg, "LogBarrier", self._scale_fixed) + + return LogBarrierCost(self.system, boundedStates) + + \ No newline at end of file From 8bf8213c00a501c901bc5efbb5c839b112733a98 Mon Sep 17 00:00:00 2001 From: ttchalakov Date: Mon, 20 Jun 2022 23:41:02 -0400 Subject: [PATCH 02/41] Added log barrier cost and old factory --- autompc/costs/barrier_cost.py | 137 ++++++++++++++++++++++++++++ autompc/ocp/barrier_cost_factory.py | 122 +++++++++++++++++++++++++ 2 files changed, 259 insertions(+) create mode 100644 autompc/costs/barrier_cost.py create mode 100644 autompc/ocp/barrier_cost_factory.py diff --git a/autompc/costs/barrier_cost.py b/autompc/costs/barrier_cost.py new file mode 100644 index 0000000..57a9016 --- /dev/null +++ b/autompc/costs/barrier_cost.py @@ -0,0 +1,137 @@ +# Created by Teodor Tchalakov, (ttcha2@illinois.edu) + +import numpy as np +import numpy.linalg as la + +from .cost import Cost + +class LogBarrierCost(Cost): + def __init__(self, system, boundedStates): + """ + Create barrier cost that approximates an inequality constraint. + Function does not exist outside the limit. + where : - b * ln ( a - x ) for upper limit + - b * ln ( a + x ) for lower limit + Parameters + ---------- + system : System + Robot system object. + boundedState : dict + Dictionary of { "observation/control name" : (limit, scale, upper)} + observation/control (x) : String + Observation/control name for which limit is specified. + limit (a) : double + limit value a that barrier is placed at. + scale (b) : double + Positive scalar to magnify the cost function. + scale: (0, inf) + upper : boolean + True if the limit is an upper limit. + False if the limit is a lower limit. + """ + super().__init__(system) + self.obsConfiguration = [] + self.ctrlsConfiguration = [] + + for variable in boundedStates.keys(): + config = boundedStates[variable] + # Check that scale is positive + if(config[1] < 0): + raise ValueError(f"{variable}'s log barrier must be positive, was {config[1]}") + elif(variable in system.observations): + self.obsConfiguration.append([variable, config]) + elif(variable in system.controls): + self.ctrlsConfiguration.append([variable, config]) + else: + raise ValueError(f"Variable {variable} is not in the given system") + + # Configs + self._is_quad = False + self._is_convex = True + self._is_diff = True + self._is_twice_diff = True + self._has_goal = False + + #Cost Function: + # - b * ln ( a - x ) upper limit + # - b * ln ( a + x ) lower limit + def eval_obs_cost(self, obs): + sum = 0 + for boundedObs in self.obsConfiguration: + variable, config = boundedObs + index = self.system.observations.index(variable) + limit, scale, upper = config + self._direction = -1 + if(upper): + direction = 1 + sum = sum + -scale * np.log(limit - (direction * obs[index])) + return sum + + #Jacobian: + # b / (a - x) upper limit + # -b / (a - x) lower limit + def eval_obs_cost_diff(self, obs): + jacobian = np.zeros(self.system.obs_dim) + for boundedObs in self.obsConfiguration: + variable, config = boundedObs + index = self.system.observations.index(variable) + limit, scale, upper = config + self._direction = -1 + if(upper): + direction = 1 + jacobian[boundedObs[0]] = direction * scale / (limit - obs[index]) + return self.eval_obs_cost(obs), jacobian + + #Hessian: + # b / (a - x)^2 upper limit + # b / (a - x)^2 lower limit + def eval_obs_cost_hess(self, obs): + hessian = np.zeros((self.system.obs_dim, self.system.obs_dim)) + for boundedObs in self.obsConfiguration: + variable, config = boundedObs + index = self.system.observations.index(variable) + limit, scale, _ = config + hessian[boundedObs[0]][boundedObs[0]] = scale / ((limit - obs[index])**2) + return *self.eval_obs_cost_diff, hessian + + def eval_ctrl_cost(self, ctrl): + sum = 0 + for boundedCtrl in self.ctrlsConfiguration: + variable, config = boundedCtrl + index = self.system.controls.index(variable) + limit, scale, upper = config + self._direction = -1 + if(upper): + direction = 1 + sum = sum + -scale * np.log(limit - (direction * ctrl[index])) + return sum + + def eval_ctrl_cost_diff(self, ctrl): + jacobian = np.zeros(self.system.ctrl_dim) + for boundedCtrl in self.ctrlsConfiguration: + variable, config = boundedCtrl + index = self.system.controls.index(variable) + limit, scale, upper = config + self._direction = -1 + if(upper): + direction = 1 + jacobian[boundedCtrl[0]] = direction * scale / (limit - ctrl[index]) + return self.eval_ctrl_cost(), jacobian + + def eval_ctrl_cost_hess(self, ctrl): + hessian = np.zeros((self.system.ctrl_dim, self.system.ctrl_dim)) + for boundedCtrl in self.ctrlsConfiguration: + variable, config = boundedCtrl + index = self.system.controls.index(variable) + limit, scale, _ = config + hessian[boundedCtrl[0]][boundedCtrl[0]] = scale / ((limit - ctrl[index])**2) + return *self.eval_ctrl_cost_diff(), hessian + + def eval_term_obs_cost(self, obs): + return 0 + + def eval_term_obs_cost_diff(self, obs): + return 0 + + def eval_term_obs_cost_hess(self, obs): + return 0 \ No newline at end of file diff --git a/autompc/ocp/barrier_cost_factory.py b/autompc/ocp/barrier_cost_factory.py new file mode 100644 index 0000000..10d66e2 --- /dev/null +++ b/autompc/ocp/barrier_cost_factory.py @@ -0,0 +1,122 @@ +# Created by Teodor Tchalakov (tcha2@illinois.edu), 2022-04-18 + +# Standard library includes +from collections import defaultdict + +# Internal library includes +from .cost_factory import CostFactory +from . import LogBarrierCost + +# External library includes +import numpy as np +import ConfigSpace as CS +import ConfigSpace.hyperparameters as CSH +import ConfigSpace.conditions as CSC + +def construct_default_bounds(): + return (1e-3, 1e4, 1.0, False) + +class LogBarrierCostFactory(CostFactory): + def __init__(self, system): + super().__init__(system) + + self._scale_bounds = defaultdict(construct_default_bounds) # Key: obsname, Value: (lower, upper, default, log_scale) + self._limits = [] # Key: obs/ctrlname, Value: (limit, upper) + self._scale_fixed = [] # Key: obs/ctrlname, Value: limit + + """ + boundedState : String + Name of observation or control in the system. + + limit : double + limit value that barrier is placed at. + + upper : boolean + True if the limit is an upper limit. + False if the limit is a lower limit. + """ + def set_limit(self, boundedState, limit, upper): + if(boundedState in self.system.observations or boundedState in self.system.controls): + self._limits[boundedState] = (limit, upper) + else: + raise ValueError(str(boundedState) + " is not in system") + + """ + boundedState : String + Name of observation or control in the system. + + limit : double + limit value that barrier is placed at. + + lower_bound : double + lower bound of configuration space + + upper_bound : double + upper bound of configuration space + + default : double + default value of configuration space + """ + def set_bounds(self, boundedState, lower_bound, upper_bound, default, log=False): + if(boundedState in self.system.observations or boundedState in self.system.controls): + if(boundedState in self._limits): + self._scale_bounds[boundedState] = (lower_bound, upper_bound, default, log) + else: + raise ValueError(str(boundedState) + " does not have a configured limit use set_limit") + else: + raise ValueError(str(boundedState) + " is not in system") + + def set_fixed_scale(self, boundedState, scale): + if(boundedState in self.system.observations or boundedState in self.system.controls): + if(boundedState in self._limits): + self._scale_fixed[boundedState] = scale + else: + raise ValueError(str(boundedState) + " does not have a configured limit use set_limit") + else: + raise ValueError(str(boundedState) + " is not in system") + + + def _get_hyperparameters(self, label, bounds_dict, fixed_dict): + hyperparameters = [] + for name in (self.system.controls + self.system.observations): + if name in fixed_dict or name not in self._limits: + continue + limit, upper = self._limits[name] + upper_string = "Upper" + if(not upper): + upper_string = "Lower" + lower_scale, upper_scale, default, log = bounds_dict[name] + hyper = CSH.UniformFloatHyperparameter("{}_{}_{}".format(name, upper_string, label), + lower=lower_scale, upper=upper_scale, default_value=default, log=log) + hyperparameters.append(hyper) + return hyperparameters + + def get_configuration_space(self): + cs = CS.ConfigurationSpace() + hypers = self._get_hyperparameters("LogBarrier", self._scale_bounds, self._scale_fixed) + cs.add_hyperparameters(hypers) + return cs + + def _get_boundedState(self, cfg, label, fixed_dict): + boundedStates = dict() + for name in (self.system.controls + self.system.observations): + if name in fixed_dict: + limit, upper = self._limits[name] + scale = self._scale_fixed[name] + boundedStates[name] = (limit, scale, upper) + elif name in self._limits: + limit, upper = self._limits[name] + upper_string = "Upper" + if(not upper): + upper_string = "Lower" + hyper_name = f"{name}_{upper_string}_{label}" + scale = cfg(hyper_name) + boundedStates[name] = (limit, scale, upper) + return boundedStates + + def __call__(self, cfg, task, trajs): + boundedStates = self._get_boundedState(cfg, "LogBarrier", self._scale_fixed) + + return LogBarrierCost(self.system, boundedStates) + + \ No newline at end of file From 8994cc41f044053db946291598ed53c7e1e5b643 Mon Sep 17 00:00:00 2001 From: Dohun Date: Tue, 19 Jul 2022 18:46:57 -0500 Subject: [PATCH 03/41] fix_hyperparameter error fixed ConfigSpace doesn't allow lower bound and upper bound to be equal. Fixed hyperparameters are replaced as CSH.Constant object --- autompc/tunable.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/autompc/tunable.py b/autompc/tunable.py index 273ebab..f9acc61 100644 --- a/autompc/tunable.py +++ b/autompc/tunable.py @@ -82,8 +82,7 @@ def fix_hyperparameters(self,**kwargs) -> None: if isinstance(hyperparam,CSH.CategoricalHyperparameter): hyperparam.choices = (value,) else: - hyperparam.lower=value - hyperparam.upper=value + self._configuration_space._hyperparameters[key] = CSH.Constant(key, value) hyperparam.default_value = value config = self.get_config() From 1b932a6385b2594eff564a46071d927b8fc35d18 Mon Sep 17 00:00:00 2001 From: Dohun Date: Wed, 20 Jul 2022 11:02:23 -0500 Subject: [PATCH 04/41] minor bug fix in bounds transformer and controller --- autompc/controller.py | 1 + autompc/costs/zero_cost.py | 2 +- autompc/ocp/bounds_transformer.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/autompc/controller.py b/autompc/controller.py index 5ff17b6..3f430ed 100644 --- a/autompc/controller.py +++ b/autompc/controller.py @@ -481,6 +481,7 @@ def reset_history(self) -> None: from influencing current model predictions. """ self.model_state = None + self.last_control = None def reset_optimizer(self) -> None: """ diff --git a/autompc/costs/zero_cost.py b/autompc/costs/zero_cost.py index f0b94d5..99687c8 100644 --- a/autompc/costs/zero_cost.py +++ b/autompc/costs/zero_cost.py @@ -29,5 +29,5 @@ def terminal_diff(self, obs): return 0.0,np.zeros(len(obs)) def terminal_hess(self, obs): - return 0.0,np.zeros(len(obs)),np.zeros(len(obs),len(obs)) + return 0.0,np.zeros(len(obs)),np.zeros((len(obs),len(obs))) diff --git a/autompc/ocp/bounds_transformer.py b/autompc/ocp/bounds_transformer.py index 9a25407..de91382 100644 --- a/autompc/ocp/bounds_transformer.py +++ b/autompc/ocp/bounds_transformer.py @@ -58,7 +58,7 @@ def ocp_requirements(self) -> dict: def __call__(self, ocp : OCP) -> OCP: res = copy.deepcopy(ocp) - res.set_cost(ZeroCost()) + res.set_cost(ZeroCost(self.system)) return res def get_prototype(self, config, ocp): From 8e6cfda202997e34f5b241642dada7e519d77569 Mon Sep 17 00:00:00 2001 From: Dohun Date: Wed, 20 Jul 2022 14:08:41 -0500 Subject: [PATCH 05/41] fix_hyperparameter error fix Can't set lower and upper bound of hyperparam to be the same value. Using constant hyperparam instead --- autompc/tunable.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/autompc/tunable.py b/autompc/tunable.py index 273ebab..f8ff508 100644 --- a/autompc/tunable.py +++ b/autompc/tunable.py @@ -82,8 +82,7 @@ def fix_hyperparameters(self,**kwargs) -> None: if isinstance(hyperparam,CSH.CategoricalHyperparameter): hyperparam.choices = (value,) else: - hyperparam.lower=value - hyperparam.upper=value + self._configuration_space._hyperparameters[key] = CSH.Constant(key,value) hyperparam.default_value = value config = self.get_config() From d616734ed6e717d28633d1f720497f05b74e4f99 Mon Sep 17 00:00:00 2001 From: Dohun Date: Wed, 20 Jul 2022 14:13:23 -0500 Subject: [PATCH 06/41] Changing cost factory to transformer in Controller_Tuning example --- examples/6_Controller_Tuning.ipynb | 109 +++++++++-------------------- 1 file changed, 32 insertions(+), 77 deletions(-) diff --git a/examples/6_Controller_Tuning.ipynb b/examples/6_Controller_Tuning.ipynb index 6f5515e..00b4258 100644 --- a/examples/6_Controller_Tuning.ipynb +++ b/examples/6_Controller_Tuning.ipynb @@ -108,7 +108,19 @@ "cell_type": "code", "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "ImportError", + "evalue": "cannot import name 'QuadCostFactory' from 'autompc.ocp' (/home/dohun/Repos/autompc/autompc/ocp/__init__.py)", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m/home/dohun/Repos/autompc/examples/6_Controller_Tuning.ipynb Cell 10'\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mautompc\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39msysid\u001b[39;00m \u001b[39mimport\u001b[39;00m MLP, SINDy\n\u001b[1;32m 2\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mautompc\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39moptim\u001b[39;00m \u001b[39mimport\u001b[39;00m IterativeLQR, MPPI\n\u001b[0;32m----> 3\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mautompc\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mocp\u001b[39;00m \u001b[39mimport\u001b[39;00m QuadCostFactory\n\u001b[1;32m 5\u001b[0m controller \u001b[39m=\u001b[39m ampc\u001b[39m.\u001b[39mController(system)\n\u001b[1;32m 6\u001b[0m controller\u001b[39m.\u001b[39madd_model(MLP(system))\n", + "\u001b[0;31mImportError\u001b[0m: cannot import name 'QuadCostFactory' from 'autompc.ocp' (/home/dohun/Repos/autompc/autompc/ocp/__init__.py)" + ] + } + ], "source": [ "from autompc.sysid import MLP, SINDy\n", "from autompc.optim import IterativeLQR, MPPI\n", @@ -119,7 +131,7 @@ "controller.add_model(SINDy(system))\n", "controller.add_optimizer(IterativeLQR(system))\n", "controller.add_optimizer(MPPI(system))\n", - "controller.add_ocp_factory(QuadCostFactory(system))" + "controller.add_ocp_transformer(QuadCostFactory(system))" ] }, { @@ -135,77 +147,15 @@ "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "Configuration space object:\n", - " Hyperparameters:\n", - " IterativeLQR:horizon, Type: UniformInteger, Range: [5, 25], Default: 20\n", - " MLP:hidden_size_1, Type: UniformInteger, Range: [16, 256], Default: 128\n", - " MLP:hidden_size_2, Type: UniformInteger, Range: [16, 256], Default: 128\n", - " MLP:hidden_size_3, Type: UniformInteger, Range: [16, 256], Default: 128\n", - " MLP:hidden_size_4, Type: UniformInteger, Range: [16, 256], Default: 128\n", - " MLP:lr, Type: UniformFloat, Range: [1e-05, 1.0], Default: 0.001, on log-scale\n", - " MLP:n_hidden_layers, Type: Categorical, Choices: {1, 2, 3, 4}, Default: 2\n", - " MLP:nonlintype, Type: Categorical, Choices: {relu, tanh, sigmoid, selu}, Default: relu\n", - " MPPI:horizon, Type: UniformInteger, Range: [5, 30], Default: 20\n", - " MPPI:lmda, Type: UniformFloat, Range: [0.1, 2.0], Default: 1.0\n", - " MPPI:num_path, Type: UniformInteger, Range: [100, 1000], Default: 200\n", - " MPPI:sigma, Type: UniformFloat, Range: [0.0001, 2.0], Default: 1.0\n", - " QuadCostFactory:dx_F, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", - " QuadCostFactory:dx_Q, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", - " QuadCostFactory:omega_F, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", - " QuadCostFactory:omega_Q, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", - " QuadCostFactory:theta_F, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", - " QuadCostFactory:theta_Q, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", - " QuadCostFactory:u_R, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", - " QuadCostFactory:x_F, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", - " QuadCostFactory:x_Q, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", - " SINDy:poly_basis, Type: Categorical, Choices: {true, false}, Default: false\n", - " SINDy:poly_cross_terms, Type: Categorical, Choices: {false}, Default: false\n", - " SINDy:poly_degree, Type: UniformInteger, Range: [2, 8], Default: 3\n", - " SINDy:threshold, Type: UniformFloat, Range: [1e-05, 10.0], Default: 0.01, on log-scale\n", - " SINDy:time_mode, Type: Categorical, Choices: {discrete, continuous}, Default: discrete\n", - " SINDy:trig_basis, Type: Categorical, Choices: {true, false}, Default: false\n", - " SINDy:trig_freq, Type: UniformInteger, Range: [1, 8], Default: 1\n", - " SINDy:trig_interaction, Type: Categorical, Choices: {false}, Default: false\n", - " model, Type: Categorical, Choices: {MLP, SINDy}, Default: MLP\n", - " ocp_factory, Type: Categorical, Choices: {Identity, QuadCostFactory}, Default: Identity\n", - " optimizer, Type: Categorical, Choices: {IterativeLQR, MPPI}, Default: IterativeLQR\n", - " Conditions:\n", - " IterativeLQR:horizon | optimizer == 'IterativeLQR'\n", - " MLP:hidden_size_1 | model == 'MLP'\n", - " MLP:hidden_size_2 | MLP:n_hidden_layers in {'2', '3', '4'}\n", - " MLP:hidden_size_3 | MLP:n_hidden_layers in {'3', '4'}\n", - " MLP:hidden_size_4 | MLP:n_hidden_layers in {'4'}\n", - " MLP:lr | model == 'MLP'\n", - " MLP:n_hidden_layers | model == 'MLP'\n", - " MLP:nonlintype | model == 'MLP'\n", - " MPPI:horizon | optimizer == 'MPPI'\n", - " MPPI:lmda | optimizer == 'MPPI'\n", - " MPPI:num_path | optimizer == 'MPPI'\n", - " MPPI:sigma | optimizer == 'MPPI'\n", - " QuadCostFactory:dx_F | ocp_factory == 'QuadCostFactory'\n", - " QuadCostFactory:dx_Q | ocp_factory == 'QuadCostFactory'\n", - " QuadCostFactory:omega_F | ocp_factory == 'QuadCostFactory'\n", - " QuadCostFactory:omega_Q | ocp_factory == 'QuadCostFactory'\n", - " QuadCostFactory:theta_F | ocp_factory == 'QuadCostFactory'\n", - " QuadCostFactory:theta_Q | ocp_factory == 'QuadCostFactory'\n", - " QuadCostFactory:u_R | ocp_factory == 'QuadCostFactory'\n", - " QuadCostFactory:x_F | ocp_factory == 'QuadCostFactory'\n", - " QuadCostFactory:x_Q | ocp_factory == 'QuadCostFactory'\n", - " SINDy:poly_basis | model == 'SINDy'\n", - " SINDy:poly_cross_terms | model == 'SINDy'\n", - " SINDy:poly_degree | SINDy:poly_basis in {'true'}\n", - " SINDy:threshold | model == 'SINDy'\n", - " SINDy:time_mode | model == 'SINDy'\n", - " SINDy:trig_basis | model == 'SINDy'\n", - " SINDy:trig_freq | SINDy:trig_basis in {'true'}\n", - " SINDy:trig_interaction | SINDy:trig_basis in {'true'}" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" + "ename": "NameError", + "evalue": "name 'controller' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m/home/dohun/Repos/autompc/examples/6_Controller_Tuning.ipynb Cell 12'\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m controller\u001b[39m.\u001b[39mget_config_space()\n", + "\u001b[0;31mNameError\u001b[0m: name 'controller' is not defined" + ] } ], "source": [ @@ -409,7 +359,7 @@ " controller,\n", " task,\n", " trajs,\n", - " n_iters=200,\n", + " n_iters=2,\n", " rng=np.random.default_rng(100),\n", " output_dir=\"6_controller_tuning_example_output/\",\n", " truedyn=benchmark.dynamics\n", @@ -432,7 +382,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de5xVdb3/8dd7hpsKCMJIKuCggooICKPmLTHM0krTTqBZQVoey+spK82fShw1szxlmXmoY1giXvJGGlpaapq3AVERNRFRQe5m4A1k5vP7Y689boa5rBlmz57Z+/18PPZj1v6u22fWhvns72V9lyICMzMzgLJCB2BmZh2Hk4KZmdVxUjAzszpOCmZmVsdJwczM6jgpmJlZHScFK1mSFks6vNBxAEj6vqTfFDoOMycFa1OSviipWtLbkpZJmi3p4C04XkjarZltFkt6LznnCknTJfVs7TmbOdeJyXneTs5Zm/P+7dYeNyIujYivtWWsWco4U9J8Se9IWiLpFkl75+N81rk5KVibkfQt4GfApcAAYDBwNXBMK47VpYW7fDYiegJjgCrg/7X0nGlExIyI6Jmc60jgjez7pKwjuhI4CzgT2A4YBtwBfLqlB2rF52KdjJOCtQlJ2wJTgdMi4raIeCciPoiIP0bEd5Jt9pP0qKS3klrEVZK65RwjJJ0m6SXgJUkPJaueTr6JT2wujohYCswGRiTHPFrSc8k5H5C0ZyPxl0k6V9LLktZIulnSdi28Bps0R0maIun6ZLky+f0mSXpN0mpJ57dy260kXSfpX5Kel/RdSUsaiWkocBpwQkT8NSLWR8S7SXK7LNnmAUlfy9lnsqSHc97X/1x+Jekn9c5zZ/KlAEk7SrpV0ipJr0g6syXX0QrLScHaygFAD+D2JrapAf4L6J9sPx74Zr1tPgfsDwyPiI8lZaOSb+I3NReEpEHAUcBTkoYBM4GzgQrgT8AfcxNRjjOScx8K7Aj8C/hlc+drhYOB3cn87hc2lqSa2fYioBLYBfgE8KUmjjEeWBIRT2xh3HWfC5lrOlGSACT1BY4AbpRUBvwReBrYKTn/2ZI+uYXnt3bipGBtpR+wOiI2NrZBRMyJiMciYmNELAb+l8wf4Vw/jIg3I+K9Fp7/DklvAQ8DD5JpwpoI3B0Rf4mID4CfAFsBBzaw/6nA+RGxJCLWA1OA/8hDc8kPIuK9iHiazB/OUa3YdgJwaUT8KyKWAD9v4hj9gGVtEHfu5/J3IIBDknX/ATwaEW8A+wIVETE1IjZExCLg18DxbRCDtQO3D1pbWQP0l9SlscSQfHP/HzJt/luT+fc3p95mrzd1Ekmz+fCP0X9GxIxk+XMRcV+9bXcEXs2+j4haSa+T+QZb387A7ZJqc8pqyPSNLG0qphZanrP8LtBUP0Rj2+7IptepqWu2BtihJQE2ou4cERGSbgROAB4Cvghcn6zeGdgxSdBZ5WQSiXUCrilYW3kUWE+mmaExvwJeAIZGRG/g+4DqbdPktL0RcWROx+6MprYF3iDzRwrIjMIBBtHwH/nXgSMjok/Oq0fSR5HWO2SSXdZHWrBvSywDBua8H9TEtvcDAyVVNbFNmrjrfy4zydSkdibTrHRrUv468Eq969grIo5q4vzWgTgpWJuIiH8DFwK/lPQ5SVtL6irpSEmXJ5v1AtYCb0vaA/hGikOvINN23ho3A5+WNF5SV+DbZBLXPxrY9hrgkuSPHJIqJLV01NQ84Pjk964i06ySDzcD50nqK2kn4PTGNoyIl8iMAJspaZykbpJ6SDpe0rk5cR+XfGa7ASc3F0BEPAWsBn4D3BsR2ZrBE8A6Sd9LOsTLJY2QtG/rf11rT04K1mYi4grgW2SGg64i863xdDLDHwHOIdPUsI5MO3OzHcdk2vavS0YPTWhhPC+S6YT9BZk/YJ8lM3R1QwObXwnMAv4saR3wGJlvwC1xAbArmU7qHwA3tHD/tKYCS4BXgPuAP5BJdo05E7iKTMf5W8DLwLFkOoQBfgpsIJOArwOaq4Fl3QAcTs7vGRE1wGeA0Ul82cSxbcpjWoHJD9kx69wkfQM4PiLqd9qbtZhrCmadjKQdJB2U3FuxO5lmsaaGApul5tFHZp1PNzLDeYeQaQ66kUy/gdkWc/ORmZnVcfORmZnV6dTNR/3794/KyspCh2Fm1qnMmTNndURUNLSuUyeFyspKqqurCx2GmVmnIunVxta5+cjMzOo4KZiZWR0nBTMzq9Op+xTMrP198MEHLFmyhPfff7/QoVgzevTowcCBA+natWvqfZwUzKxFlixZQq9evaisrCR5zo51QBHBmjVrWLJkCUOGDEm9X96ajyQNkvQ3SQuSxyGelZRvJ+kvkl5KfvZNyiXp55IWSnpG0ph8xWZmrff+++/Tr18/J4QOThL9+vVrcY0un30KG4FvR8Rw4KPAaZKGA+cC90fEUDJzvWen7z0SGJq8TiEz976ZdUBOCJ1Daz6nvDUfRcQykscARsQ6Sc+TeeLVMcC4ZLPrgAeA7yXlv4vMvBuPSeojaYfkOG1q5dJXWDT7F5uVv9e1D88MOJaass0f4fuxYRVUVbboOe5mZp1Ou/QpSKoE9gEeBwbk/KFfTuZxh5BJGLmPFVySlG2SFCSdQqYmweDBg1sVz1srXmO/16/drLxMwYsv/ZMf1ZywSXkEVL/6L274+kdbdT4zaxtr1qxh/PjxACxfvpzy8nIqKjI35j7xxBN067b5F7qWGjduHMuWLaN79+5s2LCBww8/nIsvvpg+ffps8bHrmz17NhdccAHvvvsu3bt35+Mf/zhXXHFFi44xb9483njjDY46qm0ebpf3pCCpJ5lH9Z0dEWtzqzPJs15bNCNfREwDpgFUVVW1aja/YWMOhTFvbb5i1pmcOvd3nDppEmw/vK74tBvmsqam0efRm1k76devH/PmzQNgypQp9OzZk3POOadu/caNG+nSZcv/rM2YMYOqqio2bNjAeeedxzHHHMODDz64xcfNNX/+fE4//XTuvvtu9thjD2pqapg2bVqLjzNv3jyqq6s7R1JIHoF4KzAjIm5Lildkm4Uk7QCsTMqXsumzZgfStg9Mb94nL4FFD8CMTZ+i+Evg4e6HAge3azhm1rzJkyfTo0cPnnrqKQ466CB69+69SbIYMWIEd911F5WVlVx//fX8/Oc/Z8OGDey///5cffXVlJeXN3rsbt26cfnll7Pbbrvx9NNPc+utt7Lddttx9tlnA3D++eez/fbbM2rUKKZMmUL//v2ZP38+Y8eO5frrr2+yTf/yyy/n/PPPZ4899gCgvLycb3wj84TaxYsXc9JJJ7F69WoqKir47W9/y+DBg7nlllv4wQ9+QHl5Odtuuy333XcfF154Ie+99x4PP/ww5513HhMnTtyi65m3pJA8JP3/gOcj4n9yVs0CJgGXJT/vzCk/XdKNZB6D+O989Cc0qXsv+OpsWHjfJsXLZl9O39o17RqKWWfwgz8+x4I31rbpMYfv2JuLPrtXi/ZZsmQJ//jHPygvL2fKlCkNbvP8889z00038cgjj9C1a1e++c1vMmPGDL7yla80eezy8nJGjRrFCy+8wEknncRxxx3H2WefTW1tLTfeeCNPPPEEzz77LE899RTPPfccO+64IwcddBCPPPIIBx/c+BfJ+fPn8+1vf7vBdWeccQaTJk1i0qRJXHvttZx55pnccccdTJ06lXvvvZeddtqJt956i27dujF16lSqq6u56qqrUl+vpuSzpnAQ8GXgWUnzkrLvk0kGN0s6GXgVyD5390/AUcBC4F3gq3mMrXHb7gRjJ21S9Oafr0U1HxQkHDNr3he+8IUmv/ED3H///cyZM4d9990XgPfee4/tt98+1fGzz52prKykX79+PPXUU6xYsYJ99tmHfv36AbDffvsxcOBAAEaPHs3ixYubTApNefTRR7nttkzjype//GW++93vAnDQQQcxefJkJkyYwHHHHdeqYzcnn6OPHgYaqzuNb2D7AE7LVzxbIlRGGbWFDsOsw2npN/p82WabbeqWu3TpQm3th/9fs+P0I4JJkybxwx/+sEXHrqmp4dlnn2XPPfcE4Gtf+xrTp09n+fLlnHTSSXXbde/evW65vLycjRub7ofca6+9mDNnDqNGjUodyzXXXMPjjz/O3XffzdixY5kzZ06Lfpc0PPdRCkEZclIw6xQqKyuZO3cuAHPnzuWVV14BYPz48fzhD39g5cpMN+abb77Jq682OoM0kJnS47zzzmPQoEGMHDkSgGOPPZZ77rmHJ598kk9+8pOtjvM73/kOl156Kf/85z8BqK2t5ZprrgHgwAMP5MYbbwQynd6HHHIIAC+//DL7778/U6dOpaKigtdff51evXqxbt26VsdRn5NCGipD4aRg1hl8/vOf580332SvvfbiqquuYtiwYQAMHz6ciy++mCOOOIKRI0fyiU98gmXLGu62PPHEExk5ciQjRozgnXfe4c4776xb161bNw477DAmTJjQbJMVwIUXXsisWbM2Kx85ciQ/+9nPOOGEE9hzzz0ZMWIEixYtAuAXv/gFv/3tbxk5ciS///3vufLKK4FMItl7770ZMWIEBx54IKNGjeKwww5jwYIFjB49mptuuqnF16u+Tv2M5qqqqmiPh+w8c/kR9Fi/hmEXtH1Vzayzef755+uaUkpRbW0tY8aM4ZZbbmHo0KGFDqdZDX1ekuZERFVD27umkEJQRplrCmYlb8GCBey2226MHz++UySE1vAsqSmE3KdgZpkmqGwTT7FyTSENlSE6bzObmVlaTgopZJqPagodhplZ3jkppBCuKZhZiXBSSMM3r5lZiXBSSMF3NJt1HJdccgl77bUXI0eOZPTo0Tz++ON5Oc8HH3zAueeey9ChQxkzZgwHHHAAs2fPbvFxpk+fzhtvvJGHCPPDo49SCJWhTnw/h1mxePTRR7nrrruYO3cu3bt3Z/Xq1WzYsCH1/vWn1m5qqu0LLriAZcuWMX/+fLp3786KFStaNX329OnTGTFiBDvuuGOL9y0EJ4U0PCTVrENYtmwZ/fv3r5tnqH///nXrKisrqa6upn///lRXV3POOefwwAMPMGXKFF5++WUWLVrE4MGD2X333Td5P3PmzM3O8+677/LrX/+aV155pe5cAwYMYMKEzPydM2fO5NJLLyUi+PSnP82PfvQjampqOPnkk6murkYSJ510EoMGDaK6upoTTzyRrbbaikcffZStttqqHa5U6zkppBCUUeaOZrPNzT4Xlj/btsf8yN5w5GUNrjriiCOYOnUqw4YN4/DDD2fixIkceuihzR5ywYIFPPzww2y11VZMmTJlk/cNWbhwIYMHD6Z3796brXvjjTf43ve+x5w5c+jbty9HHHEEd9xxB4MGDWLp0qXMnz8fgLfeeos+ffpw1VVX8ZOf/ISqqgZvIO5w3KeQhvsUzDqEnj17MmfOHKZNm0ZFRQUTJ05k+vTpze539NFHb5IA6r9viSeffJJx48ZRUVFBly5dOPHEE3nooYfYZZddWLRoEWeccQb33HNPgwmlM3BNIQ03H5k1rJFv9PlUXl7OuHHjGDduHHvvvTfXXXcdkydP3mTK7Ox02Vm5U2s39L6+3Xbbjddee421a9em/uPet29fnn76ae69916uueYabr75Zq69dvNnwXd0rimkITcfmXUEL774Ii+99FLd+3nz5rHzzjsDmT6F7PMFbr311i06z9Zbb83JJ5/MWWedVdeRvWrVKm655Rb2228/HnzwQVavXk1NTQ0zZ87k0EMPZfXq1dTW1vL5z3+eiy++uG767rae2jrfnBRSCJW7+cisA3j77beZNGkSw4cPZ+TIkSxYsKDu8ZsXXXQRZ511FlVVVammtM4aPXp0g+UXX3wxFRUVDB8+nBEjRvCZz3yG3r17s8MOO3DZZZdx2GGHMWrUKMaOHcsxxxzD0qVLGTduHKNHj+ZLX/pS3cN8Jk+ezKmnnsro0aN57733tvga5Junzk7h8au/zh4r7mbbH3SescZm+VLqU2d3Np46Ox8k1xTMrCQ4KaTh5iMzKxFOCmm4o9lsE5252bmUtOZzyltSkHStpJWS5ueU3SRpXvJaLGleUl4p6b2cddfkK65WcVIwq9OjRw/WrFnjxNDBRQRr1qyhR48eLdovn/cpTAeuAn6XLYiIidllSVcA/87Z/uWIaHgYQKH55jWzOgMHDmTJkiWsWrWq0KFYM3r06MHAgQNbtE/ekkJEPCSpsqF1kgRMAD6er/O3Jc+Savahrl27MmTIkEKHYXlSqD6FQ4AVEfFSTtkQSU9JelDSIY3tKOkUSdWSqtvrm4pUTrmC2lpXl82suBUqKZwA5E5NuAwYHBH7AN8CbpDU4L3lETEtIqoioqqioqIdQgUkAGpr/UhOMytu7Z4UJHUBjgNuypZFxPqIWJMszwFeBoa1d2yNibLM3ZHZeVXMzIpVIWoKhwMvRMSSbIGkCknlyfIuwFBgUQFia5gyl8k1BTMrdvkckjoTeBTYXdISSScnq45n06YjgI8BzyRDVP8AnBoRb+YrtpZSNinUbCxwJGZm+ZXP0UcnNFI+uYGyW4Etm9Ywn+TmIzMrDb6jOQ13NJtZiXBSSCNpPooa1xTMrLg5KaSRjD4K1xTMrMg5KaSQ7WiucVIwsyLnpJBGWXZIqkcfmVlxc1JIIxl9hKe5MLMi56SQRvY+hXDzkZkVNyeFFD68ec1JwcyKm5NCGnWjjzwk1cyKm5NCCko6mj0k1cyKnZNCGnUT4rmmYGbFzUkhjewdzR6SamZFzkkhBfl5CmZWIpwUUpA7ms2sRDgppKBkllR3NJtZsXNSSKMs89gJJwUzK3ZOCilkb16LcPORmRU3J4UUsvcp+I5mMyt2Tgpp+HkKZlYinBRScPORmZUKJ4UUss1HuKZgZkUub0lB0rWSVkqan1M2RdJSSfOS11E5686TtFDSi5I+ma+4WsM3r5lZqchnTWE68KkGyn8aEaOT158AJA0Hjgf2Sva5Wso+2abwsqH45jUzK3Z5SwoR8RDwZsrNjwFujIj1EfEKsBDYL1+xtdSHzUee+8jMilsh+hROl/RM0rzUNynbCXg9Z5slSdlmJJ0iqVpS9apVq/Ida+ac2dFH4cdxmllxa++k8CtgV2A0sAy4oqUHiIhpEVEVEVUVFRVtHV+D/DwFMysV7ZoUImJFRNREZmznr/mwiWgpMChn04FJWcfgZzSbWYlo16QgaYect8cC2ZFJs4DjJXWXNAQYCjzRnrE1pSxpPsIdzWZW5Lrk68CSZgLjgP6SlgAXAeMkjQYCWAz8J0BEPCfpZmABsBE4LaLjfC331NlmVirylhQi4oQGiv+vie0vAS7JVzxbom70UcfJU2ZmeeE7mlP48D4FJwUzK25OCimoPDv3kYekmllxc1JIoazuPgXXFMysuDkppJCdJdWjj8ys2DkppJAdfeSOZjMrdk4KKZR5SKqZlQgnhRTKyp0UzKw0OCmk4PsUzKxUOCmk8GGfgmsKZlbcnBRSKJPnPjKz0uCkkMKHN6+5+cjMipuTQgqeJdXMSoWTQgpl7lMwsxLhpJCCyj3NhZmVBieFFDzNhZmVCieFFMrq7lNwUjCz4uakkEJ5ufsUzKw0OCmkUFaePKDOScHMipyTQgqe5sLMSoWTQgq+T8HMSoWTQgrldfcp+HGcZlbc8pYUJF0raaWk+TllP5b0gqRnJN0uqU9SXinpPUnzktc1+YqrNSSoCbn5yMyKXj5rCtOBT9Ur+wswIiJGAv8EzstZ93JEjE5ep+YxrhaTRC1l7mg2s6KXt6QQEQ8Bb9Yr+3NEbEzePgYMzNf525qTgpmVgkL2KZwEzM55P0TSU5IelHRIYztJOkVStaTqVatW5T/KRC1uPjKz4pcqKUj6fZqytCSdD2wEZiRFy4DBEbEP8C3gBkm9G9o3IqZFRFVEVFVUVLQ2hBbLJAXXFMysuKWtKeyV+0ZSOTC2NSeUNBn4DHBiRGY4T0Ssj4g1yfIc4GVgWGuOny+1lCGPPjKzItdkUpB0nqR1wEhJa5PXOmAlcGdLTybpU8B3gaMj4t2c8ook0SBpF2AosKilx88n9ymYWSloMilExA8johfw44jonbx6RUS/iDivqX0lzQQeBXaXtETSycBVQC/gL/WGnn4MeEbSPOAPwKkR8WaDBy6QcJ+CmZWALim3u0vSNhHxjqQvAWOAKyPi1cZ2iIgTGij+v0a2vRW4NWUsBVGrMuSagpkVubR9Cr8C3pU0Cvg2mTb/3+Utqg7IHc1mVgrSJoWNSafwMcBVEfFLMs1AJSNwTcHMil/a5qN1ks4DvgwcosyjyLrmL6yOpxYBTgpmVtzS1hQmAuuBkyJiOZk7kX+ct6g6II8+MrNSkCopJIlgBrCtpM8A70dESfUpuPnIzEpB2juaJwBPAF8AJgCPS/qPfAbW0YTc0WxmxS9tn8L5wL4RsRIyN5sB95G5p6Ak1LqmYGYlIG2fQlk2ISTWtGDfouCkYGalIG1N4R5J9wIzk/cTgT/lJ6SOKXyfgpmVgCaTgqTdgAER8R1JxwEHJ6se5cMZTktCrcqRh6SaWZFrrqbwM5Kno0XEbcBtAJL2TtZ9Nq/RdSCB3HxkZkWvuX6BARHxbP3CpKwyLxF1UE4KZlYKmksKfZpYt1VbBtLR1aoc39FsZsWuuaRQLenr9QslfQ2Yk5+QOibXFMysFDTXp3A2cLukE/kwCVQB3YBj8xlYRxOeOtvMSkCTSSEiVgAHSjoMGJEU3x0Rf817ZB1MUIbw4zjNrLiluk8hIv4G/C3PsXRovnnNzEpBSd2VvEUk36dgZkXPSSGlWspdUzCzouekkFLIo4/MrPg5KaSU6Wh2UjCz4pbXpCDpWkkrJc3PKdtO0l8kvZT87JuUS9LPJS2U9IykMfmMrcUkjz4ys6KX75rCdOBT9crOBe6PiKHA/cl7gCOBocnrFOBXeY6tRWrlPgUzK355TQoR8RDwZr3iY4DrkuXrgM/llP8uMh4D+kjaIZ/xtYxHH5lZ8StEn8KAiFiWLC8HBiTLOwGv52y3JCnbhKRTJFVLql61alV+I82RuaPZzUdmVtwK2tEcEQEta6iPiGkRURURVRUVFXmKrIHzUkaZawpmVuQKkRRWZJuFkp/Zx3wuBQblbDcwKesQQh59ZGbFrxBJYRYwKVmeBNyZU/6VZBTSR4F/5zQzFZwnxDOzUpD2Gc2tImkmMA7oL2kJcBFwGXCzpJOBV4EJyeZ/Ao4CFgLvAl/NZ2wtFSqjzENSzazI5TUpRMQJjawa38C2AZyWz3i2TBmiptBBmJnlle9oTsujj8ysBDgppFTr5iMzKwFOCqmVUebmIzMrck4KaclPXjOz4uekkJLvaDazUuCkkFKojHI3H5lZkXNSSMtTZ5tZCXBSSClU7qRgZkXPSSEtlVHuuY/MrMg5KaTlCfHMrAQ4KaTlm9fMrAQ4KaSUmRDPNQUzK25OCmmpjDJPnW1mRc5JITXf0Wxmxc9JIa2ycrrINQUzK25OCimFlCy4tmBmxctJIS0ll8r9CmZWxJwU0lJ55met5z8ys+LlpJCWawpmVgKcFNJKkkKEawpmVrycFNJKmo9qa11TMLPi1aW9Tyhpd+CmnKJdgAuBPsDXgVVJ+fcj4k/tHF7jktFHtTUbKS9wKGZm+dLuSSEiXgRGA0gqB5YCtwNfBX4aET9p75hSKXNNwcyKX6Gbj8YDL0fEqwWOo3nZ5qMaJwUzK16FTgrHAzNz3p8u6RlJ10rq29AOkk6RVC2petWqVQ1tkhfKNh/Vbmy3c5qZtbeCJQVJ3YCjgVuSol8Bu5JpWloGXNHQfhExLSKqIqKqoqKiXWIFPhx95PsUzKyIFbKmcCQwNyJWAETEioioiYha4NfAfgWMbXNJUnCfgpkVs0ImhRPIaTqStEPOumOB+e0eUVOSjmbXFMysmLX76CMASdsAnwD+M6f4ckmjgQAW11tXcCpLago1TgpmVrwKkhQi4h2gX72yLxciltSyzUee5sLMilihRx91HtkJ8VxTMLMi5qSQUl3zkWsKZlbEnBRSkoekmlkJcFJIKZLRRzVuPjKzIuakkFK2poDvUzCzIuakkJLqRh+5pmBmxctJISWV++Y1Myt+TgppyUnBzIqfk0JK2VlSw30KZlbEnBTSKsvc/F3rmoKZFTEnhZRcUzCzUuCkkJI8S6qZlQAnhZScFMysFDgppJSd+yg895GZFTEnhbTq5j5yUjCz4uWkkFJZtvnIdzSbWRFzUkiprk+hxjUFMyteTgop1U2d7ZqCmRUxJ4W0ytynYGbFz0khJWUfx+magpkVMSeFlMrKXVMws+LXpVAnlrQYWAfUABsjokrSdsBNQCWwGJgQEf8qVIyb8CypZlYCCl1TOCwiRkdEVfL+XOD+iBgK3J+87xCyQ1LxzWtmVsQKnRTqOwa4Llm+DvhcAWPZhNzRbGYloGDNR0AAf5YUwP9GxDRgQEQsS9YvBwbU30nSKcApAIMHD26vWD+8T8E1BTNrA88u+Tc3PPEqEa3bf+TAPnxx/7b/G1jIpHBwRCyVtD3wF0kv5K6MiEgSBvXKpwHTAKqqqlp5OVtOSUcz7lMwszZw/WOvcsuc16no1b1V+5eXqY0jyihYUoiIpcnPlZJuB/YDVkjaISKWSdoBWFmo+OorSzqat1r3KiyZU+BoiogEH9kbyrsWOhKzdrVi3fsM37E3d51xSKFD2URBkoKkbYCyiFiXLB8BTAVmAZOAy5KfdxYivgZ12waA3V64Gl64usDBFJlDvg3jLyx0FGbtauXa9Xxk2x6FDmMzhaopDABuT55m1gW4ISLukfQkcLOkk4FXgQkFim8zsXV/jl7/35w/roL9h/QrdDjF44n/hSd+Awf/F3TvVehozNrNynXrGTlw20KHsZmCJIWIWASMaqB8DTC+/SNqniSeiV1ZPmA0DNup0OEUj637wW8+DnN/BwecVuhozNrFxppa1ryznu1b2Z+QT4XsaO5Usp06ta0dKmANGzgWdj4IHv4prFhQ6Gjyq0t3OPhs6NN+o+asYf96ZwNX/OVF1n/QstGEPbqWc84Ru7Pt1lvWB7bmnQ1EQEVvNx91WizkYZsAAApjSURBVNmOft+m0PbWfvQ7lM06Db1wX6FDyaseG96EN+ZRfvKfodz/9QrpznlLuf6x19hh2x6kHcNTE8GKtesZs3Mfjt1n4Badf+Xa9QCuKXRmZZn+DxaveYe5r3WMmTeKwcq173PRrBpWrP1xoUPJu8+W/YNfvHEV/HUq7PHZQodTuiQee/F9hvTfhr+dMy71bus31rDnBffwyqp3tjiEleveB5wUOrWtu2WGpP7irwv5xV8XFjia4rJL/2247Ztj2Xm7rQsdSl6d8vu+PPjmPA595Ep45MpCh1PSzo0dmL37f8PGDan36Q5U9u3Oy6u3PCmsWpfUFNx81Hn169mdu844mNVvry90KEWlvEyM3bkvW3cr/n+KR4/akZNnncIDXziNgb27FTqckvXSq6/R8+//zTf++TW4+Gst2ne2enDJG98DxmxRDCuTpFDR0zWFTm3ETh1v+Jh1Hkfu/RF+8MfnuHnNrnxr7O6FDqdk3bHoBW784DIe+dRyevBBi/Zd+4/rOWPdz6h9exJlPVs/NH3luvfpu3VXunXpaNPPOSmYtZvte/Xgo7v044YnXmfpW+8XOpyS9cjC1ewyeBA9Pjaxxfs+/v5IjnjkeDb+/li6fWSvVsfwiUVrWbb1p1u9fz45KZi1o5MOGsLUuxbw2KI1hQ6lZHUpFyfs17phwdvtOoaLHpzMRe/8GRY/3OoY9lu3kv11HzzwGmzVt3UH2W5XGHp4q2NojJOCWTs6fPgADh++2eS/1knsWtGTmTXj2fOgM/nKAZWtPs7xP7yFH5dfzbAHftj6YPY6zknBzKyQtu/VnW26lbNoC4alRgTPv92T2w66mnPHbcEXhDxNIumkYGaWkiR2qejJy6vebvUx3nr3AzbU1GaGo269XRtG1zacFMzMWmCXim2YPX85n/ifB1u1/wc1mWkRtu/d8YajgpOCmVmLfOmjO7OxJghaPw/amJ37cuCu/dswqrbjpGBm1gL7Vm7HvpUdr9mnrXS8OyfMzKxgnBTMzKyOk4KZmdVxUjAzszpOCmZmVsdJwczM6jgpmJlZHScFMzOro4jW35VXaJJWAa9uwSH6A6vbKJx86OjxgWNsK46xbTjGdHaOiIqGVnTqpLClJFVHRFWh42hMR48PHGNbcYxtwzFuOTcfmZlZHScFMzOrU+pJYVqhA2hGR48PHGNbcYxtwzFuoZLuUzAzs02Vek3BzMxyOCmYmVmdkkwKkj4l6UVJCyWdW+h4ACQNkvQ3SQskPSfprKR8iqSlkuYlr6MKHOdiSc8msVQnZdtJ+oukl5KffQsY3+4512qepLWSzi70dZR0raSVkubnlDV43ZTx8+Tf5zOSxhQwxh9LeiGJ43ZJfZLySknv5VzPawoYY6OfraTzkuv4oqRPFii+m3JiWyxpXlJekGvYrIgoqRdQDrwM7AJ0A54GhneAuHYAxiTLvYB/AsOBKcA5hY4vJ87FQP96ZZcD5ybL5wI/KnScOZ/1cmDnQl9H4GPAGGB+c9cNOAqYDQj4KPB4AWM8AuiSLP8oJ8bK3O0KfB0b/GyT/z9PA92BIcn/+/L2jq/e+iuACwt5DZt7lWJNYT9gYUQsiogNwI3AMQWOiYhYFhFzk+V1wPPAToWNKrVjgOuS5euAzxUwllzjgZcjYkvuem8TEfEQ8Ga94sau2zHA7yLjMaCPpB0KEWNE/DkiNiZvHwMG5juOpjRyHRtzDHBjRKyPiFeAhWT+/+dNU/FJEjABmJnPGLZUKSaFnYDXc94voYP98ZVUCewDPJ4UnZ5U368tZNNMIoA/S5oj6ZSkbEBELEuWlwMDChPaZo5n0/+AHek6QuPXraP+Gz2JTA0ma4ikpyQ9KOmQQgWVaOiz7WjX8RBgRUS8lFPWka4hUJpJoUOT1BO4FTg7ItYCvwJ2BUYDy8hUPwvp4IgYAxwJnCbpY7krI1MvLvg4Z0ndgKOBW5KijnYdN9FRrltjJJ0PbARmJEXLgMERsQ/wLeAGSb0LFF6H/mxznMCmX1I60jWsU4pJYSkwKOf9wKSs4CR1JZMQZkTEbQARsSIiaiKiFvg1ea7+NiciliY/VwK3J/GsyDZvJD9XFi7COkcCcyNiBXS865ho7Lp1qH+jkiYDnwFOTJIXSZPMmmR5Dpn2+mGFiK+Jz7bDXEdJXYDjgJuyZR3pGuYqxaTwJDBU0pDk2+TxwKwCx5Rtb/w/4PmI+J+c8ty25GOB+fX3bS+StpHUK7tMphNyPpnrNynZbBJwZ2Ei3MQm38o60nXM0dh1mwV8JRmF9FHg3znNTO1K0qeA7wJHR8S7OeUVksqT5V2AocCiAsXY2Gc7CzheUndJQ8jE+ER7x5c4HHghIpZkCzrSNdxEoXu6C/EiM7rjn2Qy8/mFjieJ6WAyzQfPAPOS11HA74Fnk/JZwA4FjHEXMqM5ngaey147oB9wP/AScB+wXYGv5TbAGmDbnLKCXkcyCWoZ8AGZtu2TG7tuZEYd/TL59/ksUFXAGBeSaZfP/pu8Jtn288m/gXnAXOCzBYyx0c8WOD+5ji8CRxYivqR8OnBqvW0Lcg2be3maCzMzq1OKzUdmZtYIJwUzM6vjpGBmZnWcFMzMrI6TgpmZ1XFSsE5NUr+cWSaX15sts1sLjvMbScPbKKa3k5+Vkr7YFsfMOfb3673/R1se38xDUq1oSJoCvB0RPylwHG9HRE9J48jM3vmZFuzbJT6cgK7RY7dFnGYNcU3Bio6k6ZL+I+d99pv7OEkPSPpD8oyAGcmd5CTlVdntJV0i6WlJj0kakJTvmrx/VtLF2eM24TLgkKTW8l+SypV5PsGTyeRt/5kT198lzQIWJGV3JJMOPpedeFDSZcBWyfFm1PvdlBx7fhLfxOZ+Z7OGOClYqdkHOJvMXPu7AAc1sM02wGMRMQp4CPh6Un4lcGVE7E3mbtXmnAv8PSJGR8RPydx9+++I2BfYF/h6Mv0CZObgPysisnPfnBQRY4Eq4ExJ/SLiXOC95Hgn1jvXcWQmhBtFZkqFH+dM/5DmdzYDnBSs9DwREUsiM3naPDIPOqlvA3BXsjwnZ5sD+HDW1Rtace4jyMxpNI/MtOj9yMx3k43rlZxtz5T0NJlnGAzK2a4xBwMzIzMx3ArgQTKJJ3vs5n5nMwC6FDoAszzYSPKFR1IZmSfsZa3PWa6h4f8DH8SHnW2NbdMaAs6IiHs3Kcz0PbxT7/3hwAER8a6kB4AeW3DeNL+zGeCaghWnxcDYZPlooGsbHfcxMpOYQWZ23easI/No1ax7gW8kU6QjaVgy22x92wL/ShLCHmQeyZn1QXb/ev4OTEz6LSrIPBayUDOCWifmpGDF6NfAoUnzywHkfAvfQmcD35L0DLAb8O9mtn8GqEk6rP8L+A2ZjuS5yjzY/X9p+Fv7PUAXSc+T6ax+LGfdNOCZbEdzjtuT8z0N/BX4bkQsb9FvZ4aHpJqlJmlrMh29Iel44ISIKPjzvc3aktsWzdIbC1yVDOl8i8wzi82KimsKZmZWx30KZmZWx0nBzMzqOCmYmVkdJwUzM6vjpGBmZnX+P0e/WTw9HJovAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de5xVdb3/8dd7hpsKCMJIKuCggooICKPmLTHM0krTTqBZQVoey+spK82fShw1szxlmXmoY1giXvJGGlpaapq3AVERNRFRQe5m4A1k5vP7Y689boa5rBlmz57Z+/18PPZj1v6u22fWhvns72V9lyICMzMzgLJCB2BmZh2Hk4KZmdVxUjAzszpOCmZmVsdJwczM6jgpmJlZHScFK1mSFks6vNBxAEj6vqTfFDoOMycFa1OSviipWtLbkpZJmi3p4C04XkjarZltFkt6LznnCknTJfVs7TmbOdeJyXneTs5Zm/P+7dYeNyIujYivtWWsWco4U9J8Se9IWiLpFkl75+N81rk5KVibkfQt4GfApcAAYDBwNXBMK47VpYW7fDYiegJjgCrg/7X0nGlExIyI6Jmc60jgjez7pKwjuhI4CzgT2A4YBtwBfLqlB2rF52KdjJOCtQlJ2wJTgdMi4raIeCciPoiIP0bEd5Jt9pP0qKS3klrEVZK65RwjJJ0m6SXgJUkPJaueTr6JT2wujohYCswGRiTHPFrSc8k5H5C0ZyPxl0k6V9LLktZIulnSdi28Bps0R0maIun6ZLky+f0mSXpN0mpJ57dy260kXSfpX5Kel/RdSUsaiWkocBpwQkT8NSLWR8S7SXK7LNnmAUlfy9lnsqSHc97X/1x+Jekn9c5zZ/KlAEk7SrpV0ipJr0g6syXX0QrLScHaygFAD+D2JrapAf4L6J9sPx74Zr1tPgfsDwyPiI8lZaOSb+I3NReEpEHAUcBTkoYBM4GzgQrgT8AfcxNRjjOScx8K7Aj8C/hlc+drhYOB3cn87hc2lqSa2fYioBLYBfgE8KUmjjEeWBIRT2xh3HWfC5lrOlGSACT1BY4AbpRUBvwReBrYKTn/2ZI+uYXnt3bipGBtpR+wOiI2NrZBRMyJiMciYmNELAb+l8wf4Vw/jIg3I+K9Fp7/DklvAQ8DD5JpwpoI3B0Rf4mID4CfAFsBBzaw/6nA+RGxJCLWA1OA/8hDc8kPIuK9iHiazB/OUa3YdgJwaUT8KyKWAD9v4hj9gGVtEHfu5/J3IIBDknX/ATwaEW8A+wIVETE1IjZExCLg18DxbRCDtQO3D1pbWQP0l9SlscSQfHP/HzJt/luT+fc3p95mrzd1Ekmz+fCP0X9GxIxk+XMRcV+9bXcEXs2+j4haSa+T+QZb387A7ZJqc8pqyPSNLG0qphZanrP8LtBUP0Rj2+7IptepqWu2BtihJQE2ou4cERGSbgROAB4Cvghcn6zeGdgxSdBZ5WQSiXUCrilYW3kUWE+mmaExvwJeAIZGRG/g+4DqbdPktL0RcWROx+6MprYF3iDzRwrIjMIBBtHwH/nXgSMjok/Oq0fSR5HWO2SSXdZHWrBvSywDBua8H9TEtvcDAyVVNbFNmrjrfy4zydSkdibTrHRrUv468Eq969grIo5q4vzWgTgpWJuIiH8DFwK/lPQ5SVtL6irpSEmXJ5v1AtYCb0vaA/hGikOvINN23ho3A5+WNF5SV+DbZBLXPxrY9hrgkuSPHJIqJLV01NQ84Pjk964i06ySDzcD50nqK2kn4PTGNoyIl8iMAJspaZykbpJ6SDpe0rk5cR+XfGa7ASc3F0BEPAWsBn4D3BsR2ZrBE8A6Sd9LOsTLJY2QtG/rf11rT04K1mYi4grgW2SGg64i863xdDLDHwHOIdPUsI5MO3OzHcdk2vavS0YPTWhhPC+S6YT9BZk/YJ8lM3R1QwObXwnMAv4saR3wGJlvwC1xAbArmU7qHwA3tHD/tKYCS4BXgPuAP5BJdo05E7iKTMf5W8DLwLFkOoQBfgpsIJOArwOaq4Fl3QAcTs7vGRE1wGeA0Ul82cSxbcpjWoHJD9kx69wkfQM4PiLqd9qbtZhrCmadjKQdJB2U3FuxO5lmsaaGApul5tFHZp1PNzLDeYeQaQ66kUy/gdkWc/ORmZnVcfORmZnV6dTNR/3794/KyspCh2Fm1qnMmTNndURUNLSuUyeFyspKqqurCx2GmVmnIunVxta5+cjMzOo4KZiZWR0nBTMzq9Op+xTMrP198MEHLFmyhPfff7/QoVgzevTowcCBA+natWvqfZwUzKxFlixZQq9evaisrCR5zo51QBHBmjVrWLJkCUOGDEm9X96ajyQNkvQ3SQuSxyGelZRvJ+kvkl5KfvZNyiXp55IWSnpG0ph8xWZmrff+++/Tr18/J4QOThL9+vVrcY0un30KG4FvR8Rw4KPAaZKGA+cC90fEUDJzvWen7z0SGJq8TiEz976ZdUBOCJ1Daz6nvDUfRcQykscARsQ6Sc+TeeLVMcC4ZLPrgAeA7yXlv4vMvBuPSeojaYfkOG1q5dJXWDT7F5uVv9e1D88MOJaass0f4fuxYRVUVbboOe5mZp1Ou/QpSKoE9gEeBwbk/KFfTuZxh5BJGLmPFVySlG2SFCSdQqYmweDBg1sVz1srXmO/16/drLxMwYsv/ZMf1ZywSXkEVL/6L274+kdbdT4zaxtr1qxh/PjxACxfvpzy8nIqKjI35j7xxBN067b5F7qWGjduHMuWLaN79+5s2LCBww8/nIsvvpg+ffps8bHrmz17NhdccAHvvvsu3bt35+Mf/zhXXHFFi44xb9483njjDY46qm0ebpf3pCCpJ5lH9Z0dEWtzqzPJs15bNCNfREwDpgFUVVW1aja/YWMOhTFvbb5i1pmcOvd3nDppEmw/vK74tBvmsqam0efRm1k76devH/PmzQNgypQp9OzZk3POOadu/caNG+nSZcv/rM2YMYOqqio2bNjAeeedxzHHHMODDz64xcfNNX/+fE4//XTuvvtu9thjD2pqapg2bVqLjzNv3jyqq6s7R1JIHoF4KzAjIm5Lildkm4Uk7QCsTMqXsumzZgfStg9Mb94nL4FFD8CMTZ+i+Evg4e6HAge3azhm1rzJkyfTo0cPnnrqKQ466CB69+69SbIYMWIEd911F5WVlVx//fX8/Oc/Z8OGDey///5cffXVlJeXN3rsbt26cfnll7Pbbrvx9NNPc+utt7Lddttx9tlnA3D++eez/fbbM2rUKKZMmUL//v2ZP38+Y8eO5frrr2+yTf/yyy/n/PPPZ4899gCgvLycb3wj84TaxYsXc9JJJ7F69WoqKir47W9/y+DBg7nlllv4wQ9+QHl5Odtuuy333XcfF154Ie+99x4PP/ww5513HhMnTtyi65m3pJA8JP3/gOcj4n9yVs0CJgGXJT/vzCk/XdKNZB6D+O989Cc0qXsv+OpsWHjfJsXLZl9O39o17RqKWWfwgz8+x4I31rbpMYfv2JuLPrtXi/ZZsmQJ//jHPygvL2fKlCkNbvP8889z00038cgjj9C1a1e++c1vMmPGDL7yla80eezy8nJGjRrFCy+8wEknncRxxx3H2WefTW1tLTfeeCNPPPEEzz77LE899RTPPfccO+64IwcddBCPPPIIBx/c+BfJ+fPn8+1vf7vBdWeccQaTJk1i0qRJXHvttZx55pnccccdTJ06lXvvvZeddtqJt956i27dujF16lSqq6u56qqrUl+vpuSzpnAQ8GXgWUnzkrLvk0kGN0s6GXgVyD5390/AUcBC4F3gq3mMrXHb7gRjJ21S9Oafr0U1HxQkHDNr3he+8IUmv/ED3H///cyZM4d9990XgPfee4/tt98+1fGzz52prKykX79+PPXUU6xYsYJ99tmHfv36AbDffvsxcOBAAEaPHs3ixYubTApNefTRR7nttkzjype//GW++93vAnDQQQcxefJkJkyYwHHHHdeqYzcnn6OPHgYaqzuNb2D7AE7LVzxbIlRGGbWFDsOsw2npN/p82WabbeqWu3TpQm3th/9fs+P0I4JJkybxwx/+sEXHrqmp4dlnn2XPPfcE4Gtf+xrTp09n+fLlnHTSSXXbde/evW65vLycjRub7ofca6+9mDNnDqNGjUodyzXXXMPjjz/O3XffzdixY5kzZ06Lfpc0PPdRCkEZclIw6xQqKyuZO3cuAHPnzuWVV14BYPz48fzhD39g5cpMN+abb77Jq682OoM0kJnS47zzzmPQoEGMHDkSgGOPPZZ77rmHJ598kk9+8pOtjvM73/kOl156Kf/85z8BqK2t5ZprrgHgwAMP5MYbbwQynd6HHHIIAC+//DL7778/U6dOpaKigtdff51evXqxbt26VsdRn5NCGipD4aRg1hl8/vOf580332SvvfbiqquuYtiwYQAMHz6ciy++mCOOOIKRI0fyiU98gmXLGu62PPHEExk5ciQjRozgnXfe4c4776xb161bNw477DAmTJjQbJMVwIUXXsisWbM2Kx85ciQ/+9nPOOGEE9hzzz0ZMWIEixYtAuAXv/gFv/3tbxk5ciS///3vufLKK4FMItl7770ZMWIEBx54IKNGjeKwww5jwYIFjB49mptuuqnF16u+Tv2M5qqqqmiPh+w8c/kR9Fi/hmEXtH1Vzayzef755+uaUkpRbW0tY8aM4ZZbbmHo0KGFDqdZDX1ekuZERFVD27umkEJQRplrCmYlb8GCBey2226MHz++UySE1vAsqSmE3KdgZpkmqGwTT7FyTSENlSE6bzObmVlaTgopZJqPagodhplZ3jkppBCuKZhZiXBSSMM3r5lZiXBSSMF3NJt1HJdccgl77bUXI0eOZPTo0Tz++ON5Oc8HH3zAueeey9ChQxkzZgwHHHAAs2fPbvFxpk+fzhtvvJGHCPPDo49SCJWhTnw/h1mxePTRR7nrrruYO3cu3bt3Z/Xq1WzYsCH1/vWn1m5qqu0LLriAZcuWMX/+fLp3786KFStaNX329OnTGTFiBDvuuGOL9y0EJ4U0PCTVrENYtmwZ/fv3r5tnqH///nXrKisrqa6upn///lRXV3POOefwwAMPMGXKFF5++WUWLVrE4MGD2X333Td5P3PmzM3O8+677/LrX/+aV155pe5cAwYMYMKEzPydM2fO5NJLLyUi+PSnP82PfvQjampqOPnkk6murkYSJ510EoMGDaK6upoTTzyRrbbaikcffZStttqqHa5U6zkppBCUUeaOZrPNzT4Xlj/btsf8yN5w5GUNrjriiCOYOnUqw4YN4/DDD2fixIkceuihzR5ywYIFPPzww2y11VZMmTJlk/cNWbhwIYMHD6Z3796brXvjjTf43ve+x5w5c+jbty9HHHEEd9xxB4MGDWLp0qXMnz8fgLfeeos+ffpw1VVX8ZOf/ISqqgZvIO5w3KeQhvsUzDqEnj17MmfOHKZNm0ZFRQUTJ05k+vTpze539NFHb5IA6r9viSeffJJx48ZRUVFBly5dOPHEE3nooYfYZZddWLRoEWeccQb33HNPgwmlM3BNIQ03H5k1rJFv9PlUXl7OuHHjGDduHHvvvTfXXXcdkydP3mTK7Ox02Vm5U2s39L6+3Xbbjddee421a9em/uPet29fnn76ae69916uueYabr75Zq69dvNnwXd0rimkITcfmXUEL774Ii+99FLd+3nz5rHzzjsDmT6F7PMFbr311i06z9Zbb83JJ5/MWWedVdeRvWrVKm655Rb2228/HnzwQVavXk1NTQ0zZ87k0EMPZfXq1dTW1vL5z3+eiy++uG767rae2jrfnBRSCJW7+cisA3j77beZNGkSw4cPZ+TIkSxYsKDu8ZsXXXQRZ511FlVVVammtM4aPXp0g+UXX3wxFRUVDB8+nBEjRvCZz3yG3r17s8MOO3DZZZdx2GGHMWrUKMaOHcsxxxzD0qVLGTduHKNHj+ZLX/pS3cN8Jk+ezKmnnsro0aN57733tvga5Junzk7h8au/zh4r7mbbH3SescZm+VLqU2d3Np46Ox8k1xTMrCQ4KaTh5iMzKxFOCmm4o9lsE5252bmUtOZzyltSkHStpJWS5ueU3SRpXvJaLGleUl4p6b2cddfkK65WcVIwq9OjRw/WrFnjxNDBRQRr1qyhR48eLdovn/cpTAeuAn6XLYiIidllSVcA/87Z/uWIaHgYQKH55jWzOgMHDmTJkiWsWrWq0KFYM3r06MHAgQNbtE/ekkJEPCSpsqF1kgRMAD6er/O3Jc+Savahrl27MmTIkEKHYXlSqD6FQ4AVEfFSTtkQSU9JelDSIY3tKOkUSdWSqtvrm4pUTrmC2lpXl82suBUqKZwA5E5NuAwYHBH7AN8CbpDU4L3lETEtIqoioqqioqIdQgUkAGpr/UhOMytu7Z4UJHUBjgNuypZFxPqIWJMszwFeBoa1d2yNibLM3ZHZeVXMzIpVIWoKhwMvRMSSbIGkCknlyfIuwFBgUQFia5gyl8k1BTMrdvkckjoTeBTYXdISSScnq45n06YjgI8BzyRDVP8AnBoRb+YrtpZSNinUbCxwJGZm+ZXP0UcnNFI+uYGyW4Etm9Ywn+TmIzMrDb6jOQ13NJtZiXBSSCNpPooa1xTMrLg5KaSRjD4K1xTMrMg5KaSQ7WiucVIwsyLnpJBGWXZIqkcfmVlxc1JIIxl9hKe5MLMi56SQRvY+hXDzkZkVNyeFFD68ec1JwcyKm5NCGnWjjzwk1cyKm5NCCko6mj0k1cyKnZNCGnUT4rmmYGbFzUkhjewdzR6SamZFzkkhBfl5CmZWIpwUUpA7ms2sRDgppKBkllR3NJtZsXNSSKMs89gJJwUzK3ZOCilkb16LcPORmRU3J4UUsvcp+I5mMyt2Tgpp+HkKZlYinBRScPORmZUKJ4UUss1HuKZgZkUub0lB0rWSVkqan1M2RdJSSfOS11E5686TtFDSi5I+ma+4WsM3r5lZqchnTWE68KkGyn8aEaOT158AJA0Hjgf2Sva5Wso+2abwsqH45jUzK3Z5SwoR8RDwZsrNjwFujIj1EfEKsBDYL1+xtdSHzUee+8jMilsh+hROl/RM0rzUNynbCXg9Z5slSdlmJJ0iqVpS9apVq/Ida+ac2dFH4cdxmllxa++k8CtgV2A0sAy4oqUHiIhpEVEVEVUVFRVtHV+D/DwFMysV7ZoUImJFRNREZmznr/mwiWgpMChn04FJWcfgZzSbWYlo16QgaYect8cC2ZFJs4DjJXWXNAQYCjzRnrE1pSxpPsIdzWZW5Lrk68CSZgLjgP6SlgAXAeMkjQYCWAz8J0BEPCfpZmABsBE4LaLjfC331NlmVirylhQi4oQGiv+vie0vAS7JVzxbom70UcfJU2ZmeeE7mlP48D4FJwUzK25OCimoPDv3kYekmllxc1JIoazuPgXXFMysuDkppJCdJdWjj8ys2DkppJAdfeSOZjMrdk4KKZR5SKqZlQgnhRTKyp0UzKw0OCmk4PsUzKxUOCmk8GGfgmsKZlbcnBRSKJPnPjKz0uCkkMKHN6+5+cjMipuTQgqeJdXMSoWTQgpl7lMwsxLhpJCCyj3NhZmVBieFFDzNhZmVCieFFMrq7lNwUjCz4uakkEJ5ufsUzKw0OCmkUFaePKDOScHMipyTQgqe5sLMSoWTQgq+T8HMSoWTQgrldfcp+HGcZlbc8pYUJF0raaWk+TllP5b0gqRnJN0uqU9SXinpPUnzktc1+YqrNSSoCbn5yMyKXj5rCtOBT9Ur+wswIiJGAv8EzstZ93JEjE5ep+YxrhaTRC1l7mg2s6KXt6QQEQ8Bb9Yr+3NEbEzePgYMzNf525qTgpmVgkL2KZwEzM55P0TSU5IelHRIYztJOkVStaTqVatW5T/KRC1uPjKz4pcqKUj6fZqytCSdD2wEZiRFy4DBEbEP8C3gBkm9G9o3IqZFRFVEVFVUVLQ2hBbLJAXXFMysuKWtKeyV+0ZSOTC2NSeUNBn4DHBiRGY4T0Ssj4g1yfIc4GVgWGuOny+1lCGPPjKzItdkUpB0nqR1wEhJa5PXOmAlcGdLTybpU8B3gaMj4t2c8ook0SBpF2AosKilx88n9ymYWSloMilExA8johfw44jonbx6RUS/iDivqX0lzQQeBXaXtETSycBVQC/gL/WGnn4MeEbSPOAPwKkR8WaDBy6QcJ+CmZWALim3u0vSNhHxjqQvAWOAKyPi1cZ2iIgTGij+v0a2vRW4NWUsBVGrMuSagpkVubR9Cr8C3pU0Cvg2mTb/3+Utqg7IHc1mVgrSJoWNSafwMcBVEfFLMs1AJSNwTcHMil/a5qN1ks4DvgwcosyjyLrmL6yOpxYBTgpmVtzS1hQmAuuBkyJiOZk7kX+ct6g6II8+MrNSkCopJIlgBrCtpM8A70dESfUpuPnIzEpB2juaJwBPAF8AJgCPS/qPfAbW0YTc0WxmxS9tn8L5wL4RsRIyN5sB95G5p6Ak1LqmYGYlIG2fQlk2ISTWtGDfouCkYGalIG1N4R5J9wIzk/cTgT/lJ6SOKXyfgpmVgCaTgqTdgAER8R1JxwEHJ6se5cMZTktCrcqRh6SaWZFrrqbwM5Kno0XEbcBtAJL2TtZ9Nq/RdSCB3HxkZkWvuX6BARHxbP3CpKwyLxF1UE4KZlYKmksKfZpYt1VbBtLR1aoc39FsZsWuuaRQLenr9QslfQ2Yk5+QOibXFMysFDTXp3A2cLukE/kwCVQB3YBj8xlYRxOeOtvMSkCTSSEiVgAHSjoMGJEU3x0Rf817ZB1MUIbw4zjNrLiluk8hIv4G/C3PsXRovnnNzEpBSd2VvEUk36dgZkXPSSGlWspdUzCzouekkFLIo4/MrPg5KaSU6Wh2UjCz4pbXpCDpWkkrJc3PKdtO0l8kvZT87JuUS9LPJS2U9IykMfmMrcUkjz4ys6KX75rCdOBT9crOBe6PiKHA/cl7gCOBocnrFOBXeY6tRWrlPgUzK355TQoR8RDwZr3iY4DrkuXrgM/llP8uMh4D+kjaIZ/xtYxHH5lZ8StEn8KAiFiWLC8HBiTLOwGv52y3JCnbhKRTJFVLql61alV+I82RuaPZzUdmVtwK2tEcEQEta6iPiGkRURURVRUVFXmKrIHzUkaZawpmVuQKkRRWZJuFkp/Zx3wuBQblbDcwKesQQh59ZGbFrxBJYRYwKVmeBNyZU/6VZBTSR4F/5zQzFZwnxDOzUpD2Gc2tImkmMA7oL2kJcBFwGXCzpJOBV4EJyeZ/Ao4CFgLvAl/NZ2wtFSqjzENSzazI5TUpRMQJjawa38C2AZyWz3i2TBmiptBBmJnlle9oTsujj8ysBDgppFTr5iMzKwFOCqmVUebmIzMrck4KaclPXjOz4uekkJLvaDazUuCkkFKojHI3H5lZkXNSSMtTZ5tZCXBSSClU7qRgZkXPSSEtlVHuuY/MrMg5KaTlCfHMrAQ4KaTlm9fMrAQ4KaSUmRDPNQUzK25OCmmpjDJPnW1mRc5JITXf0Wxmxc9JIa2ycrrINQUzK25OCimFlCy4tmBmxctJIS0ll8r9CmZWxJwU0lJ55met5z8ys+LlpJCWawpmVgKcFNJKkkKEawpmVrycFNJKmo9qa11TMLPi1aW9Tyhpd+CmnKJdgAuBPsDXgVVJ+fcj4k/tHF7jktFHtTUbKS9wKGZm+dLuSSEiXgRGA0gqB5YCtwNfBX4aET9p75hSKXNNwcyKX6Gbj8YDL0fEqwWOo3nZ5qMaJwUzK16FTgrHAzNz3p8u6RlJ10rq29AOkk6RVC2petWqVQ1tkhfKNh/Vbmy3c5qZtbeCJQVJ3YCjgVuSol8Bu5JpWloGXNHQfhExLSKqIqKqoqKiXWIFPhx95PsUzKyIFbKmcCQwNyJWAETEioioiYha4NfAfgWMbXNJUnCfgpkVs0ImhRPIaTqStEPOumOB+e0eUVOSjmbXFMysmLX76CMASdsAnwD+M6f4ckmjgQAW11tXcCpLago1TgpmVrwKkhQi4h2gX72yLxciltSyzUee5sLMilihRx91HtkJ8VxTMLMi5qSQUl3zkWsKZlbEnBRSkoekmlkJcFJIKZLRRzVuPjKzIuakkFK2poDvUzCzIuakkJLqRh+5pmBmxctJISWV++Y1Myt+TgppyUnBzIqfk0JK2VlSw30KZlbEnBTSKsvc/F3rmoKZFTEnhZRcUzCzUuCkkJI8S6qZlQAnhZScFMysFDgppJSd+yg895GZFTEnhbTq5j5yUjCz4uWkkFJZtvnIdzSbWRFzUkiprk+hxjUFMyteTgop1U2d7ZqCmRUxJ4W0ytynYGbFz0khJWUfx+magpkVMSeFlMrKXVMws+LXpVAnlrQYWAfUABsjokrSdsBNQCWwGJgQEf8qVIyb8CypZlYCCl1TOCwiRkdEVfL+XOD+iBgK3J+87xCyQ1LxzWtmVsQKnRTqOwa4Llm+DvhcAWPZhNzRbGYloGDNR0AAf5YUwP9GxDRgQEQsS9YvBwbU30nSKcApAIMHD26vWD+8T8E1BTNrA88u+Tc3PPEqEa3bf+TAPnxx/7b/G1jIpHBwRCyVtD3wF0kv5K6MiEgSBvXKpwHTAKqqqlp5OVtOSUcz7lMwszZw/WOvcsuc16no1b1V+5eXqY0jyihYUoiIpcnPlZJuB/YDVkjaISKWSdoBWFmo+OorSzqat1r3KiyZU+BoiogEH9kbyrsWOhKzdrVi3fsM37E3d51xSKFD2URBkoKkbYCyiFiXLB8BTAVmAZOAy5KfdxYivgZ12waA3V64Gl64usDBFJlDvg3jLyx0FGbtauXa9Xxk2x6FDmMzhaopDABuT55m1gW4ISLukfQkcLOkk4FXgQkFim8zsXV/jl7/35w/roL9h/QrdDjF44n/hSd+Awf/F3TvVehozNrNynXrGTlw20KHsZmCJIWIWASMaqB8DTC+/SNqniSeiV1ZPmA0DNup0OEUj637wW8+DnN/BwecVuhozNrFxppa1ryznu1b2Z+QT4XsaO5Usp06ta0dKmANGzgWdj4IHv4prFhQ6Gjyq0t3OPhs6NN+o+asYf96ZwNX/OVF1n/QstGEPbqWc84Ru7Pt1lvWB7bmnQ1EQEVvNx91WizkYZsAAApjSURBVNmOft+m0PbWfvQ7lM06Db1wX6FDyaseG96EN+ZRfvKfodz/9QrpznlLuf6x19hh2x6kHcNTE8GKtesZs3Mfjt1n4Badf+Xa9QCuKXRmZZn+DxaveYe5r3WMmTeKwcq173PRrBpWrP1xoUPJu8+W/YNfvHEV/HUq7PHZQodTuiQee/F9hvTfhr+dMy71bus31rDnBffwyqp3tjiEleveB5wUOrWtu2WGpP7irwv5xV8XFjia4rJL/2247Ztj2Xm7rQsdSl6d8vu+PPjmPA595Ep45MpCh1PSzo0dmL37f8PGDan36Q5U9u3Oy6u3PCmsWpfUFNx81Hn169mdu844mNVvry90KEWlvEyM3bkvW3cr/n+KR4/akZNnncIDXziNgb27FTqckvXSq6/R8+//zTf++TW4+Gst2ne2enDJG98DxmxRDCuTpFDR0zWFTm3ETh1v+Jh1Hkfu/RF+8MfnuHnNrnxr7O6FDqdk3bHoBW784DIe+dRyevBBi/Zd+4/rOWPdz6h9exJlPVs/NH3luvfpu3VXunXpaNPPOSmYtZvte/Xgo7v044YnXmfpW+8XOpyS9cjC1ewyeBA9Pjaxxfs+/v5IjnjkeDb+/li6fWSvVsfwiUVrWbb1p1u9fz45KZi1o5MOGsLUuxbw2KI1hQ6lZHUpFyfs17phwdvtOoaLHpzMRe/8GRY/3OoY9lu3kv11HzzwGmzVt3UH2W5XGHp4q2NojJOCWTs6fPgADh++2eS/1knsWtGTmTXj2fOgM/nKAZWtPs7xP7yFH5dfzbAHftj6YPY6zknBzKyQtu/VnW26lbNoC4alRgTPv92T2w66mnPHbcEXhDxNIumkYGaWkiR2qejJy6vebvUx3nr3AzbU1GaGo269XRtG1zacFMzMWmCXim2YPX85n/ifB1u1/wc1mWkRtu/d8YajgpOCmVmLfOmjO7OxJghaPw/amJ37cuCu/dswqrbjpGBm1gL7Vm7HvpUdr9mnrXS8OyfMzKxgnBTMzKyOk4KZmdVxUjAzszpOCmZmVsdJwczM6jgpmJlZHScFMzOro4jW35VXaJJWAa9uwSH6A6vbKJx86OjxgWNsK46xbTjGdHaOiIqGVnTqpLClJFVHRFWh42hMR48PHGNbcYxtwzFuOTcfmZlZHScFMzOrU+pJYVqhA2hGR48PHGNbcYxtwzFuoZLuUzAzs02Vek3BzMxyOCmYmVmdkkwKkj4l6UVJCyWdW+h4ACQNkvQ3SQskPSfprKR8iqSlkuYlr6MKHOdiSc8msVQnZdtJ+oukl5KffQsY3+4512qepLWSzi70dZR0raSVkubnlDV43ZTx8+Tf5zOSxhQwxh9LeiGJ43ZJfZLySknv5VzPawoYY6OfraTzkuv4oqRPFii+m3JiWyxpXlJekGvYrIgoqRdQDrwM7AJ0A54GhneAuHYAxiTLvYB/AsOBKcA5hY4vJ87FQP96ZZcD5ybL5wI/KnScOZ/1cmDnQl9H4GPAGGB+c9cNOAqYDQj4KPB4AWM8AuiSLP8oJ8bK3O0KfB0b/GyT/z9PA92BIcn/+/L2jq/e+iuACwt5DZt7lWJNYT9gYUQsiogNwI3AMQWOiYhYFhFzk+V1wPPAToWNKrVjgOuS5euAzxUwllzjgZcjYkvuem8TEfEQ8Ga94sau2zHA7yLjMaCPpB0KEWNE/DkiNiZvHwMG5juOpjRyHRtzDHBjRKyPiFeAhWT+/+dNU/FJEjABmJnPGLZUKSaFnYDXc94voYP98ZVUCewDPJ4UnZ5U368tZNNMIoA/S5oj6ZSkbEBELEuWlwMDChPaZo5n0/+AHek6QuPXraP+Gz2JTA0ma4ikpyQ9KOmQQgWVaOiz7WjX8RBgRUS8lFPWka4hUJpJoUOT1BO4FTg7ItYCvwJ2BUYDy8hUPwvp4IgYAxwJnCbpY7krI1MvLvg4Z0ndgKOBW5KijnYdN9FRrltjJJ0PbARmJEXLgMERsQ/wLeAGSb0LFF6H/mxznMCmX1I60jWsU4pJYSkwKOf9wKSs4CR1JZMQZkTEbQARsSIiaiKiFvg1ea7+NiciliY/VwK3J/GsyDZvJD9XFi7COkcCcyNiBXS865ho7Lp1qH+jkiYDnwFOTJIXSZPMmmR5Dpn2+mGFiK+Jz7bDXEdJXYDjgJuyZR3pGuYqxaTwJDBU0pDk2+TxwKwCx5Rtb/w/4PmI+J+c8ty25GOB+fX3bS+StpHUK7tMphNyPpnrNynZbBJwZ2Ei3MQm38o60nXM0dh1mwV8JRmF9FHg3znNTO1K0qeA7wJHR8S7OeUVksqT5V2AocCiAsXY2Gc7CzheUndJQ8jE+ER7x5c4HHghIpZkCzrSNdxEoXu6C/EiM7rjn2Qy8/mFjieJ6WAyzQfPAPOS11HA74Fnk/JZwA4FjHEXMqM5ngaey147oB9wP/AScB+wXYGv5TbAGmDbnLKCXkcyCWoZ8AGZtu2TG7tuZEYd/TL59/ksUFXAGBeSaZfP/pu8Jtn288m/gXnAXOCzBYyx0c8WOD+5ji8CRxYivqR8OnBqvW0Lcg2be3maCzMzq1OKzUdmZtYIJwUzM6vjpGBmZnWcFMzMrI6TgpmZ1XFSsE5NUr+cWSaX15sts1sLjvMbScPbKKa3k5+Vkr7YFsfMOfb3673/R1se38xDUq1oSJoCvB0RPylwHG9HRE9J48jM3vmZFuzbJT6cgK7RY7dFnGYNcU3Bio6k6ZL+I+d99pv7OEkPSPpD8oyAGcmd5CTlVdntJV0i6WlJj0kakJTvmrx/VtLF2eM24TLgkKTW8l+SypV5PsGTyeRt/5kT198lzQIWJGV3JJMOPpedeFDSZcBWyfFm1PvdlBx7fhLfxOZ+Z7OGOClYqdkHOJvMXPu7AAc1sM02wGMRMQp4CPh6Un4lcGVE7E3mbtXmnAv8PSJGR8RPydx9+++I2BfYF/h6Mv0CZObgPysisnPfnBQRY4Eq4ExJ/SLiXOC95Hgn1jvXcWQmhBtFZkqFH+dM/5DmdzYDnBSs9DwREUsiM3naPDIPOqlvA3BXsjwnZ5sD+HDW1Rtace4jyMxpNI/MtOj9yMx3k43rlZxtz5T0NJlnGAzK2a4xBwMzIzMx3ArgQTKJJ3vs5n5nMwC6FDoAszzYSPKFR1IZmSfsZa3PWa6h4f8DH8SHnW2NbdMaAs6IiHs3Kcz0PbxT7/3hwAER8a6kB4AeW3DeNL+zGeCaghWnxcDYZPlooGsbHfcxMpOYQWZ23easI/No1ax7gW8kU6QjaVgy22x92wL/ShLCHmQeyZn1QXb/ev4OTEz6LSrIPBayUDOCWifmpGDF6NfAoUnzywHkfAvfQmcD35L0DLAb8O9mtn8GqEk6rP8L+A2ZjuS5yjzY/X9p+Fv7PUAXSc+T6ax+LGfdNOCZbEdzjtuT8z0N/BX4bkQsb9FvZ4aHpJqlJmlrMh29Iel44ISIKPjzvc3aktsWzdIbC1yVDOl8i8wzi82KimsKZmZWx30KZmZWx0nBzMzqOCmYmVkdJwUzM6vjpGBmZnX+P0e/WTw9HJovAAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -482,7 +432,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.9.12 ('base')", "language": "python", "name": "python3" }, @@ -496,7 +446,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.9.12" }, "latex_envs": { "LaTeX_envs_menu_present": true, @@ -528,6 +478,11 @@ "toc_position": {}, "toc_section_display": true, "toc_window_display": false + }, + "vscode": { + "interpreter": { + "hash": "8205f83729152a74c9afc694ae4592a9e3038e45d9cbfebf1ba348127e2980f0" + } } }, "nbformat": 4, From a796e512201fe7f2d644a483615352560bdb5bb3 Mon Sep 17 00:00:00 2001 From: Dohun Date: Thu, 21 Jul 2022 19:03:01 -0500 Subject: [PATCH 07/41] get_traj implemented for RoundedOptimizer --- autompc/optim/rounded_optimizer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/autompc/optim/rounded_optimizer.py b/autompc/optim/rounded_optimizer.py index 46e5309..cdfd817 100644 --- a/autompc/optim/rounded_optimizer.py +++ b/autompc/optim/rounded_optimizer.py @@ -33,4 +33,9 @@ def get_state(self): def set_state(self, state): self.optimizer.set_state(state) + + def get_traj(self): + traj = self.optimizer.get_traj() + traj.ctrls=np.around(traj.ctrls) + return traj \ No newline at end of file From dfaf603f5079cf57c2accd1065ff622722482e65 Mon Sep 17 00:00:00 2001 From: Dohun Date: Thu, 21 Jul 2022 19:03:40 -0500 Subject: [PATCH 08/41] Fix string formatting bug --- autompc/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autompc/system.py b/autompc/system.py index 4e61aaf..0d5415f 100644 --- a/autompc/system.py +++ b/autompc/system.py @@ -46,7 +46,7 @@ def __str__(self): if self._dt is None: return '{}({},{})'.format(self.__class__.__name__,observation_str,control_str) else: - dt_str = "dt={.3f}".format(self._dt) + dt_str = "dt={:.3f}".format(self._dt) return '{}({},{},{})'.format(self.__class__.__name__,observation_str,control_str,dt_str) From 1f6dae0245455ea10e958ae569fdd5b47316c08e Mon Sep 17 00:00:00 2001 From: Dohun Date: Thu, 21 Jul 2022 19:04:12 -0500 Subject: [PATCH 09/41] Fix bug converting namedtuple to dict --- autompc/tuning/control_evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autompc/tuning/control_evaluator.py b/autompc/tuning/control_evaluator.py index 33bdf1c..11f166f 100644 --- a/autompc/tuning/control_evaluator.py +++ b/autompc/tuning/control_evaluator.py @@ -15,7 +15,7 @@ "cost","traj","term_cond","eval_time"]) def trial_to_json(trial : ControlEvaluationTrial): - res = dict(trial) + res = dict() res['policy'] = str(trial.policy) res['task'] = str(trial.task) res['dynamics'] = str(trial.dynamics) From ab9e92558ec21e943abf61bc90fa757cbe6ad94d Mon Sep 17 00:00:00 2001 From: Dohun Date: Wed, 27 Jul 2022 17:26:02 -0500 Subject: [PATCH 10/41] Minor bug fixes --- autompc/controller.py | 3 +++ autompc/tuning/control_performance_metric.py | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/autompc/controller.py b/autompc/controller.py index 3f430ed..70239cc 100644 --- a/autompc/controller.py +++ b/autompc/controller.py @@ -143,6 +143,8 @@ def set_ocp_transformer(self, ocp_transformer : OCPTransformer) -> None: OCPTransformer to set. """ self.ocp_transformers = [ocp_transformer] + self.set_component("cost_transformer",self.ocp_transformers) + def set_ocp_transformers(self, ocp_transformers : List[OCPTransformer]) -> None: """ @@ -154,6 +156,7 @@ def set_ocp_transformers(self, ocp_transformers : List[OCPTransformer]) -> None: Set of OCP transformers which can be selected. """ self.ocp_transformers = ocp_transformers + self.set_component("cost_transformer",self.ocp_transformers) def add_ocp_transformer(self, ocp_transformer): """ diff --git a/autompc/tuning/control_performance_metric.py b/autompc/tuning/control_performance_metric.py index feb47f5..0fba244 100644 --- a/autompc/tuning/control_performance_metric.py +++ b/autompc/tuning/control_performance_metric.py @@ -8,8 +8,7 @@ class ControlPerformanceMetric: for tuning. Default implementation just averages the cost. """ def __call__(self,trials : List[ControlEvaluationTrial]) -> float: - return np.mean(t.cost for t in trials) - + return np.mean([t.cost for t in trials]) class ConfidenceBoundPerformanceMetric(ControlPerformanceMetric): """A performance metric that uses a quantile of some statistical From 036fb72cd12f79444aebe81bce77771070fafe58 Mon Sep 17 00:00:00 2001 From: Dohun Date: Wed, 3 Aug 2022 07:51:57 -0500 Subject: [PATCH 11/41] Cost minor syntax error fixed. ThresholdCost goal convention fixed --- autompc/costs/__init__.py | 1 + autompc/costs/barrier_cost.py | 48 +++++++++++++++++++++++++++++------ autompc/costs/cost.py | 4 +-- autompc/costs/quad_cost.py | 3 ++- autompc/costs/thresh_cost.py | 31 +++++++++------------- autompc/costs/zero_cost.py | 2 +- 6 files changed, 58 insertions(+), 31 deletions(-) diff --git a/autompc/costs/__init__.py b/autompc/costs/__init__.py index e67e25a..3b78487 100644 --- a/autompc/costs/__init__.py +++ b/autompc/costs/__init__.py @@ -1,3 +1,4 @@ from .quad_cost import QuadCost from .thresh_cost import ThresholdCost, BoxThresholdCost +from .barrier_cost import LogBarrierCost from .cost import Cost diff --git a/autompc/costs/barrier_cost.py b/autompc/costs/barrier_cost.py index 57a9016..0bf4401 100644 --- a/autompc/costs/barrier_cost.py +++ b/autompc/costs/barrier_cost.py @@ -52,6 +52,38 @@ def __init__(self, system, boundedStates): self._is_twice_diff = True self._has_goal = False + def incremental(self, obs, control): + return self.eval_obs_cost(obs) + self.eval_ctrl_cost(control) + + def incremental_diff(self, obs, control): + return self.incremental(obs, control), self.eval_obs_cost_diff(obs), self.eval_ctrl_cost_diff(control) + + def incremental_hess(self, obs, control): # TODO: Tuple unpacking only supported for python>=3.8 + hess_obs_ctrl = np.zeros((self.system.obs_dim, self.system.ctrl_dim)) + return *self.incremental_diff(obs, control), self.eval_obs_cost_hess(obs), hess_obs_ctrl, self.eval_ctrl_cost_hess(control) + + def terminal(self, obs): + return 0 + + def terminal_diff(self, obs): + return 0, 0 + + def terminal_hess(self, obs): + return 0, 0, 0 + + def __add__(self, rhs): + if isinstance(rhs, LogBarrierCost): + if (self.goal is None and rhs.goal is None) or np.all(self.goal == rhs.goal): + return LogBarrierCost(self.system, self.boundedStates+rhs.boundedStates) + return Cost.__add__(self, rhs) + + def __mul__(self, rhs): + if not isinstance(rhs, (float, int)): + raise ValueError("* only supports product with numbers") + new_cost = LogBarrierCost(self.system, self.boundedStates) + return new_cost + + #Cost Function: # - b * ln ( a - x ) upper limit # - b * ln ( a + x ) lower limit @@ -79,8 +111,8 @@ def eval_obs_cost_diff(self, obs): self._direction = -1 if(upper): direction = 1 - jacobian[boundedObs[0]] = direction * scale / (limit - obs[index]) - return self.eval_obs_cost(obs), jacobian + jacobian[index] = direction * scale / (limit - obs[index]) + return jacobian #Hessian: # b / (a - x)^2 upper limit @@ -91,8 +123,8 @@ def eval_obs_cost_hess(self, obs): variable, config = boundedObs index = self.system.observations.index(variable) limit, scale, _ = config - hessian[boundedObs[0]][boundedObs[0]] = scale / ((limit - obs[index])**2) - return *self.eval_obs_cost_diff, hessian + hessian[index][index] = scale / ((limit - obs[index])**2) + return hessian def eval_ctrl_cost(self, ctrl): sum = 0 @@ -115,8 +147,8 @@ def eval_ctrl_cost_diff(self, ctrl): self._direction = -1 if(upper): direction = 1 - jacobian[boundedCtrl[0]] = direction * scale / (limit - ctrl[index]) - return self.eval_ctrl_cost(), jacobian + jacobian[index] = direction * scale / (limit - ctrl[index]) + return jacobian def eval_ctrl_cost_hess(self, ctrl): hessian = np.zeros((self.system.ctrl_dim, self.system.ctrl_dim)) @@ -124,8 +156,8 @@ def eval_ctrl_cost_hess(self, ctrl): variable, config = boundedCtrl index = self.system.controls.index(variable) limit, scale, _ = config - hessian[boundedCtrl[0]][boundedCtrl[0]] = scale / ((limit - ctrl[index])**2) - return *self.eval_ctrl_cost_diff(), hessian + hessian[index][index] = scale / ((limit - ctrl[index])**2) + return hessian def eval_term_obs_cost(self, obs): return 0 diff --git a/autompc/costs/cost.py b/autompc/costs/cost.py index d161092..c4b46b2 100644 --- a/autompc/costs/cost.py +++ b/autompc/costs/cost.py @@ -201,12 +201,12 @@ def __add__(self, other): def __mul__(self, rhs): if not isinstance(rhs,(int,float)): raise ValueError("Can only multiply by a float") - return MulCost(self.system, [self, rhs]) + return MulCost(self.system, self, rhs) def __rmul__(self, lhs): if not isinstance(lhs,(int,float)): raise ValueError("Can only multiply by a float") - return MulCost(self.system, [self, lhs]) + return MulCost(self.system, self, lhs) class SumCost(Cost): diff --git a/autompc/costs/quad_cost.py b/autompc/costs/quad_cost.py index 4c17a9b..1926d2f 100644 --- a/autompc/costs/quad_cost.py +++ b/autompc/costs/quad_cost.py @@ -81,7 +81,8 @@ def incremental_hess(self, obs, control): obst = obs QQt = (self._Q + self._Q.T) RRt = (self._R + self._R) - return obst.T @ self._Q @ obst + control.T @ self._R @control, QQt @ obst, RRt @ control, QQt, None, RRt + hess_obs_ctrl = np.zeros((self.system.obs_dim, self.system.ctrl_dim)) + return obst.T @ self._Q @ obst + control.T @ self._R @control, QQt @ obst, RRt @ control, QQt, hess_obs_ctrl, RRt def terminal(self, obs): try: diff --git a/autompc/costs/thresh_cost.py b/autompc/costs/thresh_cost.py index cff8e56..3dcb9f7 100644 --- a/autompc/costs/thresh_cost.py +++ b/autompc/costs/thresh_cost.py @@ -10,22 +10,21 @@ def __init__(self, system, goal, threshold, obs_range=None, observations=None): """ Create threshold cost. Returns 1 for every time steps where :math:`||x - x_\\textrm{goal}||_\\infty > \\textrm{threshold}`. - The check is performed only over the observation dimensions from - obs_range[0] to obs_range[1]. - + + The norm is performed only over the observation dimensions from + obs_range[0] to obs_range[1], or the observations named in + `observations`. Parameters ---------- system : System Robot system object - goal : Numpy array - Goal position - + Goal position. Can either be length system.obs_dim or + # of observations in obs_range : (int, int) First (inclusive and last (exclusive) index of observations for which goal is specified. If neither this field nor observations is set, default is full observation range. - observations : [str] List of observation names for which goal is specified. Supersedes obs_range when present. @@ -39,17 +38,14 @@ def __init__(self, system, goal, threshold, obs_range=None, observations=None): self._obs_idxs = [system.observations.index(obs) for obs in observations] if self._obs_idxs is None: self._obs_idxs = list(range(0, system.obs_dim)) - self.set_goal(goal) - - def set_goal(self, goal): if len(goal) < self.system.obs_dim: - self._goal = np.zeros(self.system.obs_dim) - self._goal[self._obs_idxs] = goal - else: - self._goal = np.copy(goal) + full_goal = np.zeros(self.system.obs_dim) + full_goal[self._obs_idxs] = goal + goal = full_goal + self.set_goal(goal) def incremental(self, obs, ctrl): - if (la.norm(obs[self._obs_idxs] - self._goal[self._obs_idxs], np.inf) + if (la.norm(obs[self._obs_idxs] - self.goal[self._obs_idxs], np.inf) > self._threshold): return 1.0 else: @@ -64,16 +60,13 @@ def __init__(self, system, limits, goal=None): """ Create Box threshold cost. Returns 1 for every time steps where observation is outisde of limits. - Paramters --------- system : System System cost is computed for - limits : numpy array of shape (system.obs_dim, 2) Upper and lower limits. Use +np.inf or -np.inf to allow certain dimensions unbounded. - goal : numpy array of size system.obs_dim Goal state. Not used directly for computing cost, but may be used by downstream cost factories. @@ -91,4 +84,4 @@ def incremental(self, obs, ctrl): return 0.0 def terminal(self, obs): - return 0.0 + return 0.0 \ No newline at end of file diff --git a/autompc/costs/zero_cost.py b/autompc/costs/zero_cost.py index 99687c8..fdeee20 100644 --- a/autompc/costs/zero_cost.py +++ b/autompc/costs/zero_cost.py @@ -20,7 +20,7 @@ def incremental_diff(self, obs, ctrl): return 0.0,np.zeros(len(obs)),np.zeros(len(ctrl)) def incremental_hess(self, obs, ctrl): - return 0.0,np.zeros(len(obs)),np.zeros(len(ctrl)),np.zeros(len(obs),len(obs)),None,np.zeros(len(ctrl),len(ctrl)) + return 0.0,np.zeros(len(obs)),np.zeros(len(ctrl)),np.zeros((len(obs),len(obs))),np.zeros((len(obs),len(ctrl))),np.zeros((len(ctrl),len(ctrl))) def terminal(self, obs) -> float: return 0.0 From c5cc14873024c8dd104791817790783df5f769e6 Mon Sep 17 00:00:00 2001 From: Dohun Date: Wed, 3 Aug 2022 07:54:43 -0500 Subject: [PATCH 12/41] Barrier Cost changed from factory to transformer --- ...factory.py => barrier_cost_transformer.py} | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) rename autompc/ocp/{barrier_cost_factory.py => barrier_cost_transformer.py} (81%) diff --git a/autompc/ocp/barrier_cost_factory.py b/autompc/ocp/barrier_cost_transformer.py similarity index 81% rename from autompc/ocp/barrier_cost_factory.py rename to autompc/ocp/barrier_cost_transformer.py index 10d66e2..22ae393 100644 --- a/autompc/ocp/barrier_cost_factory.py +++ b/autompc/ocp/barrier_cost_transformer.py @@ -1,11 +1,12 @@ # Created by Teodor Tchalakov (tcha2@illinois.edu), 2022-04-18 # Standard library includes +import copy from collections import defaultdict # Internal library includes -from .cost_factory import CostFactory -from . import LogBarrierCost +from .ocp_transformer import OCPTransformer, PrototypeOCP +from ..costs import LogBarrierCost # External library includes import numpy as np @@ -16,13 +17,13 @@ def construct_default_bounds(): return (1e-3, 1e4, 1.0, False) -class LogBarrierCostFactory(CostFactory): +class LogBarrierCostTransformer(OCPTransformer): def __init__(self, system): - super().__init__(system) + super().__init__(system, 'LogBarrierCostTransformer') self._scale_bounds = defaultdict(construct_default_bounds) # Key: obsname, Value: (lower, upper, default, log_scale) - self._limits = [] # Key: obs/ctrlname, Value: (limit, upper) - self._scale_fixed = [] # Key: obs/ctrlname, Value: limit + self._limits = {} # Key: obs/ctrlname, Value: (limit, upper) + self._scale_fixed = {} # Key: obs/ctrlname, Value: limit """ boundedState : String @@ -110,13 +111,27 @@ def _get_boundedState(self, cfg, label, fixed_dict): if(not upper): upper_string = "Lower" hyper_name = f"{name}_{upper_string}_{label}" - scale = cfg(hyper_name) + scale = cfg[hyper_name] boundedStates[name] = (limit, scale, upper) return boundedStates - def __call__(self, cfg, task, trajs): - boundedStates = self._get_boundedState(cfg, "LogBarrier", self._scale_fixed) + def get_default_config_space(self): + return CS.ConfigurationSpace() - return LogBarrierCost(self.system, boundedStates) + def get_prototype(self, config, ocp): + return PrototypeOCP(ocp, cost=LogBarrierCost) + + def is_compatible(self, ocp): + return True + + def ocp_requirements(self) -> dict: + return {} + + def __call__(self, ocp): + boundedStates = self._get_boundedState(self.get_config(), "LogBarrier", self._scale_fixed) + new_cost = LogBarrierCost(self.system, boundedStates) + new_ocp = copy.deepcopy(ocp) + new_ocp.set_cost(new_cost) + return new_ocp \ No newline at end of file From a7b57e0d3f7526e331e1f58221c995f23b53540f Mon Sep 17 00:00:00 2001 From: Dohun Date: Wed, 3 Aug 2022 07:55:14 -0500 Subject: [PATCH 13/41] Barrier cost transformer added to __init__.py --- autompc/ocp/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/autompc/ocp/__init__.py b/autompc/ocp/__init__.py index d5c7a2b..c4081c2 100644 --- a/autompc/ocp/__init__.py +++ b/autompc/ocp/__init__.py @@ -3,3 +3,4 @@ from .quad_cost_transformer import QuadCostTransformer from .gauss_reg_transformer import GaussRegTransformer from .bounds_transformer import KeepBoundsTransformer,DeleteBoundsTransformer +from .barrier_cost_transformer import LogBarrierCostTransformer \ No newline at end of file From ef2a93b479aa38f87d54abeba87b2d0039b46634 Mon Sep 17 00:00:00 2001 From: Dohun Date: Wed, 3 Aug 2022 07:56:22 -0500 Subject: [PATCH 14/41] ILQR stop gap solution --- autompc/optim/ilqr.py | 160 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 157 insertions(+), 3 deletions(-) diff --git a/autompc/optim/ilqr.py b/autompc/optim/ilqr.py index ebf336e..221a4a1 100644 --- a/autompc/optim/ilqr.py +++ b/autompc/optim/ilqr.py @@ -91,7 +91,7 @@ def cost_requirements(self): def set_state(self, state): self._guess = copy.deepcopy(state["guess"]) - def compute_ilqr(self, x0, uguess, u_threshold=1e-3, + '''def compute_ilqr(self, x0, uguess, u_threshold=1e-3, ls_max_iter=10, ls_discount=0.2, ls_cost_threshold=0.3): """Use equations from https://medium.com/@jonathan_hui/rl-lqr-ilqr-linear-quadratic-regulator-a5de5104c750 . A better version is https://homes.cs.washington.edu/~todorov/papers/TassaIROS12.pdf @@ -145,7 +145,7 @@ def eval_obj(xs, us): vn[:obsdim] = cost_jac lin_cost_reduce = quad_cost_reduce = 0 for t in range(H, 0, -1): # so run for H steps... - # first assemble Ct and ct, they are linearized at current state + # first assemble Ct and ct, they are linearized at current state\ _, Qx[:obsdim], Ru, Q[:obsdim, :obsdim], _, R = cost.incremental_hess(states[t - 1, :obsdim],ctrls[t - 1]) Ct[:dimx, :dimx] = Q * dt Ct[dimx:, dimx:] = R * dt @@ -159,6 +159,8 @@ def eval_obj(xs, us): Qux = Qt[dimx:, :dimx] qx = qt[:dimx] qu = qt[dimx:] + print(cost) + print(cost.incremental_hess(states[t - 1, :obsdim],ctrls[t - 1])) QuuInv = inverse_semidefinite(Quu,1e-2) K = -QuuInv @ Qux k = -QuuInv @ qu @@ -236,7 +238,7 @@ def eval_obj(xs, us): best_alpha = 1.0 else: if self.verbose : - print("Line search fails, best obj",best_obj,"obj",obj,"best_alpha",best_alpha) + print(" fails, best obj",best_obj,"obj",obj,"best_alpha",best_alpha) break # return since update of action is small @@ -266,6 +268,158 @@ def eval_obj(xs, us): if self.verbose and not converged : print('ilqr fails to converge, try a new guess? Last u update is %f ks norm is %f' % (du_norm, ks_norm)) print('ilqr is not converging...') + return converged, states, ctrls, Ks''' + + def compute_ilqr(self, x0, uguess, u_threshold=1e-3, max_iter=50, + ls_max_iter=10, ls_discount=0.2, ls_cost_threshold=0.3, silent=True): + """Use equations from https://medium.com/@jonathan_hui/rl-lqr-ilqr-linear-quadratic-regulator-a5de5104c750 . + A better version is https://homes.cs.washington.edu/~todorov/papers/TassaIROS12.pdf + Here I do not have Hessian correction since I'm certain all my matrices are SPD + """ + cost = self.ocp.get_cost() + H = self.horizon + dt = self.system.dt + + def eval_obj(xs, us): + obj = 0 + for i in range(H): + obj += dt * cost.incremental(xs[i, :self.system.obs_dim],us[i]) + obj += cost.terminal(xs[-1, :self.system.obs_dim]) + return obj + + dimx, dimu = self.model.state_dim, self.system.ctrl_dim + obsdim = self.system.obs_dim + # handy variables... + states, new_states = np.zeros((2, H + 1, dimx)) + ctrls, new_ctrls = np.zeros((2, H, dimu)) + ls_states = np.zeros((ls_max_iter, H + 1, dimx)) + ls_ctrls = np.zeros((ls_max_iter, H, dimu)) + Ks = np.zeros((H, dimu, dimx)) + ks = np.zeros((H, dimu)) + Jacs = np.zeros((H, dimx, dimx + dimu)) # Jacobian from dynamics... + # first forward simulation + states[0] = x0 + ctrls[:] = uguess + for i in range(H): + states[i + 1], jx, ju = self.model.pred_diff(states[i], ctrls[i]) + Jacs[i, :, :dimx] = jx + Jacs[i, :, dimx:] = ju + obj = eval_obj(states, ctrls) + initcost = obj + # start iteration from here + Ct = np.zeros((dimx + dimu, dimx + dimu)) + ct = np.zeros(dimx + dimu) + converged = False + du_norm = np.inf + for itr in range(max_iter): + if self.verbose: + print('At iteration %d' % itr) + # compute at the last step, Vn and vn, just hessian and gradient at the last state + _, cost_jac, cost_hess = cost.terminal_hess(states[H, :obsdim]) + Vn = np.zeros((dimx, dimx)) + vn = np.zeros(dimx) + Vn[:obsdim, :obsdim] = cost_hess + vn[:obsdim] = cost_jac + lin_cost_reduce = quad_cost_reduce = 0 + for t in range(H, 0, -1): # so run for H steps... + # first assemble Ct and ct, they are linearized at current state + Q = np.zeros((dimx, dimx)) + Qx = np.zeros(dimx) + _, Qx[:obsdim], Ru, Q[:obsdim, :obsdim], _, R = cost.incremental_hess(states[t - 1, :obsdim],ctrls[t - 1]) + Ct[:dimx, :dimx] = Q * dt + Ct[dimx:, dimx:] = R * dt + ct[:dimx] = Qx * dt + ct[dimx:] = Ru * dt + Qt = Ct + Jacs[t - 1].T @ Vn @ Jacs[t - 1] + qt = ct + Jacs[t - 1].T @ (vn) # here Vn @ states[t] may be necessary + # ready to compute feedback + # Qtinv = np.linalg.inv(Qt) + Ks[t - 1] = -np.linalg.solve(Qt[dimx:, dimx:], Qt[dimx:, :dimx]) + ks[t - 1] = -np.linalg.solve(Qt[dimx:, dimx:], qt[dimx:]) + lin_cost_reduce += qt[dimx:].dot(ks[t - 1]) + quad_cost_reduce += ks[t - 1] @ Qt[dimx:, dimx:] @ ks[t - 1] + # Ks[t - 1] = -Qtinv[dimx:, dimx:] @ Qt[dimx:, :dimx] + # ks[t - 1] = -Qtinv[dimx:, dimx:] @ qt[dimx:] + # update Vn and vn + Vn = Qt[:dimx, :dimx] + Qt[:dimx, dimx:] @ Ks[t - 1] + Ks[t - 1].T @ Qt[dimx:, :dimx] + Ks[t - 1].T @ Qt[dimx:, dimx:] @ Ks[t - 1] + vn = qt[:dimx] + Qt[:dimx, dimx:] @ ks[t - 1] + Ks[t - 1].T @ (qt[dimx:] + Qt[dimx:, dimx:] @ ks[t - 1]) + # redo forward simulation and record actions... + ls_success = False + best_alpha = None + best_obj = np.inf + best_obj_estimate_reduction = None + ks_norm = np.linalg.norm(ks) + # print('norm of ks = ', np.linalg.norm(ks)) + + # Compute rollout for all possible alphas + alphas = np.array([ls_discount**i for i in range(ls_max_iter)]) + for i in range(ls_max_iter): + ls_states[i,0,:] = x0 + for i in range(H): + for j, alpha in enumerate(alphas): + ls_ctrls[j, i, :] = alpha * ks[i] + ctrls[i] + Ks[i] @ (ls_states[j, i, :] - states[i, :]) + if self.ubounds is not None: + ls_ctrls[j, i, :] = np.clip(ls_ctrls[j, i, :], self.ubounds[0], self.ubounds[1]) + ls_states[:, i + 1, :] = self.model.pred_batch(ls_states[:, i, :], ls_ctrls[:, i, :]) + + # Now do backtrack line search. + for lsitr, ls_alpha in enumerate(alphas): + new_states = ls_states[lsitr, :, :] + new_ctrls = ls_ctrls[lsitr, :, :] + new_obj = eval_obj(new_states, new_ctrls) + expect_cost_reduction = ls_alpha * lin_cost_reduce + ls_alpha ** 2 * quad_cost_reduce / 2 + #print((obj - new_obj) / (-expect_cost_reduction)) + if (obj - new_obj) / (-expect_cost_reduction) > ls_cost_threshold: + best_obj = new_obj + best_alpha = ls_alpha + best_alpha_idx = lsitr + break + if new_obj < best_obj: + best_obj = new_obj + best_alpha = ls_alpha + best_alpha_idx = lsitr + #ls_alpha *= ls_discount + if ks_norm < u_threshold: + break + if self.verbose: + print('line search obj %f to %f at alpha = %f' % (obj, new_obj, ls_alpha)) + if best_obj < obj or ks_norm < u_threshold: + ls_success = True + new_ctrls = ls_ctrls[best_alpha_idx, :, :] + new_states = ls_states[best_alpha_idx, :, :] + _, jxs, jus = self.model.pred_diff_batch(new_states[:-1,:], new_ctrls) + Jacs[:, :, :dimx] = jxs + Jacs[:, :, dimx:] = jus + new_obj = eval_obj(new_states, new_ctrls) + if (not ls_success and new_obj > obj + 1e-3) or best_alpha is None: + if not silent: + print('Line search fails...') + break + else: + if self.verbose and not silent: + print('alpha is successful at %f with cost from %f to %f' % (best_alpha, obj, new_obj)) + pass + # return since update of action is small + if self.verbose and not silent: + print('u update', np.linalg.norm(new_ctrls - ctrls)) + du_norm = np.linalg.norm(new_ctrls - ctrls) + if du_norm < u_threshold: + if self.verbose and not silent: + print('Break since update of control is small at %f' % (np.linalg.norm(new_ctrls - ctrls))) + converged = True + # ready to swap... + states = np.copy(new_states) + ctrls = np.copy(new_ctrls) + obj = new_obj + if converged: + if not silent: + print('Convergence achieved within %d iterations' % itr) + print('Cost update from %f to %f' % (initcost, obj)) + print('Final state is ', states[-1]) + break + if not converged and not silent: + print('ilqr fails to converge, try a new guess? Last u update is %f ks norm is %f' % (du_norm, ks_norm)) + print('ilqr is not converging...') return converged, states, ctrls, Ks def set_guess(self, guess : Trajectory) -> None: From a61459ce25584835620695b9bd59d8abcb0b21e8 Mon Sep 17 00:00:00 2001 From: Dohun Date: Sun, 7 Aug 2022 21:49:24 -0500 Subject: [PATCH 15/41] Barrier Cost lower bound fixed --- autompc/costs/barrier_cost.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/autompc/costs/barrier_cost.py b/autompc/costs/barrier_cost.py index 0bf4401..9d2f481 100644 --- a/autompc/costs/barrier_cost.py +++ b/autompc/costs/barrier_cost.py @@ -93,10 +93,11 @@ def eval_obs_cost(self, obs): variable, config = boundedObs index = self.system.observations.index(variable) limit, scale, upper = config - self._direction = -1 if(upper): - direction = 1 - sum = sum + -scale * np.log(limit - (direction * obs[index])) + self._direction = 1 + else: + self._direction = -1 + sum = sum + -scale * np.log(limit - (self._direction * obs[index])) return sum #Jacobian: @@ -110,8 +111,8 @@ def eval_obs_cost_diff(self, obs): limit, scale, upper = config self._direction = -1 if(upper): - direction = 1 - jacobian[index] = direction * scale / (limit - obs[index]) + self._direction = 1 + jacobian[index] = self._direction * scale / (limit - obs[index]) return jacobian #Hessian: @@ -134,8 +135,8 @@ def eval_ctrl_cost(self, ctrl): limit, scale, upper = config self._direction = -1 if(upper): - direction = 1 - sum = sum + -scale * np.log(limit - (direction * ctrl[index])) + self._direction = 1 + sum = sum + -scale * np.log(limit - (self._direction * ctrl[index])) return sum def eval_ctrl_cost_diff(self, ctrl): @@ -146,8 +147,8 @@ def eval_ctrl_cost_diff(self, ctrl): limit, scale, upper = config self._direction = -1 if(upper): - direction = 1 - jacobian[index] = direction * scale / (limit - ctrl[index]) + self._direction = 1 + jacobian[index] = self._direction * scale / (limit - ctrl[index]) return jacobian def eval_ctrl_cost_hess(self, ctrl): From 153861ada19bd3af1273971def7ad42d704a2a39 Mon Sep 17 00:00:00 2001 From: Dohun Date: Sun, 7 Aug 2022 21:49:41 -0500 Subject: [PATCH 16/41] Fixed goal setter syntax --- autompc/costs/cost.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/autompc/costs/cost.py b/autompc/costs/cost.py index c4b46b2..3024dcc 100644 --- a/autompc/costs/cost.py +++ b/autompc/costs/cost.py @@ -186,6 +186,12 @@ def goal(self) -> Optional[np.ndarray]: return np.copy(self.properties['goal']) @goal.setter + def goal(self, goal): + """Sets the cost's goal state. (Note: not all costs actually act to + drive the system toward a goal). + """ + self.properties['goal'] = np.copy(goal) + def set_goal(self,goal): """Sets the cost's goal state. (Note: not all costs actually act to drive the system toward a goal). @@ -279,6 +285,11 @@ def goal(self): return super().goal @goal.setter + def goal(self, goal): + super().goal=goal + for cost in self.costs: + cost.goal = goal + def set_goal(self,goal): super().set_goal(goal) for cost in self.costs: @@ -354,9 +365,15 @@ def goal(self): return super().goal @goal.setter + def goal(self, goal): + super().goal=goal + for cost in self.costs: + cost.goal = goal + def set_goal(self,goal): super().set_goal(goal) - self._cost.goal = goal + for cost in self.costs: + cost.goal = goal def __mul__(self, scale): if not isinstance(scale,(float,int)): From 50a19d2856ba66b882bfd400fd36eb1b9cc4e1da Mon Sep 17 00:00:00 2001 From: Dohun Date: Sun, 7 Aug 2022 21:50:00 -0500 Subject: [PATCH 17/41] Barrier Cost Transformer tunable --- autompc/ocp/barrier_cost_transformer.py | 38 ++++++++++++++----------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/autompc/ocp/barrier_cost_transformer.py b/autompc/ocp/barrier_cost_transformer.py index 22ae393..7c8a990 100644 --- a/autompc/ocp/barrier_cost_transformer.py +++ b/autompc/ocp/barrier_cost_transformer.py @@ -15,16 +15,16 @@ import ConfigSpace.conditions as CSC def construct_default_bounds(): - return (1e-3, 1e4, 1.0, False) - + return (0, 100, 1.0, False) +BARRIER_COST_DEFAULT_BOUNDS = (0, 100) +BARRIER_COST_DEFAULT_VALUE = 1.0 +BARRIER_COST_DEFAULT_LOG = False class LogBarrierCostTransformer(OCPTransformer): def __init__(self, system): - super().__init__(system, 'LogBarrierCostTransformer') - self._scale_bounds = defaultdict(construct_default_bounds) # Key: obsname, Value: (lower, upper, default, log_scale) self._limits = {} # Key: obs/ctrlname, Value: (limit, upper) self._scale_fixed = {} # Key: obs/ctrlname, Value: limit - + super().__init__(system, 'LogBarrierCostTransformer') """ boundedState : String Name of observation or control in the system. @@ -46,9 +46,6 @@ def set_limit(self, boundedState, limit, upper): boundedState : String Name of observation or control in the system. - limit : double - limit value that barrier is placed at. - lower_bound : double lower bound of configuration space @@ -57,11 +54,15 @@ def set_limit(self, boundedState, limit, upper): default : double default value of configuration space + + log : boolean + Whether hyperparameter should use logarithmic scale. (Default: False) """ def set_bounds(self, boundedState, lower_bound, upper_bound, default, log=False): if(boundedState in self.system.observations or boundedState in self.system.controls): if(boundedState in self._limits): self._scale_bounds[boundedState] = (lower_bound, upper_bound, default, log) + else: raise ValueError(str(boundedState) + " does not have a configured limit use set_limit") else: @@ -75,7 +76,6 @@ def set_fixed_scale(self, boundedState, scale): raise ValueError(str(boundedState) + " does not have a configured limit use set_limit") else: raise ValueError(str(boundedState) + " is not in system") - def _get_hyperparameters(self, label, bounds_dict, fixed_dict): hyperparameters = [] @@ -92,12 +92,6 @@ def _get_hyperparameters(self, label, bounds_dict, fixed_dict): hyperparameters.append(hyper) return hyperparameters - def get_configuration_space(self): - cs = CS.ConfigurationSpace() - hypers = self._get_hyperparameters("LogBarrier", self._scale_bounds, self._scale_fixed) - cs.add_hyperparameters(hypers) - return cs - def _get_boundedState(self, cfg, label, fixed_dict): boundedStates = dict() for name in (self.system.controls + self.system.observations): @@ -116,7 +110,19 @@ def _get_boundedState(self, cfg, label, fixed_dict): return boundedStates def get_default_config_space(self): - return CS.ConfigurationSpace() + cs = CS.ConfigurationSpace() + for name in self.system.observations + self.system.controls: + hyper = CSH.UniformFloatHyperparameter(name+"_Upper_LogBarrier", + lower=BARRIER_COST_DEFAULT_BOUNDS[0], upper=BARRIER_COST_DEFAULT_BOUNDS[1], + default_value=BARRIER_COST_DEFAULT_VALUE, log=BARRIER_COST_DEFAULT_LOG + ) + cs.add_hyperparameter(hyper) + hyper = CSH.UniformFloatHyperparameter(name+"_Lower_LogBarrier", + lower=BARRIER_COST_DEFAULT_BOUNDS[0], upper=BARRIER_COST_DEFAULT_BOUNDS[1], + default_value=BARRIER_COST_DEFAULT_VALUE, log=BARRIER_COST_DEFAULT_LOG + ) + cs.add_hyperparameter(hyper) + return cs def get_prototype(self, config, ocp): return PrototypeOCP(ocp, cost=LogBarrierCost) From b4b09ec5cd4bb5a5defad166345e5d9e6a6f7eb4 Mon Sep 17 00:00:00 2001 From: Dohun Date: Sun, 7 Aug 2022 21:59:11 -0500 Subject: [PATCH 18/41] Controls weren't clipping when multi-stepping --- autompc/optim/ilqr.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/autompc/optim/ilqr.py b/autompc/optim/ilqr.py index 221a4a1..387b0c3 100644 --- a/autompc/optim/ilqr.py +++ b/autompc/optim/ilqr.py @@ -437,10 +437,11 @@ def step(self, obs): self._traj = Trajectory(self.model.state_system, states, np.vstack([ctrls, np.zeros(self.system.ctrl_dim)])) self._gains = Ks - return ctrls[0] + return np.clip(ctrls[0], self.ubounds[0], self.ubounds[1]) else: #just reuse prior strajectory and gains - return self._traj.ctrls[substep]+self._gains[substep]@(obs-self._traj.obs[substep]) + return np.clip(self._traj.ctrls[substep]+self._gains[substep]@(obs-self._traj.obs[substep]), + self.ubounds[0], self.ubounds[1]) def get_traj(self): return self._traj.clone() From a7b2bcf902eafe5d94d84af0a8e8d0a990eecebc Mon Sep 17 00:00:00 2001 From: Dohun Date: Sun, 7 Aug 2022 21:59:49 -0500 Subject: [PATCH 19/41] REVISIT THIS: Solution to fix hyperparameters in twostage tuning? --- autompc/tunable.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/autompc/tunable.py b/autompc/tunable.py index f9acc61..cefe052 100644 --- a/autompc/tunable.py +++ b/autompc/tunable.py @@ -252,7 +252,10 @@ def set_config(self, config): if opt is None: continue opt_config = create_subspace_configuration(config,opt.name,opt.get_config_space()) - opt.set_config(opt_config) + try: + opt.set_config(opt_config) + except: + print('Skipping set_config. Frozen hyperparameters?') def get_config(self): return self._config From 659798159ed8ee89f1369b8149e9ed8a76149fd7 Mon Sep 17 00:00:00 2001 From: Dohun Date: Sun, 7 Aug 2022 22:05:19 -0500 Subject: [PATCH 20/41] Twostage tuning fixes --- autompc/tuning/control_tuner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/autompc/tuning/control_tuner.py b/autompc/tuning/control_tuner.py index fe0879b..5d4a230 100644 --- a/autompc/tuning/control_tuner.py +++ b/autompc/tuning/control_tuner.py @@ -296,7 +296,7 @@ def _get_tuning_data(self, controller : Controller, task : List[Task], trajs : L print("------------------------------------------------------------------") print("Beginning two-stage tuning") print("------------------------------------------------------------------") - if 'surr_tune_result' in self.tuning_mode and isinstance(surrogate,AutoSelectModel): + if 'surr_tune_result' in tuning_data.keys() and isinstance(surrogate,AutoSelectModel): print("Reusing surrogate model tuning for SysID model to save time") model = surrogate else: @@ -310,7 +310,9 @@ def _get_tuning_data(self, controller : Controller, task : List[Task], trajs : L print("Cost",tune_result.inc_costs[-1]) print("------------------------------------------------------------------") controller.fix_option('model',model.name) - controller.set_model_hyper_values(model.name,**model.get_config()) + controller.set_model_hyper_values(model.name,**tune_result.inc_cfg) + controller.model.freeze_hyperparameters() + print() tuning_data["control_evaluator"] = control_evaluator From 7d71418de4e7d8e748b22ac8bd212a00b1c0b12a Mon Sep 17 00:00:00 2001 From: Dohun Date: Sun, 7 Aug 2022 22:05:57 -0500 Subject: [PATCH 21/41] Changing default metric to ControlPerformanceMetric() --- autompc/tuning/control_tuner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/autompc/tuning/control_tuner.py b/autompc/tuning/control_tuner.py index 5d4a230..d6f8ba2 100644 --- a/autompc/tuning/control_tuner.py +++ b/autompc/tuning/control_tuner.py @@ -220,7 +220,8 @@ def __init__(self, tuning_mode="default", surrogate : Optional[Model] = None, self.max_trials_per_evaluation = max_trials_per_evaluation self.control_evaluator = control_evaluator if performance_metric is None: - performance_metric = ConfidenceBoundPerformanceMetric(quantile=performance_quantile,eval_time_weight=performance_eval_time_weight,infeasible_cost=performance_infeasible_cost) + performance_metric = ControlPerformanceMetric() # DEBUG + #performance_metric = ConfidenceBoundPerformanceMetric(quantile=performance_quantile,eval_time_weight=performance_eval_time_weight,infeasible_cost=performance_infeasible_cost) self.performance_metric = performance_metric def _get_tuning_data(self, controller : Controller, task : List[Task], trajs : List[Trajectory], From 8f4787b7677651b98f392883fa4916a3307f99ca Mon Sep 17 00:00:00 2001 From: Dohun Date: Mon, 8 Aug 2022 01:11:51 -0500 Subject: [PATCH 22/41] New Barrier Transformer API --- autompc/costs/barrier_cost.py | 61 ++++++++++++++----------- autompc/ocp/barrier_cost_transformer.py | 45 ++++++++---------- 2 files changed, 52 insertions(+), 54 deletions(-) diff --git a/autompc/costs/barrier_cost.py b/autompc/costs/barrier_cost.py index 9d2f481..83df236 100644 --- a/autompc/costs/barrier_cost.py +++ b/autompc/costs/barrier_cost.py @@ -92,39 +92,43 @@ def eval_obs_cost(self, obs): for boundedObs in self.obsConfiguration: variable, config = boundedObs index = self.system.observations.index(variable) - limit, scale, upper = config - if(upper): - self._direction = 1 - else: - self._direction = -1 - sum = sum + -scale * np.log(limit - (self._direction * obs[index])) + lower, upper, scale = config + if lower > -np.inf: + sum = sum + -scale * np.log(-lower + obs[index]) + if upper < np.inf: + sum = sum + -scale * np.log(upper - obs[index]) return sum #Jacobian: # b / (a - x) upper limit - # -b / (a - x) lower limit + # -b / (-a + x) lower limit def eval_obs_cost_diff(self, obs): jacobian = np.zeros(self.system.obs_dim) for boundedObs in self.obsConfiguration: variable, config = boundedObs index = self.system.observations.index(variable) - limit, scale, upper = config - self._direction = -1 - if(upper): - self._direction = 1 - jacobian[index] = self._direction * scale / (limit - obs[index]) + lower, upper, scale = config + if lower > -np.inf: + jacobian[index] += -scale / (-lower + obs[index]) + if upper < np.inf: + jacobian[index] += scale / (upper - obs[index]) + return jacobian #Hessian: # b / (a - x)^2 upper limit - # b / (a - x)^2 lower limit + # b / (-a + x)^2 lower limit def eval_obs_cost_hess(self, obs): hessian = np.zeros((self.system.obs_dim, self.system.obs_dim)) for boundedObs in self.obsConfiguration: variable, config = boundedObs index = self.system.observations.index(variable) - limit, scale, _ = config - hessian[index][index] = scale / ((limit - obs[index])**2) + lower, upper, scale = config + if lower > -np.inf: + hessian[index][index] += scale / ((lower - obs[index])**2) + if upper < np.inf: + hessian[index][index] += scale / ((upper - obs[index])**2) + return hessian def eval_ctrl_cost(self, ctrl): @@ -132,11 +136,11 @@ def eval_ctrl_cost(self, ctrl): for boundedCtrl in self.ctrlsConfiguration: variable, config = boundedCtrl index = self.system.controls.index(variable) - limit, scale, upper = config - self._direction = -1 - if(upper): - self._direction = 1 - sum = sum + -scale * np.log(limit - (self._direction * ctrl[index])) + lower, upper, scale = config + if lower != -np.inf: + sum = sum + -scale * np.log(lower + ctrl[index]) + if upper != np.inf: + sum = sum + -scale * np.log(upper - ctrl[index]) return sum def eval_ctrl_cost_diff(self, ctrl): @@ -144,11 +148,11 @@ def eval_ctrl_cost_diff(self, ctrl): for boundedCtrl in self.ctrlsConfiguration: variable, config = boundedCtrl index = self.system.controls.index(variable) - limit, scale, upper = config - self._direction = -1 - if(upper): - self._direction = 1 - jacobian[index] = self._direction * scale / (limit - ctrl[index]) + lower, upper, scale = config + if lower != -np.inf: + jacobian[index] += -scale / (lower + ctrl[index]) + if upper != np.inf: + jacobian[index] += scale / (upper - ctrl[index]) return jacobian def eval_ctrl_cost_hess(self, ctrl): @@ -156,8 +160,11 @@ def eval_ctrl_cost_hess(self, ctrl): for boundedCtrl in self.ctrlsConfiguration: variable, config = boundedCtrl index = self.system.controls.index(variable) - limit, scale, _ = config - hessian[index][index] = scale / ((limit - ctrl[index])**2) + lower, upper, scale = config + if lower != -np.inf: + hessian[index][index] += scale / ((lower + ctrl[index])**2) + if upper != np.inf: + hessian[index][index] += scale / ((upper - ctrl[index])**2) return hessian def eval_term_obs_cost(self, obs): diff --git a/autompc/ocp/barrier_cost_transformer.py b/autompc/ocp/barrier_cost_transformer.py index 7c8a990..a3a066b 100644 --- a/autompc/ocp/barrier_cost_transformer.py +++ b/autompc/ocp/barrier_cost_transformer.py @@ -29,16 +29,18 @@ def __init__(self, system): boundedState : String Name of observation or control in the system. - limit : double - limit value that barrier is placed at. + lower : double + lower limit value that barrier is placed at. + Default is -np.inf and the barrier isn't placed + + upper : double + upper limit value that barrier is placed at. + Default is -np.inf and the barrier isn't placed - upper : boolean - True if the limit is an upper limit. - False if the limit is a lower limit. """ - def set_limit(self, boundedState, limit, upper): + def set_limit(self, boundedState, lower=-np.inf, upper=np.inf): if(boundedState in self.system.observations or boundedState in self.system.controls): - self._limits[boundedState] = (limit, upper) + self._limits[boundedState] = (lower, upper) else: raise ValueError(str(boundedState) + " is not in system") @@ -82,42 +84,31 @@ def _get_hyperparameters(self, label, bounds_dict, fixed_dict): for name in (self.system.controls + self.system.observations): if name in fixed_dict or name not in self._limits: continue - limit, upper = self._limits[name] - upper_string = "Upper" - if(not upper): - upper_string = "Lower" + lower, upper = self._limits[name] lower_scale, upper_scale, default, log = bounds_dict[name] - hyper = CSH.UniformFloatHyperparameter("{}_{}_{}".format(name, upper_string, label), + hyper = CSH.UniformFloatHyperparameter("{}_{}".format(name, label), lower=lower_scale, upper=upper_scale, default_value=default, log=log) - hyperparameters.append(hyper) + hyperparameters.append(hyper) return hyperparameters def _get_boundedState(self, cfg, label, fixed_dict): boundedStates = dict() for name in (self.system.controls + self.system.observations): if name in fixed_dict: - limit, upper = self._limits[name] + lower, upper = self._limits[name] scale = self._scale_fixed[name] - boundedStates[name] = (limit, scale, upper) + boundedStates[name] = (lower, upper, scale) elif name in self._limits: - limit, upper = self._limits[name] - upper_string = "Upper" - if(not upper): - upper_string = "Lower" - hyper_name = f"{name}_{upper_string}_{label}" + lower, upper = self._limits[name] + hyper_name = f"{name}_{label}" scale = cfg[hyper_name] - boundedStates[name] = (limit, scale, upper) + boundedStates[name] = (lower, upper, scale) return boundedStates def get_default_config_space(self): cs = CS.ConfigurationSpace() for name in self.system.observations + self.system.controls: - hyper = CSH.UniformFloatHyperparameter(name+"_Upper_LogBarrier", - lower=BARRIER_COST_DEFAULT_BOUNDS[0], upper=BARRIER_COST_DEFAULT_BOUNDS[1], - default_value=BARRIER_COST_DEFAULT_VALUE, log=BARRIER_COST_DEFAULT_LOG - ) - cs.add_hyperparameter(hyper) - hyper = CSH.UniformFloatHyperparameter(name+"_Lower_LogBarrier", + hyper = CSH.UniformFloatHyperparameter(name+"_LogBarrier", lower=BARRIER_COST_DEFAULT_BOUNDS[0], upper=BARRIER_COST_DEFAULT_BOUNDS[1], default_value=BARRIER_COST_DEFAULT_VALUE, log=BARRIER_COST_DEFAULT_LOG ) From d1fd354b848d8599d1067612e658baaa5df20dbc Mon Sep 17 00:00:00 2001 From: Dohun Date: Mon, 8 Aug 2022 22:09:43 -0500 Subject: [PATCH 23/41] Costs fixed to work with barriers --- autompc/costs/barrier_cost.py | 55 +++++++++++++++++++++++------------ autompc/costs/thresh_cost.py | 4 +-- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/autompc/costs/barrier_cost.py b/autompc/costs/barrier_cost.py index 83df236..fd12ab4 100644 --- a/autompc/costs/barrier_cost.py +++ b/autompc/costs/barrier_cost.py @@ -85,8 +85,9 @@ def __mul__(self, rhs): #Cost Function: - # - b * ln ( a - x ) upper limit - # - b * ln ( a + x ) lower limit + # b = scale + # - b * ln ( a - x ) upper limit x < a + # - b * ln ( a + x ) lower limit x > a def eval_obs_cost(self, obs): sum = 0 for boundedObs in self.obsConfiguration: @@ -94,9 +95,15 @@ def eval_obs_cost(self, obs): index = self.system.observations.index(variable) lower, upper, scale = config if lower > -np.inf: - sum = sum + -scale * np.log(-lower + obs[index]) + if lower >= obs[index]: + sum += np.inf + else: + sum = sum + -scale * np.log(-lower + obs[index]) if upper < np.inf: - sum = sum + -scale * np.log(upper - obs[index]) + if obs[index] >= upper: + sum += np.inf + else: + sum = sum + -scale * np.log(upper - obs[index]) return sum #Jacobian: @@ -109,9 +116,15 @@ def eval_obs_cost_diff(self, obs): index = self.system.observations.index(variable) lower, upper, scale = config if lower > -np.inf: - jacobian[index] += -scale / (-lower + obs[index]) + if lower >= obs[index]: + jacobian[index] += -np.inf + else: + jacobian[index] += -scale / (-lower + obs[index]) if upper < np.inf: - jacobian[index] += scale / (upper - obs[index]) + if obs[index] >= upper: + jacobian[index] += np.inf + else: + jacobian[index] += scale / (upper - obs[index]) return jacobian @@ -125,9 +138,15 @@ def eval_obs_cost_hess(self, obs): index = self.system.observations.index(variable) lower, upper, scale = config if lower > -np.inf: - hessian[index][index] += scale / ((lower - obs[index])**2) + if lower >= obs[index]: + hessian[index][index] += np.inf + else: + hessian[index][index] += scale / ((lower - obs[index])**2) if upper < np.inf: - hessian[index][index] += scale / ((upper - obs[index])**2) + if obs[index] >= upper: + hessian[index][index] += np.inf + else: + hessian[index][index] += scale / ((upper - obs[index])**2) return hessian @@ -137,10 +156,10 @@ def eval_ctrl_cost(self, ctrl): variable, config = boundedCtrl index = self.system.controls.index(variable) lower, upper, scale = config - if lower != -np.inf: - sum = sum + -scale * np.log(lower + ctrl[index]) - if upper != np.inf: - sum = sum + -scale * np.log(upper - ctrl[index]) + if lower > -np.inf: + sum = sum + -scale * np.log(-lower + ctrl[index]) + if upper < np.inf: + sum = sum + -scale * np.log(upper - ctrl[index]) return sum def eval_ctrl_cost_diff(self, ctrl): @@ -149,9 +168,9 @@ def eval_ctrl_cost_diff(self, ctrl): variable, config = boundedCtrl index = self.system.controls.index(variable) lower, upper, scale = config - if lower != -np.inf: - jacobian[index] += -scale / (lower + ctrl[index]) - if upper != np.inf: + if lower > -np.inf: + jacobian[index] += -scale / (-lower + ctrl[index]) + if upper < np.inf: jacobian[index] += scale / (upper - ctrl[index]) return jacobian @@ -161,9 +180,9 @@ def eval_ctrl_cost_hess(self, ctrl): variable, config = boundedCtrl index = self.system.controls.index(variable) lower, upper, scale = config - if lower != -np.inf: - hessian[index][index] += scale / ((lower + ctrl[index])**2) - if upper != np.inf: + if lower > -np.inf: + hessian[index][index] += scale / ((lower - ctrl[index])**2) + if upper < np.inf: hessian[index][index] += scale / ((upper - ctrl[index])**2) return hessian diff --git a/autompc/costs/thresh_cost.py b/autompc/costs/thresh_cost.py index 3dcb9f7..9318b47 100644 --- a/autompc/costs/thresh_cost.py +++ b/autompc/costs/thresh_cost.py @@ -45,8 +45,8 @@ def __init__(self, system, goal, threshold, obs_range=None, observations=None): self.set_goal(goal) def incremental(self, obs, ctrl): - if (la.norm(obs[self._obs_idxs] - self.goal[self._obs_idxs], np.inf) - > self._threshold): + max_dist_to_goal = la.norm(obs[self._obs_idxs] - self.goal[self._obs_idxs], np.inf) + if (max_dist_to_goal > self._threshold or np.isnan(max_dist_to_goal)): return 1.0 else: return 0.0 From 9bf63e4bd770b02b785b47bff4a68d198174f660 Mon Sep 17 00:00:00 2001 From: Dohun Date: Mon, 8 Aug 2022 23:06:15 -0500 Subject: [PATCH 24/41] Parallel Control Evaluator --- autompc/tuning/control_evaluator.py | 28 +++++++++++++++++++++++++++- autompc/tuning/control_tuner.py | 4 ++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/autompc/tuning/control_evaluator.py b/autompc/tuning/control_evaluator.py index 11f166f..d9f1b5a 100644 --- a/autompc/tuning/control_evaluator.py +++ b/autompc/tuning/control_evaluator.py @@ -1,6 +1,7 @@ from abc import ABC, abstractmethod from collections import namedtuple import time +import os import numpy as np from ..system import System from ..dynamics import Dynamics @@ -10,7 +11,7 @@ from ..task import Task from ..utils import simulate from typing import Union,List,Tuple,Callable - +from joblib import Parallel, delayed ControlEvaluationTrial = namedtuple("ControlEvaluationTrial", ["policy","task","dynamics","weight", "cost","traj","term_cond","eval_time"]) @@ -119,3 +120,28 @@ def evaluate_for_task(self, controller : Policy, task : Task): res = self.evaluate_for_task_dynamics(controller,task,self.dynamics) print("Resulting cost",res.cost) return res + +class ParallelStandardEvaluator(StandardEvaluator): + def __call__(self, policy : Union[Policy,Controller]) -> List[ControlEvaluationTrial]: + """ + Evaluates policy on all tasks in parallel. Default just runs evaluate_for_task + on all tasks. + + Returns + -------- + trial_info (List[ControlEvaluationTrial]): + A list of trials evaluated. + """ + if callable(self.tasks): + raise ValueError("Can't use a task sampling function in evaluator class {}, must override __call__".format(self.__class__.__name__)) + + results = [] + def f(task): + i, task = task + if hasattr(policy,'set_ocp'): #it's a Controller + policy.set_ocp(task) + policy.reset() + print(f"Evaluating Task {i}") + return self.evaluate_for_task(policy, task) + results = Parallel(n_jobs=os.cpu_count())(delayed(f)(task) for task in enumerate(self.tasks)) + return results diff --git a/autompc/tuning/control_tuner.py b/autompc/tuning/control_tuner.py index d6f8ba2..3be6a3a 100644 --- a/autompc/tuning/control_tuner.py +++ b/autompc/tuning/control_tuner.py @@ -19,7 +19,7 @@ from ..dynamics import Dynamics from .model_tuner import ModelTuner from .model_evaluator import ModelEvaluator -from .control_evaluator import ControlEvaluator, StandardEvaluator, ControlEvaluationTrial, trial_to_json +from .control_evaluator import ControlEvaluator, ParallelStandardEvaluator, StandardEvaluator, ControlEvaluationTrial, trial_to_json from .control_performance_metric import ControlPerformanceMetric,ConfidenceBoundPerformanceMetric from .bootstrap_evaluator import BootstrapSurrogateEvaluator @@ -251,7 +251,7 @@ def _get_tuning_data(self, controller : Controller, task : List[Task], trajs : L print("Skipping surrogate tuning, surrogate is a trained model") print("------------------------------------------------------------------") if control_evaluator is None: - control_evaluator = StandardEvaluator(controller.system, task, surrogate, 'surr_') + control_evaluator = ParallelStandardEvaluator(controller.system, task, surrogate, 'surr_') else: assert not isinstance(control_evaluator,BootstrapSurrogateEvaluator),'Need an evaluator that does not train' else: From 6b2c4ab734f0c22d4a0a2bf70009e0c3b621822c Mon Sep 17 00:00:00 2001 From: Dohun Date: Thu, 11 Aug 2022 19:05:17 -0500 Subject: [PATCH 25/41] BarrierCostTransformer works directly with Configs and not separate dict --- autompc/ocp/barrier_cost_transformer.py | 59 ++++++++++--------------- 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/autompc/ocp/barrier_cost_transformer.py b/autompc/ocp/barrier_cost_transformer.py index a3a066b..410b9cc 100644 --- a/autompc/ocp/barrier_cost_transformer.py +++ b/autompc/ocp/barrier_cost_transformer.py @@ -14,16 +14,12 @@ import ConfigSpace.hyperparameters as CSH import ConfigSpace.conditions as CSC -def construct_default_bounds(): - return (0, 100, 1.0, False) BARRIER_COST_DEFAULT_BOUNDS = (0, 100) BARRIER_COST_DEFAULT_VALUE = 1.0 BARRIER_COST_DEFAULT_LOG = False class LogBarrierCostTransformer(OCPTransformer): def __init__(self, system): - self._scale_bounds = defaultdict(construct_default_bounds) # Key: obsname, Value: (lower, upper, default, log_scale) self._limits = {} # Key: obs/ctrlname, Value: (limit, upper) - self._scale_fixed = {} # Key: obs/ctrlname, Value: limit super().__init__(system, 'LogBarrierCostTransformer') """ boundedState : String @@ -40,7 +36,19 @@ def __init__(self, system): """ def set_limit(self, boundedState, lower=-np.inf, upper=np.inf): if(boundedState in self.system.observations or boundedState in self.system.controls): + # Setting the lower and upper limit self._limits[boundedState] = (lower, upper) + + # Changing the scale hyperparameter from Constant to default UniformFloatHyperparameter + cs = self.get_config_space() + hp = cs.get_hyperparameter(boundedState+'_LogBarrier') + name = hp.name + new_hp = CS.UniformFloatHyperparameter(name=hp.name, + lower=BARRIER_COST_DEFAULT_BOUNDS[0], + upper=BARRIER_COST_DEFAULT_BOUNDS[1], + default_value=BARRIER_COST_DEFAULT_VALUE, + log=BARRIER_COST_DEFAULT_LOG) + cs._hyperparameters[name] = new_hp else: raise ValueError(str(boundedState) + " is not in system") @@ -63,8 +71,9 @@ def set_limit(self, boundedState, lower=-np.inf, upper=np.inf): def set_bounds(self, boundedState, lower_bound, upper_bound, default, log=False): if(boundedState in self.system.observations or boundedState in self.system.controls): if(boundedState in self._limits): - self._scale_bounds[boundedState] = (lower_bound, upper_bound, default, log) - + self.set_hyperparameter_bounds(**{boundedState+"_LogBarrier": (lower_bound,upper_bound)}) + self.set_hyperparameter_defaults(**{boundedState+"_LogBarrier": default}) + self.set_hyperparameter_log(**{boundedState+"_LogBarrier": log}) else: raise ValueError(str(boundedState) + " does not have a configured limit use set_limit") else: @@ -73,45 +82,25 @@ def set_bounds(self, boundedState, lower_bound, upper_bound, default, log=False) def set_fixed_scale(self, boundedState, scale): if(boundedState in self.system.observations or boundedState in self.system.controls): if(boundedState in self._limits): - self._scale_fixed[boundedState] = scale + self.fix_hyperparameters(**{boundedState+"_LogBarrier": scale}) else: raise ValueError(str(boundedState) + " does not have a configured limit use set_limit") else: raise ValueError(str(boundedState) + " is not in system") - def _get_hyperparameters(self, label, bounds_dict, fixed_dict): - hyperparameters = [] - for name in (self.system.controls + self.system.observations): - if name in fixed_dict or name not in self._limits: - continue - lower, upper = self._limits[name] - lower_scale, upper_scale, default, log = bounds_dict[name] - hyper = CSH.UniformFloatHyperparameter("{}_{}".format(name, label), - lower=lower_scale, upper=upper_scale, default_value=default, log=log) - hyperparameters.append(hyper) - return hyperparameters - - def _get_boundedState(self, cfg, label, fixed_dict): + def _get_boundedState(self, cfg, label): boundedStates = dict() - for name in (self.system.controls + self.system.observations): - if name in fixed_dict: - lower, upper = self._limits[name] - scale = self._scale_fixed[name] - boundedStates[name] = (lower, upper, scale) - elif name in self._limits: - lower, upper = self._limits[name] - hyper_name = f"{name}_{label}" - scale = cfg[hyper_name] - boundedStates[name] = (lower, upper, scale) + for name in self._limits.keys(): + lower, upper = self._limits[name] + hyper_name = f"{name}_{label}" + scale = cfg[hyper_name] + boundedStates[name] = (lower, upper, scale) return boundedStates def get_default_config_space(self): cs = CS.ConfigurationSpace() for name in self.system.observations + self.system.controls: - hyper = CSH.UniformFloatHyperparameter(name+"_LogBarrier", - lower=BARRIER_COST_DEFAULT_BOUNDS[0], upper=BARRIER_COST_DEFAULT_BOUNDS[1], - default_value=BARRIER_COST_DEFAULT_VALUE, log=BARRIER_COST_DEFAULT_LOG - ) + hyper = CS.Constant(name+"_LogBarrier", BARRIER_COST_DEFAULT_VALUE) cs.add_hyperparameter(hyper) return cs @@ -125,7 +114,7 @@ def ocp_requirements(self) -> dict: return {} def __call__(self, ocp): - boundedStates = self._get_boundedState(self.get_config(), "LogBarrier", self._scale_fixed) + boundedStates = self._get_boundedState(self.get_config(), "LogBarrier") new_cost = LogBarrierCost(self.system, boundedStates) new_ocp = copy.deepcopy(ocp) new_ocp.set_cost(new_cost) From d89aa649a858666b448f9f0e7eb9ff7697473cf1 Mon Sep 17 00:00:00 2001 From: Dohun Date: Thu, 11 Aug 2022 19:07:09 -0500 Subject: [PATCH 26/41] Added ability to change default_value and log when setting hyper_bounds --- autompc/utils/cs_utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/autompc/utils/cs_utils.py b/autompc/utils/cs_utils.py index e058e05..1c61510 100644 --- a/autompc/utils/cs_utils.py +++ b/autompc/utils/cs_utils.py @@ -162,20 +162,23 @@ def add_configuration_space(self, conditions_to_add.append(condition) self.add_conditions(conditions_to_add) -def set_hyper_bounds(cs, hp_name, lower, upper): +def set_hyper_bounds(cs, hp_name, lower, upper, default_value=None, log=None): hp = cs.get_hyperparameter(hp_name) if not isinstance(hp, NumericalHyperparameter): raise ValueError("Can only call set_hyper_bounds for NumericalHyperparameter") name = hp.name - default_value = hp.default_value + if default_value is None: + default_value = hp.default_value + if log is None: + log = hp.log if not (lower < default_value < upper): default_value = lower if isinstance(hp, UniformFloatHyperparameter): new_hp = CS.UniformFloatHyperparameter(name=name, lower=lower, - upper=upper, default_value=default_value) + upper=upper, default_value=default_value, log=log) if isinstance(hp, UniformIntegerHyperparameter): new_hp = CS.UniformIntegerHyperparameter(name=name, lower=lower, - upper=upper, default_value=default_value) + upper=upper, default_value=default_value, log=log) cs._hyperparameters[name] = new_hp def set_hyper_choices(cs, hp_name, choices): From e339ef22516147c6c9f06b979921656b0d2e1f26 Mon Sep 17 00:00:00 2001 From: Dohun Date: Thu, 11 Aug 2022 22:20:40 -0500 Subject: [PATCH 27/41] change hyperparameter log scale setting --- autompc/tunable.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/autompc/tunable.py b/autompc/tunable.py index cefe052..08ab146 100644 --- a/autompc/tunable.py +++ b/autompc/tunable.py @@ -115,6 +115,14 @@ def set_hyperparameter_defaults(self,**kwargs) -> None: for key,defaults in kwargs.items(): hyperparams[key].default_value = defaults + def set_hyperparameter_log(self,**kwargs) -> None: + """Sets the configuration space log values for a set of + hyperparameters, as keyword arguments. + """ + hyperparams = self._configuration_space.get_hyperparameters_dict() + for key,log in kwargs.items(): + hyperparams[key].log = log + def freeze_hyperparameters(self): """Denotes that this instance should no longer be tunable.""" self._configuration_space = ConfigurationSpace() From aabc895f03ea844a64d020d796f251ef73c153f4 Mon Sep 17 00:00:00 2001 From: Dohun Date: Sun, 14 Aug 2022 01:20:19 -0500 Subject: [PATCH 28/41] max_iter fixed in stop gap solution --- autompc/optim/ilqr.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/autompc/optim/ilqr.py b/autompc/optim/ilqr.py index 387b0c3..0c8b65b 100644 --- a/autompc/optim/ilqr.py +++ b/autompc/optim/ilqr.py @@ -270,8 +270,7 @@ def eval_obj(xs, us): print('ilqr is not converging...') return converged, states, ctrls, Ks''' - def compute_ilqr(self, x0, uguess, u_threshold=1e-3, max_iter=50, - ls_max_iter=10, ls_discount=0.2, ls_cost_threshold=0.3, silent=True): + def compute_ilqr(self, x0, uguess, u_threshold=1e-3, ls_max_iter=10, ls_discount=0.2, ls_cost_threshold=0.3, silent=True): """Use equations from https://medium.com/@jonathan_hui/rl-lqr-ilqr-linear-quadratic-regulator-a5de5104c750 . A better version is https://homes.cs.washington.edu/~todorov/papers/TassaIROS12.pdf Here I do not have Hessian correction since I'm certain all my matrices are SPD @@ -311,7 +310,7 @@ def eval_obj(xs, us): ct = np.zeros(dimx + dimu) converged = False du_norm = np.inf - for itr in range(max_iter): + for itr in range(self.max_iter): if self.verbose: print('At iteration %d' % itr) # compute at the last step, Vn and vn, just hessian and gradient at the last state From c1f8d33e87ab4c6ce2f8f34c00ac0b37c3913940 Mon Sep 17 00:00:00 2001 From: Dohun Date: Sun, 14 Aug 2022 01:31:45 -0500 Subject: [PATCH 29/41] Auto stash before merge of "soft_water_robot_devel" and "origin/soft_water_robot_devel" --- autompc/controller.py | 2 +- autompc/optim/ilqr.py | 59 ++++++++++++++++++++++----------- autompc/sysid/mlp.py | 3 +- autompc/tuning/control_tuner.py | 7 ++-- autompc/utils/simulation.py | 1 + 5 files changed, 49 insertions(+), 23 deletions(-) diff --git a/autompc/controller.py b/autompc/controller.py index 70239cc..1299c67 100644 --- a/autompc/controller.py +++ b/autompc/controller.py @@ -208,7 +208,7 @@ def get_config_space(self): regularizers.append(transformer) elif transformer.name != 'Identity': cost_transformers.append(transformer) - cost_transformers.append(dummy) + #cost_transformers.append(dummy) if len(cost_transformers) > 1: self.set_component("cost_transformer", cost_transformers) diff --git a/autompc/optim/ilqr.py b/autompc/optim/ilqr.py index 0c8b65b..7deee97 100644 --- a/autompc/optim/ilqr.py +++ b/autompc/optim/ilqr.py @@ -91,7 +91,7 @@ def cost_requirements(self): def set_state(self, state): self._guess = copy.deepcopy(state["guess"]) - '''def compute_ilqr(self, x0, uguess, u_threshold=1e-3, + def compute_ilqr(self, x0, uguess, u_threshold=1e-3, ls_max_iter=10, ls_discount=0.2, ls_cost_threshold=0.3): """Use equations from https://medium.com/@jonathan_hui/rl-lqr-ilqr-linear-quadratic-regulator-a5de5104c750 . A better version is https://homes.cs.washington.edu/~todorov/papers/TassaIROS12.pdf @@ -159,13 +159,14 @@ def eval_obj(xs, us): Qux = Qt[dimx:, :dimx] qx = qt[:dimx] qu = qt[dimx:] - print(cost) - print(cost.incremental_hess(states[t - 1, :obsdim],ctrls[t - 1])) - QuuInv = inverse_semidefinite(Quu,1e-2) - K = -QuuInv @ Qux - k = -QuuInv @ qu - # K = -np.linalg.solve(Quu,Qux) - # k = -np.linalg.solve(Quu,qu) + if self.verbose: + print(cost) + print(cost.incremental_hess(states[t - 1, :obsdim],ctrls[t - 1])) + #QuuInv = inverse_semidefinite(Quu,1e-2) + #K = -QuuInv @ Qux + #k = -QuuInv @ qu + K = -np.linalg.solve(Quu,Qux) + k = -np.linalg.solve(Quu,qu) lin_cost_reduce += qu.dot(k) quad_cost_reduce += k @ Quu @ k # update Vn and vn @@ -268,13 +269,13 @@ def eval_obj(xs, us): if self.verbose and not converged : print('ilqr fails to converge, try a new guess? Last u update is %f ks norm is %f' % (du_norm, ks_norm)) print('ilqr is not converging...') - return converged, states, ctrls, Ks''' - - def compute_ilqr(self, x0, uguess, u_threshold=1e-3, ls_max_iter=10, ls_discount=0.2, ls_cost_threshold=0.3, silent=True): + return converged, states, ctrls, Ks + def compute_ilqr_old(self, x0, uguess, u_threshold=1e-3, ls_max_iter=10, ls_discount=0.2, ls_cost_threshold=0.3, silent=True): """Use equations from https://medium.com/@jonathan_hui/rl-lqr-ilqr-linear-quadratic-regulator-a5de5104c750 . A better version is https://homes.cs.washington.edu/~todorov/papers/TassaIROS12.pdf Here I do not have Hessian correction since I'm certain all my matrices are SPD """ + start = time.time() cost = self.ocp.get_cost() H = self.horizon dt = self.system.dt @@ -310,7 +311,12 @@ def eval_obj(xs, us): ct = np.zeros(dimx + dimu) converged = False du_norm = np.inf + backward_duration=0 + rollout_duration=0 + ls_duration=0 + preloop_duration = time.time()-start for itr in range(self.max_iter): + start = time.time() if self.verbose: print('At iteration %d' % itr) # compute at the last step, Vn and vn, just hessian and gradient at the last state @@ -332,13 +338,13 @@ def eval_obj(xs, us): Qt = Ct + Jacs[t - 1].T @ Vn @ Jacs[t - 1] qt = ct + Jacs[t - 1].T @ (vn) # here Vn @ states[t] may be necessary # ready to compute feedback - # Qtinv = np.linalg.inv(Qt) + # Qtinv = inverse_semidefinite(Qt[dimx:, dimx:]) Ks[t - 1] = -np.linalg.solve(Qt[dimx:, dimx:], Qt[dimx:, :dimx]) ks[t - 1] = -np.linalg.solve(Qt[dimx:, dimx:], qt[dimx:]) lin_cost_reduce += qt[dimx:].dot(ks[t - 1]) quad_cost_reduce += ks[t - 1] @ Qt[dimx:, dimx:] @ ks[t - 1] - # Ks[t - 1] = -Qtinv[dimx:, dimx:] @ Qt[dimx:, :dimx] - # ks[t - 1] = -Qtinv[dimx:, dimx:] @ qt[dimx:] + # Ks[t - 1] = -Qtinv @ Qt[dimx:, :dimx] + # ks[t - 1] = -Qtinv @ qt[dimx:] # update Vn and vn Vn = Qt[:dimx, :dimx] + Qt[:dimx, dimx:] @ Ks[t - 1] + Ks[t - 1].T @ Qt[dimx:, :dimx] + Ks[t - 1].T @ Qt[dimx:, dimx:] @ Ks[t - 1] vn = qt[:dimx] + Qt[:dimx, dimx:] @ ks[t - 1] + Ks[t - 1].T @ (qt[dimx:] + Qt[dimx:, dimx:] @ ks[t - 1]) @@ -349,7 +355,9 @@ def eval_obj(xs, us): best_obj_estimate_reduction = None ks_norm = np.linalg.norm(ks) # print('norm of ks = ', np.linalg.norm(ks)) - + backward_duration+=time.time()-start + #print('Backward pass', duration) + start = time.time() # Compute rollout for all possible alphas alphas = np.array([ls_discount**i for i in range(ls_max_iter)]) for i in range(ls_max_iter): @@ -357,10 +365,13 @@ def eval_obj(xs, us): for i in range(H): for j, alpha in enumerate(alphas): ls_ctrls[j, i, :] = alpha * ks[i] + ctrls[i] + Ks[i] @ (ls_states[j, i, :] - states[i, :]) - if self.ubounds is not None: - ls_ctrls[j, i, :] = np.clip(ls_ctrls[j, i, :], self.ubounds[0], self.ubounds[1]) + if self.ubounds is not None: + ls_ctrls[:, i, :] = np.clip(ls_ctrls[:, i, :], self.ubounds[0], self.ubounds[1]) ls_states[:, i + 1, :] = self.model.pred_batch(ls_states[:, i, :], ls_ctrls[:, i, :]) + rollout_duration += time.time()-start + #print('Rollout: ', duration) + start = time.time() # Now do backtrack line search. for lsitr, ls_alpha in enumerate(alphas): new_states = ls_states[lsitr, :, :] @@ -380,6 +391,9 @@ def eval_obj(xs, us): #ls_alpha *= ls_discount if ks_norm < u_threshold: break + ls_duration += time.time()-start + #print('Line search: ', duration) + if self.verbose: print('line search obj %f to %f at alpha = %f' % (obj, new_obj, ls_alpha)) if best_obj < obj or ks_norm < u_threshold: @@ -419,7 +433,7 @@ def eval_obj(xs, us): if not converged and not silent: print('ilqr fails to converge, try a new guess? Last u update is %f ks norm is %f' % (du_norm, ks_norm)) print('ilqr is not converging...') - return converged, states, ctrls, Ks + return converged, states, ctrls, Ks, itr, preloop_duration, backward_duration, rollout_duration, ls_duration # DEBUG, no itr def set_guess(self, guess : Trajectory) -> None: assert len(guess) == self.horizon,"Invalid guess provided" @@ -431,7 +445,14 @@ def step(self, obs): if self._guess is None: self._guess = np.zeros((self.horizon, self.system.ctrl_dim)) if substep == 0: - converged, states, ctrls, Ks = self.compute_ilqr(obs, self._guess) + start = time.time() + #converged, states, ctrls, Ks, itr, preloop_duration, backward_duration, rollout_duration, ls_duration = self.compute_ilqr_old(obs, self._guess) # DEBUG, no itr + converged, states, ctrls, Ks = self.compute_ilqr(obs, self._guess) # DEBUG, no itr + end = time.time() + if (end-start > 0.3): + print("ILQR timeout: ", end-start) + #print('Num iter: ', itr) + #print('Breakdown: ', preloop_duration, backward_duration, rollout_duration, ls_duration, preloop_duration+backward_duration+rollout_duration+ls_duration) self._guess = np.concatenate((ctrls[1:], ctrls[-1:]*0), axis=0) self._traj = Trajectory(self.model.state_system, states, np.vstack([ctrls, np.zeros(self.system.ctrl_dim)])) diff --git a/autompc/sysid/mlp.py b/autompc/sysid/mlp.py index f971ce2..21e826a 100644 --- a/autompc/sysid/mlp.py +++ b/autompc/sysid/mlp.py @@ -102,7 +102,7 @@ class MLP(FullyObservableModel): Type of activation function. - **lr** *(Type: float, Low: 1e-5, High: 1, Default: 1e-3)*: Adam learning rate for the network. """ - def __init__(self, system, n_train_iters=200, n_batch=64, use_cuda=True): + def __init__(self, system, n_train_iters=200, n_batch=64, use_cuda=False): super().__init__(system, "MLP") self.n_train_iters = n_train_iters self.n_batch = n_batch @@ -224,6 +224,7 @@ def train(self, trajs, silent=False, seed=100): t0 = time.time() for i in tqdm(range(self.n_train_iters), file=sys.stdout): self._step_train() + self.train_time_budget=500 if self.train_time_budget is not None and time.time()-t0 > self.train_time_budget: print("Reached timeout of %.2fs"%self.train_time_budget) break diff --git a/autompc/tuning/control_tuner.py b/autompc/tuning/control_tuner.py index 3be6a3a..854386a 100644 --- a/autompc/tuning/control_tuner.py +++ b/autompc/tuning/control_tuner.py @@ -507,6 +507,7 @@ def run(self, controller, tasks, trajs, n_iters, rng, truedyn=None, return eval_cfg smac_rng = np.random.RandomState(seed=rng.integers(1 << 31)) + print(controller.get_config_space()) scenario = Scenario({"run_obj" : "quality", "runcount-limit" : n_iters, "cs" : controller.get_config_space(), @@ -577,7 +578,8 @@ def get_tuning_data(self): def __call__(self, cfg): self.eval_number += 1 - if self.timeout is None: + #if self.timeout is None: + if True: result = self.run(cfg) else: #p = multiprocessing.Process(target=self.run_mp, args=(cfg,)) @@ -651,6 +653,7 @@ def run(self, cfg): performance = performance_metric(trajs) info["surr_cost"] = performance info["surr_info"] = trajs + print('info') # DEBUG if not truedyn_evaluator is None: trajs = truedyn_evaluator(controller) performance = performance_metric(trajs) @@ -665,5 +668,5 @@ def run(self, cfg): # if not self.log_file_name is None: # sys.stdout.close() # sys.stderr.close() - + print('returning') # DEBUG return info diff --git a/autompc/utils/simulation.py b/autompc/utils/simulation.py index 2c22c72..a391744 100644 --- a/autompc/utils/simulation.py +++ b/autompc/utils/simulation.py @@ -10,6 +10,7 @@ # External library includes import numpy as np from tqdm import tqdm +from time import time def rollout(dynamics : Dynamics, traj : Trajectory, start : int = 0, horizon : int = None) -> Trajectory: """ From 49301f127765ac1c15121da09b6517dc6d47cf91 Mon Sep 17 00:00:00 2001 From: Dohun Date: Tue, 16 Aug 2022 09:03:50 -0500 Subject: [PATCH 30/41] ILQR faster again, saves step duration simulation --- autompc/optim/ilqr.py | 20 +++++--------------- autompc/utils/simulation.py | 6 +++++- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/autompc/optim/ilqr.py b/autompc/optim/ilqr.py index 7deee97..6c73ad6 100644 --- a/autompc/optim/ilqr.py +++ b/autompc/optim/ilqr.py @@ -91,7 +91,7 @@ def cost_requirements(self): def set_state(self, state): self._guess = copy.deepcopy(state["guess"]) - def compute_ilqr(self, x0, uguess, u_threshold=1e-3, + '''def compute_ilqr(self, x0, uguess, u_threshold=1e-3, ls_max_iter=10, ls_discount=0.2, ls_cost_threshold=0.3): """Use equations from https://medium.com/@jonathan_hui/rl-lqr-ilqr-linear-quadratic-regulator-a5de5104c750 . A better version is https://homes.cs.washington.edu/~todorov/papers/TassaIROS12.pdf @@ -270,12 +270,13 @@ def eval_obj(xs, us): print('ilqr fails to converge, try a new guess? Last u update is %f ks norm is %f' % (du_norm, ks_norm)) print('ilqr is not converging...') return converged, states, ctrls, Ks - def compute_ilqr_old(self, x0, uguess, u_threshold=1e-3, ls_max_iter=10, ls_discount=0.2, ls_cost_threshold=0.3, silent=True): + ''' + + def compute_ilqr(self, x0, uguess, u_threshold=1e-3, ls_max_iter=10, ls_discount=0.2, ls_cost_threshold=0.3, silent=True): """Use equations from https://medium.com/@jonathan_hui/rl-lqr-ilqr-linear-quadratic-regulator-a5de5104c750 . A better version is https://homes.cs.washington.edu/~todorov/papers/TassaIROS12.pdf Here I do not have Hessian correction since I'm certain all my matrices are SPD """ - start = time.time() cost = self.ocp.get_cost() H = self.horizon dt = self.system.dt @@ -311,12 +312,7 @@ def eval_obj(xs, us): ct = np.zeros(dimx + dimu) converged = False du_norm = np.inf - backward_duration=0 - rollout_duration=0 - ls_duration=0 - preloop_duration = time.time()-start for itr in range(self.max_iter): - start = time.time() if self.verbose: print('At iteration %d' % itr) # compute at the last step, Vn and vn, just hessian and gradient at the last state @@ -355,9 +351,7 @@ def eval_obj(xs, us): best_obj_estimate_reduction = None ks_norm = np.linalg.norm(ks) # print('norm of ks = ', np.linalg.norm(ks)) - backward_duration+=time.time()-start #print('Backward pass', duration) - start = time.time() # Compute rollout for all possible alphas alphas = np.array([ls_discount**i for i in range(ls_max_iter)]) for i in range(ls_max_iter): @@ -368,10 +362,8 @@ def eval_obj(xs, us): if self.ubounds is not None: ls_ctrls[:, i, :] = np.clip(ls_ctrls[:, i, :], self.ubounds[0], self.ubounds[1]) ls_states[:, i + 1, :] = self.model.pred_batch(ls_states[:, i, :], ls_ctrls[:, i, :]) - rollout_duration += time.time()-start #print('Rollout: ', duration) - start = time.time() # Now do backtrack line search. for lsitr, ls_alpha in enumerate(alphas): new_states = ls_states[lsitr, :, :] @@ -391,7 +383,6 @@ def eval_obj(xs, us): #ls_alpha *= ls_discount if ks_norm < u_threshold: break - ls_duration += time.time()-start #print('Line search: ', duration) if self.verbose: @@ -433,7 +424,7 @@ def eval_obj(xs, us): if not converged and not silent: print('ilqr fails to converge, try a new guess? Last u update is %f ks norm is %f' % (du_norm, ks_norm)) print('ilqr is not converging...') - return converged, states, ctrls, Ks, itr, preloop_duration, backward_duration, rollout_duration, ls_duration # DEBUG, no itr + return converged, states, ctrls, Ks def set_guess(self, guess : Trajectory) -> None: assert len(guess) == self.horizon,"Invalid guess provided" @@ -446,7 +437,6 @@ def step(self, obs): self._guess = np.zeros((self.horizon, self.system.ctrl_dim)) if substep == 0: start = time.time() - #converged, states, ctrls, Ks, itr, preloop_duration, backward_duration, rollout_duration, ls_duration = self.compute_ilqr_old(obs, self._guess) # DEBUG, no itr converged, states, ctrls, Ks = self.compute_ilqr(obs, self._guess) # DEBUG, no itr end = time.time() if (end-start > 0.3): diff --git a/autompc/utils/simulation.py b/autompc/utils/simulation.py index a391744..1ba18f9 100644 --- a/autompc/utils/simulation.py +++ b/autompc/utils/simulation.py @@ -92,6 +92,7 @@ def simulate(policy : Policy, init_obs, dynamics : Dynamics, term_cond=None, max raise ValueError("Trying to do simulate() on an untrained dynamics model?") x = init_obs traj = DynamicTrajectory(dynamics.system) + step_times = [] simstate = dynamics.init_state(init_obs) if silent: @@ -99,8 +100,11 @@ def simulate(policy : Policy, init_obs, dynamics : Dynamics, term_cond=None, max else: itr = tqdm(range(max_steps), file=sys.stdout) for _ in itr: + start = time() u = policy.step(x) + end = time() traj.append(x,u) + step_times.append(end-start) if term_cond is not None and term_cond(traj): break if len(traj) == max_steps: @@ -111,4 +115,4 @@ def simulate(policy : Policy, init_obs, dynamics : Dynamics, term_cond=None, max simstate = simstate2 x = dynamics.get_obs(simstate) #finalize Trajectory - return traj.freeze() + return traj.freeze(), step_times From 541043213257d23abb1416b985edcb0e694b3e92 Mon Sep 17 00:00:00 2001 From: dohunnim Date: Tue, 16 Aug 2022 11:57:49 -0500 Subject: [PATCH 31/41] Fix conversion of ControlEvaluationTrial to dict --- autompc/tuning/control_evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autompc/tuning/control_evaluator.py b/autompc/tuning/control_evaluator.py index d9f1b5a..0e3d03c 100644 --- a/autompc/tuning/control_evaluator.py +++ b/autompc/tuning/control_evaluator.py @@ -16,7 +16,7 @@ "cost","traj","term_cond","eval_time"]) def trial_to_json(trial : ControlEvaluationTrial): - res = dict() + res = trial._asdict() res['policy'] = str(trial.policy) res['task'] = str(trial.task) res['dynamics'] = str(trial.dynamics) From 93595f92b999a561695ca50a0f48177f6afe4ace Mon Sep 17 00:00:00 2001 From: Dohun Date: Tue, 16 Aug 2022 13:22:08 -0500 Subject: [PATCH 32/41] Merge of "soft_water_robot_devel" and "origin/soft_water_robot_devel" --- autompc/optim/ilqr.py | 9 +- autompc/sysid/mlp.py | 4 +- autompc/tunable.py | 4 +- autompc/tuning/control_evaluator.py | 8 +- autompc/tuning/control_performance_metric.py | 14 + autompc/tuning/control_tuner.py | 12 +- autompc/tuning/model_tuner.py | 10 +- examples/0_MainDemo.ipynb | 174 ++++++++- examples/6_Controller_Tuning.ipynb | 352 +++++++++++-------- 9 files changed, 402 insertions(+), 185 deletions(-) diff --git a/autompc/optim/ilqr.py b/autompc/optim/ilqr.py index 6c73ad6..99e9a74 100644 --- a/autompc/optim/ilqr.py +++ b/autompc/optim/ilqr.py @@ -91,8 +91,8 @@ def cost_requirements(self): def set_state(self, state): self._guess = copy.deepcopy(state["guess"]) - '''def compute_ilqr(self, x0, uguess, u_threshold=1e-3, - ls_max_iter=10, ls_discount=0.2, ls_cost_threshold=0.3): + def compute_ilqr(self, x0, uguess, u_threshold=1e-3, + ls_max_iter=10, ls_discount=0.2, ls_cost_threshold=0.3, timeout=np.inf): """Use equations from https://medium.com/@jonathan_hui/rl-lqr-ilqr-linear-quadratic-regulator-a5de5104c750 . A better version is https://homes.cs.washington.edu/~todorov/papers/TassaIROS12.pdf """ @@ -134,6 +134,7 @@ def eval_obj(xs, us): converged = False du_norm = np.inf ls_alpha = 1.0 + start = time.time() for itr in range(self.max_iter): if self.verbose: print('At iteration %d' % itr) @@ -265,6 +266,8 @@ def eval_obj(xs, us): print('Cost update from %f to %f' % (initcost, obj)) print('Final state is ', states[-1]) break + if time.time()-start > timeout: + break if self.verbose and not converged : print('ilqr fails to converge, try a new guess? Last u update is %f ks norm is %f' % (du_norm, ks_norm)) @@ -447,7 +450,7 @@ def step(self, obs): self._traj = Trajectory(self.model.state_system, states, np.vstack([ctrls, np.zeros(self.system.ctrl_dim)])) self._gains = Ks - return np.clip(ctrls[0], self.ubounds[0], self.ubounds[1]) + return ctrls[0] else: #just reuse prior strajectory and gains return np.clip(self._traj.ctrls[substep]+self._gains[substep]@(obs-self._traj.obs[substep]), diff --git a/autompc/sysid/mlp.py b/autompc/sysid/mlp.py index 21e826a..8dec1c5 100644 --- a/autompc/sysid/mlp.py +++ b/autompc/sysid/mlp.py @@ -28,7 +28,7 @@ def transform_output(xu_means, xu_std, XU): return np.vstack(XUt).T class ForwardNet(torch.nn.Module): - def __init__(self, n_in, n_out, hidden_sizes, nonlintype, batchnorm=False): + def __init__(self, n_in, n_out, hidden_sizes, nonlintype, batchnorm=True): """Specify the feedforward neuro network size and nonlinearity""" assert len(hidden_sizes) > 0 torch.nn.Module.__init__(self) @@ -224,7 +224,7 @@ def train(self, trajs, silent=False, seed=100): t0 = time.time() for i in tqdm(range(self.n_train_iters), file=sys.stdout): self._step_train() - self.train_time_budget=500 + self.train_time_budget=500 #DEBUG if self.train_time_budget is not None and time.time()-t0 > self.train_time_budget: print("Reached timeout of %.2fs"%self.train_time_budget) break diff --git a/autompc/tunable.py b/autompc/tunable.py index 08ab146..2f4b6a2 100644 --- a/autompc/tunable.py +++ b/autompc/tunable.py @@ -77,6 +77,7 @@ def fix_hyperparameters(self,**kwargs) -> None: AND in the current configuration. Hyperparameters are given as keyword arguments. """ + config = self.get_config() for key,value in kwargs.items(): hyperparam = self._configuration_space.get_hyperparameters_dict()[key] if isinstance(hyperparam,CSH.CategoricalHyperparameter): @@ -84,9 +85,6 @@ def fix_hyperparameters(self,**kwargs) -> None: else: self._configuration_space._hyperparameters[key] = CSH.Constant(key, value) hyperparam.default_value = value - - config = self.get_config() - for (key,value) in kwargs.items(): config[key] = value self.set_config(config) diff --git a/autompc/tuning/control_evaluator.py b/autompc/tuning/control_evaluator.py index 0e3d03c..b2f7b7c 100644 --- a/autompc/tuning/control_evaluator.py +++ b/autompc/tuning/control_evaluator.py @@ -122,6 +122,12 @@ def evaluate_for_task(self, controller : Policy, task : Task): return res class ParallelStandardEvaluator(StandardEvaluator): + def __init__(self, system, tasks, dynamics, prefix='',max_jobs=None): + super().__init__(system, tasks, dynamics, prefix) + if max_jobs is None: + self.max_jobs = len(tasks) + else: + self.max_jobs = max_jobs def __call__(self, policy : Union[Policy,Controller]) -> List[ControlEvaluationTrial]: """ Evaluates policy on all tasks in parallel. Default just runs evaluate_for_task @@ -143,5 +149,5 @@ def f(task): policy.reset() print(f"Evaluating Task {i}") return self.evaluate_for_task(policy, task) - results = Parallel(n_jobs=os.cpu_count())(delayed(f)(task) for task in enumerate(self.tasks)) + results = Parallel(n_jobs=self.max_jobs)(delayed(f)(task) for task in enumerate(self.tasks)) return results diff --git a/autompc/tuning/control_performance_metric.py b/autompc/tuning/control_performance_metric.py index 0fba244..c208b47 100644 --- a/autompc/tuning/control_performance_metric.py +++ b/autompc/tuning/control_performance_metric.py @@ -10,6 +10,20 @@ class ControlPerformanceMetric: def __call__(self,trials : List[ControlEvaluationTrial]) -> float: return np.mean([t.cost for t in trials]) +class DistToGoal(ControlPerformanceMetric): + def __call__(self,trials : List[ControlEvaluationTrial]) -> float: + dists = [] + for trial in trials: + task = trial.task + obs_idxs = task.get_cost()._obs_idxs + dist = 0 + for t in range(len(trial.traj)): + dist += np.linalg.norm(np.array(trial.traj[t].obs[obs_idxs]) - trial.task.get_cost().goal[obs_idxs]) + dists.append(dist) + performance_metric = np.sum(dists) + print("DistToGoal: ", performance_metric) + return performance_metric + class ConfidenceBoundPerformanceMetric(ControlPerformanceMetric): """A performance metric that uses a quantile of some statistical distribution, and also allows incorporating evaluation time and diff --git a/autompc/tuning/control_tuner.py b/autompc/tuning/control_tuner.py index 854386a..a1d3712 100644 --- a/autompc/tuning/control_tuner.py +++ b/autompc/tuning/control_tuner.py @@ -22,6 +22,7 @@ from .control_evaluator import ControlEvaluator, ParallelStandardEvaluator, StandardEvaluator, ControlEvaluationTrial, trial_to_json from .control_performance_metric import ControlPerformanceMetric,ConfidenceBoundPerformanceMetric from .bootstrap_evaluator import BootstrapSurrogateEvaluator +from .parallel_evaluator import ParallelEvaluator # External library includes import numpy as np @@ -251,6 +252,7 @@ def _get_tuning_data(self, controller : Controller, task : List[Task], trajs : L print("Skipping surrogate tuning, surrogate is a trained model") print("------------------------------------------------------------------") if control_evaluator is None: + #control_evaluator = ParallelEvaluator(StandardEvaluator(controller.system, task, surrogate, 'surr_'), dynamics=surrogate, max_jobs=5) control_evaluator = ParallelStandardEvaluator(controller.system, task, surrogate, 'surr_') else: assert not isinstance(control_evaluator,BootstrapSurrogateEvaluator),'Need an evaluator that does not train' @@ -505,7 +507,6 @@ def run(self, controller, tasks, trajs, n_iters, rng, truedyn=None, if debug_return_evaluator: return eval_cfg - smac_rng = np.random.RandomState(seed=rng.integers(1 << 31)) print(controller.get_config_space()) scenario = Scenario({"run_obj" : "quality", @@ -578,8 +579,7 @@ def get_tuning_data(self): def __call__(self, cfg): self.eval_number += 1 - #if self.timeout is None: - if True: + if True: # DEBUG self.timeout is None result = self.run(cfg) else: #p = multiprocessing.Process(target=self.run_mp, args=(cfg,)) @@ -649,11 +649,11 @@ def run(self, cfg): controller.set_config(cfg) controller.build(sysid_trajs) + start = time.time() trajs = control_evaluator(controller) - performance = performance_metric(trajs) - info["surr_cost"] = performance info["surr_info"] = trajs - print('info') # DEBUG + info["surr_cost"] = performance + performance = performance_metric(trajs) if not truedyn_evaluator is None: trajs = truedyn_evaluator(controller) performance = performance_metric(trajs) diff --git a/autompc/tuning/model_tuner.py b/autompc/tuning/model_tuner.py index 1b5d43d..759ebe4 100644 --- a/autompc/tuning/model_tuner.py +++ b/autompc/tuning/model_tuner.py @@ -60,7 +60,7 @@ class ModelTuner: def __init__(self, system : System, trajs : List[Trajectory], model : Optional[Model] = None, eval_holdout=0.25, eval_folds=3, eval_metric="rmse", eval_horizon=1, eval_quantile=None, evaluator : Optional[ModelEvaluator] = None, - multi_fidelity=True, verbose=0): + multi_fidelity=False, verbose=0): """ Parameters ---------- @@ -178,10 +178,10 @@ def run(self, rng=None, n_iters=10, min_train_time=None, max_train_time=None, else: if max_train_time is not None: self.model.set_train_budget(max_train_time) - #smac = SMAC4HPO(scenario=scenario, rng=smac_rng, - # tae_runner=self._evaluate) - smac = SMAC4AC(scenario=scenario, rng=smac_rng, - tae_runner=self._evaluate) + smac = SMAC4HPO(scenario=scenario, rng=smac_rng, + tae_runner=self._evaluate) + # smac = SMAC4AC(scenario=scenario, rng=smac_rng, + # tae_runner=self._evaluate) incumbent = smac.optimize() diff --git a/examples/0_MainDemo.ipynb b/examples/0_MainDemo.ipynb index b07bfd8..c6e3fd1 100644 --- a/examples/0_MainDemo.ipynb +++ b/examples/0_MainDemo.ipynb @@ -26,9 +26,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading AutoMPC...\n", + "Finished loading AutoMPC\n" + ] + } + ], "source": [ "import autompc as ampc\n", "import numpy as np\n", @@ -45,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -63,7 +72,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -81,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -98,7 +107,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -119,7 +128,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -136,11 +145,132 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Forbidding model MLP to be used with LQR due to property is_linear\n", + "Forbidding model SINDy to be used with LQR due to property is_linear\n", + "Checking compatibility with OCP\n", + "Requiring cost transformer for ocp to be used with IterativeLQR due to property is_twice_diff\n", + "Requiring cost transformer for ocp to be used with LQR due to property is_quad\n", + "cost_transformer _ optimizer LQR\n", + "cost_transformer _ optimizer IterativeLQR\n", + "model MLP optimizer LQR\n", + "model SINDy optimizer LQR\n", + "Configuration space object:\n", + " Hyperparameters:\n", + " ARX:history, Type: UniformInteger, Range: [1, 10], Default: 4\n", + " DirectTranscription:horizon, Type: UniformInteger, Range: [1, 30], Default: 10\n", + " GaussRegTransformer:reg_weight, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " IterativeLQR:frequency, Type: UniformInteger, Range: [1, 5], Default: 1\n", + " IterativeLQR:horizon, Type: UniformInteger, Range: [5, 25], Default: 20\n", + " IterativeLQR:max_iter, Type: UniformInteger, Range: [10, 50], Default: 20\n", + " Koopman:lasso_alpha, Type: UniformFloat, Range: [1e-10, 100.0], Default: 1.0, on log-scale\n", + " Koopman:method, Type: Categorical, Choices: {lstsq, lasso, stable}, Default: lstsq\n", + " Koopman:poly_basis, Type: Categorical, Choices: {true, false}, Default: false\n", + " Koopman:poly_cross_terms, Type: Categorical, Choices: {false}, Default: false\n", + " Koopman:poly_degree, Type: UniformInteger, Range: [2, 8], Default: 3\n", + " Koopman:trig_basis, Type: Categorical, Choices: {true, false}, Default: false\n", + " Koopman:trig_freq, Type: UniformInteger, Range: [1, 8], Default: 1\n", + " Koopman:trig_interaction, Type: Categorical, Choices: {false}, Default: false\n", + " LQR:finite_horizon, Type: Categorical, Choices: {true, false}, Default: true\n", + " LQR:horizon, Type: UniformInteger, Range: [1, 1000], Default: 10\n", + " MLP:batchnorm, Type: Categorical, Choices: {False, True}, Default: False\n", + " MLP:hidden_size_1, Type: UniformInteger, Range: [16, 256], Default: 128\n", + " MLP:hidden_size_2, Type: UniformInteger, Range: [16, 256], Default: 128\n", + " MLP:hidden_size_3, Type: UniformInteger, Range: [16, 256], Default: 128\n", + " MLP:hidden_size_4, Type: UniformInteger, Range: [16, 256], Default: 128\n", + " MLP:lr, Type: UniformFloat, Range: [1e-05, 1.0], Default: 0.001, on log-scale\n", + " MLP:n_hidden_layers, Type: Categorical, Choices: {1, 2, 3, 4}, Default: 2\n", + " MLP:nonlintype, Type: Categorical, Choices: {relu, tanh, sigmoid, selu}, Default: relu\n", + " MPPI:horizon, Type: UniformInteger, Range: [5, 30], Default: 20\n", + " MPPI:lmda, Type: UniformFloat, Range: [0.1, 2.0], Default: 1.0\n", + " MPPI:num_path, Type: UniformInteger, Range: [100, 1000], Default: 200\n", + " MPPI:sigma, Type: UniformFloat, Range: [0.0001, 2.0], Default: 1.0\n", + " QuadCostTransformer:dx_F, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " QuadCostTransformer:dx_Q, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " QuadCostTransformer:omega_F, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " QuadCostTransformer:omega_Q, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " QuadCostTransformer:theta_F, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " QuadCostTransformer:theta_Q, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " QuadCostTransformer:u_R, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " QuadCostTransformer:x_F, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " QuadCostTransformer:x_Q, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " SINDy:poly_basis, Type: Categorical, Choices: {true, false}, Default: false\n", + " SINDy:poly_cross_terms, Type: Categorical, Choices: {false}, Default: false\n", + " SINDy:poly_degree, Type: UniformInteger, Range: [2, 8], Default: 3\n", + " SINDy:threshold, Type: UniformFloat, Range: [1e-05, 10.0], Default: 0.01, on log-scale\n", + " SINDy:time_mode, Type: Categorical, Choices: {discrete, continuous}, Default: discrete\n", + " SINDy:trig_basis, Type: Categorical, Choices: {true, false}, Default: false\n", + " SINDy:trig_freq, Type: UniformInteger, Range: [1, 8], Default: 1\n", + " SINDy:trig_interaction, Type: Categorical, Choices: {false}, Default: false\n", + " constraint_transformer, Type: Categorical, Choices: {_}, Default: _\n", + " cost_transformer, Type: Categorical, Choices: {QuadCostTransformer, _}, Default: QuadCostTransformer\n", + " model, Type: Categorical, Choices: {MLP, SINDy, ARX, Koopman}, Default: MLP\n", + " optimizer, Type: Categorical, Choices: {IterativeLQR, MPPI, LQR, DirectTranscription}, Default: IterativeLQR\n", + " regularizer, Type: Categorical, Choices: {_, GaussRegTransformer}, Default: _\n", + " Conditions:\n", + " ARX:history | model == 'ARX'\n", + " DirectTranscription:horizon | optimizer == 'DirectTranscription'\n", + " GaussRegTransformer:reg_weight | regularizer == 'GaussRegTransformer'\n", + " IterativeLQR:frequency | optimizer == 'IterativeLQR'\n", + " IterativeLQR:horizon | optimizer == 'IterativeLQR'\n", + " IterativeLQR:max_iter | optimizer == 'IterativeLQR'\n", + " Koopman:lasso_alpha | Koopman:method in {'lasso'}\n", + " Koopman:method | model == 'Koopman'\n", + " Koopman:poly_basis | model == 'Koopman'\n", + " Koopman:poly_cross_terms | Koopman:poly_basis in {'true'}\n", + " Koopman:poly_degree | Koopman:poly_basis in {'true'}\n", + " Koopman:trig_basis | model == 'Koopman'\n", + " Koopman:trig_freq | Koopman:trig_basis in {'true'}\n", + " Koopman:trig_interaction | Koopman:trig_basis in {'true'}\n", + " LQR:finite_horizon | optimizer == 'LQR'\n", + " LQR:horizon | LQR:finite_horizon in {'true'}\n", + " MLP:batchnorm | model == 'MLP'\n", + " MLP:hidden_size_1 | model == 'MLP'\n", + " MLP:hidden_size_2 | MLP:n_hidden_layers in {'2', '3', '4'}\n", + " MLP:hidden_size_3 | MLP:n_hidden_layers in {'3', '4'}\n", + " MLP:hidden_size_4 | MLP:n_hidden_layers in {'4'}\n", + " MLP:lr | model == 'MLP'\n", + " MLP:n_hidden_layers | model == 'MLP'\n", + " MLP:nonlintype | model == 'MLP'\n", + " MPPI:horizon | optimizer == 'MPPI'\n", + " MPPI:lmda | optimizer == 'MPPI'\n", + " MPPI:num_path | optimizer == 'MPPI'\n", + " MPPI:sigma | optimizer == 'MPPI'\n", + " QuadCostTransformer:dx_F | cost_transformer == 'QuadCostTransformer'\n", + " QuadCostTransformer:dx_Q | cost_transformer == 'QuadCostTransformer'\n", + " QuadCostTransformer:omega_F | cost_transformer == 'QuadCostTransformer'\n", + " QuadCostTransformer:omega_Q | cost_transformer == 'QuadCostTransformer'\n", + " QuadCostTransformer:theta_F | cost_transformer == 'QuadCostTransformer'\n", + " QuadCostTransformer:theta_Q | cost_transformer == 'QuadCostTransformer'\n", + " QuadCostTransformer:u_R | cost_transformer == 'QuadCostTransformer'\n", + " QuadCostTransformer:x_F | cost_transformer == 'QuadCostTransformer'\n", + " QuadCostTransformer:x_Q | cost_transformer == 'QuadCostTransformer'\n", + " SINDy:poly_basis | model == 'SINDy'\n", + " SINDy:poly_cross_terms | model == 'SINDy'\n", + " SINDy:poly_degree | SINDy:poly_basis in {'true'}\n", + " SINDy:threshold | model == 'SINDy'\n", + " SINDy:time_mode | model == 'SINDy'\n", + " SINDy:trig_basis | model == 'SINDy'\n", + " SINDy:trig_freq | SINDy:trig_basis in {'true'}\n", + " SINDy:trig_interaction | SINDy:trig_basis in {'true'}\n", + " Forbidden Clauses:\n", + " (Forbidden: cost_transformer == '_' && Forbidden: optimizer == 'LQR')\n", + " (Forbidden: cost_transformer == '_' && Forbidden: optimizer == 'IterativeLQR')\n", + " (Forbidden: model == 'MLP' && Forbidden: optimizer == 'LQR')\n", + " (Forbidden: model == 'SINDy' && Forbidden: optimizer == 'LQR')\n", + "\n" + ] + } + ], "source": [ "controller.set_ocp(benchmark.task.get_ocp())\n", + "#print(controller._hyperparameters)\n", "print(controller.get_config_space())" ] }, @@ -164,7 +294,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -187,13 +317,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "ename": "SyntaxError", + "evalue": "invalid syntax (2055630436.py, line 1)", + "output_type": "error", + "traceback": [ + "\u001b[0;36m Input \u001b[0;32mIn [10]\u001b[0;36m\u001b[0m\n\u001b[0;31m tuned_controller, tune_result = tuner.run(controller, task, trajs, n_iters=2, surrogate_tune_iters=2 rng=np.random.default_rng(100),\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" + ] + } + ], "source": [ - "tuned_controller, tune_result = tuner.run(controller, task, trajs, n_iters=100, rng=np.random.default_rng(100), \n", + "tuned_controller, tune_result = tuner.run(controller, task, trajs, n_iters=2, surrogate_tune_iters=2, rng=np.random.default_rng(100), \n", " truedyn=benchmark.dynamics)\n", " #restore_dir=\"autompc-output_2022-02-10T17:10:43\")" ] @@ -310,7 +449,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.9.12 ('base')", "language": "python", "name": "python3" }, @@ -324,7 +463,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.9.12" }, "latex_envs": { "LaTeX_envs_menu_present": true, @@ -356,6 +495,11 @@ "toc_position": {}, "toc_section_display": true, "toc_window_display": false + }, + "vscode": { + "interpreter": { + "hash": "8205f83729152a74c9afc694ae4592a9e3038e45d9cbfebf1ba348127e2980f0" + } } }, "nbformat": 4, diff --git a/examples/6_Controller_Tuning.ipynb b/examples/6_Controller_Tuning.ipynb index 00b4258..946d25f 100644 --- a/examples/6_Controller_Tuning.ipynb +++ b/examples/6_Controller_Tuning.ipynb @@ -108,30 +108,18 @@ "cell_type": "code", "execution_count": 5, "metadata": {}, - "outputs": [ - { - "ename": "ImportError", - "evalue": "cannot import name 'QuadCostFactory' from 'autompc.ocp' (/home/dohun/Repos/autompc/autompc/ocp/__init__.py)", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m/home/dohun/Repos/autompc/examples/6_Controller_Tuning.ipynb Cell 10'\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mautompc\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39msysid\u001b[39;00m \u001b[39mimport\u001b[39;00m MLP, SINDy\n\u001b[1;32m 2\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mautompc\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39moptim\u001b[39;00m \u001b[39mimport\u001b[39;00m IterativeLQR, MPPI\n\u001b[0;32m----> 3\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mautompc\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mocp\u001b[39;00m \u001b[39mimport\u001b[39;00m QuadCostFactory\n\u001b[1;32m 5\u001b[0m controller \u001b[39m=\u001b[39m ampc\u001b[39m.\u001b[39mController(system)\n\u001b[1;32m 6\u001b[0m controller\u001b[39m.\u001b[39madd_model(MLP(system))\n", - "\u001b[0;31mImportError\u001b[0m: cannot import name 'QuadCostFactory' from 'autompc.ocp' (/home/dohun/Repos/autompc/autompc/ocp/__init__.py)" - ] - } - ], + "outputs": [], "source": [ "from autompc.sysid import MLP, SINDy\n", "from autompc.optim import IterativeLQR, MPPI\n", - "from autompc.ocp import QuadCostFactory\n", + "from autompc.ocp import QuadCostTransformer\n", "\n", "controller = ampc.Controller(system)\n", - "controller.add_model(MLP(system))\n", - "controller.add_model(SINDy(system))\n", - "controller.add_optimizer(IterativeLQR(system))\n", - "controller.add_optimizer(MPPI(system))\n", - "controller.add_ocp_transformer(QuadCostFactory(system))" + "controller.set_model(MLP(system))\n", + "#controller.add_model(SINDy(system))\n", + "controller.set_optimizer(IterativeLQR(system))\n", + "#controller.add_optimizer(MPPI(system))\n", + "controller.set_ocp_transformer(QuadCostTransformer(system))" ] }, { @@ -147,15 +135,60 @@ "metadata": {}, "outputs": [ { - "ename": "NameError", - "evalue": "name 'controller' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m/home/dohun/Repos/autompc/examples/6_Controller_Tuning.ipynb Cell 12'\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m controller\u001b[39m.\u001b[39mget_config_space()\n", - "\u001b[0;31mNameError\u001b[0m: name 'controller' is not defined" - ] + "data": { + "text/plain": [ + "Configuration space object:\n", + " Hyperparameters:\n", + " IterativeLQR:frequency, Type: UniformInteger, Range: [1, 5], Default: 1\n", + " IterativeLQR:horizon, Type: UniformInteger, Range: [5, 25], Default: 20\n", + " IterativeLQR:max_iter, Type: UniformInteger, Range: [10, 50], Default: 20\n", + " MLP:batchnorm, Type: Categorical, Choices: {False, True}, Default: False\n", + " MLP:hidden_size_1, Type: UniformInteger, Range: [16, 256], Default: 128\n", + " MLP:hidden_size_2, Type: UniformInteger, Range: [16, 256], Default: 128\n", + " MLP:hidden_size_3, Type: UniformInteger, Range: [16, 256], Default: 128\n", + " MLP:hidden_size_4, Type: UniformInteger, Range: [16, 256], Default: 128\n", + " MLP:lr, Type: UniformFloat, Range: [1e-05, 1.0], Default: 0.001, on log-scale\n", + " MLP:n_hidden_layers, Type: Categorical, Choices: {1, 2, 3, 4}, Default: 2\n", + " MLP:nonlintype, Type: Categorical, Choices: {relu, tanh, sigmoid, selu}, Default: relu\n", + " QuadCostTransformer:dx_F, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " QuadCostTransformer:dx_Q, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " QuadCostTransformer:omega_F, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " QuadCostTransformer:omega_Q, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " QuadCostTransformer:theta_F, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " QuadCostTransformer:theta_Q, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " QuadCostTransformer:u_R, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " QuadCostTransformer:x_F, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " QuadCostTransformer:x_Q, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale\n", + " constraint_transformer, Type: Categorical, Choices: {_}, Default: _\n", + " cost_transformer, Type: Categorical, Choices: {QuadCostTransformer, _}, Default: QuadCostTransformer\n", + " model, Type: Categorical, Choices: {MLP}, Default: MLP\n", + " optimizer, Type: Categorical, Choices: {IterativeLQR}, Default: IterativeLQR\n", + " Conditions:\n", + " IterativeLQR:frequency | optimizer == 'IterativeLQR'\n", + " IterativeLQR:horizon | optimizer == 'IterativeLQR'\n", + " IterativeLQR:max_iter | optimizer == 'IterativeLQR'\n", + " MLP:batchnorm | model == 'MLP'\n", + " MLP:hidden_size_1 | model == 'MLP'\n", + " MLP:hidden_size_2 | MLP:n_hidden_layers in {'2', '3', '4'}\n", + " MLP:hidden_size_3 | MLP:n_hidden_layers in {'3', '4'}\n", + " MLP:hidden_size_4 | MLP:n_hidden_layers in {'4'}\n", + " MLP:lr | model == 'MLP'\n", + " MLP:n_hidden_layers | model == 'MLP'\n", + " MLP:nonlintype | model == 'MLP'\n", + " QuadCostTransformer:dx_F | cost_transformer == 'QuadCostTransformer'\n", + " QuadCostTransformer:dx_Q | cost_transformer == 'QuadCostTransformer'\n", + " QuadCostTransformer:omega_F | cost_transformer == 'QuadCostTransformer'\n", + " QuadCostTransformer:omega_Q | cost_transformer == 'QuadCostTransformer'\n", + " QuadCostTransformer:theta_F | cost_transformer == 'QuadCostTransformer'\n", + " QuadCostTransformer:theta_Q | cost_transformer == 'QuadCostTransformer'\n", + " QuadCostTransformer:u_R | cost_transformer == 'QuadCostTransformer'\n", + " QuadCostTransformer:x_F | cost_transformer == 'QuadCostTransformer'\n", + " QuadCostTransformer:x_Q | cost_transformer == 'QuadCostTransformer'" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -175,21 +208,12 @@ "cell_type": "code", "execution_count": 7, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/lib/python3/dist-packages/pyparsing.py:1745: FutureWarning: Possible set intersection at position 3\n", - " self.re = re.compile( self.reString )\n" - ] - } - ], + "outputs": [], "source": [ "from autompc.tuning import ControlTuner\n", "from autompc.sysid import MLP\n", "\n", - "tuner = ControlTuner(surrogate_mode=\"default\", surrogate=MLP(system), surrogate_split=0.5)" + "tuner = ControlTuner(tuning_mode=\"default\", surrogate=MLP(system), surrogate_split=0.5)" ] }, { @@ -216,29 +240,45 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/home/william/.local/lib/python3.8/site-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n", - " return array(a, dtype, copy=False, order=order)\n" + "INFO:smac.utils.io.cmd_reader.CMDReader:Output to smac3-output_2022-07-24_02:28:41_935779\n", + "INFO:smac.facade.smac_hpo_facade.SMAC4HPO:Optimizing a deterministic scenario for quality without a tuner timeout - will make SMAC deterministic and only evaluate one configuration per iteration!\n", + "INFO:smac.initial_design.sobol_design.SobolDesign:Running initial design for 1 configurations\n", + "INFO:smac.facade.smac_hpo_facade.SMAC4HPO:\n", + "INFO:smac.optimizer.smbo.SMBO:Running initial design\n", + "INFO:smac.intensification.intensification.Intensifier:First run, no incumbent provided; challenger is assumed to be the incumbent\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "100%|██████████| 50/50 [00:53<00:00, 1.06s/it]" + "------------------------------------------------------------------\n", + "Beginning surrogate tuning with model class MLP\n", + "------------------------------------------------------------------\n", + " 53%|█████▎ | 106/200 [03:04<02:43, 1.74s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "Adding unsupported scenario options: {'save_results_instantly': True}\n" + "INFO:smac.stats.stats.Stats:##########################################################\n", + "INFO:smac.stats.stats.Stats:Statistics:\n", + "INFO:smac.stats.stats.Stats:#Incumbent changed: -1\n", + "INFO:smac.stats.stats.Stats:#Submitted target algorithm runs: 0 / 2.0\n", + "INFO:smac.stats.stats.Stats:#Finished target algorithm runs: 0 / 2.0\n", + "INFO:smac.stats.stats.Stats:#Configurations: 0\n", + "INFO:smac.stats.stats.Stats:Used wallclock time: 185.78 / inf sec \n", + "INFO:smac.stats.stats.Stats:Used target algorithm runtime: 0.00 / inf sec\n", + "INFO:smac.stats.stats.Stats:##########################################################\n", + "INFO:smac.facade.smac_hpo_facade.SMAC4HPO:Final Incumbent: None\n" ] }, { @@ -249,108 +289,29 @@ ] }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "Target Algorithm returned NaN or inf as quality. Algorithm run is treated as CRASHED, cost is set to 2147483647.0 for quality scenarios. (Change value through \"cost_for_crash\"-option.)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CfgRunner: Exception during evaluation\n", - "Exit code: 1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Target Algorithm returned NaN or inf as quality. Algorithm run is treated as CRASHED, cost is set to 2147483647.0 for quality scenarios. (Change value through \"cost_for_crash\"-option.)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CfgRunner: Exception during evaluation\n", - "Exit code: 1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Target Algorithm returned NaN or inf as quality. Algorithm run is treated as CRASHED, cost is set to 2147483647.0 for quality scenarios. (Change value through \"cost_for_crash\"-option.)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CfgRunner: Exception during evaluation\n", - "Exit code: 1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Target Algorithm returned NaN or inf as quality. Algorithm run is treated as CRASHED, cost is set to 2147483647.0 for quality scenarios. (Change value through \"cost_for_crash\"-option.)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CfgRunner: Exception during evaluation\n", - "Exit code: 1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Target Algorithm returned NaN or inf as quality. Algorithm run is treated as CRASHED, cost is set to 2147483647.0 for quality scenarios. (Change value through \"cost_for_crash\"-option.)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CfgRunner: Exception during evaluation\n", - "Exit code: 1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Target Algorithm returned NaN or inf as quality. Algorithm run is treated as CRASHED, cost is set to 2147483647.0 for quality scenarios. (Change value through \"cost_for_crash\"-option.)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CfgRunner: Exception during evaluation\n", - "Exit code: 1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Target Algorithm returned NaN or inf as quality. Algorithm run is treated as CRASHED, cost is set to 2147483647.0 for quality scenarios. (Change value through \"cost_for_crash\"-option.)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CfgRunner: Exception during evaluation\n", - "Exit code: 1\n" + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m/home/dohun/Repos/autompc/examples/6_Controller_Tuning.ipynb Cell 17'\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m tuned_controller, tune_result \u001b[39m=\u001b[39m tuner\u001b[39m.\u001b[39;49mrun(\n\u001b[1;32m 2\u001b[0m controller,\n\u001b[1;32m 3\u001b[0m task,\n\u001b[1;32m 4\u001b[0m trajs,\n\u001b[1;32m 5\u001b[0m n_iters\u001b[39m=\u001b[39;49m\u001b[39m2\u001b[39;49m,\n\u001b[1;32m 6\u001b[0m surrogate_tune_iters\u001b[39m=\u001b[39;49m\u001b[39m2\u001b[39;49m,\n\u001b[1;32m 7\u001b[0m rng\u001b[39m=\u001b[39;49mnp\u001b[39m.\u001b[39;49mrandom\u001b[39m.\u001b[39;49mdefault_rng(\u001b[39m100\u001b[39;49m),\n\u001b[1;32m 8\u001b[0m output_dir\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39m6_controller_tuning_example_output/\u001b[39;49m\u001b[39m\"\u001b[39;49m,\n\u001b[1;32m 9\u001b[0m truedyn\u001b[39m=\u001b[39;49mbenchmark\u001b[39m.\u001b[39;49mdynamics\n\u001b[1;32m 10\u001b[0m )\n", + "File \u001b[0;32m~/Repos/autompc/autompc/tuning/control_tuner.py:492\u001b[0m, in \u001b[0;36mControlTuner.run\u001b[0;34m(self, controller, tasks, trajs, n_iters, rng, truedyn, surrogate_tune_iters, eval_timeout, output_dir, restore_dir, save_all_controllers, use_default_initial_design, debug_return_evaluator, max_eval_jobs)\u001b[0m\n\u001b[1;32m 490\u001b[0m controller \u001b[39m=\u001b[39m controller\u001b[39m.\u001b[39mclone()\n\u001b[1;32m 491\u001b[0m controller\u001b[39m.\u001b[39mset_ocp(tasks[\u001b[39m0\u001b[39m]\u001b[39m.\u001b[39mget_ocp())\n\u001b[0;32m--> 492\u001b[0m tuning_data \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_get_tuning_data(controller, tasks, trajs, truedyn, rng, surrogate_tune_iters)\n\u001b[1;32m 493\u001b[0m \u001b[39mwith\u001b[39;00m \u001b[39mopen\u001b[39m(os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mjoin(run_dir, \u001b[39m\"\u001b[39m\u001b[39mtuning_data.pkl\u001b[39m\u001b[39m\"\u001b[39m), \u001b[39m\"\u001b[39m\u001b[39mwb\u001b[39m\u001b[39m\"\u001b[39m) \u001b[39mas\u001b[39;00m f:\n\u001b[1;32m 494\u001b[0m pickle\u001b[39m.\u001b[39mdump(tuning_data, f)\n", + "File \u001b[0;32m~/Repos/autompc/autompc/tuning/control_tuner.py:265\u001b[0m, in \u001b[0;36mControlTuner._get_tuning_data\u001b[0;34m(self, controller, task, trajs, truedyn, rng, surrogate_tune_iters)\u001b[0m\n\u001b[1;32m 262\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39m------------------------------------------------------------------\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 263\u001b[0m tuner \u001b[39m=\u001b[39m ModelTuner(controller\u001b[39m.\u001b[39msystem, surr_trajs, surrogate, eval_holdout\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msurrogate_tune_holdout, eval_folds\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msurrogate_tune_folds,\n\u001b[1;32m 264\u001b[0m eval_metric\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msurrogate_tune_metric,eval_horizon\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msurrogate_tune_horizon,eval_quantile\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msurrogate_tune_quantile,evaluator\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msurrogate_evaluator)\n\u001b[0;32m--> 265\u001b[0m model, tune_result \u001b[39m=\u001b[39m tuner\u001b[39m.\u001b[39;49mrun(rng, surrogate_tune_iters, retrain_full\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m)\n\u001b[1;32m 266\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39m------------------------------------------------------------------\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 267\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mAuto-tuned surrogate model class:\u001b[39m\u001b[39m\"\u001b[39m)\n", + "File \u001b[0;32m~/Repos/autompc/autompc/tuning/model_tuner.py:186\u001b[0m, in \u001b[0;36mModelTuner.run\u001b[0;34m(self, rng, n_iters, min_train_time, max_train_time, retrain_full)\u001b[0m\n\u001b[1;32m 181\u001b[0m smac \u001b[39m=\u001b[39m SMAC4HPO(scenario\u001b[39m=\u001b[39mscenario, rng\u001b[39m=\u001b[39msmac_rng,\n\u001b[1;32m 182\u001b[0m tae_runner\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_evaluate)\n\u001b[1;32m 183\u001b[0m \u001b[39m# smac = SMAC4AC(scenario=scenario, rng=smac_rng,\u001b[39;00m\n\u001b[1;32m 184\u001b[0m \u001b[39m# tae_runner=self._evaluate)\u001b[39;00m\n\u001b[0;32m--> 186\u001b[0m incumbent \u001b[39m=\u001b[39m smac\u001b[39m.\u001b[39;49moptimize()\n\u001b[1;32m 188\u001b[0m inc_cost \u001b[39m=\u001b[39m \u001b[39mfloat\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39minf\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 189\u001b[0m inc_costs \u001b[39m=\u001b[39m []\n", + "File \u001b[0;32m~/Downloads/ENTER/lib/python3.9/site-packages/smac/facade/smac_ac_facade.py:597\u001b[0m, in \u001b[0;36mSMAC4AC.optimize\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 595\u001b[0m incumbent \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 596\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 597\u001b[0m incumbent \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msolver\u001b[39m.\u001b[39;49mrun()\n\u001b[1;32m 598\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 599\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msolver\u001b[39m.\u001b[39mstats\u001b[39m.\u001b[39msave()\n", + "File \u001b[0;32m~/Downloads/ENTER/lib/python3.9/site-packages/smac/optimizer/smbo.py:259\u001b[0m, in \u001b[0;36mSMBO.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 247\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrunhistory\u001b[39m.\u001b[39madd(\n\u001b[1;32m 248\u001b[0m config\u001b[39m=\u001b[39mrun_info\u001b[39m.\u001b[39mconfig,\n\u001b[1;32m 249\u001b[0m cost\u001b[39m=\u001b[39m\u001b[39mfloat\u001b[39m(MAXINT),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 254\u001b[0m budget\u001b[39m=\u001b[39mrun_info\u001b[39m.\u001b[39mbudget,\n\u001b[1;32m 255\u001b[0m )\n\u001b[1;32m 257\u001b[0m run_info\u001b[39m.\u001b[39mconfig\u001b[39m.\u001b[39mconfig_id \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrunhistory\u001b[39m.\u001b[39mconfig_ids[run_info\u001b[39m.\u001b[39mconfig]\n\u001b[0;32m--> 259\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtae_runner\u001b[39m.\u001b[39;49msubmit_run(run_info\u001b[39m=\u001b[39;49mrun_info)\n\u001b[1;32m 261\u001b[0m \u001b[39m# There are 2 criteria that the stats object uses to know\u001b[39;00m\n\u001b[1;32m 262\u001b[0m \u001b[39m# if the budged was exhausted.\u001b[39;00m\n\u001b[1;32m 263\u001b[0m \u001b[39m# The budget time, which can only be known when the run finishes,\u001b[39;00m\n\u001b[1;32m 264\u001b[0m \u001b[39m# And the number of ta executions. Because we submit the job at this point,\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[39m# we count this submission as a run. This prevent for using more\u001b[39;00m\n\u001b[1;32m 266\u001b[0m \u001b[39m# runner runs than what the scenario allows\u001b[39;00m\n\u001b[1;32m 267\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstats\u001b[39m.\u001b[39msubmitted_ta_runs \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m \u001b[39m1\u001b[39m\n", + "File \u001b[0;32m~/Downloads/ENTER/lib/python3.9/site-packages/smac/tae/serial_runner.py:82\u001b[0m, in \u001b[0;36mSerialRunner.submit_run\u001b[0;34m(self, run_info)\u001b[0m\n\u001b[1;32m 64\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39msubmit_run\u001b[39m(\u001b[39mself\u001b[39m, run_info: RunInfo) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 65\u001b[0m \u001b[39m\"\"\"This function submits a run_info object\u001b[39;00m\n\u001b[1;32m 66\u001b[0m \u001b[39m in a serial fashion.\u001b[39;00m\n\u001b[1;32m 67\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 79\u001b[0m \n\u001b[1;32m 80\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[1;32m 81\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mresults\u001b[39m.\u001b[39mappend(\n\u001b[0;32m---> 82\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mrun_wrapper(run_info)\n\u001b[1;32m 83\u001b[0m )\n", + "File \u001b[0;32m~/Downloads/ENTER/lib/python3.9/site-packages/smac/tae/base.py:207\u001b[0m, in \u001b[0;36mBaseRunner.run_wrapper\u001b[0;34m(self, run_info)\u001b[0m\n\u001b[1;32m 204\u001b[0m cutoff \u001b[39m=\u001b[39m \u001b[39mint\u001b[39m(math\u001b[39m.\u001b[39mceil(run_info\u001b[39m.\u001b[39mcutoff))\n\u001b[1;32m 206\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 207\u001b[0m status, cost, runtime, additional_info \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mrun(\n\u001b[1;32m 208\u001b[0m config\u001b[39m=\u001b[39;49mrun_info\u001b[39m.\u001b[39;49mconfig,\n\u001b[1;32m 209\u001b[0m instance\u001b[39m=\u001b[39;49mrun_info\u001b[39m.\u001b[39;49minstance,\n\u001b[1;32m 210\u001b[0m cutoff\u001b[39m=\u001b[39;49mcutoff,\n\u001b[1;32m 211\u001b[0m seed\u001b[39m=\u001b[39;49mrun_info\u001b[39m.\u001b[39;49mseed,\n\u001b[1;32m 212\u001b[0m budget\u001b[39m=\u001b[39;49mrun_info\u001b[39m.\u001b[39;49mbudget,\n\u001b[1;32m 213\u001b[0m instance_specific\u001b[39m=\u001b[39;49mrun_info\u001b[39m.\u001b[39;49minstance_specific\n\u001b[1;32m 214\u001b[0m )\n\u001b[1;32m 215\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m 216\u001b[0m status \u001b[39m=\u001b[39m StatusType\u001b[39m.\u001b[39mCRASHED\n", + "File \u001b[0;32m~/Downloads/ENTER/lib/python3.9/site-packages/smac/tae/execute_func.py:207\u001b[0m, in \u001b[0;36mAbstractTAFunc.run\u001b[0;34m(self, config, instance, cutoff, seed, budget, instance_specific)\u001b[0m\n\u001b[1;32m 205\u001b[0m \u001b[39m# call ta\u001b[39;00m\n\u001b[1;32m 206\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 207\u001b[0m rval \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_ta(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_ta, config, obj_kwargs)\n\u001b[1;32m 208\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(rval, \u001b[39mtuple\u001b[39m):\n\u001b[1;32m 209\u001b[0m result \u001b[39m=\u001b[39m rval[\u001b[39m0\u001b[39m]\n", + "File \u001b[0;32m~/Downloads/ENTER/lib/python3.9/site-packages/smac/tae/execute_func.py:296\u001b[0m, in \u001b[0;36mExecuteTAFuncDict._call_ta\u001b[0;34m(self, obj, config, obj_kwargs)\u001b[0m\n\u001b[1;32m 289\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_call_ta\u001b[39m(\n\u001b[1;32m 290\u001b[0m \u001b[39mself\u001b[39m,\n\u001b[1;32m 291\u001b[0m obj: typing\u001b[39m.\u001b[39mCallable,\n\u001b[1;32m 292\u001b[0m config: Configuration,\n\u001b[1;32m 293\u001b[0m obj_kwargs: typing\u001b[39m.\u001b[39mDict[\u001b[39mstr\u001b[39m, typing\u001b[39m.\u001b[39mUnion[\u001b[39mint\u001b[39m, \u001b[39mstr\u001b[39m, \u001b[39mfloat\u001b[39m, \u001b[39mNone\u001b[39;00m]],\n\u001b[1;32m 294\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m typing\u001b[39m.\u001b[39mUnion[\u001b[39mfloat\u001b[39m, typing\u001b[39m.\u001b[39mTuple[\u001b[39mfloat\u001b[39m, typing\u001b[39m.\u001b[39mDict]]:\n\u001b[0;32m--> 296\u001b[0m \u001b[39mreturn\u001b[39;00m obj(config, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mobj_kwargs)\n", + "File \u001b[0;32m~/Repos/autompc/autompc/tuning/model_tuner.py:119\u001b[0m, in \u001b[0;36mModelTuner._evaluate\u001b[0;34m(self, cfg, seed, budget)\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel\u001b[39m.\u001b[39mset_config(cfg)\n\u001b[1;32m 118\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel\u001b[39m.\u001b[39mset_train_budget(budget)\n\u001b[0;32m--> 119\u001b[0m value \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mevaluator(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmodel)\n\u001b[1;32m 120\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mverbose:\n\u001b[1;32m 121\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mModel Score \u001b[39m\u001b[39m\"\u001b[39m, value)\n", + "File \u001b[0;32m~/Repos/autompc/autompc/tuning/model_evaluator.py:160\u001b[0m, in \u001b[0;36mCrossValidationModelEvaluator.__call__\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m 158\u001b[0m \u001b[39mfor\u001b[39;00m (train,test) \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfolds:\n\u001b[1;32m 159\u001b[0m m \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39mclone()\n\u001b[0;32m--> 160\u001b[0m m\u001b[39m.\u001b[39;49mtrain(train)\n\u001b[1;32m 161\u001b[0m metric_value \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmetric(m, test)\n\u001b[1;32m 162\u001b[0m \u001b[39mif\u001b[39;00m np\u001b[39m.\u001b[39misinf(metric_value):\n", + "File \u001b[0;32m~/Repos/autompc/autompc/sysid/mlp.py:227\u001b[0m, in \u001b[0;36mMLP.train\u001b[0;34m(self, trajs, silent, seed)\u001b[0m\n\u001b[1;32m 225\u001b[0m t0 \u001b[39m=\u001b[39m time\u001b[39m.\u001b[39mtime()\n\u001b[1;32m 226\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m tqdm(\u001b[39mrange\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mn_train_iters), file\u001b[39m=\u001b[39msys\u001b[39m.\u001b[39mstdout):\n\u001b[0;32m--> 227\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_step_train()\n\u001b[1;32m 228\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtrain_time_budget\u001b[39m=\u001b[39m\u001b[39m500\u001b[39m \u001b[39m#DEBUG\u001b[39;00m\n\u001b[1;32m 229\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtrain_time_budget \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m time\u001b[39m.\u001b[39mtime()\u001b[39m-\u001b[39mt0 \u001b[39m>\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtrain_time_budget:\n", + "File \u001b[0;32m~/Repos/autompc/autompc/sysid/mlp.py:202\u001b[0m, in \u001b[0;36mMLP._step_train\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 200\u001b[0m predy \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mnet(x)\n\u001b[1;32m 201\u001b[0m loss \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mlossfun(predy, y\u001b[39m.\u001b[39mto(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_device))\n\u001b[0;32m--> 202\u001b[0m loss\u001b[39m.\u001b[39;49mbackward()\n\u001b[1;32m 203\u001b[0m cum_loss \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m loss\u001b[39m.\u001b[39mitem()\n\u001b[1;32m 204\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39moptim\u001b[39m.\u001b[39mstep()\n", + "File \u001b[0;32m~/Downloads/ENTER/lib/python3.9/site-packages/torch/_tensor.py:363\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 354\u001b[0m \u001b[39mif\u001b[39;00m has_torch_function_unary(\u001b[39mself\u001b[39m):\n\u001b[1;32m 355\u001b[0m \u001b[39mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 356\u001b[0m Tensor\u001b[39m.\u001b[39mbackward,\n\u001b[1;32m 357\u001b[0m (\u001b[39mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 361\u001b[0m create_graph\u001b[39m=\u001b[39mcreate_graph,\n\u001b[1;32m 362\u001b[0m inputs\u001b[39m=\u001b[39minputs)\n\u001b[0;32m--> 363\u001b[0m torch\u001b[39m.\u001b[39;49mautograd\u001b[39m.\u001b[39;49mbackward(\u001b[39mself\u001b[39;49m, gradient, retain_graph, create_graph, inputs\u001b[39m=\u001b[39;49minputs)\n", + "File \u001b[0;32m~/Downloads/ENTER/lib/python3.9/site-packages/torch/autograd/__init__.py:173\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 168\u001b[0m retain_graph \u001b[39m=\u001b[39m create_graph\n\u001b[1;32m 170\u001b[0m \u001b[39m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[1;32m 171\u001b[0m \u001b[39m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 172\u001b[0m \u001b[39m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 173\u001b[0m Variable\u001b[39m.\u001b[39;49m_execution_engine\u001b[39m.\u001b[39;49mrun_backward( \u001b[39m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m 174\u001b[0m tensors, grad_tensors_, retain_graph, create_graph, inputs,\n\u001b[1;32m 175\u001b[0m allow_unreachable\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, accumulate_grad\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], @@ -360,12 +321,105 @@ " task,\n", " trajs,\n", " n_iters=2,\n", + " surrogate_tune_iters=2,\n", " rng=np.random.default_rng(100),\n", " output_dir=\"6_controller_tuning_example_output/\",\n", " truedyn=benchmark.dynamics\n", ")" ] }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "ic| cs._hyperparameters: OrderedDict([('constraint_transformer',\n", + " constraint_transformer, Type: Categorical, Choices: {_}, Default: _),\n", + " ('cost_transformer',\n", + " cost_transformer, Type: Categorical, Choices: {QuadCostTransformer, _}, Default: QuadCostTransformer),\n", + " ('model', model, Type: Categorical, Choices: {MLP}, Default: MLP),\n", + " ('optimizer',\n", + " optimizer, Type: Categorical, Choices: {IterativeLQR}, Default: IterativeLQR),\n", + " ('IterativeLQR:frequency',\n", + " IterativeLQR:frequency, Type: UniformInteger, Range: [1, 5], Default: 1),\n", + " ('IterativeLQR:horizon',\n", + " IterativeLQR:horizon, Type: UniformInteger, Range: [5, 25], Default: 20),\n", + " ('IterativeLQR:max_iter',\n", + " IterativeLQR:max_iter, Type: UniformInteger, Range: [10, 50], Default: 20),\n", + " ('MLP:batchnorm',\n", + " MLP:batchnorm, Type: Categorical, Choices: {False, True}, Default: False),\n", + " ('MLP:hidden_size_1',\n", + " MLP:hidden_size_1, Type: UniformInteger, Range: [16, 256], Default: 128),\n", + " ('MLP:lr',\n", + " MLP:lr, Type: UniformFloat, Range: [1e-05, 1.0], Default: 0.001, on log-scale),\n", + " ('MLP:n_hidden_layers',\n", + " MLP:n_hidden_layers, Type: Categorical, Choices: {1, 2, 3, 4}, Default: 2),\n", + " ('MLP:nonlintype',\n", + " MLP:nonlintype, Type: Categorical, Choices: {relu, tanh, sigmoid, selu}, Default: relu),\n", + " ('QuadCostTransformer:dx_F',\n", + " QuadCostTransformer:dx_F, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale),\n", + " ('QuadCostTransformer:dx_Q',\n", + " QuadCostTransformer:dx_Q, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale),\n", + " ('QuadCostTransformer:omega_F',\n", + " QuadCostTransformer:omega_F, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale),\n", + " ('QuadCostTransformer:omega_Q',\n", + " QuadCostTransformer:omega_Q, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale),\n", + " ('QuadCostTransformer:theta_F',\n", + " QuadCostTransformer:theta_F, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale),\n", + " ('QuadCostTransformer:theta_Q',\n", + " QuadCostTransformer:theta_Q, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale),\n", + " ('QuadCostTransformer:u_R',\n", + " QuadCostTransformer:u_R, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale),\n", + " ('QuadCostTransformer:x_F',\n", + " QuadCostTransformer:x_F, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale),\n", + " ('QuadCostTransformer:x_Q',\n", + " QuadCostTransformer:x_Q, Type: UniformFloat, Range: [0.001, 10000.0], Default: 1.0, on log-scale),\n", + " ('MLP:hidden_size_2',\n", + " MLP:hidden_size_2, Type: UniformInteger, Range: [16, 256], Default: 128),\n", + " ('MLP:hidden_size_3',\n", + " MLP:hidden_size_3, Type: UniformInteger, Range: [16, 256], Default: 128),\n", + " ('MLP:hidden_size_4',\n", + " MLP:hidden_size_4, Type: UniformInteger, Range: [16, 256], Default: 128)])\n", + "ic| id_: '1'\n", + " values: {'batchnorm': False,\n", + " 'hidden_size_1': 128,\n", + " 'hidden_size_2': 128,\n", + " 'lr': 0.001,\n", + " 'n_hidden_layers': '2',\n", + " 'nonlintype': 'relu'}\n", + " config_origins.get(id_, None): 'Default'\n" + ] + }, + { + "ename": "ValueError", + "evalue": "Tried to specify unknown hyperparameter batchnorm", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m/home/dohun/Repos/autompc/examples/6_Controller_Tuning.ipynb Cell 18'\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[39mfor\u001b[39;00m id_, values \u001b[39min\u001b[39;00m all_data[\u001b[39m'\u001b[39m\u001b[39mconfigs\u001b[39m\u001b[39m'\u001b[39m]\u001b[39m.\u001b[39mitems():\n\u001b[1;32m 16\u001b[0m ic(id_, values, config_origins\u001b[39m.\u001b[39mget(id_, \u001b[39mNone\u001b[39;00m))\n\u001b[0;32m---> 17\u001b[0m \u001b[39mprint\u001b[39m(Configuration(cs, values\u001b[39m=\u001b[39;49mvalues, origin\u001b[39m=\u001b[39;49mconfig_origins\u001b[39m.\u001b[39;49mget(id_, \u001b[39mNone\u001b[39;49;00m)))\n\u001b[1;32m 18\u001b[0m runhistory\u001b[39m.\u001b[39mload_json(rh_path, cs)\n\u001b[1;32m 19\u001b[0m \u001b[39m#print(runhistory.data.items())\u001b[39;00m\n", + "File \u001b[0;32mConfigSpace/configuration_space.pyx:1452\u001b[0m, in \u001b[0;36mConfigSpace.configuration_space.Configuration.__init__\u001b[0;34m()\u001b[0m\n", + "\u001b[0;31mValueError\u001b[0m: Tried to specify unknown hyperparameter batchnorm" + ] + } + ], + "source": [ + "from icecream import ic\n", + "import json\n", + "from smac.runhistory.runhistory import RunHistory\n", + "from smac.scenario.scenario import Scenario\n", + "from smac.configspace import Configuration\n", + "rh_path = '/home/dohun/Repos/autompc/examples/smac3-output_2022-07-20_23:09:54_019785/run_2091578153/runhistory.json'\n", + "cs = controller.get_config_space()\n", + "runhistory = RunHistory()\n", + "runhistory.load_json(rh_path, cs)\n", + "tuner._get_tune_result(None, runhistory)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -394,14 +448,12 @@ } ], "source": [ - "from autompc.graphs import TuningCurveGraph\n", + "from autompc.graphs import plot_tuning_curve\n", "import matplotlib.pyplot as plt\n", "\n", - "graph = TuningCurveGraph()\n", - "\n", "fig = plt.figure() \n", "ax = fig.gca()\n", - "graph(ax, tune_result)\n", + "plot_tuning_curve(tune_result, ax)\n", "ax.set_title(\"Cart-Pole Tuning Curve\")\n", "plt.show()" ] From 905ef8415320fb775795a0fe8581368f848e76f8 Mon Sep 17 00:00:00 2001 From: Dohun Date: Tue, 16 Aug 2022 23:51:08 -0500 Subject: [PATCH 33/41] Timeout feature added to ILQR --- autompc/optim/ilqr.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/autompc/optim/ilqr.py b/autompc/optim/ilqr.py index 99e9a74..b10365a 100644 --- a/autompc/optim/ilqr.py +++ b/autompc/optim/ilqr.py @@ -90,7 +90,7 @@ def cost_requirements(self): def set_state(self, state): self._guess = copy.deepcopy(state["guess"]) - + ''' def compute_ilqr(self, x0, uguess, u_threshold=1e-3, ls_max_iter=10, ls_discount=0.2, ls_cost_threshold=0.3, timeout=np.inf): """Use equations from https://medium.com/@jonathan_hui/rl-lqr-ilqr-linear-quadratic-regulator-a5de5104c750 . @@ -275,11 +275,12 @@ def eval_obj(xs, us): return converged, states, ctrls, Ks ''' - def compute_ilqr(self, x0, uguess, u_threshold=1e-3, ls_max_iter=10, ls_discount=0.2, ls_cost_threshold=0.3, silent=True): + def compute_ilqr(self, x0, uguess, u_threshold=1e-3, ls_max_iter=10, ls_discount=0.2, ls_cost_threshold=0.3, timeout=np.inf, silent=True): """Use equations from https://medium.com/@jonathan_hui/rl-lqr-ilqr-linear-quadratic-regulator-a5de5104c750 . A better version is https://homes.cs.washington.edu/~todorov/papers/TassaIROS12.pdf Here I do not have Hessian correction since I'm certain all my matrices are SPD """ + start = time.time() cost = self.ocp.get_cost() H = self.horizon dt = self.system.dt @@ -316,6 +317,10 @@ def eval_obj(xs, us): converged = False du_norm = np.inf for itr in range(self.max_iter): + if time.time()-start > timeout: + if self.verbose: + print('Timeout') + break if self.verbose: print('At iteration %d' % itr) # compute at the last step, Vn and vn, just hessian and gradient at the last state @@ -440,7 +445,7 @@ def step(self, obs): self._guess = np.zeros((self.horizon, self.system.ctrl_dim)) if substep == 0: start = time.time() - converged, states, ctrls, Ks = self.compute_ilqr(obs, self._guess) # DEBUG, no itr + converged, states, ctrls, Ks = self.compute_ilqr(obs, self._guess, timeout=self.system.dt) # DEBUG, no itr end = time.time() if (end-start > 0.3): print("ILQR timeout: ", end-start) From 825209ca9cf91354f52f9eebede8232b01c57722 Mon Sep 17 00:00:00 2001 From: Dohun Date: Tue, 16 Aug 2022 23:52:32 -0500 Subject: [PATCH 34/41] Merge of "soft_water_robot_devel" and "origin/soft_water_robot_devel" --- autompc/optim/ilqr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autompc/optim/ilqr.py b/autompc/optim/ilqr.py index b10365a..c025d6b 100644 --- a/autompc/optim/ilqr.py +++ b/autompc/optim/ilqr.py @@ -90,8 +90,8 @@ def cost_requirements(self): def set_state(self, state): self._guess = copy.deepcopy(state["guess"]) - ''' - def compute_ilqr(self, x0, uguess, u_threshold=1e-3, + + '''def compute_ilqr(self, x0, uguess, u_threshold=1e-3, ls_max_iter=10, ls_discount=0.2, ls_cost_threshold=0.3, timeout=np.inf): """Use equations from https://medium.com/@jonathan_hui/rl-lqr-ilqr-linear-quadratic-regulator-a5de5104c750 . A better version is https://homes.cs.washington.edu/~todorov/papers/TassaIROS12.pdf From 1149260f8e5d9d88f3ad888e2fd29779cbb6bde0 Mon Sep 17 00:00:00 2001 From: Dohun Date: Thu, 18 Aug 2022 15:56:18 -0500 Subject: [PATCH 35/41] Removing debugging stuff --- autompc/optim/ilqr.py | 8 +------- autompc/sysid/mlp.py | 2 +- autompc/tuning/control_tuner.py | 6 ++---- autompc/utils/simulation.py | 6 +----- 4 files changed, 5 insertions(+), 17 deletions(-) diff --git a/autompc/optim/ilqr.py b/autompc/optim/ilqr.py index c025d6b..3701cfd 100644 --- a/autompc/optim/ilqr.py +++ b/autompc/optim/ilqr.py @@ -444,13 +444,7 @@ def step(self, obs): if self._guess is None: self._guess = np.zeros((self.horizon, self.system.ctrl_dim)) if substep == 0: - start = time.time() - converged, states, ctrls, Ks = self.compute_ilqr(obs, self._guess, timeout=self.system.dt) # DEBUG, no itr - end = time.time() - if (end-start > 0.3): - print("ILQR timeout: ", end-start) - #print('Num iter: ', itr) - #print('Breakdown: ', preloop_duration, backward_duration, rollout_duration, ls_duration, preloop_duration+backward_duration+rollout_duration+ls_duration) + converged, states, ctrls, Ks = self.compute_ilqr(obs, self._guess) self._guess = np.concatenate((ctrls[1:], ctrls[-1:]*0), axis=0) self._traj = Trajectory(self.model.state_system, states, np.vstack([ctrls, np.zeros(self.system.ctrl_dim)])) diff --git a/autompc/sysid/mlp.py b/autompc/sysid/mlp.py index 8dec1c5..431cdf7 100644 --- a/autompc/sysid/mlp.py +++ b/autompc/sysid/mlp.py @@ -102,7 +102,7 @@ class MLP(FullyObservableModel): Type of activation function. - **lr** *(Type: float, Low: 1e-5, High: 1, Default: 1e-3)*: Adam learning rate for the network. """ - def __init__(self, system, n_train_iters=200, n_batch=64, use_cuda=False): + def __init__(self, system, n_train_iters=200, n_batch=64, use_cuda=True): super().__init__(system, "MLP") self.n_train_iters = n_train_iters self.n_batch = n_batch diff --git a/autompc/tuning/control_tuner.py b/autompc/tuning/control_tuner.py index a1d3712..5d608b0 100644 --- a/autompc/tuning/control_tuner.py +++ b/autompc/tuning/control_tuner.py @@ -649,12 +649,11 @@ def run(self, cfg): controller.set_config(cfg) controller.build(sysid_trajs) - start = time.time() trajs = control_evaluator(controller) + performance = performance_metric(trajs) info["surr_info"] = trajs info["surr_cost"] = performance - performance = performance_metric(trajs) - if not truedyn_evaluator is None: + if truedyn_evaluator is not None: trajs = truedyn_evaluator(controller) performance = performance_metric(trajs) info["truedyn_cost"] = performance @@ -668,5 +667,4 @@ def run(self, cfg): # if not self.log_file_name is None: # sys.stdout.close() # sys.stderr.close() - print('returning') # DEBUG return info diff --git a/autompc/utils/simulation.py b/autompc/utils/simulation.py index 1ba18f9..a391744 100644 --- a/autompc/utils/simulation.py +++ b/autompc/utils/simulation.py @@ -92,7 +92,6 @@ def simulate(policy : Policy, init_obs, dynamics : Dynamics, term_cond=None, max raise ValueError("Trying to do simulate() on an untrained dynamics model?") x = init_obs traj = DynamicTrajectory(dynamics.system) - step_times = [] simstate = dynamics.init_state(init_obs) if silent: @@ -100,11 +99,8 @@ def simulate(policy : Policy, init_obs, dynamics : Dynamics, term_cond=None, max else: itr = tqdm(range(max_steps), file=sys.stdout) for _ in itr: - start = time() u = policy.step(x) - end = time() traj.append(x,u) - step_times.append(end-start) if term_cond is not None and term_cond(traj): break if len(traj) == max_steps: @@ -115,4 +111,4 @@ def simulate(policy : Policy, init_obs, dynamics : Dynamics, term_cond=None, max simstate = simstate2 x = dynamics.get_obs(simstate) #finalize Trajectory - return traj.freeze(), step_times + return traj.freeze() From 17b440303dfb9b27bdb43cdadf2360f410b50843 Mon Sep 17 00:00:00 2001 From: Dohun Date: Thu, 18 Aug 2022 15:57:05 -0500 Subject: [PATCH 36/41] Remove unrolling w * since it's not compatible with python <3.8 --- autompc/costs/barrier_cost.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/autompc/costs/barrier_cost.py b/autompc/costs/barrier_cost.py index fd12ab4..7fc7292 100644 --- a/autompc/costs/barrier_cost.py +++ b/autompc/costs/barrier_cost.py @@ -60,7 +60,7 @@ def incremental_diff(self, obs, control): def incremental_hess(self, obs, control): # TODO: Tuple unpacking only supported for python>=3.8 hess_obs_ctrl = np.zeros((self.system.obs_dim, self.system.ctrl_dim)) - return *self.incremental_diff(obs, control), self.eval_obs_cost_hess(obs), hess_obs_ctrl, self.eval_ctrl_cost_hess(control) + return self.incremental(obs, control), self.eval_obs_cost_diff(obs), self.eval_ctrl_cost_diff(control), self.eval_obs_cost_hess(obs), hess_obs_ctrl, self.eval_ctrl_cost_hess(control) def terminal(self, obs): return 0 @@ -184,13 +184,4 @@ def eval_ctrl_cost_hess(self, ctrl): hessian[index][index] += scale / ((lower - ctrl[index])**2) if upper < np.inf: hessian[index][index] += scale / ((upper - ctrl[index])**2) - return hessian - - def eval_term_obs_cost(self, obs): - return 0 - - def eval_term_obs_cost_diff(self, obs): - return 0 - - def eval_term_obs_cost_hess(self, obs): - return 0 \ No newline at end of file + return hessian \ No newline at end of file From 3977ba5521c20a6e023857ff7ab33430cffe3563 Mon Sep 17 00:00:00 2001 From: Dohun Date: Thu, 18 Aug 2022 16:26:02 -0500 Subject: [PATCH 37/41] preparing 0.2-dev pull request --- autompc/controller.py | 18 +++++++----------- autompc/optim/ilqr.py | 2 +- autompc/sysid/mlp.py | 4 ++-- autompc/tunable.py | 7 ++----- autompc/tuning/control_tuner.py | 9 ++++----- autompc/tuning/model_tuner.py | 8 ++++---- 6 files changed, 20 insertions(+), 28 deletions(-) diff --git a/autompc/controller.py b/autompc/controller.py index 1299c67..50a48cc 100644 --- a/autompc/controller.py +++ b/autompc/controller.py @@ -143,8 +143,6 @@ def set_ocp_transformer(self, ocp_transformer : OCPTransformer) -> None: OCPTransformer to set. """ self.ocp_transformers = [ocp_transformer] - self.set_component("cost_transformer",self.ocp_transformers) - def set_ocp_transformers(self, ocp_transformers : List[OCPTransformer]) -> None: """ @@ -156,7 +154,6 @@ def set_ocp_transformers(self, ocp_transformers : List[OCPTransformer]) -> None: Set of OCP transformers which can be selected. """ self.ocp_transformers = ocp_transformers - self.set_component("cost_transformer",self.ocp_transformers) def add_ocp_transformer(self, ocp_transformer): """ @@ -176,13 +173,6 @@ def set_ocp(self, ocp): """ self.ocp = ocp - if self.ocp_transformer: - self.transformed_ocp = self.ocp_transformer(self.ocp) - else: - self.transformed_ocp = self.ocp - if self.optimizer: - self.optimizer.set_ocp(self.transformed_ocp) - def get_config_space(self): """ Returns the joint controller configuration space. @@ -208,7 +198,7 @@ def get_config_space(self): regularizers.append(transformer) elif transformer.name != 'Identity': cost_transformers.append(transformer) - #cost_transformers.append(dummy) + cost_transformers.append(dummy) if len(cost_transformers) > 1: self.set_component("cost_transformer", cost_transformers) @@ -456,6 +446,12 @@ def build(self, trajs : List[Trajectory] = None) -> None: self.ocp_transformer.train(trajs) else: raise ControllerStateError("Specified OCP transformer requires learning from trajectories.") + + if self.ocp_transformer: + self.transformed_ocp = self.ocp_transformer(self.ocp) + else: + self.transformed_ocp = self.ocp + self.optimizer.set_ocp(self.transformed_ocp) self.reset() diff --git a/autompc/optim/ilqr.py b/autompc/optim/ilqr.py index 3701cfd..82960a8 100644 --- a/autompc/optim/ilqr.py +++ b/autompc/optim/ilqr.py @@ -240,7 +240,7 @@ def eval_obj(xs, us): best_alpha = 1.0 else: if self.verbose : - print(" fails, best obj",best_obj,"obj",obj,"best_alpha",best_alpha) + print("Line search fails, best obj",best_obj,"obj",obj,"best_alpha",best_alpha) break # return since update of action is small diff --git a/autompc/sysid/mlp.py b/autompc/sysid/mlp.py index 431cdf7..ec8ccaa 100644 --- a/autompc/sysid/mlp.py +++ b/autompc/sysid/mlp.py @@ -28,7 +28,7 @@ def transform_output(xu_means, xu_std, XU): return np.vstack(XUt).T class ForwardNet(torch.nn.Module): - def __init__(self, n_in, n_out, hidden_sizes, nonlintype, batchnorm=True): + def __init__(self, n_in, n_out, hidden_sizes, nonlintype, batchnorm=False): """Specify the feedforward neuro network size and nonlinearity""" assert len(hidden_sizes) > 0 torch.nn.Module.__init__(self) @@ -224,7 +224,7 @@ def train(self, trajs, silent=False, seed=100): t0 = time.time() for i in tqdm(range(self.n_train_iters), file=sys.stdout): self._step_train() - self.train_time_budget=500 #DEBUG + # self.train_time_budget=500 #DEBUG if self.train_time_budget is not None and time.time()-t0 > self.train_time_budget: print("Reached timeout of %.2fs"%self.train_time_budget) break diff --git a/autompc/tunable.py b/autompc/tunable.py index 2f4b6a2..2d3066f 100644 --- a/autompc/tunable.py +++ b/autompc/tunable.py @@ -113,7 +113,7 @@ def set_hyperparameter_defaults(self,**kwargs) -> None: for key,defaults in kwargs.items(): hyperparams[key].default_value = defaults - def set_hyperparameter_log(self,**kwargs) -> None: + def set_hyperparameter_logs(self,**kwargs) -> None: """Sets the configuration space log values for a set of hyperparameters, as keyword arguments. """ @@ -258,10 +258,7 @@ def set_config(self, config): if opt is None: continue opt_config = create_subspace_configuration(config,opt.name,opt.get_config_space()) - try: - opt.set_config(opt_config) - except: - print('Skipping set_config. Frozen hyperparameters?') + opt.set_config(opt_config) def get_config(self): return self._config diff --git a/autompc/tuning/control_tuner.py b/autompc/tuning/control_tuner.py index 5d608b0..fcfe5be 100644 --- a/autompc/tuning/control_tuner.py +++ b/autompc/tuning/control_tuner.py @@ -221,8 +221,8 @@ def __init__(self, tuning_mode="default", surrogate : Optional[Model] = None, self.max_trials_per_evaluation = max_trials_per_evaluation self.control_evaluator = control_evaluator if performance_metric is None: - performance_metric = ControlPerformanceMetric() # DEBUG - #performance_metric = ConfidenceBoundPerformanceMetric(quantile=performance_quantile,eval_time_weight=performance_eval_time_weight,infeasible_cost=performance_infeasible_cost) + performance_metric = ControlPerformanceMetric() + #performance_metric = ConfidenceBoundPerformanceMetric(quantile=performance_quantile,eval_time_weight=performance_eval_time_weight,infeasible_cost=performance_infeasible_cost) # TODO: Fix this self.performance_metric = performance_metric def _get_tuning_data(self, controller : Controller, task : List[Task], trajs : List[Trajectory], @@ -508,7 +508,6 @@ def run(self, controller, tasks, trajs, n_iters, rng, truedyn=None, if debug_return_evaluator: return eval_cfg smac_rng = np.random.RandomState(seed=rng.integers(1 << 31)) - print(controller.get_config_space()) scenario = Scenario({"run_obj" : "quality", "runcount-limit" : n_iters, "cs" : controller.get_config_space(), @@ -579,7 +578,7 @@ def get_tuning_data(self): def __call__(self, cfg): self.eval_number += 1 - if True: # DEBUG self.timeout is None + if self.timeout is None: result = self.run(cfg) else: #p = multiprocessing.Process(target=self.run_mp, args=(cfg,)) @@ -651,8 +650,8 @@ def run(self, cfg): controller.build(sysid_trajs) trajs = control_evaluator(controller) performance = performance_metric(trajs) - info["surr_info"] = trajs info["surr_cost"] = performance + info["surr_info"] = trajs if truedyn_evaluator is not None: trajs = truedyn_evaluator(controller) performance = performance_metric(trajs) diff --git a/autompc/tuning/model_tuner.py b/autompc/tuning/model_tuner.py index 759ebe4..0468da9 100644 --- a/autompc/tuning/model_tuner.py +++ b/autompc/tuning/model_tuner.py @@ -178,10 +178,10 @@ def run(self, rng=None, n_iters=10, min_train_time=None, max_train_time=None, else: if max_train_time is not None: self.model.set_train_budget(max_train_time) - smac = SMAC4HPO(scenario=scenario, rng=smac_rng, - tae_runner=self._evaluate) - # smac = SMAC4AC(scenario=scenario, rng=smac_rng, - # tae_runner=self._evaluate) + #smac = SMAC4HPO(scenario=scenario, rng=smac_rng, + # tae_runner=self._evaluate) + smac = SMAC4AC(scenario=scenario, rng=smac_rng, + tae_runner=self._evaluate) incumbent = smac.optimize() From a3e2f145263cd9bd4d636b5ad6d084bb7d590fb8 Mon Sep 17 00:00:00 2001 From: Dohun Date: Thu, 18 Aug 2022 16:29:12 -0500 Subject: [PATCH 38/41] OCP needs to be transformed when being set --- autompc/controller.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/autompc/controller.py b/autompc/controller.py index 50a48cc..dde2293 100644 --- a/autompc/controller.py +++ b/autompc/controller.py @@ -173,6 +173,15 @@ def set_ocp(self, ocp): """ self.ocp = ocp + if len(self.ocp_transformers) == 1: + self.ocp_transformer = self.ocp_transformers[0] + if self.ocp_transformer: + self.transformed_ocp = self.ocp_transformer(self.ocp) + else: + self.transformed_ocp = self.ocp + if self.optimizer: + self.optimizer.set_ocp(self.transformed_ocp) + def get_config_space(self): """ Returns the joint controller configuration space. From 40bff3f5bfc4b5083f3bb1df4cec998de103a414 Mon Sep 17 00:00:00 2001 From: Dohun Date: Thu, 18 Aug 2022 19:00:18 -0500 Subject: [PATCH 39/41] Fixing ParallelEvaluator --- autompc/tuning/control_evaluator.py | 33 ++-------------------------- autompc/tuning/control_tuner.py | 5 ++--- autompc/tuning/parallel_evaluator.py | 2 +- 3 files changed, 5 insertions(+), 35 deletions(-) diff --git a/autompc/tuning/control_evaluator.py b/autompc/tuning/control_evaluator.py index b2f7b7c..3a1bd25 100644 --- a/autompc/tuning/control_evaluator.py +++ b/autompc/tuning/control_evaluator.py @@ -117,37 +117,8 @@ def __init__(self, system, tasks, dynamics, prefix=''): def evaluate_for_task(self, controller : Policy, task : Task): print("Simulating Trajectory...",end='') + controller.set_ocp(task) + controller.reset() res = self.evaluate_for_task_dynamics(controller,task,self.dynamics) print("Resulting cost",res.cost) return res - -class ParallelStandardEvaluator(StandardEvaluator): - def __init__(self, system, tasks, dynamics, prefix='',max_jobs=None): - super().__init__(system, tasks, dynamics, prefix) - if max_jobs is None: - self.max_jobs = len(tasks) - else: - self.max_jobs = max_jobs - def __call__(self, policy : Union[Policy,Controller]) -> List[ControlEvaluationTrial]: - """ - Evaluates policy on all tasks in parallel. Default just runs evaluate_for_task - on all tasks. - - Returns - -------- - trial_info (List[ControlEvaluationTrial]): - A list of trials evaluated. - """ - if callable(self.tasks): - raise ValueError("Can't use a task sampling function in evaluator class {}, must override __call__".format(self.__class__.__name__)) - - results = [] - def f(task): - i, task = task - if hasattr(policy,'set_ocp'): #it's a Controller - policy.set_ocp(task) - policy.reset() - print(f"Evaluating Task {i}") - return self.evaluate_for_task(policy, task) - results = Parallel(n_jobs=self.max_jobs)(delayed(f)(task) for task in enumerate(self.tasks)) - return results diff --git a/autompc/tuning/control_tuner.py b/autompc/tuning/control_tuner.py index fcfe5be..c0a6dfa 100644 --- a/autompc/tuning/control_tuner.py +++ b/autompc/tuning/control_tuner.py @@ -19,7 +19,7 @@ from ..dynamics import Dynamics from .model_tuner import ModelTuner from .model_evaluator import ModelEvaluator -from .control_evaluator import ControlEvaluator, ParallelStandardEvaluator, StandardEvaluator, ControlEvaluationTrial, trial_to_json +from .control_evaluator import ControlEvaluator, StandardEvaluator, ControlEvaluationTrial, trial_to_json from .control_performance_metric import ControlPerformanceMetric,ConfidenceBoundPerformanceMetric from .bootstrap_evaluator import BootstrapSurrogateEvaluator from .parallel_evaluator import ParallelEvaluator @@ -252,8 +252,7 @@ def _get_tuning_data(self, controller : Controller, task : List[Task], trajs : L print("Skipping surrogate tuning, surrogate is a trained model") print("------------------------------------------------------------------") if control_evaluator is None: - #control_evaluator = ParallelEvaluator(StandardEvaluator(controller.system, task, surrogate, 'surr_'), dynamics=surrogate, max_jobs=5) - control_evaluator = ParallelStandardEvaluator(controller.system, task, surrogate, 'surr_') + control_evaluator = ParallelEvaluator(StandardEvaluator(controller.system, task, surrogate, 'surr_'), dynamics=surrogate, max_jobs=len(task)) else: assert not isinstance(control_evaluator,BootstrapSurrogateEvaluator),'Need an evaluator that does not train' else: diff --git a/autompc/tuning/parallel_evaluator.py b/autompc/tuning/parallel_evaluator.py index 4e5d0db..84cf2ef 100644 --- a/autompc/tuning/parallel_evaluator.py +++ b/autompc/tuning/parallel_evaluator.py @@ -28,7 +28,7 @@ def num_jobs(self) -> int: return min(self.max_jobs,len(self.dynamics_models)*len(self.tasks)) def run_job(self, controller, job_idx) -> ControlEvaluationTrial: - model_idx, task_idx = job_idx % len(self.tasks), job_idx // len(self.tasks) + task_idx, model_idx = job_idx % len(self.tasks), job_idx // len(self.tasks) surrogate = self.dynamics_models[model_idx] print("Simulating Surrogate Trajectory for Model {}, Task {}: ".format(model_idx, task_idx)) self.evaluator.dynamics = surrogate From 50850b68d6c41025b78d35818299fc4dd4ef7616 Mon Sep 17 00:00:00 2001 From: Dohun Date: Thu, 18 Aug 2022 19:01:33 -0500 Subject: [PATCH 40/41] Minor change to Barrier Cost Transformer --- autompc/ocp/barrier_cost_transformer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autompc/ocp/barrier_cost_transformer.py b/autompc/ocp/barrier_cost_transformer.py index 410b9cc..3238458 100644 --- a/autompc/ocp/barrier_cost_transformer.py +++ b/autompc/ocp/barrier_cost_transformer.py @@ -73,7 +73,7 @@ def set_bounds(self, boundedState, lower_bound, upper_bound, default, log=False) if(boundedState in self._limits): self.set_hyperparameter_bounds(**{boundedState+"_LogBarrier": (lower_bound,upper_bound)}) self.set_hyperparameter_defaults(**{boundedState+"_LogBarrier": default}) - self.set_hyperparameter_log(**{boundedState+"_LogBarrier": log}) + self.set_hyperparameter_logs(**{boundedState+"_LogBarrier": log}) else: raise ValueError(str(boundedState) + " does not have a configured limit use set_limit") else: @@ -100,7 +100,7 @@ def _get_boundedState(self, cfg, label): def get_default_config_space(self): cs = CS.ConfigurationSpace() for name in self.system.observations + self.system.controls: - hyper = CS.Constant(name+"_LogBarrier", BARRIER_COST_DEFAULT_VALUE) + hyper = CS.Constant(name+"_LogBarrier", 0.0) cs.add_hyperparameter(hyper) return cs From 9cad9a1f757c8cb43d788333393ab685efadc385 Mon Sep 17 00:00:00 2001 From: Dohun Date: Mon, 22 Aug 2022 16:35:50 -0500 Subject: [PATCH 41/41] Timeout enabled --- autompc/optim/ilqr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autompc/optim/ilqr.py b/autompc/optim/ilqr.py index 82960a8..908d445 100644 --- a/autompc/optim/ilqr.py +++ b/autompc/optim/ilqr.py @@ -444,7 +444,7 @@ def step(self, obs): if self._guess is None: self._guess = np.zeros((self.horizon, self.system.ctrl_dim)) if substep == 0: - converged, states, ctrls, Ks = self.compute_ilqr(obs, self._guess) + converged, states, ctrls, Ks = self.compute_ilqr(obs, self._guess, timeout=self.system.dt) self._guess = np.concatenate((ctrls[1:], ctrls[-1:]*0), axis=0) self._traj = Trajectory(self.model.state_system, states, np.vstack([ctrls, np.zeros(self.system.ctrl_dim)]))