diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..c58d2d9 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,14 @@ +language: python +python: +- "3.6" +cache: pip +install: +- pip install --upgrade pip +- pip install -e . +- pip install -r requirements.txt -U --upgrade-strategy eager +- pip install -r test-requirements.txt +script: +- flake8 . +- travis_wait 50 nosetests --with-coverage --cover-package=active_learning +after_success: +- coveralls diff --git a/README.md b/README.md index c64ef8d..9461c92 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,23 @@ # Active Learning for Python +[![Build Status](https://travis-ci.org/globus-labs/active-learning.svg?branch=master)](https://travis-ci.org/globus-labs/active-learning) +[![Coverage Status](https://coveralls.io/repos/github/globus-labs/active-learning/badge.svg?branch=master)](https://coveralls.io/github/globus-labs/active-learning?branch=master) -This is a toolkit for active learning in python designed to be used in conjunction -with scikit-learn models. Its structure comes from Roman Garnett's active learning [toolbox for Matlab](https://github.com/rmgarnett/active_learning). +Toolkit for active learning in Python designed to be used in conjunction with scikit-learn models. ## Installation -You can install by cloning with `git clone https://github.com/theodore-ando/active-learning` followed +You can install by cloning with `git clone https://github.com/globus-labs/active-learning` followed by `pip install -e ./active-learning` -## Basic usage +## Usage -See [example.ipynb](example.ipynb) for the basic usage of the API and a simple comparison of some query strategies. -[example_live.ipynb](example_live.ipynb) shows how easy it is to integrate a real person into the labeling loop. +Examples and tutorials TBD. -## Advanced Usage +## See Also -In the works. +Roman Garnett's active learning [toolbox for Matlab](https://github.com/rmgarnett/active_learning). -# License +## License Copyright 2018 Theodore Ando @@ -31,4 +31,4 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file +limitations under the License. diff --git a/active_learning/__init__.py b/active_learning/__init__.py index d227f53..e69de29 100644 --- a/active_learning/__init__.py +++ b/active_learning/__init__.py @@ -1,6 +0,0 @@ -from . import query_strats -from . import scoring -from . import selectors -from . import utils - -name = "active-learning" \ No newline at end of file diff --git a/active_learning/active_learning.py b/active_learning/active_learning.py deleted file mode 100644 index 273dedb..0000000 --- a/active_learning/active_learning.py +++ /dev/null @@ -1,113 +0,0 @@ -import logging -import numpy as np -from collections import Iterable -from tqdm import tqdm - -from active_learning.query_strats import uncertainty_sampling -from active_learning.selectors import identity_selector - - -def _actively_learn(problem, train_ixs, obs_labels, oracle, **kwargs): - """ - The main active learning loop - :param problem: dictionary that defines the problem, containing keys: - * points: an (n_samples, n_dim) matrix of points in the space - * num_classes: the number of different classes [0, num_classes) - * batch_size: number of points to query each iteration - * num_queries: the max number of queries we can make on the data - * model: the sk-learn model we are training - Optional: - - partition: list of np.arrays of indices into problem['points'] partitioning the space. - This can restrict the batch to be from one partition! - :param train_ixs: index into `points` of the training examples - :param obs_labels: labels for the training examples - :param oracle: gets the true labels for a set of points - :param selector: gets the indexes of all available points to be tested - :param query_strat: gets the indexes of points we wish to test next - :param callback: function or list of funcs to call at end of each iteration (retrain the model?) - :return: None (yet) - """ - selector = kwargs.get("selector", identity_selector) - query_strat = kwargs.get("query_strat", uncertainty_sampling) - callback = kwargs.get("callback") - - problem['num_initial'] = len(train_ixs) - num_queries = problem['num_queries'] - batch_size = problem['batch_size'] - - for i in tqdm(range(num_queries)): - # print(f"Query {i} / {num_queries}") - - # get the available points we might want to query - unlabeled_ixs = selector(problem, train_ixs, obs_labels, **kwargs) - logging.debug(f"{len(unlabeled_ixs)} available unlabeled points") - - if len(unlabeled_ixs) == 0: - logging.debug("No available unlabeled points") - return - - # choose points from the available points - selected_ixs = query_strat(problem, train_ixs, obs_labels, unlabeled_ixs, batch_size, budget=num_queries - i, - **kwargs) - - # get the true labels from the oracle - true_labels = oracle(problem, train_ixs, obs_labels, selected_ixs, **kwargs) - logging.debug(selected_ixs) - logging.debug(true_labels) - - # add the new labeled points to the training set - train_ixs = np.concatenate([train_ixs, selected_ixs]) - obs_labels = np.concatenate([obs_labels, true_labels]) - - # presumably the call back will update the model - if callback is None: - continue - elif isinstance(callback, Iterable): - for func in callback: - func(problem, train_ixs, obs_labels, query=selected_ixs, **kwargs) - else: - callback(problem, train_ixs, obs_labels, query=selected_ixs, **kwargs) - - -def actively_learn_holdout(problem, train_ixs, obs_labels, oracle, holdout_points, holdout_iter, **kwargs): - selector = kwargs.get("selector", identity_selector) - query_strat = kwargs.get("query_strat", uncertainty_sampling) - callback = kwargs.get("callback") - - problem['num_initial'] = len(train_ixs) - num_queries = problem['num_queries'] - batch_size = problem['batch_size'] - - for i in tqdm(range(num_queries)): - if i == holdout_iter: - problem['points'] = np.vstack([problem['points'], holdout_points]) - - # get the available points we might want to query - unlabeled_ixs = selector(problem, train_ixs, obs_labels, **kwargs) - logging.debug(f"{len(unlabeled_ixs)} available unlabeled points") - - if len(unlabeled_ixs) == 0: - logging.debug("No available unlabeled points") - return - - # choose points from the available points - selected_ixs = query_strat(problem, train_ixs, obs_labels, unlabeled_ixs, batch_size, budget=num_queries - i, - **kwargs) - - # get the true labels from the oracle - true_labels = oracle(problem, train_ixs, obs_labels, selected_ixs, **kwargs) - logging.debug(selected_ixs) - logging.debug(true_labels) - - # add the new labeled points to the training set - train_ixs = np.concatenate([train_ixs, selected_ixs]) - obs_labels = np.concatenate([obs_labels, true_labels]) - - # presumably the call back will update the model - if callback is None: - continue - elif isinstance(callback, Iterable): - for func in callback: - func(problem, train_ixs, obs_labels, query=selected_ixs, **kwargs) - else: - callback(problem, train_ixs, obs_labels, query=selected_ixs, **kwargs) diff --git a/active_learning/objective.py b/active_learning/objective.py new file mode 100644 index 0000000..8ddd2e7 --- /dev/null +++ b/active_learning/objective.py @@ -0,0 +1,32 @@ +"""Objective functions used in defining an active learning problem""" + +from typing import List +import numpy as np + + +class ObjectiveFunction: + """Class that generates objective function scores for regression functions""" + + def score(self, y: List, y_uncert: List = None) -> List[float]: + """Generate the objective function score + + Args: + y (list): Values of a class for many entries + y_uncert (list): Any kind of uncertainty values + Returns: + ([float]): Scores where minimal values are preferred + """ + raise NotImplementedError + + +class Maximize(ObjectiveFunction): + """Find the maximum scalar value""" + + def score(self, y: List, y_uncert: List = None) -> List[float]: + return np.multiply(y, -1) + + +class Minimize(ObjectiveFunction): + + def score(self, y: List, y_uncert: List = None) -> List[float]: + return y diff --git a/active_learning/problem.py b/active_learning/problem.py new file mode 100644 index 0000000..64f78f5 --- /dev/null +++ b/active_learning/problem.py @@ -0,0 +1,103 @@ +"""Classes and methods related to defining an active learning problem""" + +from .objective import ObjectiveFunction, Minimize +from typing import List, Tuple, Union +import numpy as np + + +class ActiveLearningProblem: + """Class for defining an active learning problem. + + The main point in defining an active learning problem is to define the total search space, + which points in this space have already been labeled, and what those labels are. + + Optionally, you can define the budget of how many points are left to label. + """ + + def __init__(self, points, labeled_ixs: List[int], labels, + budget=None, target_label=1, objective_fun: ObjectiveFunction = Minimize()): + """Set up the active learning problem + + Args: + points (ndarray): Coordinates of all points in the search space + labeled_ixs ([int]): Indices of points that have been labeled + labels (ndarray): Labels for the labeled points, in same order as labeled_ixs + budget (int): How many entries are budgeted to be labeled (default: all of them) + target_label (int): Index the desired class, used in classification problems + objective_fun (ObjectiveFunction): Objective function, used in regression problems + """ + + # TODO: Add batch size and support for grouping points together -lw + self.points = points + self.labeled_ixs = labeled_ixs + self.labels = list(labels) + self.target_label = target_label + self.objective_fun = objective_fun + + # Set the budget + self.budget = budget + if budget is None: + self.budget = len(points) - len(labeled_ixs) + + @classmethod + def from_labeled_and_unlabled(cls, labeled_points, labels, unlabeled_points, **kwargs): + """Construct an active learning problem from labeled and unlabled points + + Args: + labeled_points (ndarray): Coordinates of points with labels + labels (ndarray): Labels of those points + unlabeled_points (ndarray): Points that could possibly be labeled + """ + + points = np.vstack((labeled_points, unlabeled_points)) + labeled_ixs = list(range(len(labeled_points))) + + return cls(points, labeled_ixs, labels, **kwargs) + + def get_unlabeled_ixs(self) -> List[int]: + """Get a list of the unlabeled indices + + Returns: + ([int]) Unlabeled indices + """ + return list( + set(range(len(self.points))).difference(self.labeled_ixs) + ) + + def get_labeled_ixs(self) -> List[int]: + """Get a list of the labeled indices + + Returns: + ([int]): Labeled indices + """ + return list(self.labeled_ixs) + + def add_label(self, ind: int, label: float): + """Add a label to the labeled set + + Args: + ind (int): Index of point to label + label (float): Label of that point + """ + + if ind in self.labeled_ixs: + raise AttributeError('Index already included in labeled set') + self.labeled_ixs.append(ind) + self.labels.append(label) + + def get_labeled_points(self) -> Tuple[np.ndarray, List[Union[float, int]]]: + """Get the labeled points and their labels + + Returns: + - (ndarray): Coordinates of all points with labels + - (ndarray): Labels for all labeled points + """ + return self.points[self.labeled_ixs], self.labels + + def get_unlabeled_points(self) -> np.ndarray: + """Get the coordinates of all unlabeled points + + Returns: + (list) Coordinates of all unlabeled points + """ + return self.points[self.get_unlabeled_ixs()] diff --git a/active_learning/query_strats/__init__.py b/active_learning/query_strats/__init__.py index f9b79ca..e996e38 100644 --- a/active_learning/query_strats/__init__.py +++ b/active_learning/query_strats/__init__.py @@ -1,13 +1,5 @@ +"""General active learning querying strategies""" -from .argmax import argmax +from .random_sampling import RandomQuery -from .active_search import active_search -from .batch_active_search import seq_sim_batch -from .mcal_regression import mcal_regression -from .random_sampling import random_sampling -from .rfr_balanced import rfr_balanced -from .greedy import greedy -from .greedy_regression import greedy_regression -from .rfr_variance import rfr_variance -from .three_ds import three_ds -from .uncertainty_sampling import uncertainty_sampling \ No newline at end of file +__all__ = ['RandomQuery'] diff --git a/active_learning/query_strats/active_search.py b/active_learning/query_strats/active_search.py deleted file mode 100644 index 3635595..0000000 --- a/active_learning/query_strats/active_search.py +++ /dev/null @@ -1,192 +0,0 @@ -from multiprocessing import cpu_count - -from joblib import Parallel, delayed -import numpy as np -from sklearn.base import clone - -from active_learning.utils import chunks -from . import argmax - -""" -The general strategy in this file is taken from the paper -http://proceedings.mlr.press/v70/jiang17d/jiang17d.pdf. -It tries to make an efficient, non-myopic approximation for the future utility -of every unlabeled point. -""" - - -def _lookahead(points, model, train_ixs, obs_labels, x, label): - """ - Does a lookahead at what the model would be if (x, label) were added to the - known set. If the model implements the partial_fit API from sklearn, then - that will be used. Otherwise, the model is retrained from scratch on - problem['points'] + x. - :param problem: dictionary that defines the problem, containing keys: - * points: an (n_samples, n_dim) matrix of points in the space - * num_classes: the number of different classes [0, num_classes) - * batch_size: number of points to query each iteration - * num_queries: the max number of queries we can make on the data - * model: the sk-learn model we are training - :param model: sklearn model to be trained. This is likely a copy of the - model in the problem dictionary because many copies of model trained on - different potential new points to be queried. - :param train_ixs: index into `points` of the training examples - :param obs_labels: labels for the training examples - :param x: the data point in question to lookahead at - :param label: potential label for x (not the real one!) - :return: a newly trained model - """ - if hasattr(model, "partial_fit"): - return model.partial_fit([x], [label], [0, 1]) - - X_train = np.concatenate([points[train_ixs], [x]]) - obs_labels = np.concatenate([obs_labels, [label]]) - return model.fit(X_train, obs_labels) - - -def _split_lookahead(problem, points_and_models, train_ixs, obs_labels): - """ - """ - return [ - (_lookahead(problem['points'], models[0], train_ixs, obs_labels, x, 0), - _lookahead(problem['points'], models[1], train_ixs, obs_labels, x, 1)) - for x, models in points_and_models - ] - - -def _expected_future_utility(model, points, test_set, budget): - """ - The expected future utility of all remaining points is the sum top `budget` - number of probabilities that the model predicts on the test set. This is - assuming that the utility function is the number of targets found, and that - we can only make `budget` queries. - :param model: model trained on training set + potential new point - :param points: all points in the space - :param test_set: set of unlabeled points - potential new point - :param budget: number of points that we will be able to query - :return: utility - """ - probs = model.predict_proba(points[test_set]) - positives = probs[:,1] - - # sum only the top `budget` probabilities! Even if there are more, we can - # only possibly gain `budget` more targets. - klargest = positives.argpartition(-budget)[-budget:] - u = np.sum(positives[klargest]) - return u - - -def _split_future_utility(models, points, test_set, budget): - """ - dumb utility function to evaluate _expected_future_utility on two models. - The two models come from the two possible labels {0, 1} for a potential new - point, and this function is useful for parallelizing the evaluation. - """ - return ( - _expected_future_utility(models[0], points, test_set, budget), - _expected_future_utility(models[1], points, test_set, budget) - ) - - -def _mem_saver(model, points, train_ixs, obs_labels, unlabeled_chunk, all_unlabeled_ixs, budget): - model_copy = clone(model) - chunk_scores = [] - for unlabeled_ix in unlabeled_chunk: - test_set = np.delete(all_unlabeled_ixs, np.argwhere(all_unlabeled_ixs == unlabeled_ix)) - p0, p1 = model.predict_proba(points[[unlabeled_ix]]).reshape(-1) - - m0 = _lookahead(points, model_copy, train_ixs, obs_labels, points[unlabeled_ix], label=0) - s0 = _expected_future_utility(m0, points, test_set, budget) - - m1 = _lookahead(points, model_copy, train_ixs, obs_labels, points[unlabeled_ix], label=1) - s1 = _expected_future_utility(m1, points, test_set, budget) - - chunk_scores.append(p0 * s0 + p1 * s1) - del test_set - - return np.array(chunk_scores) - - -# def _search_score(problem, train_ixs, obs_labels, unlabeled_ixs, batch_size, **kwargs): -# model = problem['model'] -# points = problem['points'] -# -# # num queries remaining -# budget = kwargs['budget'] -# assert budget > 0 -# -# # OS X requires you to use "threading" rather than "multiprocessing" -# # because it doesn't support BLAS calls on both 'sides' of a fork -# # however, we cannot just use threading because RandomForest is not thread safe... -# backend = problem.get("parallel_backend", "threading") -# -# n_cpu = cpu_count() -# n_chunks = len(unlabeled_ixs) // 100 -# -# with Parallel(n_jobs=n_cpu, max_nbytes=1e6, backend=backend) as parallel: -# real_budget = min(budget * batch_size, len(unlabeled_ixs)-1) -# expected_future_utilities = parallel( -# delayed(_mem_saver)(model, points, train_ixs, obs_labels, chunk, unlabeled_ixs, real_budget) -# for chunk in chunks(unlabeled_ixs, n_chunks) -# ) -# -# expected_future_utilities = np.concatenate(expected_future_utilities) -# -# # print(expected_future_utilities) -# return expected_future_utilities - - -def _search_score(problem, train_ixs, obs_labels, unlabeled_ixs, batch_size, **kwargs): - model = problem['model'] - points = problem['points'] - - # num queries remaining - budget = kwargs['budget'] - assert budget > 0 - - # OS X requires you to use "threading" rather than "multiprocessing" - # because it doesn't support BLAS calls on both 'sides' of a fork - # however, we cannot just use threading because RandomForest is not thread safe... - backend = problem.get("parallel_backend", "threading") - - n_cpu = cpu_count() - with Parallel(n_jobs=n_cpu, max_nbytes=1e6, backend=backend) as parallel: - # copy the model many times. Partial fit on each candidate point - # and each possible label for that point - model_copies = np.array(clone([model] * (2 * len(unlabeled_ixs)))) - model_copies = model_copies.reshape(-1, 2) - points_and_models = list(zip(points[unlabeled_ixs], model_copies)) - model_copies = parallel( - delayed(_split_lookahead)(problem, chunk, train_ixs, obs_labels) - for chunk in chunks(points_and_models, n_cpu) - ) - model_copies = sum(model_copies, []) # collapse list of lists into single list - # TODO: maybe we only need to evaluate when y=1, because y=0 cannot increase utility of remaining points?? - - # create one test set for each point: U_{i-1} \ {x} - test_sets = [ - np.delete(unlabeled_ixs, i) - for i in range(len(unlabeled_ixs)) - ] - - # sometimes the budget might be too big, so take min - real_budget = min(budget*batch_size, len(test_sets[0])) - - future_utilities = np.array( - parallel( - delayed(_split_future_utility)(models, points, test_set, real_budget) - for test_set, models in zip(test_sets, model_copies) - ) - ) - - probs = model.predict_proba(points[unlabeled_ixs]) - - E_future_utilities = np.sum(probs * future_utilities, axis=1) - - return probs[:, 1] + 0.10 * E_future_utilities - - -def active_search(problem, train_ixs, obs_labels, unlabeled_ixs, npoints, **kwargs): - score = _search_score - - return argmax(problem, train_ixs, obs_labels, unlabeled_ixs, score, npoints, **kwargs) \ No newline at end of file diff --git a/active_learning/query_strats/argmax.py b/active_learning/query_strats/argmax.py deleted file mode 100644 index 1e64ffe..0000000 --- a/active_learning/query_strats/argmax.py +++ /dev/null @@ -1,73 +0,0 @@ -import numpy as np - - -def _partition_ixs(partition, unlabeled_ixs): - """ - Splits the unlabeled_ixs along the partitions given. - :param partition: list of np.array of indices - :param unlabeled_ixs: np.array of indices - :return: mask of length unlabeled_ixs indicating which are in each partition - """ - return [ - np.in1d(unlabeled_ixs, p_i) - for p_i in partition - ] - - -def argmax(problem, train_ixs, obs_labels, unlabeled_ixs, score, batch_size, **kwargs): - """ - Generic arg-maximizer query strat that can be used with different scoring mechanisms. - :param problem: dictionary that defines the problem, containing keys: - * points: an (n_samples, n_dim) matrix of points in the space - * num_classes: the number of different classes [0, num_classes) - * batch_size: number of points to query each iteration - * num_queries: the max number of queries we can make on the data - * model: the sk-learn model we are training - Optional: - - partition: list of sets of indices into problem['points'] partitioning the space. - This can restrict the batch to be from one partition! - :param train_ixs: index into `points` of the training examples - :param obs_labels: labels for the training examples - :param unlabeled_ixs: np.array of the indices of the unlabeled examples - :param score: scoring function (see scoring API) - :param batch_size: size of the batch to select - :param kwargs: passed on to scoring function - :return: - """ - scores = score(problem, train_ixs, obs_labels, unlabeled_ixs, batch_size, **kwargs) - - # default to argmax over all unlabeled indices - if "partition" not in problem: - best_ixs = np.argsort(scores)[-batch_size:] - # print(scores[best_ixs]) - # print(scores) - return unlabeled_ixs[best_ixs] - - # user can specify partition (maybe only can take batch from one experiment at a time?) - partition = problem['partition'] - part_unlabeled_ixs = _partition_ixs(partition, unlabeled_ixs) - - # find best batch for each partition, choose max across those - max_batch_score = -np.inf - max_batch = None - for p_ixs in part_unlabeled_ixs: - # mask the scores of the elements not in partition - p_score = np.ma.masked_array(scores, mask=~p_ixs) - - # can only choose up to min(num things in partition, batch_size) items from this partition - k = min(np.sum(p_ixs), batch_size) - - # indices of top k scores - top_k_ixs = p_score.argsort(fill_value=-np.inf)[-k:] - - # score for the batch would be sum of them - batch_score = np.sum(p_score[top_k_ixs]) - if batch_score > max_batch_score: - max_batch_score = batch_score - - # batch_mask = np.zeros(p_ixs.shape, dtype=bool) - # batch_mask[top_k_ixs] = True - # max_batch = batch_mask - max_batch = top_k_ixs - - return unlabeled_ixs[max_batch] \ No newline at end of file diff --git a/active_learning/query_strats/base.py b/active_learning/query_strats/base.py new file mode 100644 index 0000000..34778cc --- /dev/null +++ b/active_learning/query_strats/base.py @@ -0,0 +1,125 @@ +"""Base classes for query strategies""" + +from functools import partial +from multiprocessing import Pool +from typing import List, Tuple + +import numpy as np +from sklearn.base import BaseEstimator + +from active_learning.problem import ActiveLearningProblem + + +class BaseQueryStrategy: + """Base class for all active learning query strategies.""" + + def select_points(self, problem: ActiveLearningProblem, n_to_select: int) -> List[int]: + """Identify which points should be queried next + + Args: + problem (ActiveLearningProblem): Problem definition + n_to_select (int): Number of points to select + Returns: + ([int]): List of which points to query next + """ + raise NotImplementedError() + + +class IndividualScoreQueryStrategy(BaseQueryStrategy): + """Base class for query strategies that rate each point independently + + Queries are determined by evaluating the score of batches of entries independently, + which allows for parallelization. + """ + + def __init__(self, n_cpus: int = 1, chunks_per_thread: int = 32): + """Initialize the query strategy + + Args: + n_cpus (int): Number of processors to use + chunks_per_thread (int): Number of chunks of indices per thread when multiprocessing + """ + self.n_cpus = n_cpus + self.chunks_per_thread = chunks_per_thread + + def _score_chunk(self, inds: List[int], problem: ActiveLearningProblem) -> List[float]: + """Score a list of indices + + Used to prevent copying the active learning problem definition many times + + Args: + inds ([int]): List of indices to score + problem (ActiveLearningProblem): Active learning problem definition + Returns: + ([float]) Scores for each entry + """ + + raise NotImplementedError() + + def select_points(self, problem: ActiveLearningProblem, n_to_select: int): + # Get the unlabeled indices as an ndarray (for easy indexing) + unlabled_ixs, scores = self.score_all(problem) + + # Get the top entries + return unlabled_ixs[np.argpartition(scores, -n_to_select)[-n_to_select:]] + + def score_all(self, problem: ActiveLearningProblem) -> Tuple[List[int], List[float]]: + """Determine the scores for all non-labeled points + + Entries with the highest scores are selected + + Args: + problem (ActiveLearningProblem): Active learning problem definition + Return: + - [int] Indices of all unlabeled points + - [float] Scores for each point + """ + # Get the unlabeled indices as an ndarray (for easy indexing) + unlabled_ixs = np.array(problem.get_unlabeled_ixs()) + + # Get the scores + if self.n_cpus == 1: + scores = self._score_chunk(unlabled_ixs, problem) + else: + # Make the chunks of indices to evaluate in parallel + chunks = np.array_split(unlabled_ixs, self.n_cpus * self.chunks_per_thread) + + # Make the function to be evaluated + fun = partial(self._score_chunk, problem=problem) + + # Score each chunk + # TODO (lw): Switch to joblib, so that we can use distributed memory executors + with Pool(self.n_cpus) as p: + scores = p.map(fun, chunks) + + # De-chunk the scores + scores = np.concatenate(scores) + + return unlabled_ixs, scores + + +class ModelBasedQueryStrategy(BaseQueryStrategy): + """Mixin for query strategies that use an model to make predictions + + Model objects must satisfy the scikit-learn API""" + + def __init__(self, model: BaseEstimator, fit_model: bool = True, **kwargs): + """ + Args: + model (BaseEstimator): Model to use for querying + fit_model (bool): Whether to fit the model before selecting points + """ + super().__init__(**kwargs) + self.model = model + self.fit_model = fit_model + + def _fit_model(self, problem: ActiveLearningProblem): + """Fit the model on the current active learning problem + + Args: + problem (ActiveLearningProblem): Description of the active learning problem + """ + X, y = problem.get_labeled_points() + if self.fit_model: + self.model.fit(X, y) + self.fit_model = True diff --git a/active_learning/query_strats/batch_active_search.py b/active_learning/query_strats/batch_active_search.py deleted file mode 100644 index edeb1ce..0000000 --- a/active_learning/query_strats/batch_active_search.py +++ /dev/null @@ -1,102 +0,0 @@ -import numpy as np -from sklearn.base import clone - -from .active_search import active_search, _lookahead -from ..selectors import identity_selector - -""" -https://bayesopt.github.io/papers/2017/12.pdf -""" - -# ----------------------------------------------------------------------------- -# Fictional Oracles to Simulate Sequential -# ----------------------------------------------------------------------------- -def _sampling_oracle(model, x: np.array) -> int: - """ - Samples a label according to bernoulli trial with probability of target - from model - :param model: sklearn model trained on the observed samples observed, - as well as the points and fictional labels in the batch so far. - :param x: point to be labeled - """ - probs = model.predict_proba([x]) - probs = probs.reshape(2) - return np.random.binomial(1, probs[1]) - - -def _most_likely_oracle(model, x): - """ - Return the most likely label for x according to model, i.e.: argmax p(y|x) - :param model: sklearn model trained on the observed samples observed, - as well as the points and fictional labels in the batch so far. - :param x: point to be labeled - :return: - """ - probs = model.predict_proba([x]) - probs = probs.reshape(2) - raise np.argmax(probs) - - -def _pessimistic_oracle(model, x) -> 0: - return 0 - - -def _optimistic_oracle(model, x) -> 1: - raise 1 - - -_FICTIONAL_ORACLES = { - "sampling": _sampling_oracle, - "most_likely": _most_likely_oracle, - "pessimistic": _pessimistic_oracle, - "optimistic": _optimistic_oracle -} - - -# ----------------------------------------------------------------------------- -# Two approximations to compute batch -# ----------------------------------------------------------------------------- - -def seq_sim_batch(problem, train_ixs, obs_labels, unlabeled_ixs, batch_size, **kwargs): - oracle_type = kwargs.get("fictional_oracle", "pessimistic") - fictional_oracle = _FICTIONAL_ORACLES[oracle_type] - - orig_model = problem['model'] - model = clone(orig_model) - points = problem['points'] - - # accumulate batch - batch_ixs = [] - - X = train_ixs - Y = obs_labels - U = unlabeled_ixs - - for i in range(batch_size): - # use the active search policy to select next point - x = active_search(problem, X, Y, U, 1, **kwargs) - # since we requested only one point, get value of singleton - x = x[0] - - # Query the fictional oracle - y = fictional_oracle(model, points[x]) - - # update the sets - batch_ixs.append(x) - X = np.append(X, x) - Y = np.append(Y, y) - U = identity_selector(problem, X, Y, **kwargs) - - problem['model'] = orig_model - return batch_ixs - - -def batch_ens_greedy(problem, train_ixs, obs_labels, unlabeled_ixs, batch_size, **kwargs): - # accumulate batch - batch_ixs = [] - - X = train_ixs - Y = obs_labels - U = unlabeled_ixs - - raise NotImplementedError() \ No newline at end of file diff --git a/active_learning/query_strats/classification/__init__.py b/active_learning/query_strats/classification/__init__.py new file mode 100644 index 0000000..9943720 --- /dev/null +++ b/active_learning/query_strats/classification/__init__.py @@ -0,0 +1,10 @@ +"""Query strategies specific to classification problems""" + +from .active_search import ActiveSearch +from .batch_active_search import SequentialSimulatedBatchSearch +from .greedy import GreedySearch +from .three_ds import ThreeDs +from .uncertainty_sampling import UncertaintySampling + +__all__ = ['ActiveSearch', 'SequentialSimulatedBatchSearch', 'GreedySearch', + 'ThreeDs', 'UncertaintySampling'] diff --git a/active_learning/query_strats/classification/active_search.py b/active_learning/query_strats/classification/active_search.py new file mode 100644 index 0000000..79d7ce1 --- /dev/null +++ b/active_learning/query_strats/classification/active_search.py @@ -0,0 +1,129 @@ +from sklearn.base import BaseEstimator, clone +from active_learning.problem import ActiveLearningProblem +from active_learning.query_strats.base import IndividualScoreQueryStrategy, ModelBasedQueryStrategy +from typing import List +import numpy as np + + +def _lookahead(points: np.ndarray, model: BaseEstimator, + train_ixs: List[int], obs_labels: List[float], + x: np.ndarray, label: float): + """ + Does a lookahead at what the model would be if (x, label) were added to the + known set. If the model implements the partial_fit API from sklearn, then + that will be used. Otherwise, the model is retrained from scratch + + Args: + model (BaseEstimator): sklearn model to be retrained + train_ixs (ndarray): Indices of currently-labeled set + obs_labels (ndarray): Labels for each labeled entry + x (ndarray): Data point to simulate being labeled + label (float): Simulated label + """ + # If partial-fit available, use it + if hasattr(model, "partial_fit"): + return model.partial_fit([x], [label], [0, 1]) + + # Update the training set + X_train = np.concatenate([points[train_ixs], [x]]) + obs_labels = np.concatenate([obs_labels, [label]]) + + # Refit the model + model.fit(X_train, obs_labels) + + +def _split_lookahead(problem, points_and_models, train_ixs, obs_labels): + """ + """ + return [ + (_lookahead(problem['points'], models[0], train_ixs, obs_labels, x, 0), + _lookahead(problem['points'], models[1], train_ixs, obs_labels, x, 1)) + for x, models in points_and_models + ] + + +def _expected_future_utility(model: BaseEstimator, test_set: np.ndarray, + budget: int, target_label: int): + """ + The expected future utility of all remaining points is the sum top `budget` + number of probabilities that the model predicts on the test set. This is + assuming that the utility function is the number of targets found, and that + we can only make `budget` queries. + + Args: + model (BaseEstimator): Model trained on training set + potential new point + test_set (ndarray): Test set for the model + budget (int): number of points that we will be able to query + target_label (int): Index of target label + + Returns: + (float) Expected utility + """ + + # Predict the probability of each entry in the test set + probs = model.predict_proba(test_set) + positives = probs[:, target_label] + + # sum only the top `budget` probabilities! Even if there are more, we can + # only possibly gain `budget` more targets. + klargest = positives.argpartition(-budget)[-budget:] + u = np.sum(positives[klargest]) + + return u + + +class ActiveSearch(ModelBasedQueryStrategy, IndividualScoreQueryStrategy): + """Efficient Non-Myopic Active Search. + + Based on an algorithm by + `Jiang et al. `_. + Automatically balances between the desire to greedily query points + highly likely to be the target class and those which, if queried, + will lead to model improvements that will lead to more + targets to be found later on.""" + + def _score_chunk(self, inds: List[int], problem: ActiveLearningProblem): + # Get a list of *all* unlabeled entries + all_unlabeled_ixs = np.array(problem.get_unlabeled_ixs()) + + # Get the probabilities for each class + probs = self.model.predict_proba(problem.points[all_unlabeled_ixs]) + + # Get scores for this chunk + chunk_scores = [] + for ix, unlabeled_ix in enumerate(inds): + # Make the new test set + test_set = problem.points[all_unlabeled_ixs[all_unlabeled_ixs != unlabeled_ix]] + + # Make a copy of the model to use in lookahead + model = clone(self.model) + + # Get the probabilities for this point + my_probs = probs[ix, :] + + # If the budget is only for one more label, the score is just the probability + if problem.budget - 1 == 0: + chunk_scores.append(my_probs[problem.target_label]) + else: + # If not, assess the effect of labeling this point + + # Get the expected utility of the entry being labeled each class + expected_util = [] + for i in range(my_probs.shape[0]): + _lookahead(problem.points, model, problem.labeled_ixs, + problem.labels, problem.points[unlabeled_ix], + label=i) + expected_util.append(_expected_future_utility(self.model, test_set, + problem.budget - 1, + problem.target_label)) + + # Compute the score for this point + # This is equal to the probability of it being positive (p1) + # plus the expected number of positives found in the + # "budget - 1" remaining entries if this point is labeled, + # which is equal to the product of the probability of "1" or "0" + # times the number of positives in the top budget-1 if "1" or "0" + chunk_scores.append(my_probs[problem.target_label] + + np.dot(my_probs, expected_util)) + + return chunk_scores diff --git a/active_learning/query_strats/classification/batch_active_search.py b/active_learning/query_strats/classification/batch_active_search.py new file mode 100644 index 0000000..2f08ea1 --- /dev/null +++ b/active_learning/query_strats/classification/batch_active_search.py @@ -0,0 +1,129 @@ +from active_learning.problem import ActiveLearningProblem +from active_learning.query_strats.base import ModelBasedQueryStrategy, BaseQueryStrategy +from sklearn.base import BaseEstimator +from copy import deepcopy +from typing import Union +import numpy as np + + +""" +https://bayesopt.github.io/papers/2017/12.pdf +""" + +# ----------------------------------------------------------------------------- +# Fictional Oracles to Simulate Sequential +# ----------------------------------------------------------------------------- + +# TODO: Add oracles for regression problems + + +class FictionalOracle: + """Class to emulate a labeling function""" + + def label(self, model: BaseEstimator, x: np.array, target: int): + """Assign a label to a certain data point + + Args: + model (BaseEstimator): Model trained on all labeled points + x (ndarray): Point to be "labeled" + target (int): Index of the target class + Returns: + (int) Label for this point + """ + raise NotImplementedError() + + +class SamplingOracle(FictionalOracle): + """Pick a random label weighted by the predicted probabilities from the model""" + def label(self, model: BaseEstimator, x: np.array, target: int): + probs = model.predict_proba([x]) + probs = probs.reshape(2) + return np.random.binomial(1, probs[1]) + + +class MostLikelyOracle(FictionalOracle): + """Return the most likely label for x according to model, i.e.: argmax p(y|x)""" + def label(self, model: BaseEstimator, x: np.array, target: int): + probs = model.predict_proba([x]) + probs = probs.reshape(2) + raise np.argmax(probs) + + +class PessimisticOracle(FictionalOracle): + """Assume the prediction is not the target class. + Assumes that the model binary classification""" + def label(self, model: BaseEstimator, x: np.array, target: int): + return 1 - target + + +class OptimisticOracle(FictionalOracle): + """Assume that the prediction is the target class""" + def label(self, model: BaseEstimator, x: np.array, target: int): + return target + + +_FICTIONAL_ORACLES = { + "sampling": SamplingOracle(), + "most_likely": MostLikelyOracle(), + "pessimistic": PessimisticOracle(), + "optimistic": OptimisticOracle() +} + + +class SequentialSimulatedBatchSearch(ModelBasedQueryStrategy): + """Batch active learning strategy where you simulate multiple, sequential steps of an + active learning process. + + TBD: Better description after reading the paper""" + + def __init__(self, model: BaseEstimator, query_strategy: BaseQueryStrategy, + fictional_oracle: Union[str, FictionalOracle], fit_model: bool = True): + """Initialize strategy + + Args: + model (BaseEstimator): Model used to guide the search + fit_model (bool): Whether to fit the model before querying + query_strategy (BaseQueryStrategy): Strategy to perform sequential selection + fictional_oracle (string): Function used to emulate labeling + """ + super().__init__(model=model, fit_model=fit_model) + self.query_strategy = query_strategy + if isinstance(fictional_oracle, str): + self.fictional_oracle = _FICTIONAL_ORACLES[fictional_oracle] + else: + self.fictional_oracle = fictional_oracle + + def select_points(self, problem: ActiveLearningProblem, n_to_select: int): + # Make a copy of the active learning problem + # TODO: Copying the entire problem could be costly. Could we just copy model/labels? -lw + problem = deepcopy(problem) + + # Start accumulation for the batch + batch_ixs = [] + + for _ in range(n_to_select - 1): + # Select a single point + x = self.query_strategy.select_points(problem, 1) + + # since we requested only one point, get value of singleton + x = x[0] + batch_ixs.append(x) + + # Query the fictional oracle + y = self.fictional_oracle.label(self.model, problem.points[x], problem.target_label) + + # Update the active learning problem + problem.add_label(x, y) + self._fit_model(problem) + + # Decrement the budget + problem.budget -= 1 + + # Select a single point + x = self.query_strategy.select_points(problem, 1) + + # since we requested only one point, get value of singleton + x = x[0] + batch_ixs.append(x) + + return batch_ixs diff --git a/active_learning/query_strats/classification/greedy.py b/active_learning/query_strats/classification/greedy.py new file mode 100644 index 0000000..498c561 --- /dev/null +++ b/active_learning/query_strats/classification/greedy.py @@ -0,0 +1,11 @@ +from active_learning.problem import ActiveLearningProblem +from active_learning.query_strats.base import IndividualScoreQueryStrategy, ModelBasedQueryStrategy +from typing import List + + +class GreedySearch(ModelBasedQueryStrategy, IndividualScoreQueryStrategy): + """Query strategy where you pick the score most likely to be the target label""" + + def _score_chunk(self, inds: List[int], problem: ActiveLearningProblem): + probs = self.model.predict_proba(problem.points[inds]) + return probs[:, problem.target_label] diff --git a/active_learning/query_strats/classification/tests/test_active_search.py b/active_learning/query_strats/classification/tests/test_active_search.py new file mode 100644 index 0000000..39d3f85 --- /dev/null +++ b/active_learning/query_strats/classification/tests/test_active_search.py @@ -0,0 +1,34 @@ +from unittest import TestCase + +import numpy as np +from sklearn.neighbors import KNeighborsClassifier + +from active_learning.query_strats.classification.active_search import ActiveSearch +from active_learning.query_strats.classification.greedy import GreedySearch +from active_learning.tests.test_problem import make_grid_problem + + +class TestActiveSearch(TestCase): + + def test_active_search(self): + # Make the grid problem with a KNN + model = KNeighborsClassifier(n_neighbors=3) + problem = make_grid_problem() + problem.positive_label = 1 + problem.budget = 1 + model.fit(*problem.get_labeled_points()) + + # For a budget of 1, active search should be equal to greedy + active_search = ActiveSearch(model) + greedy = GreedySearch(model) + inds, score = active_search.score_all(problem) + greedy_score = greedy._score_chunk(inds, problem) + self.assertTrue(np.isclose(greedy_score, score).all()) + + # Test with a larger lookahead + # Have not figured out a good case to solve by hand, so these test + # results are from a reference implement. + problem.budget = 2 + score = active_search._score_chunk(sorted(inds), problem) + self.assertTrue(np.isclose(score, + [[4./3, 4./3, 4./3, 1]]).all()) diff --git a/active_learning/query_strats/classification/tests/test_batch_active_search.py b/active_learning/query_strats/classification/tests/test_batch_active_search.py new file mode 100644 index 0000000..b1c6450 --- /dev/null +++ b/active_learning/query_strats/classification/tests/test_batch_active_search.py @@ -0,0 +1,21 @@ +from active_learning.query_strats.classification.batch_active_search\ + import SequentialSimulatedBatchSearch +from active_learning.query_strats.random_sampling import RandomQuery +from active_learning.tests.test_problem import make_grid_problem +from unittest import TestCase + +from sklearn.neighbors import KNeighborsClassifier + + +class TestBatch(TestCase): + + def test_seq_sim(self): + problem = make_grid_problem() + + # Make sure the code works. + # TODO: Figure out a batch active learning problem we can model analytically -lw + model = KNeighborsClassifier() + query_strat = RandomQuery() + seq_sim = SequentialSimulatedBatchSearch(model, query_strat, 'pessimistic') + points = seq_sim.select_points(problem, 3) + self.assertEqual(3, len(points)) diff --git a/active_learning/query_strats/classification/tests/test_greedy.py b/active_learning/query_strats/classification/tests/test_greedy.py new file mode 100644 index 0000000..edcea58 --- /dev/null +++ b/active_learning/query_strats/classification/tests/test_greedy.py @@ -0,0 +1,23 @@ +from unittest import TestCase + +import numpy as np +from sklearn.neighbors import KNeighborsClassifier + +from active_learning.query_strats.classification.greedy import GreedySearch +from active_learning.tests.test_problem import make_grid_problem + + +class TestGreedy(TestCase): + + def test_greedy(self): + # Make the grid problem with a KNN + model = KNeighborsClassifier(n_neighbors=3) + problem = make_grid_problem() + problem.positive_label = 1 + model.fit(*problem.get_labeled_points()) + + # Compute the probabilities for each test point + greedy = GreedySearch(model) + inds, scores = greedy.score_all(problem) + probs = model.predict_proba(problem.points[inds])[:, 1] + self.assertTrue(np.isclose(probs, scores).all()) diff --git a/active_learning/query_strats/classification/tests/test_three_ds.py b/active_learning/query_strats/classification/tests/test_three_ds.py new file mode 100644 index 0000000..fa6573d --- /dev/null +++ b/active_learning/query_strats/classification/tests/test_three_ds.py @@ -0,0 +1,23 @@ +from unittest import TestCase + +from sklearn.neighbors import KNeighborsClassifier + +from active_learning.query_strats.classification.three_ds import ThreeDs +from active_learning.tests.test_problem import make_grid_problem + + +class TestUncertainty(TestCase): + + def test_uncertainty(self): + # Make the grid problem with a KNN + model = KNeighborsClassifier(n_neighbors=3) + problem = make_grid_problem() + + # Run the selection + d = ThreeDs(model) + pts = d.select_points(problem, 4) + self.assertEqual(4, len(set(pts))) + + d.dwc = 0 + pts = d.select_points(problem, 4) + self.assertEqual(4, len(set(pts))) diff --git a/active_learning/query_strats/classification/tests/test_uncertainty.py b/active_learning/query_strats/classification/tests/test_uncertainty.py new file mode 100644 index 0000000..5586bf7 --- /dev/null +++ b/active_learning/query_strats/classification/tests/test_uncertainty.py @@ -0,0 +1,22 @@ +from unittest import TestCase + +import numpy as np +from sklearn.neighbors import KNeighborsClassifier + +from active_learning.query_strats.classification.uncertainty_sampling import UncertaintySampling +from active_learning.tests.test_problem import make_grid_problem + + +class TestUncertainty(TestCase): + + def test_uncertainty(self): + # Make the grid problem with a KNN + model = KNeighborsClassifier(n_neighbors=2) + problem = make_grid_problem() + model.fit(*problem.get_labeled_points()) + + # Compute the uncertainties + sampler = UncertaintySampling(model) + probs = model.predict_proba(problem.points[problem.get_unlabeled_ixs()]) + score = -1 * np.multiply(probs, np.log(probs)).sum(axis=1) + self.assertTrue(np.isclose(score, sampler.score_all(problem)[1]).all()) diff --git a/active_learning/query_strats/classification/three_ds.py b/active_learning/query_strats/classification/three_ds.py new file mode 100644 index 0000000..f6d372b --- /dev/null +++ b/active_learning/query_strats/classification/three_ds.py @@ -0,0 +1,118 @@ +from active_learning.query_strats.base import ModelBasedQueryStrategy +from active_learning.problem import ActiveLearningProblem +from sklearn.base import BaseEstimator +from sklearn.mixture.base import BaseMixture +from sklearn.mixture import GaussianMixture +from typing import List +import numpy as np + + +def _calc_diversities(gmm: BaseMixture, points: np.ndarray, + S: List[int], X_test: np.ndarray): + """Compute the diversity of points in the unlabeled set + + Args: + gmm (BaseMixture): Gaussian mixture model trained on the labeled points + points (ndarray): All points in the problem space + S ([int]): Points that have already been selected in this batch + X_test ([int]): Possible points to include in the + Returns: + (np.ndarray) Diversity score for each point + """ + S_scores = gmm.score(points[S]) + S_score = np.sum(S_scores) + + # TODO (lw): The state of S does not change the point with highest score? + log_probs = gmm.score_samples(X_test) + scores = -(log_probs + S_score) / (len(S) + 1) + + return scores + + +class ThreeDs(ModelBasedQueryStrategy): + """Select points based on distance, density, and diversity. + + Based on work by `Reitmaier and Sick `_. + + Distance is based on how far a point is from the decision boundary. The distance is computed + based on the ratio between the probability of the 1st and 2nd-most likely classes. Points + farther from the boundary are more distant. + + Density is related to whether the points are in regions with many points in the search space. + The density is determined by the probability an entry is from the search space given a Gaussian + mixture model. Points with a higher density are desirable. + + Diversity is achieved by selecting a set of points with large differences between each other. + The algorithm assumes the distribution of point in the set is equal to the search space, + so the effect of this factor is to pick points in low-density regions. + """ + + def __init__(self, model: BaseEstimator, dwc: float = 0.5, + gmm: BaseMixture = GaussianMixture()): + """Initialize the strategy + + Args: + model (BaseEstimator): Model used to generate the "distance" metric + dwc (float): Diversity weighting coefficient (larger to weigh diversity more) + gmm (BaseMixture): Strategy used to model the distribution of the search space + """ + super().__init__(model) + self.dwc = dwc + self.gmm = gmm + + def select_points(self, problem: ActiveLearningProblem, n_to_select: int): + # Get the model and search space + points = problem.points + + # Fit the model on the new problem + self._fit_model(problem) + + # Get the unlabeled indices and points + U = problem.get_unlabeled_ixs() + X_test = points[U] + + # calculate weighting factor + probs = self.model.predict_proba(X_test) + eps = 1.0 / len(probs) * np.sum(1 - np.argmax(probs, axis=1)) + + # pseudo-distance that works on models that do not have a real decision boundary + probs = np.apply_along_axis(np.sort, 1, probs) + c1 = probs[:, -1] + c2 = probs[:, -2] + distances = np.log(c1) - np.log(c2) # Only work for binary classification + + # density function via mixture model + # The paper describes traininng the model with initial unlabled set and a small labeled set + # As we do not assume the unlabled set is the initial one or that the labeled set is small, + # I use the entire search space to train a mixture model. + self.gmm.fit(points) + densities = np.exp(self.gmm.score_samples(X_test)) + + # select the first point + x = np.argmax((1-eps) * (1-distances) + eps * densities) + S = [U.pop(x)] + + # Update the tests + X_test = np.delete(X_test, x, 0) + distances = np.delete(distances, x, 0) + densities = np.delete(densities, x, 0) + + # Get the weighting factors + alpha = (1 - self.dwc) * (1 - eps) # coefficient normalization + beta = (1 - self.dwc) - alpha # coefficient normalization + + # Pick points considering density + while len(S) < n_to_select: + # Generate the densities of the test set + diversities = _calc_diversities(self.gmm, points, S, X_test) + + # Select the maximum point + x = np.argmax(alpha*(1-distances) + beta*densities + self.dwc*diversities) + S.append(U.pop(x)) + + # Update the tests + X_test = np.delete(X_test, x, 0) + distances = np.delete(distances, x, 0) + densities = np.delete(densities, x, 0) + + return S diff --git a/active_learning/query_strats/classification/uncertainty_sampling.py b/active_learning/query_strats/classification/uncertainty_sampling.py new file mode 100644 index 0000000..361c2ad --- /dev/null +++ b/active_learning/query_strats/classification/uncertainty_sampling.py @@ -0,0 +1,13 @@ +from active_learning.problem import ActiveLearningProblem +from active_learning.query_strats.base import IndividualScoreQueryStrategy, ModelBasedQueryStrategy +from typing import List +import numpy as np + + +class UncertaintySampling(ModelBasedQueryStrategy, IndividualScoreQueryStrategy): + """Sample entries with the highest uncertainty in the classification score""" + + def _score_chunk(self, inds: List[int], problem: ActiveLearningProblem): + probs = self.model.predict_proba(problem.points[inds]) + probs = probs.clip(1e-9, 1 - 1e-9) + return -1 * (probs * np.log(probs)).sum(axis=1) diff --git a/active_learning/query_strats/greedy.py b/active_learning/query_strats/greedy.py deleted file mode 100644 index 6bf2efe..0000000 --- a/active_learning/query_strats/greedy.py +++ /dev/null @@ -1,8 +0,0 @@ -from active_learning.query_strats import argmax -from active_learning.scoring import probability - - -def greedy(problem, train_ixs, obs_labels, unlabeled_ixs, npoints, **kwargs): - """Greedily choose the most probably to be a target""" - score_fn = probability - return argmax(problem, train_ixs, obs_labels, unlabeled_ixs, score_fn, npoints) \ No newline at end of file diff --git a/active_learning/query_strats/greedy_regression.py b/active_learning/query_strats/greedy_regression.py deleted file mode 100644 index 09e8ab0..0000000 --- a/active_learning/query_strats/greedy_regression.py +++ /dev/null @@ -1,28 +0,0 @@ -import numpy as np - - -def greedy_regression(problem: dict, train_ixs: np.ndarray, obs_values: np.ndarray, unlabeled_ixs: np.ndarray, - batch_size: int, **kwargs): - """ - Simple greedy approach to black box function minimization. Model predicts values for unlabeled points, choose - the points with the minimum values predicted by model. - :param problem: dictionary that defines the problem, containing keys: - * points: an (n_samples, n_dim) matrix of points in the space - * model: the sk-learn model we are training - :param train_ixs: index into `points` of the training examples - :param obs_labels: labels for the training examples - :param unlabeled_ixs: np.array of the indices of the unlabeled examples - :param batch_size: size of the batch to select - :return: - """ - points = problem['points'] - model = problem['model'] - - # delta e is what I was predicting for materials - delta_e_pred = model.predict(points[unlabeled_ixs]) - scores = -delta_e_pred - - # samples that are likely very small delta e - min_delta_e_ixs = np.argpartition(scores, -batch_size)[-batch_size:] - - return unlabeled_ixs[min_delta_e_ixs] diff --git a/active_learning/query_strats/mcal_regression.py b/active_learning/query_strats/mcal_regression.py deleted file mode 100644 index 4e37197..0000000 --- a/active_learning/query_strats/mcal_regression.py +++ /dev/null @@ -1,77 +0,0 @@ -from collections import defaultdict - -import numpy as np -from scipy.spatial.distance import pdist, squareform -from sklearn.cluster import DBSCAN -from sklearn.svm import SVR - - -def _density(pts): - """Sort of density for a set of points""" - pts = np.array(pts) - dists = pdist(pts) - return len(pts) / dists.max(), squareform(dists) - - -def mcal_regression(problem: dict, train_ixs: np.ndarray, obs_labels: np.ndarray, unlabeled_ixs: np.ndarray, - batch_size: int, **kwargs) -> np.ndarray: - """ - Multiple criteria active regression for SVMs (https://doi.org/10.1016/j.patcog.2014.02.001). - :param problem: dictionary that defines the problem, containing keys: - * points: an (n_samples, n_dim) matrix of points in the space - * model: SVM regressor we are training - :param train_ixs: index into `points` of the training examples - :param obs_labels: labels for the training examples - :param unlabeled_ixs: np.array of the indices of the unlabeled examples - :param batch_size: size of the batch to select - :return: - """ - points: np.ndarray = problem['points'] - model: SVR = problem['model'] - assert isinstance(model, SVR) - - # split training points into support vectors and non support vectors - support_mask = np.zeros(len(train_ixs), dtype=bool) - support_mask[model.support_] = True - train_sv_ixs = train_ixs[support_mask] - train_not_sv_ixs = train_ixs[~support_mask] - - # train clusterer - # extra arrays and whatnot to track indices into the points array and whether a given points was - # a training point or not - clusterer = DBSCAN(eps=1.0) - clst_ixs = np.concatenate([train_not_sv_ixs, unlabeled_ixs]) - train_mask = np.zeros(clst_ixs.shape, dtype=bool) - train_mask[:len(train_not_sv_ixs)] = True - clst_pts = points[clst_ixs] - clst_labels = clusterer.fit_predict(clst_pts) - - # group by cluster labels - clst2pts = defaultdict(list) - for pt, label, is_train, ix in zip(clst_pts, clst_labels, train_mask, clst_ixs): - clst2pts[label].append((pt, is_train, ix)) - - # find clusters that do not contain any non support vectors from training - good_clsts = [ - label - for label, pts in clst2pts.items() - if not any(is_train for pt, is_train, ix in pts) - ] - - # find the "densest" clusters - densities = [ - (i, _density([pt for pt, is_train, ix in clst2pts[i]])) - for i in good_clsts - ] - - n_samples = min(batch_size, len(good_clsts)) - k_densest = sorted(densities, key=lambda x: x[1][0], reverse=True)[:n_samples] - - # sample one point from each of the densest clusters - selected = [] - for i, (density, dists) in densities: - dists = np.mean(dists, axis=1) - dense_ix = np.argmin(dists) - selected.append(clst2pts[i][dense_ix][2]) - - return np.array(selected, dtype=int) diff --git a/active_learning/query_strats/random_sampling.py b/active_learning/query_strats/random_sampling.py index 026c164..a36fe87 100644 --- a/active_learning/query_strats/random_sampling.py +++ b/active_learning/query_strats/random_sampling.py @@ -1,8 +1,11 @@ -import numpy as np +from ..problem import ActiveLearningProblem +from active_learning.query_strats.base import IndividualScoreQueryStrategy +from random import random +from typing import List -def random_sampling(problem, train_ixs, obs_labels, unlabeled_ixs, npoints, **kwargs): - """Simple random sample of points from unlabeled indices""" - rand_ixs = np.random.randint(0, len(unlabeled_ixs), size=npoints) +class RandomQuery(IndividualScoreQueryStrategy): + """Randomly select entries from the unlabeled set""" - return unlabeled_ixs[rand_ixs] \ No newline at end of file + def _score_chunk(self, inds: List[int], problem: ActiveLearningProblem): + return [random() for _ in inds] diff --git a/active_learning/query_strats/regression/__init__.py b/active_learning/query_strats/regression/__init__.py new file mode 100644 index 0000000..0ee2773 --- /dev/null +++ b/active_learning/query_strats/regression/__init__.py @@ -0,0 +1,7 @@ +"""Query strategies specific to regression problems""" + +from .greedy import GreedySelection +from .mcal_regression import MCALSelection +from .uncertainty import UncertaintySampling + +__all__ = ['GreedySelection', 'MCALSelection', 'UncertaintySampling'] diff --git a/active_learning/query_strats/regression/bayesian.py b/active_learning/query_strats/regression/bayesian.py new file mode 100644 index 0000000..afd176d --- /dev/null +++ b/active_learning/query_strats/regression/bayesian.py @@ -0,0 +1,55 @@ +"""Bayesian active learning methods""" +from inspect import signature +from typing import List + +import numpy as np +from scipy.stats import norm + +from active_learning.problem import ActiveLearningProblem +from active_learning.query_strats.base import IndividualScoreQueryStrategy + + +# Following: http://krasserm.github.io/2018/03/21/bayesian-optimization/ + +class ExpectedImprovement(IndividualScoreQueryStrategy): + """Bayesian 'Expected Improvement' active learning + + Determines which points have the largest expected improvement over + the best labeled point to date. + + Each point is assigned a value equal to the expected/mean improvement + of that point's value over a threshold. + """ + + def __init__(self, model, refit_model: bool = True, epsilon: float = 0): + """ + Args: + model: Scikit-learn model used to make inferences + """ + super().__init__() + self.model = model + self.refit_model = refit_model + self.epsilon = epsilon + + # Check if the function supports "return_std" + if 'return_std' not in signature(self.model.predict).parameters: + raise ValueError('The model must have "return_std" in the predict methods') + + def select_points(self, problem: ActiveLearningProblem, n_to_select: int): + if self.refit_model: + self.model.fit(*problem.get_labeled_points()) + return super().select_points(problem, n_to_select) + + def _score_chunk(self, inds: List[int], problem: ActiveLearningProblem): + y_mean, y_std = self.model.predict(problem.points[inds], return_std=True) + + # Compute the EI + # TODO (wardlt): Support minimization + _, known_labels = problem.get_labeled_points() + threshold = np.max(known_labels) # f(x^+) in the + z_score = (y_mean - threshold - self.epsilon) / y_std + + ei = (y_mean - threshold - self.epsilon) * norm.cdf(z_score) + y_std * norm.pdf(z_score) + + return ei + diff --git a/active_learning/query_strats/regression/greedy.py b/active_learning/query_strats/regression/greedy.py new file mode 100644 index 0000000..7895969 --- /dev/null +++ b/active_learning/query_strats/regression/greedy.py @@ -0,0 +1,15 @@ +from ...problem import ActiveLearningProblem +from active_learning.query_strats.base import IndividualScoreQueryStrategy, ModelBasedQueryStrategy +from typing import List + + +class GreedySelection(ModelBasedQueryStrategy, IndividualScoreQueryStrategy): + """Select the points with this highest predicted values of the output function""" + + def select_points(self, problem: ActiveLearningProblem, n_to_select: int): + if self.fit_model: + self.model.fit(*problem.get_labeled_points()) + return super().select_points(problem, n_to_select) + + def _score_chunk(self, inds: List[int], problem: ActiveLearningProblem): + return -1 * problem.objective_fun.score(self.model.predict(problem.points[inds])) diff --git a/active_learning/query_strats/regression/mcal_regression.py b/active_learning/query_strats/regression/mcal_regression.py new file mode 100644 index 0000000..157bf4d --- /dev/null +++ b/active_learning/query_strats/regression/mcal_regression.py @@ -0,0 +1,115 @@ +from active_learning.query_strats.base import ModelBasedQueryStrategy +from active_learning.problem import ActiveLearningProblem + +from scipy.spatial.distance import pdist, squareform +from sklearn.cluster import DBSCAN +from sklearn.svm import SVR +from collections import defaultdict +from random import sample +from typing import List +import numpy as np + + +def _density(pts): + """Compute a density-like metric for a set of points + + Args: + pts ([[float]]): Distances for a set of points + Returns: + - (float): Density metrics + - (np.ndarray): Distances between each point + """ + pts = np.array(pts) + dists = pdist(pts) + return len(pts) / dists.max(), squareform(dists) + + +class MCALSelection(ModelBasedQueryStrategy): + """The Multiple Criteria Active Learning method for support vector regression + + Uses the methods described by + `Demir and Bruzzone `_ + to select points for evaluation based on: + + 1. *Relevancy*: Whether each point is likely to be important in model fitting + 2. *Diversity*: Whether the points are different regions of the search space + 3. *Density*: Whether the points are from regions that contain many other points + """ + + def __init__(self, svm_options: dict = None): + """Initialize the model + + Args: + svm_options (dict): Any options for the SVM + """ + # Make the SVR model + model = SVR(**(svm_options if svm_options is not None else {})) + super(MCALSelection, self).__init__(model, fit_model=True) + + def select_points(self, problem: ActiveLearningProblem, n_to_select: int) -> List[int]: + # Fit the SVR model + self._fit_model(problem) + + # split training points into support vectors and non support vectors + train_ixs = np.array(problem.get_labeled_ixs()) + support_mask = np.zeros(len(train_ixs), dtype=bool) + support_mask[self.model.support_] = True + train_not_sv_ixs = train_ixs[~support_mask] + + # train clusterer + # extra arrays and whatnot to track indices into the points array + # and whether a given points was a training point or not + clusterer = DBSCAN(eps=1.0) + unlabeled_ixs = problem.get_unlabeled_ixs() + clst_ixs = np.concatenate([train_not_sv_ixs, unlabeled_ixs]) + train_mask = np.zeros(clst_ixs.shape, dtype=bool) + train_mask[:len(train_not_sv_ixs)] = True + clst_pts = problem.points[clst_ixs] + clst_labels = clusterer.fit_predict(clst_pts) + + # group by cluster labels + clst2pts = defaultdict(list) + for pt, label, is_train, ix in zip(clst_pts, clst_labels, train_mask, clst_ixs): + clst2pts[label].append((pt, is_train, ix)) + + # find clusters that do not contain any non support vectors from training + good_clsts = [ + label + for label, pts in clst2pts.items() + if not any(is_train for pt, is_train, ix in pts) + ] + + # find the "densest" clusters + densities = [ + (i, _density([pt for pt, is_train, ix in clst2pts[i]])) + for i in good_clsts + ] + + n_samples = min(n_to_select, len(good_clsts)) + k_densest = sorted(densities, key=lambda x: x[1][0], reverse=True)[:n_samples] + + # sample one point from each of the densest clusters + selected = [] + for i, (density, dists) in k_densest: + dists = np.mean(dists, axis=1) + dense_ix = np.argmin(dists) + selected.append(clst2pts[i].pop(dense_ix)[2]) + + # Randomly select from good clusters, if selection not met + # Picking randomly from the list of unlabeled indices to target "density" + if len(selected) < n_to_select: + good_ixs = sum([list(map(lambda x: x[2], clst2pts[c])) for c in good_clsts], list()) + unselected_ixs = set(good_ixs).difference(selected) + if len(unselected_ixs) <= n_to_select - len(selected): + # Add all to the list + selected.extend(unselected_ixs) + else: + # Add a random subset + selected.extend(sample(unselected_ixs, n_to_select - len(selected))) + + # Randomly pick points from all the clusters, even the bad ones + if len(selected) < n_to_select: + unselected_ixs = set(unlabeled_ixs).difference(selected) + selected.extend(sample(unselected_ixs, n_to_select - len(selected))) + + return np.array(selected, dtype=int) diff --git a/active_learning/query_strats/regression/tests/test_bayesian.py b/active_learning/query_strats/regression/tests/test_bayesian.py new file mode 100644 index 0000000..356947a --- /dev/null +++ b/active_learning/query_strats/regression/tests/test_bayesian.py @@ -0,0 +1,17 @@ +from active_learning.query_strats.regression.bayesian import ExpectedImprovement +from active_learning.tests.test_problem import make_xsinx +from sklearn.linear_model import BayesianRidge +from unittest import TestCase + + +class TestBayesian(TestCase): + + def test_ei(self): + # Make the problem + problem = make_xsinx() + model = BayesianRidge() + + # Run the sampling. For now, let's just test that it gives the correct number of samples + uncert = ExpectedImprovement(model) + selections = uncert.select_points(problem, 2) + self.assertEqual(len(selections), 2) diff --git a/active_learning/query_strats/regression/tests/test_greedy.py b/active_learning/query_strats/regression/tests/test_greedy.py new file mode 100644 index 0000000..bc3dcb3 --- /dev/null +++ b/active_learning/query_strats/regression/tests/test_greedy.py @@ -0,0 +1,20 @@ +from unittest import TestCase +from sklearn.linear_model import LinearRegression +from active_learning.query_strats.regression.greedy import GreedySelection +from active_learning.tests.test_problem import make_xsinx + + +class TestGreedy(TestCase): + + def test_greedy(self): + problem = make_xsinx() + model = LinearRegression() + + # Should pick either the largest or the smallest index, + # depending on the slope of the linear regression line + greedy = GreedySelection(model) + selection = greedy.select_points(problem, 1)[0] + if greedy.model.coef_.sum() > 0: + self.assertEqual(0, selection) + else: + self.assertEqual(15, selection) diff --git a/active_learning/query_strats/regression/tests/test_mcal.py b/active_learning/query_strats/regression/tests/test_mcal.py new file mode 100644 index 0000000..dcc1066 --- /dev/null +++ b/active_learning/query_strats/regression/tests/test_mcal.py @@ -0,0 +1,15 @@ +from unittest import TestCase +from active_learning.query_strats.regression.mcal_regression import MCALSelection +from active_learning.tests.test_problem import make_xsinx + + +class TestMCAL(TestCase): + + def test_mcal(self): + problem = make_xsinx() + + # Not sure how this one functions yet, + # so we're just going to make sure it does not crash + mcal = MCALSelection() + selection = mcal.select_points(problem, 4) + self.assertEquals(4, len(selection)) diff --git a/active_learning/query_strats/regression/tests/test_uncertainty.py b/active_learning/query_strats/regression/tests/test_uncertainty.py new file mode 100644 index 0000000..39695b5 --- /dev/null +++ b/active_learning/query_strats/regression/tests/test_uncertainty.py @@ -0,0 +1,23 @@ +from active_learning.query_strats.regression import UncertaintySampling +from active_learning.tests.test_problem import make_xsinx +from sklearn.linear_model import BayesianRidge +from unittest import TestCase +import numpy as np + + +class TestUncertainty(TestCase): + + def test_uncertainty(self): + # Make the problem + problem = make_xsinx() + model = BayesianRidge() + + # Figure out the answer beforehand + model.fit(*problem.get_labeled_points()) + _, y_std = model.predict(problem.get_unlabeled_points(), return_std=True) + max_ix = problem.get_unlabeled_ixs()[np.argmax(y_std)] + + # Run the sampling + uncert = UncertaintySampling(model) + selections = uncert.select_points(problem, 1) + self.assertEqual(selections, max_ix) diff --git a/active_learning/query_strats/regression/uncertainty.py b/active_learning/query_strats/regression/uncertainty.py new file mode 100644 index 0000000..cf4f681 --- /dev/null +++ b/active_learning/query_strats/regression/uncertainty.py @@ -0,0 +1,24 @@ +from active_learning.query_strats.base import IndividualScoreQueryStrategy, ModelBasedQueryStrategy +from active_learning.problem import ActiveLearningProblem +from inspect import signature +from typing import List + + +class UncertaintySampling(ModelBasedQueryStrategy, IndividualScoreQueryStrategy): + """Select the entries with the largest uncertainty""" + + def __init__(self, model, refit_model: bool = True): + super().__init__(model, refit_model) + + # Check if the function supports "return_std" + if 'return_std' not in signature(self.model.predict).parameters: + raise ValueError('The model must have "return_std" in the predict methods') + + def select_points(self, problem: ActiveLearningProblem, n_to_select: int): + if self.fit_model: + self.model.fit(*problem.get_labeled_points()) + return super().select_points(problem, n_to_select) + + def _score_chunk(self, inds: List[int], problem: ActiveLearningProblem): + _, y_std = self.model.predict(problem.points[inds], return_std=True) + return y_std diff --git a/active_learning/query_strats/rfr_balanced.py b/active_learning/query_strats/rfr_balanced.py deleted file mode 100644 index c53e5c7..0000000 --- a/active_learning/query_strats/rfr_balanced.py +++ /dev/null @@ -1,81 +0,0 @@ -import numpy as np - -from sklearn.ensemble import RandomForestRegressor -from sklearn.neighbors import NearestNeighbors - - -def _score_variances(estimators, points): - # find the prediction that each estimator would place on each point - # predictions: [n_samples, n_estimators] - predictions = np.hstack([tree.predict(points).reshape(-1, 1) for tree in estimators]) - - # calculate the variance in the estimators' predictions for each sample - variances = np.var(predictions, axis=1) - - return variances - - -def _score_densities(nn: NearestNeighbors, points: np.ndarray): - """ - Average distance of each point to its ten nearest neighbors--gives estimate of density of neighborhood around - each point - """ - dists, ixs = nn.kneighbors(points, n_neighbors=11) - - # lop off first column of all 0s due to points being their own closest neighbor - dists = dists[:, 1:] - - return 1 / dists.mean(axis=1) - - -def rfr_balanced(problem: dict, train_ixs: np.ndarray, obs_values: np.ndarray, unlabeled_ixs: np.ndarray, - batch_size: int, **kwargs): - """ - Active learning for regression based on random forest regressor. Balances three criteria: uncertainty, density, - and greediness. - :param problem: dictionary that defines the problem, containing keys: - * points: an (n_samples, n_dim) matrix of points in the space - * model: RandomForestRegressor - :param train_ixs: - :param obs_values: - :param unlabeled_ixs: - :param batch_size: - :param kwargs: - :return: - """ - points = problem['points'] - model: RandomForestRegressor = problem['model'] - assert isinstance(model, RandomForestRegressor) - - # use nearest neighbors computer to estimate densities, and if possible store the model - # to avoid recomputing - nn_model: NearestNeighbors = problem.get("_nn_model") - if nn_model is None: - nn_model = NearestNeighbors().fit(points) - problem["_nn_model"] = nn_model - - trees = model.estimators_ - unlabeled_pts = points[unlabeled_ixs] - - # compute variance in predictions - measures uncertainty - variances = _score_variances(trees, unlabeled_pts) - variances = variances / variances.max() - - # potentially recompute density of neighborhood around each point - # density = how representative a point is of overall data - if "densities" not in problem: - problem['densities'] = _score_densities(nn_model, points) - densities = problem['densities'][unlabeled_ixs] - densities = densities / densities.max() - - # predict value for each point = greedy criterion - pred_vals = -1 * model.predict(unlabeled_pts) - pred_vals = pred_vals - pred_vals.min() - pred_vals = pred_vals / pred_vals.max() - - scores = variances + densities + pred_vals - - # uncertain samples - uncertain_ixs = np.argpartition(scores, -batch_size)[-batch_size:] - - return unlabeled_ixs[uncertain_ixs] diff --git a/active_learning/query_strats/rfr_variance.py b/active_learning/query_strats/rfr_variance.py deleted file mode 100644 index 570183e..0000000 --- a/active_learning/query_strats/rfr_variance.py +++ /dev/null @@ -1,44 +0,0 @@ -import numpy as np - -from sklearn.ensemble import RandomForestRegressor - - -def _score(estimators, points): - # find the prediction that each estimator would place on each point - # predictions: [n_samples, n_estimators] - predictions = np.hstack([tree.predict(points).reshape(-1, 1) for tree in estimators]) - - # calculate the variance in the estimators' predictions for each sample - variances = np.var(predictions, axis=1) - - return variances - - -def rfr_variance(problem: dict, train_ixs: np.ndarray, obs_values: np.ndarray, unlabeled_ixs: np.ndarray, - batch_size: int, **kwargs): - """ - AL for regression based on RandomForestRegressor. Like uncertainty sampling in the classification setting! - Use estimates from the trees in the forest, compute variance as uncertainty. - :param problem: dictionary that defines the problem, containing keys: - * points: an (n_samples, n_dim) matrix of points in the space - * model: the RandomForestRegressor model we are training - :param train_ixs: - :param obs_values: - :param unlabeled_ixs: - :param batch_size: - :param kwargs: - :return: - """ - points = problem['points'] - model: RandomForestRegressor = problem['model'] - assert isinstance(model, RandomForestRegressor) - - trees = model.estimators_ - unlabeled_pts = points[unlabeled_ixs] - - variances = _score(trees, unlabeled_pts) - - # uncertain samples - uncertain_ixs = np.argpartition(variances, -batch_size)[-batch_size:] - - return unlabeled_ixs[uncertain_ixs] \ No newline at end of file diff --git a/active_learning/query_strats/tests/test_random.py b/active_learning/query_strats/tests/test_random.py new file mode 100644 index 0000000..cbd758b --- /dev/null +++ b/active_learning/query_strats/tests/test_random.py @@ -0,0 +1,21 @@ +from active_learning.tests.test_problem import make_grid_problem +from active_learning.query_strats.random_sampling import RandomQuery +from unittest import TestCase + + +class RandomTest(TestCase): + + def test_random(self): + problem = make_grid_problem() + + # Make the selection tool + query = RandomQuery() + + # Test with no threads + output = query.select_points(problem, 2) + self.assertEqual(2, len(output)) + + # Test with 2 threads + query.n_cpus = 2 + output = query.select_points(problem, 3) + self.assertEqual(3, len(output)) diff --git a/active_learning/query_strats/three_ds.py b/active_learning/query_strats/three_ds.py deleted file mode 100644 index cd6cecb..0000000 --- a/active_learning/query_strats/three_ds.py +++ /dev/null @@ -1,71 +0,0 @@ -import numpy as np -from sklearn.mixture import GaussianMixture - - -def _calc_diversities(gmm, points, S, X_test): - S_scores = gmm.score(points[S]) - S_score = np.sum(S_scores) - - log_probs = gmm.score(X_test) - scores = -(log_probs + S_score) / (len(S) + 1) - - return scores - - -def three_ds(problem, L, obs_labels, U, batch_size, **kwargs): - """ - Selects points based on distance, density, and diversity as found in - https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=5949421 - Importantly: - * distance(x) = log[ p(c1|x) / p(c2|x) ] - where :math:`c1 = argmax_{c} p(c|x)` and :math:`c2 = argmax_{C\setminus\{c1\}} p(c|x)`. - * density(x) = p(x) - according to the mixture model used (Gaussian Mixture Model) - - :param problem: - :param L: np.ndarray of the indices of labeled points - :param obs_labels: - :param U: np.ndarray of the indices of unlabeled points - :param score: - :param batch_size: - :param kwargs: - :return: - """ - points = problem['points'] - model = problem['model'] - - dwc = kwargs.get("dwc", 0.5) # diversity weighting coefficient - - X_train = points[L] - X_test = points[U] - - # calculate weighting factor - probs = model.predict_proba(X_test) - eps = 1.0 / len(probs) * np.sum(1 - np.argmax(probs, axis=1)) - - # pseudo-distance that works on models that do not have a real decision boundary - log_probs = np.log(probs) - maxes = log_probs.max(axis=1) - mins = log_probs.min(axis=1) - distances = maxes - mins - - # density function via mixture model - gmm = GaussianMixture() - gmm.fit(X_train) - densities = gmm.score_samples(X_test) - - # select the first point - x = U[np.argmax((1-eps) * (1-distances) + eps * densities)] - - alpha = (1 - dwc) * (1 - eps) # coefficient normalization - beta = (1 - dwc) - alpha # coefficient normalization - - S = np.array([x], dtype=int) - - while len(S) < batch_size: - diversities = _calc_diversities(gmm, points, S, X_test) - x = U[np.argmax(alpha*(1-distances) + beta*densities + dwc*diversities)] - - S = np.append(S, x) - - return S \ No newline at end of file diff --git a/active_learning/query_strats/uncertainty_sampling.py b/active_learning/query_strats/uncertainty_sampling.py deleted file mode 100644 index d92223f..0000000 --- a/active_learning/query_strats/uncertainty_sampling.py +++ /dev/null @@ -1,10 +0,0 @@ -import numpy as np - -from . import argmax -from active_learning.scoring.marginal_entropy import marginal_entropy - - -def uncertainty_sampling(problem, train_ixs, obs_labels, unlabeled_ixs, npoints, **kwargs): - score = marginal_entropy - - return argmax(problem, train_ixs, obs_labels, unlabeled_ixs, score, npoints) \ No newline at end of file diff --git a/active_learning/scoring/__init__.py b/active_learning/scoring/__init__.py deleted file mode 100644 index 645224f..0000000 --- a/active_learning/scoring/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .information_density import information_density -from .marginal_entropy import marginal_entropy -from .probability import probability \ No newline at end of file diff --git a/active_learning/scoring/information_density.py b/active_learning/scoring/information_density.py deleted file mode 100644 index 998c91c..0000000 --- a/active_learning/scoring/information_density.py +++ /dev/null @@ -1,41 +0,0 @@ -import numpy as np -from scipy.spatial import distance -from sklearn.metrics.pairwise import pairwise_distances - - -def information_density(problem: dict, train_ixs: np.ndarray, obs_labels: np.ndarray, unlabeled_ixs: np.ndarray, - batch_size: int, **kwargs) -> np.ndarray: - """ - Score is uncertainty(x) * representativeness(x)--In particular marginal entropy of the point times its - average distance to all other points. - :param problem: dictionary that defines the problem, containing keys: - * points: an (n_samples, n_dim) matrix of points in the space - * num_classes: the number of different classes [0, num_classes) - * batch_size: number of points to query each iteration - * num_queries: the max number of queries we can make on the data - * model: the sk-learn model we are training - :param train_ixs: index into `points` of the training examples - :param obs_labels: labels for the training examples - :param unlabeled_ixs: indices into problem['points'] to score - :param batch_size: - :param kwargs: unused - :return: scores for each of selected_ixs - """ - points = problem['points'] - model = problem['model'] - - test_X = points[unlabeled_ixs] - - p_x = model.predict_proba(test_X) - p_x = p_x.clip(1e-9, 1 - 1e-9) - logp_x = np.log(p_x) - uncertainties = -1 * (p_x * logp_x).sum(axis=1) - - # TODO: rather than use average distance, use GMM to estimate density? - # pdistances = distance.pdist(points, metric="sqeuclidean") - # pdistances = distance.squareform(pdistances) - pdistances = pairwise_distances(points, metric="l1",n_jobs=-1) - avg_distances = pdistances.sum(axis=1) / (len(points) - 1.) - avg_distances = avg_distances[unlabeled_ixs] - - return avg_distances * uncertainties diff --git a/active_learning/scoring/marginal_entropy.py b/active_learning/scoring/marginal_entropy.py deleted file mode 100644 index c503afb..0000000 --- a/active_learning/scoring/marginal_entropy.py +++ /dev/null @@ -1,31 +0,0 @@ -import numpy as np - - -def marginal_entropy(problem: dict, train_ixs: np.ndarray, obs_labels: np.ndarray, unlabeled_ixs: np.ndarray, - batch_size: int, **kwargs) -> np.ndarray: - """ - Score is -p(x)log[p(x)] i.e. marginal entropy of the point. - :param problem: dictionary that defines the problem, containing keys: - * points: an (n_samples, n_dim) matrix of points in the space - * num_classes: the number of different classes [0, num_classes) - * batch_size: number of points to query each iteration - * num_queries: the max number of queries we can make on the data - * model: the sk-learn model we are training - :param train_ixs: index into `points` of the training examples - :param obs_labels: labels for the training examples - :param unlabeled_ixs: indices into problem['points'] to score - :param kwargs: unused - :return: scores for each of selected_ixs - """ - points = problem['points'] - model = problem['model'] - - test_X = points[unlabeled_ixs] - - p_x = model.predict_proba(test_X) - p_x = p_x.clip(1e-9, 1 - 1e-9) - - logp_x = np.log(p_x) - - return -1 * (p_x * logp_x).sum(axis=1) - # return 1/ np.abs(model.decision_function(test_X)) diff --git a/active_learning/scoring/probability.py b/active_learning/scoring/probability.py deleted file mode 100644 index cc10332..0000000 --- a/active_learning/scoring/probability.py +++ /dev/null @@ -1,23 +0,0 @@ -def probability(problem, train_ixs, obs_labels, selected_ixs, batch_size, **kwargs): - """ - Score is simply the probability of being a target under current model. - :param problem: dictionary that defines the problem, containing keys: - * points: an (n_samples, n_dim) matrix of points in the space - * num_classes: the number of different classes [0, num_classes) - * batch_size: number of points to query each iteration - * num_queries: the max number of queries we can make on the data - * model: the sk-learn model we are training - :param train_ixs: index into `points` of the training examples - :param obs_labels: labels for the training examples - :param selected_ixs: indices into problem['points'] to score - :param kwargs: unused - :return: scores for each of selected_ixs - """ - points = problem['points'] - model = problem['model'] - - test_X = points[selected_ixs] - - p_x = model.predict_proba(test_X) - - return p_x[:,1].reshape(-1) \ No newline at end of file diff --git a/active_learning/selectors/__init__.py b/active_learning/selectors/__init__.py deleted file mode 100644 index 13f6ebf..0000000 --- a/active_learning/selectors/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -import numpy as np - - -""" -A selector is a function that takes in the problem dictionary (described -below), the training set, and the observed labels for the training set, and -outputs the indices into problem['points'] of the unlabeled set of points for -consideration in the next round of queries. - -Selectors should be functions of the form: - - selector(problem, train_ixs, obs_labels) - -Where: - :param problem: dictionary that defines the problem, containing keys: - * points: an (n_samples, n_dim) matrix of points in the space - * num_classes: the number of different classes [0, num_classes) - * batch_size: number of points to query each iteration - * num_queries: the max number of queries we can make on the data - * model: the sk-learn model we are training - :param train_ixs: index into `points` of the training examples - :param obs_labels: labels for the training examples - :returns: np.array of ints indexing into problem['points'] -""" - - -def identity_selector(problem, train_ixs, obs_labels, **kwargs): - space = problem['points'] - num_points = space.shape[0] - ixs = np.arange(num_points) - - train_mask = np.zeros(space.shape[0], dtype=bool) - train_mask[train_ixs] = True - - # return every index not in the training mask - return ixs[~train_mask] \ No newline at end of file diff --git a/active_learning/ss_active_learning.py b/active_learning/ss_active_learning.py deleted file mode 100644 index 4ebe6d9..0000000 --- a/active_learning/ss_active_learning.py +++ /dev/null @@ -1,172 +0,0 @@ -import logging -import numpy as np - -#TODO: make this query a kNN sort of model or maybe clustering for predicting how likely -from collections import Iterable - -from sklearn.svm import SVC - -from active_learning.query_strats import uncertainty_sampling -from active_learning.selectors import identity_selector - - -def _standardize_is_ssl_round(is_ssl_round): - if not callable(is_ssl_round): - if isinstance(is_ssl_round, int): - return lambda i, n: i % is_ssl_round == 0 and i != 0 - else: - raise TypeError("SSL_every key in `problem` dict must be callable or int") - return is_ssl_round - - -def _default_ssl_config(): - return { - "is_ssl_round": (lambda i, n: i >= n // 2 and i % 3 == 0), - "clear_label_history": (lambda i, n: i >= n // 2 and i % 6 == 0) - } - - -def _ss_labeler(problem, train_ixs, obs_labels, unlabeled_ixs, **kwargs): - label_history = kwargs['label_history'] - label_change_rate = np.sum(label_history, axis=0) - - pts = problem['points'] - X_train = pts[train_ixs] - - # clf = KNeighborsClassifier(n_neighbors=4) - clf = SVC(probability=True) - clf.fit(X_train, obs_labels) - - ixs = np.arange(len(problem['points'])) - train_mask = np.zeros(len(ixs), dtype=bool) - train_mask[train_ixs] = True - # only choose among points with label change rate = 0 - test_ixs = ixs[(~train_mask) & (label_change_rate == 0)] - - if len(test_ixs) == 0: - return np.zeros(0, dtype=int), np.zeros(0, dtype=int) - - pred_labels = clf.predict(pts[test_ixs]) - pos_ixs = test_ixs[pred_labels == 1] - neg_ixs = test_ixs[pred_labels == 0] - - if len(pos_ixs) > 0: - pos_distances = clf.decision_function(pts[pos_ixs]).reshape(-1) - # log_probs = np.log(clf.predict_proba(pts[pos_ixs])) - # pos_distances = log_probs.max(axis=1) - log_probs.min(axis=1) - median_pos_ix = pos_ixs[np.argsort(pos_distances)[-1]] # [len(pos_ixs)//2]] - pos_ixs = np.array([median_pos_ix], dtype=int) - else: - pos_ixs = np.array([], dtype=int) - - if len(neg_ixs) > 0: - neg_distances = clf.decision_function(pts[neg_ixs]).reshape(-1) - # log_probs = np.log(clf.predict_proba(pts[neg_ixs])) - # neg_distances = log_probs.max(axis=1) - log_probs.min(axis=1) - median_neg_ix = neg_ixs[np.argsort(neg_distances)[-1]] # [len(neg_ixs)//2]] - neg_ixs = np.array([median_neg_ix], dtype=int) - else: - neg_ixs = np.array([], dtype=int) - - return pos_ixs, neg_ixs - - -def ss_actively_learn(problem, train_ixs, obs_labels, oracle, - query_strat=uncertainty_sampling, - ss_labeler=_ss_labeler, - callback=None, **kwargs): - """ - A variation on active learning that will use a Semi-supervised influenced approach to avoid - oversampling the dense region. - :param problem: dictionary that defines the problem, containing keys: - * points: an (n_samples, n_dim) matrix of points in the space - * num_classes: the number of different classes [0, num_classes) - * batch_size: number of points to query each iteration - * num_queries: the max number of queries we can make on the data - * model: the sk-learn model we are training - Optional: - - - partition: list of np.arrays of indices into problem['points'] partitioning the space. - This can restrict the batch to be from one partition! - - is_ssl_round: `int` or callable. If `int` then `ss_labeler` will be called every `is_ssl_round` iterations - except for the first. If callable, will be called as `is_ssl_round(query_num, num_queries)`. - - clear_label_history: `int` or callable. If `int` then label history will be cleared every x iterations - except for the first. If callable, will be called as `clear_label_history(query_num, num_queries)`. - :param train_ixs: index into `points` of the training examples - :param obs_labels: labels for the training examples - :param oracle: gets the true labels for a set of points - :param selector: gets the indexes of all available points to be tested - :param query_strat: gets the indexes of points we wish to test next - :param ss_labeler: semi-supervised labeler, returns (list of pos_ixs, list neg_ixs) - :param callback: function or list of funcs to call at end of each iteration (retrain the model?) - :return: None (yet) - """ - - problem['num_initial'] = len(train_ixs) - num_queries = problem['num_queries'] - batch_size = problem['batch_size'] - - # function to check whether ss_labeler should be queried in a given round - is_ssl_round = _standardize_is_ssl_round(problem.get('is_ssl_round', 3)) - - # function to check whether label_history should be cleared, default to every ssl_round - clear_label_history = _standardize_is_ssl_round(problem.get('clear_label_history', 3)) - - selector = kwargs.get("selector", identity_selector) - - # in order to track label change rate, store what each point would be labeled with - label_history = [] - - for i in range(num_queries): - print(f"Query {i} / {num_queries}") - - # get the available points we might want to query - unlabeled_ixs = selector(problem, train_ixs, obs_labels, **kwargs) - logging.debug(f"{len(unlabeled_ixs)} available unlabeled points") - - # to track label change rate - label_history.append(problem['model'].predict(problem['points'])) - - if len(unlabeled_ixs) == 0: - logging.debug("No available unlabeled points") - return - - # ------------------ - # choose points from the available points - selected_ixs = query_strat(problem, train_ixs, obs_labels, unlabeled_ixs, batch_size, - budget=num_queries - i, **kwargs) - - # get the true labels from the oracle - true_labels = oracle(problem, train_ixs, obs_labels, selected_ixs, **kwargs) - - # add the new labeled points to the training set - train_ixs = np.concatenate([train_ixs, selected_ixs]) - obs_labels = np.concatenate([obs_labels, true_labels]) - # ------------------- - - # ------------------- - if is_ssl_round(i, num_queries): - # choose points from a model that should be targets - pos_ixs, neg_ixs = ss_labeler(problem, train_ixs, obs_labels, unlabeled_ixs, label_history=label_history, **kwargs) - proactive_ixs = np.concatenate([pos_ixs, neg_ixs]) - ss_labels = np.concatenate([ - np.ones(len(pos_ixs), dtype=int), - np.zeros(len(neg_ixs), dtype=int) - ]) - - # just fake it till you make it and add these to the training set - train_ixs = np.concatenate([train_ixs, proactive_ixs]) - obs_labels = np.concatenate([obs_labels, ss_labels]) - - if clear_label_history(i, num_queries): - label_history = [] - # ------------------- - - # presumably the call back will update the model - if callback is None: - continue - elif isinstance(callback, Iterable): - for func in callback: - func(problem, train_ixs, obs_labels, query=selected_ixs, ** kwargs) - else: - callback(problem, train_ixs, obs_labels, query=selected_ixs, **kwargs) diff --git a/active_learning/tests/__init__.py b/active_learning/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/active_learning/tests/test_problem.py b/active_learning/tests/test_problem.py new file mode 100644 index 0000000..ac2f2f5 --- /dev/null +++ b/active_learning/tests/test_problem.py @@ -0,0 +1,46 @@ +from ..problem import ActiveLearningProblem +from unittest import TestCase +import numpy as np + + +def make_grid_problem(): + """Makes a test active learning problem based on a grid + + 0 1 2 + ------------- + 0 | x | ? | ? | + ------------- + 1 | ? | o | x | + ------------- + 2 | x | ? | o | + ------------- + + The grid is designed to make computing the utilities of each point easier. + """ + + # Print out the points + x_known = [(0, 0), (1, 1), (1, 2), (2, 0), (2, 2)] + x_labels = [1, 0, 1, 1, 0] + x_unlabeled = [(0, 1), (0, 2), (1, 0), (2, 1)] + + # Make the active learning problem + return ActiveLearningProblem.from_labeled_and_unlabled(x_known, x_labels, x_unlabeled, + target_label=1) + + +def make_xsinx(): + points = np.arange(0, 16)[:, None] + y = np.squeeze(points * np.sin(points)) + selection = [1, 4, 11, 6] + return ActiveLearningProblem(points, selection, y[selection]) + + +class TestProblem(TestCase): + + def test_grid(self): + prob = make_grid_problem() + + self.assertEqual((9, 2), prob.points.shape) + self.assertListEqual(list(range(5)), prob.labeled_ixs) + self.assertListEqual(list(range(5, 9)), prob.get_unlabeled_ixs()) + self.assertEqual(1, prob.target_label) diff --git a/active_learning/utils.py b/active_learning/utils.py deleted file mode 100644 index 2b2c0cd..0000000 --- a/active_learning/utils.py +++ /dev/null @@ -1,266 +0,0 @@ -import numpy as np -from sklearn.base import ClassifierMixin -from sklearn.metrics import accuracy_score -from sklearn.model_selection import train_test_split -from sklearn.svm import SVC - -from active_learning.selectors import identity_selector -from active_learning.ss_active_learning import ss_actively_learn, _default_ssl_config -from .active_learning import _actively_learn -from active_learning.query_strats.uncertainty_sampling import uncertainty_sampling - - -def chunks(l: list, n: int) -> list: - """ - Chunks a list into parts - :param l: - :param n: - """ - chunk_size = max(len(l) // n, 1) - while len(l): - yield l[:chunk_size] - l = l[chunk_size:] - - -def make_training_set(estimator, y: np.ndarray, size: int = 5, n_targets: int = 1): - """ - Produces a training set for active learning problems. - - :param estimator: estimator we are training - :param y: array of labels for points in the problem space - :param size: size of the training set to produce - :param n_targets: number of targets (y=1) to include in training set - :return: array of indices into ``y`` - """ - assert size >= 0 and 0 <= n_targets <= size - - train_ixs = np.random.randint(low=0, high=len(y), size=size) - - # if classification problem, we only want to include so many targets in the input - if isinstance(estimator, ClassifierMixin): - while sum(y[train_ixs]) != n_targets: - train_ixs = np.random.randint(low=0, high=len(y), size=size) - - # otherwise we can just use random selection - return train_ixs - - -# ----------------------------------------------------------------------------- -# Useful callback functions -# ----------------------------------------------------------------------------- - - -def retrain_model(problem, train_ixs, obs_labels): - """ - Retrain the problem['model'] on train_ixs and obs_labels. - :param problem: - :param train_ixs: - :param obs_labels: - :return: - """ - points = problem['points'] - model = problem['model'] - problem['model'] = model.fit(points[train_ixs], obs_labels) - - -def make_save_scores(scores, score_fn, X_test, y_test): - """ - Callback to save the score of the model on a test set (i.e.: accuracy, F1, etc) - :param X_test: - :param y_test: - :return: - """ - def save_scores(problem, train_ixs, obs_labels, **kwargs): - model = problem['model'] - pred_labels = model.predict(X_test) - score = score_fn(y_test, pred_labels) - scores.append(score) - - return save_scores - - -def make_save_queries(queries_list): - def save_queries(problem, train_ixs, obs_labels, **kwargs): - queries_list.append(kwargs['query']) - - return save_queries - - -def make_history_retrain(history): - """ - Makes a `callback` function that can be passed to `actively_learn` that - accumulates the number of targets seen so far in each iteration in a list. - :return: (pointer to the list, callback function) - """ - - def plot_retrain_model(problem, train_ixs, obs_labels, **kwargs): - history.append(np.sum(obs_labels)) - retrain_model(problem, train_ixs, obs_labels) - - return plot_retrain_model - - -def make_callback_retrain(func): - def retrain_callback(problem, train_ixs, obs_labels, **kwargs): - func(problem, train_ixs, obs_labels, **kwargs) - retrain_model(problem, train_ixs, obs_labels) - - return retrain_callback - - -# ----------------------------------------------------------------------------- -# Oracles -# ----------------------------------------------------------------------------- - - -def make_training_oracle(y_true: np.array): - """ - Makes a training oracle: - training_oracle(problem, train_ixs, obs_labels, selected_ixs) - That returns y_true[selected_ixs]. Useful for training active learning - models. - :param y_true: np array of the true labels - :return: - """ - - def training_oracle(problem, train_ixs, obs_labels, selected_ixs, **kwargs): - return y_true[selected_ixs] - - return training_oracle - - -# ----------------------------------------------------------------------------- -# Testing Stuff -# ----------------------------------------------------------------------------- - - -def _standardize_score_fns(score_fns) -> dict: - if isinstance(score_fns, dict): - return score_fns - - if callable(score_fns): - return { - "score": score_fns - } - - if score_fns is None: - return { - "accuracy": accuracy_score - } - - if isinstance(score_fns, list) or isinstance(score_fns, tuple): - if callable(score_fns[0]): - return { - f"score{i}": score_fn - for i, score_fn in enumerate(score_fns) - } - - return dict(score_fns) - - raise TypeError("score_fns bad") - - -def perform_experiment(X: np.ndarray, y: np.ndarray, X_test: np.ndarray=None, y_test: np.ndarray=None, - L=None, - base_estimator=SVC(probability=True), - init_L_size: int=2, n_queries: int=40, batch_size: int=1, - semisupervised: bool=False, - selector=identity_selector, query_strat=uncertainty_sampling, oracle=None, - score_fns=None, parallel_backend='threading', random_state=None, **kwargs): - """ - A function that presents a simple API to users who don't need the most fine grained control. - :param X: numpy array of inputs: (n_samples, n_features) - :param y: numpy array of targets: (n_samples,) - :param X_test: numpy array of test set inputs. Will be used rather than splitting if there are score functions - :param y_test: numpy array of test set targets. Will be used rather than splitting if there are score functions - :param L: - numpy array of ints for the initial labeled set that indexes into X. If None, then one will be selected and - will assume that all of the labels in y are correct. If given, then y will only be assumed to be correct at - `y[L]`, otherwise it will be assumed to be entirely correct. - :param base_estimator: the scikit-learn compatible model to be trained - :param n_queries: number of queries to make of the oracle / iterations of active learning - :param batch_size: number of points that can be made in each query - :param semisupervised: if true, will use model to guess labels for points on which it is very confident - :param init_L_size: size of the initial labeled set to train the model - :param selector: selects what points are eligible in the next round - :param query_strat: selects points for the oracle to label - :param oracle: function to label any point in X - :param score_fns: - score functions to track after each round of AL. Can be dict of name->function, list/tuple of functions, or - a single function. if left to None, then accuracy will be used. If there are any scoring functions (i.e.: you - do not explicitly set it to an empty dict) then a test set will be separated out from X, y. - :param parallel_backend: 'threading' or 'multiprocessing' to use in active search strategy - :param random_state: to use for train/test split for consistency across experiments - :param kwargs: passed to train_test_split - :return: - dictionary of data from the experiment - """ - score_fns = _standardize_score_fns(score_fns) - - # These are the fields which can be filled in - experiment_data = { - "n_targets": [], - "queries": [], - "history": [] - } - - callbacks = [] - - callbacks += [make_save_queries(experiment_data["queries"])] - callbacks += [make_history_retrain(experiment_data["history"])] - - # if score_fn, then - if len(score_fns) > 0: - if X_test is None or y_test is None: - X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=random_state, **kwargs) - X, y = X_train, y_train - - for name, score_fn in score_fns.items(): - experiment_data[name] = [] - score_callback = make_save_scores(experiment_data[name], score_fn, X_test, y_test) - callbacks.append(score_callback) - - if L is None: - L = make_training_set(base_estimator, y, size=init_L_size) - - if oracle is None: - oracle = make_training_oracle(y) - - problem = { - "model": base_estimator, - "num_queries": n_queries, - "batch_size": batch_size, - "points": X, - "training_set_size": init_L_size, - "parallel_backend": parallel_backend - } - - retrain_model(problem, L, y[L]) - - if not semisupervised: - _actively_learn( - problem, - L, y[L], - oracle=oracle, - selector=selector, - query_strat=query_strat, - callback=callbacks - ) - else: - # load some good defaults - default_ssl_config = _default_ssl_config() - ssl_config = default_ssl_config.copy() - if isinstance(semisupervised, dict): - ssl_config.update(semisupervised) - ssl_config.update(problem) - problem = ssl_config - - ss_actively_learn( - problem, - L, y[L], - oracle=oracle, - query_strat=query_strat, - callback=callbacks - ) - - return {**experiment_data, **problem} diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..298ea9e --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,19 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/api-docs.rst b/docs/api-docs.rst new file mode 100644 index 0000000..9f6c765 --- /dev/null +++ b/docs/api-docs.rst @@ -0,0 +1,42 @@ +API Documentation +================= + +This part of the documentation holds the detailed documentation for all parts of the ``active-learning`` API. + +active_learning.problem +----------------------- + +.. automodule:: active_learning.problem + :members: + + +active_learning.objective +------------------------- + +.. automodule:: active_learning.objective + :members: + + +active_learning.query_strats +---------------------------- + +.. automodule:: active_learning.query_strats + :members: + +active_learning.query_strats.base +--------------------------------- + +.. automodule:: active_learning.query_strats.base + :members: + +active_learning.query_strats.classification +------------------------------------------- + +.. automodule:: active_learning.query_strats.classification + :members: + +active_learning.query_strats.regression +--------------------------------------- + +.. automodule:: active_learning.query_strats.regression + :members: diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..ac1b2ce --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'active-learning' +copyright = '2019, Globus Labs' +author = 'Globus Labs' + +# The short X.Y version +version = '' +# The full version, including alpha/beta/rc tags +release = '' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.napoleon', + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.imgmath', + 'sphinx.ext.viewcode', + 'sphinx.ext.coverage' +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = None + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'active-learningdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'active-learning.tex', 'active-learning Documentation', + 'Globus Labs', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'active-learning', 'active-learning Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'active-learning', 'active-learning Documentation', + author, 'active-learning', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Options for Epub output ------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + + +# -- Extension configuration ------------------------------------------------- + +autoclass_content = 'both' + +# -- Options for intersphinx extension --------------------------------------- + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/': None} + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..e6b78c5 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,33 @@ +Welcome to active-learning's documentation! +=========================================== + +``active-learning`` is a library of different active learning methods. +Active learning methods are designed to identify what new data to capture, +and a variety of algorithms exist depending on what goals you have. +There are algorithms for when your objectives are to build the best machine-learning model. +There are also algorithms for when your objective is optimization (i.e., collecting the "best" data), +or even a combination of these two objectives. +Some algorithms are designed to identify a single experiment to run, and others optimize selecting a batch of new entries. +We created ``active-learning`` to make it easy to try out these different algorithms. + +The key algorithms implemented in ``active-learning`` are known as "query strategies." +Query strategies are designed to identify which new experiments to perform provided a list of +experiments have been performed, the results of those experiments, and a list of valid new experiments. +Each query strategy is + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + problem-definition + query-strategies + api-docs + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..7893348 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/problem-definition.rst b/docs/problem-definition.rst new file mode 100644 index 0000000..d0b5418 --- /dev/null +++ b/docs/problem-definition.rst @@ -0,0 +1,20 @@ +Problem Definition +================== + +``active-learning`` is built around a core class, :class:`active_learning.problem.ActiveLearningProblem`, that defines the active learning problem. + +Problem definitions require two features to be defined: + +#. *Search Space*: All possible experiments that could be performed +#. *Labels*: A set of experiments that have been perform, and their results (i.e., labels) + +Some active learning algorithms also make use of further information about the active learning problem: + +#. *Budget*: How many new experiments can be performed +#. *Objective*: Whether there are any desired experimental outcomes (e.g., successes, optimized properties) + +The ``ActiveLearningProblem`` class stores all of this information in a single object to simplify defining and performing active learning: + +.. autoclass:: active_learning.problem.ActiveLearningProblem + :members: + :noindex: diff --git a/docs/query-strategies.rst b/docs/query-strategies.rst new file mode 100644 index 0000000..dd1b1c7 --- /dev/null +++ b/docs/query-strategies.rst @@ -0,0 +1,44 @@ +Query Strategies +================ + +This portion of the documentation details the query strategies that are available in ``active-learning`` + +Base API +-------- + +All of the query strategies are based on the ``BaseQueryStrategy``, +which provides a consistent API between all strategies. + +.. autoclass:: active_learning.query_strats.base.BaseQueryStrategy + :members: + :noindex: + +General Strategies +------------------ + +Several active learning strategies are agnostic to the type of problem being solved (e.g., classification, regression). + +.. automodule:: active_learning.query_strats + :members: + :noindex: + +Classification +-------------- + +``active-learning`` provides several algorithms for active learning in classification tasks. + + +.. automodule:: active_learning.query_strats.classification + :members: + :exclude-members: select_points + :noindex: + +Regression +---------- + +There are also many algorithms for supporting regression tasks. + +.. automodule:: active_learning.query_strats.regression + :members: + :exclude-members: select_points + :noindex: diff --git a/example.ipynb b/example.ipynb deleted file mode 100644 index cb2a86d..0000000 --- a/example.ipynb +++ /dev/null @@ -1,337 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Example of simple use of active learning API\n", - "Compare 3 query strategies: random sampling, uncertainty sampling, and active search.\n", - "Observe how we trade off between finding targets and accuracy." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Imports" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import warnings\n", - "warnings.filterwarnings(action='ignore', category=RuntimeWarning)\n", - "\n", - "from matplotlib import pyplot as plt\n", - "import numpy as np\n", - "\n", - "from sklearn.base import clone\n", - "from sklearn.datasets import make_moons\n", - "from sklearn.svm import SVC\n", - "\n", - "import active_learning\n", - "from active_learning.utils import *\n", - "from active_learning.query_strats import random_sampling, uncertainty_sampling, active_search\n", - "\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "np.random.seed(0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Load toy data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Have a little binary classification task that is not linearly separable." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "X, y = make_moons(noise=0.1, n_samples=200)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnX20XXV55z9PLhe5VYcESRUuieA0C7SKRO+gLV1d8iKgtiSjDqJ9wRGatko7pR0WYVyLMkw7Blkjras6bUqp2iqQAsZYYVLkZVwLi+WmgfCiFIQKuaWSGsJocxvy8swfZ59kn3P3+8vZ+5zz/ax11zln733Ofs6++/ye3+95NXdHCCGE6LKoaQGEEEK0CykGIYQQPUgxCCGE6EGKQQghRA9SDEIIIXqQYhBCCNGDFIMQQogepBiEEEL0UIliMLMbzOx5M3skZv8vmNk2M3vYzL5pZm8O7fvHYPuDZjZbhTxCCCGKY1VkPpvZzwI/Ar7g7m+M2P/TwLfd/QUzexdwlbu/Ldj3j8CMu/9L1vMdffTRfvzxx5eWWwghxoktW7b8i7svTTvusCpO5u7fMLPjE/Z/M/TyfuC4Muc7/vjjmZ3V4kIIIfJgZt/LclwTPoaLgDtCrx34GzPbYmZr4t5kZmvMbNbMZnfs2FG7kEIIMa5UsmLIipmdTkcx/Exo88+4+5yZ/Thwp5l9x92/0f9ed18PrAeYmZlR5T8hhKiJga0YzOxk4Hpglbv/oLvd3eeCx+eBLwOnDkomIYQQCxmIYjCz5cBtwC+5+z+Etr/czF7ZfQ6cDURGNgkhhBgMlZiSzOxG4B3A0Wa2HfhdYBLA3f8YuBJ4FfBZMwPY5+4zwKuBLwfbDgO+5O7/pwqZhBBCFKOqqKQPpuy/GLg4YvtTwJsXvkMIIURTKPNZCCFEDwONShKiSjZunePazY/zT7vmOXbxFJedcyKrV043LZYQQ48UgxhKNm6d44rbHmZ+734A5nbNc8VtDwNIOQhREikGkUpdM/Myn3vt5scPKoUu83v3c+3mx6UYhCiJFINIpK6ZednP/add87m2CyGyI+ezSCRpZt7k5x67eCrXdiFEdqQYRCJZZ+Ybt85x2rq7OWHt1zht3d1s3DpXyefGcdk5JzI1OdGzbWpygsvOOTHT+4UQ8UgxiESyzMy7ZqG5XfM4h8xCScqh7Ix/9cppPvHeNzG9eAoDphdP8Yn3vkn+BSEqQD4Gkchl55zY4wuAhTPzIo7gLJ+bxuqV01IEQtSAFINIpDvwJkUPFTELZflcIUQzSDGIVNJm5scunmIuQgmkmYXaMuNXopwQvcjHIEozzI7gIv4RIUYdrRhEaQZtFqpyhq9EOSEWIsUgKiHKLFSHiabqhDslygmxEJmSRpy8+QVVnrcOE03VCXdKlBNiIVIMI0yT9vO6MqbzzPCzKMVh9o8IURcyJY0wTdrP6zLRZI2AympySvKPtLF4oBCDoJIVg5ndYGbPm1lkv2br8Gkze9LMtpnZW0L7LjSzJ4K/C6uQR3SoanDOY47qHusx+8uaaLLO8LOsWLqyXnrzgwBc94FTuG/tGQeVQh2rLUVBiWGgKlPS54BzE/a/C1gR/K0B/jeAmR1Fpz/024BTgd81syUVyTT2xA3CDpn9DXkGsvCxURQ10YQV07WbH+d9b51OLYWRphTTvlceU1gexVmXiU2IKqlEMbj7N4CdCYesAr7gHe4HFpvZMcA5wJ3uvtPdXwDuJFnBiBxEza67ZJ2p5hnIoo7tUrSWUdQAfuuWOS4750SeXveegzP8ftKcymnfK0/xwDwrAEVBiWFgUM7naeDZ0Ovtwba47QswszVmNmtmszt27KhN0FEiXGguiqgBvn/2Gzf7jxrI4gY3g9gBPI2iM+w0k1PaAJ01WimvfIqCEsPA0EQluft6d59x95mlS5c2Lc7QsHrlNPetPQOL2R8eIKNmv3HvixrI6hj0is6w06qvpsma1ZeRVz5FQYlhYFBRSXPAstDr44Jtc8A7+rbfOyCZxoos0TxRs1+nM+MPO5PjBrK4iqmnn7SU09bdXSgKJ4vccVE+SbWY0qq7Zs3mPnJqkl3zexPlC6PigWIYGJRi2ARcYmY30XE0v+juz5nZZuB/hhzOZwNXDEimsSJLmeu4Wa7TmXGnDWRRg97pJy3l1i1zhTOV0+QumgmdZYBOK/K3cesc//rSvgXbJxdZ4gqgLcUDhYijEsVgZjfSmfkfbWbb6UQaTQK4+x8DtwPvBp4EdgP/Odi308z+B/BA8FFXu3uSE3voaEvMepaBMG52Pr14ivvWnhH72Unf8bR1d5fKpUiTu0yuRtkB+trNj7N3/8LA3FcccZgGfjHUVKIY3P2DKfsd+FjMvhuAG6qQo21UXdenLGkDYZHmOWnfsYoonCS580QPVa2g4869a/dC05IQw8TQOJ+HkWGLWS/SLjPtO9YdhVNX69Gqzi3EMCLFUCPDGLPejWJKyhEIk/Yd647CyfL5dSnoMt+tqeKGQmRBtZJqpGhns2Ei7TvWHYVTV+vRvOee2zXPhFmPwon7jm0zMQrRjxRDjVTR8H5QFLXBZ/mOdUfh1NV6NOu5gVwDvZoDibYjU1KNFLHZN0EZG/wwfMciJp866x/FrVTmds3LrCRagVYMNdNEzHre2X/ZGWzb4/LzmrPymnrymqriVjBZziXEIJBiGDGK2K+H0UmelzzKK6ui7CrgvCXGo8xvaecSYpDIlDRiFInAiRvAFpmNpVkjS+HAMiXG04ob9p9LiEEjxTBiFJn9x5Xn3u8+dk1kNm6dy1Q4sGyJ8W5YcJxyyNMzQ4iqkSmpYarOyC0SgdM93+9seIj93msYGRWzRtbrHGcaMshUV6pbYjwrSWYl+RtEU2jF0CB1ZOQWTbpavXKaAx5tLa/brFF3slee65xUSLC/rlQUeUNgi/TMEKJupBgapI6M3DLho02UeKhCOaYpljzXOe679g/cVYbA5umZIcQgkCmpQerMyC1iemgiIa9sqGyWKKw81znrNagjBHYcMuXFcCDF0CBZB4Ik+3iVPoommsiUVY5ZFEueATfPNag6BHaYMuXFaCPF0CBZBoKomeZv3fwg//2rj/Kek48p1QQnikEnq5WdJWdRLHkH3DquQRY5V6+cZvZ7O7nxW8+y350JM9731nYnD4rRRD6GBsniD4gLi3xh916+eP8zQ1XWO4qy1Vez+EXaULYja3nwW7fMHYwM2+/OrVvmFLIqBo5WDA2TNjtNMqnEZdwOk7OyrPkqj0+gyZl3FjlVXE+0hapae54L/CEwAVzv7uv69l8HnB68/DHgx919cbBvP/BwsO8Zdz+vCplGhaS6OknvGSbKDNpN+EWK0GR5cCHyUloxmNkE8BngncB24AEz2+Tuj3WPcfdLQ8f/BrAy9BHz7n5KWTlGlbS6OkbvymEcnZVNrway0mR5cCHyUIWP4VTgSXd/yt1fAm4CViUc/0HgxgrOOxZ07eOLpyYX7JuanOAX3r681SWvx5UiSXt1d7sTIitVmJKmgWdDr7cDb4s60MxeC5wA3B3afISZzQL7gHXuvjHmvWuANQDLly+vQOzhoTvTrKOhfRSDOs+oUrRD27CYxcToYx5TBiHzB5i9HzjX3S8OXv8S8DZ3vyTi2MuB49z9N0Lbpt19zsxeR0dhnOnu300658zMjM/OzpaSuwmGYcDtH9SgM2vVSiQ7p627O9IkNL14KlcdJSGqxsy2uPtM2nFVmJLmgGWh18cF26K4gD4zkrvPBY9PAffS638YGeqoi1QHdZTpGDcG4USuu76UGG+qMCU9AKwwsxPoKIQLgA/1H2RmJwFLgL8NbVsC7Hb3PWZ2NHAa8MkKZGoVG7fODbxyadHVSZZBbRhWPk1StxO5qKlKiKyUXjG4+z7gEmAz8G1gg7s/amZXm1k49PQC4CbvtV29Hpg1s4eAe+j4GB5jhOj+iPuVQpc6QhHLrE7SErGGZeXTJG3rMS1EXirJY3D324Hb+7Zd2ff6qoj3fRN4UxUytJWkhi5QTyhimUSptEQsJWGl07Ye00LkRZnPNZP0Y60rFLHMwJE2qGlQykYdPaa7KN9B1I0UQ83E/YgnzGqL9Ck7cCQNahqU4onyvUD6yiGvslUVVlE3KqJXM3H25v91/ptrM73UmSilJKxoonwvl93yEJf91UOp/pi8DZLaUBRQjDZaMdRME0lLdZ5TSVjRRJmD9u5fGHAQZSIqsgIYljIgYjgpneDWBMOa4FYEhYYOByes/Vpstdt+DHh63Xt6tun/LAZB1gQ3rRhaSHeQmNs131MkT/Hq7SVPFdy4znFF/qeD6u4nxgsphpbRH7rYPwtVaGg7iTIHTU4YOOw9cOi/WKU/JinMFVASnCiMFEPLSMt7AIWGtpE430vUtqoG5rREN+WbiKJIMbSMLIO+QkPbSZw5qK6BuEhOiSYVIgsKV20ZaYN+q0JDt22A694IVy3uPG7b0LREY0VSmGveEFghwkgxtIyoPAELHlsVr75tA3z1N+HFZwHvPH71N6UcBkhSTonyTUQZZEoaMGmRIkOTJ3DX1bC3zyyxd76z/eTzm5FpzMhyr0T5PE5bd3e77y3ROMpjGCB5muC0PtTwqsUsjJkCMLhq16ClERlQE6aWsW1DZyL14nY48jg488raJ1WDbNQzdhRtkpK1XPJQlLY+8rh820XjqFx3i2i5KVampBT6Z+6nn7SUW7fMFYoPzxpFMhSlrc+8snMjh81Jk1Od7aKVqDJui2i5KVYrhgSiZu5fvP+ZwrOurJEiST/g1rR0PPl8+PlPw5HLAOs8/vynW3FTi2gUqdQiXtyeb/uAkWJIIGrmHueRyVIOIWukSNwPdfGPTbbLxHTy+XDpIx2fwqWPSCm0HEUqtYg4k+vUklaEgFeiGMzsXDN73MyeNLO1Efs/bGY7zOzB4O/i0L4LzeyJ4O/CKuSpijxLbIPUATprueS4H7B7fDbrSKH8iFxkXUWqXHcO6r4Hz7yyY3oNM3E47Plhr9/htjXw179d7bkzUNrHYGYTwGeAdwLbgQfMbFNE7+ab3f2SvvceBfwuMENnMr4leO8LZeWqgjyF0Rwy+QCyFEuLC0O89OYHI48fKRtx1ynXtb92nXKgFUkEeduC9t9b3UmFlEOIQdyD3c8JRyW99K8wv7PvQIfZG2D52wd6/1exYjgVeNLdn3L3l4CbgFUZ33sOcKe77wyUwZ3AuRXIlJmk2VbczD2OKgfo1SunuW/tGTy97j3ct/YMVq+cHg8bcZJTTiwgb6TRUES8Nc2g7sF+U+x83HzYB37/V6EYpoFnQ6+3B9v6eZ+ZbTOzW8xsWc731kLajyRu6T0dMxAvMqv1BzYWNuKWO+XaRt5II4WsZqCpezAp1HvA9/+gwlW/Ctzo7nvM7FeBzwNn5PkAM1sDrAFYvnx5JUJlCQuNM/30JwoB7HevtbTx0GRFl+HI4wIba8R2sYC8PbgVspqBpu7BM6/s+BSiQlwGfP9XsWKYA5aFXh8XbDuIu//A3fcEL68H3pr1vaHPWO/uM+4+s3Tp0grELv4j6a4kJswW7EubfZUNN40yMY0UUU455UfEErWKnFxk7H5pX+Q9NhbmyLI0dQ+efD7MfIRD1dEGeO4+qlAMDwArzOwEMzscuADYFD7AzI4JvTwP+HbwfDNwtpktMbMlwNnBtoFQ5keyeuU0B2LKicQplrz23dbkLAwS5Ufkot/cuXhqEgxe2L038h4bC3NkWZq8B3/uU/De9Y3f/5XUSjKzdwN/AEwAN7j775vZ1cCsu28ys0/QUQj7gJ3Ar7v7d4L3fgT4b8FH/b67/3na+aqqlVS2dszKq/+GF3bvXbB9evEU961daCk7bd3dkcv+qONV10YUIcs91vo6XKI2Btrz2d1vB27v23Zl6PkVwBUx770BuKEKOfJSxma/cescP/q3fQu2T05Y7Owrj+lqKMpiiFopMoDHhVeHtxftLy3Gh7GvlVT0R3Lt5sd7evl2efnhh8V+Xh5HoZyE403e/IQuE2bsj7ACRPnDhIhDJTEKEjdAvzi/0LTUJY99V07C8aZoWGmUUkjaLkQUUgwFiRugk3IZ8pQkGHknocpeJFJ0xRiXYxO3XYgoxt6UVJTLzjkxVy5Dv734ug+ckmgSGOmchTpKDjTQ9KRO8uYndDn9pKX85f3PRG6PIs2PIUf1eCLFUJDuj+N3Njy0YJne7yQuai8eWSdh1bXoR7C+UtTEI8uK8Z7v7Mi8Pe2+LHrfiuFHpqQSZM1liLMX/9bND45PfkKYqksOjGB9paKVUKuKfMuyX1RAS02qWjGUJMuSP8kuPJazsKpLDoxofaUiK8a4+9Hp5DiETUFpSkSRcTXT4pWuVgwlyeIkTrMLj90srOqSA+o/fZCo+7FLfxb0kVOTkccdu3iKjVvnWBQT4qrIuIpo8UpXiqEkWZb8ST/WLmMzC+s6iffOgwXXJJz2X2RprfpKBwnfj1F0JyEbt87xry9FJGguMk4/aSlX3PZwZIjrSEXGNU3Uqjlp+wCRKakC0pb84QijuMzUsZiF9S+dff+hAbyrFIosraOangx5VFIZuvfjCWu/FtmKttugZ+/+hXtfccRh3POdHQt8C9BJklNJlgqxic5vIGp7w2jFMCC6VVH/4AOnjHZ+QhJpS+cyS2v1n15AUpJk3Ap11+69sfsOuEspVEmUUuhub9gRLcUwYMa6726ak3hEnchNkeT/SlIayrqvkbCpNGll0F0tN6QcZEpqgJHNT0gjLRpJTXoqJS1JMilPokgOhUghypSaRJncnpJIMfShTM8aOfPK3h8GdHwMK87uzKJefJZOkxLv3T+GTuSqiJuEZMms1++gYqJMpRDva4DGVstSDCGU6VkzUU7iFWfDQ18K/WCcg8rhyGVj7USum6SV69iuauskbpD3A517vUWrZSmGEOqBMABOPr93oL/ujRGzqEApXPrIQEUTolaSTKVxq+mGVstyPodQpmcDyOHcOGPZQrYJkvJtWtbStpIVg5mdC/whndae17v7ur79vw1cTKe15w7gI+7+vWDffuDh4NBn3P28KmSKIs1/ULSipSjB1BKY3xm9XdRC+Hew+Mcm+dG/7TvYdErm0xpJy7fpX003SGnFYGYTwGeAdwLbgQfMbJO7PxY6bCsw4+67zezXgU8CHwj2zbv7KWXlSCOL/6BoRUshhoX+30FUz3KZT2ukRYN/ElWYkk4FnnT3p9z9JeAmYFX4AHe/x913By/vBwbuUclSKXKscwyaYv6FfNtFKaJ+B1HIfFqSllZNzUoVpqRpIOxR2Q68LeH4i4A7Qq+PMLNZOmamde6+sQKZFpDVfxCOxuguuS+9+UGF7NVFltyFbRvgjssPmZymjoJ3XTMUM6+2kXXAl/m0BC2umpqVgTqfzewXgRng2tDm17r7DPAh4A/M7N/HvHeNmc2a2eyOHdHNSJLIm83ZXXLP7ZrHWViZUlREWgG8bRvgKx/r9UPM74SNHx26WVgbyDLgy3xakhZXTc1KFYphDlgWen1csK0HMzsL+Dhwnrvv6W5397ng8SngXmBl1Encfb27z7j7zNKl0W0Kk8jbQ1lNSgZEWjTGXVfD/pcWvu/A3qH6obWFqN/B5ISxeGpS5tM8JJmKYiPtnh0a01IVpqQHgBVmdgIdhXABndn/QcxsJfAnwLnu/nxo+xJgt7vvMbOjgdPoOKYrJ28PZYWuDpAkh1xS2KpCWnOT9DsIm06v3fy4TKdxpJmK4syjAPhQmJZKKwZ332dmlwCb6YSr3uDuj5rZ1cCsu2+iYzp6BfBX1mn+0Q1LfT3wJ2Z2gM7qZV1fNFOl5MnmVOhqS0j6kamGUiGifgfK+s9BWs/yqGS1fhqsg5SFSvIY3P124Pa+bVeGnp8V875vAm+qQoaqOf2kpXzx/md66tnL9toAZ17Z8TH0m5MWTaqGUoUo6z8HaUmZ/fkKkV0xQsd3m1e1qJeISmJEsHHrHLdumev5dxrwvreqfszA6f5AFJVUKzKd5iBLJF3YPHqwQGQfU0vgmhN6AytaYmaSYoggavbkwD3fyR8NJQoQNYO6/OmmpRppZDrNQd66RlHHTxwOe37YCaLopwVmJtVKikCzpwbpOvZefJYeR104imPIk4faSN6ovbEmb12jqOMPf0W0UujScGCFVgwRaPbUIGmOvRFIHmojeaP2xp68pS36j79qcfLxDQdWSDFEoJpJDZLm2EtTHKIwaVF7amJVIUnRdi1oTiVTUgSqmdQgcTOl7naV6W4EVQKomKiMf4DJl8NhU3DbmkbNpFoxxKAOVg2R5tirui90C0MF24jCWSsmqZthN0qpQTOpFINoF2k166vsdCV/RSJh01FMJL4CMsqQpZthQ2ZSKQbRPpIce2mKIw/yV8TSnwkdhwIyKqRFZlIphpzIAdcCqmp20qIfYtvI0rdBARkVU7WZtARyPudADrgRoZsHEWcgGYMaTGl9npNMRArIqIm0EvQDRCuGHMgBNwL0+xX6aUGoYN1kKZgXl8szvXiK+9aeMThhx4kqzaQlGUvFUNQcVEVGtExRDRPlV+hy5LKxiErKMsFRLk9DtKQn9NgphjLlhctmRKu0cQmqCiuN9R8YXPpIKRGHhSwTHGVCjzdjpxjKmIPKzqJkiipI1rDSLMqjRQ6+psg6wVEuTwtoKM9m7JzPZcxBZTOiVZyvIHFhpbf9yqHs0CzF96CTRIT1bhsDv0IYFcwbErLe0zUwdiuGsuagMrMoFecrSGJ7z+DHcthUtPK44/JDM66pJZ1Sx/3RSMed2gq77iDo+rjm9+5nwoz97kzLTFQtVc3yG8yzqWTFYGbnmtnjZvakma2N2P8yM7s52P8tMzs+tO+KYPvjZnZOFfIk0eRsSTO1gqSZefbO9zY7CTO/89CMa35ndKnjp78xFqW7w+HWAPvdD95/UgoVkXeWn1RCvsE8m9KKwcwmgM8A7wLeAHzQzN7Qd9hFwAvu/hPAdcA1wXvfAFwA/CRwLvDZ4PNqo8kCeSrOV5C4gmOV4Z1Z2IiT5OMSFZE0y+8nTYmkFZSskSpMSacCT7r7UwBmdhOwCngsdMwq4Krg+S3AH5mZBdtvcvc9wNNm9mTweX9bgVyxNOlUk0OvAD3x3TGliqeOgn3zyQ3YkxixbOeosGj5uAZAnll+mqmoyrpgOanClDQNhH+t24Ntkce4+z7gReBVGd8rROeHcukj8N4/jc4Ofdc1C7tkTR2V/fNHKCopLkP/yKnJyOPHysdVd/e/PLP8NCWSt1NchQyN89nM1gBrAJYvX96wNKIx+rNDp5Z0Xt+2ZqGjLy3LucuIRSXFmYyOmFzE1OTE+CatDaKabp5ZfpbQ6YYS3qpYMcwBy0Kvjwu2RR5jZocBRwI/yPheANx9vbvPuPvM0qVLKxBbDC0HVw/rO+aj+Z1E2mh7Zlwx2MTAZmGDIs40tGv33vH2ceWx/xclzyy/RbWR+qlixfAAsMLMTqAzqF8AfKjvmE3AhXR8B+8H7nZ3N7NNwJfM7FPAscAK4O8qkEmMA1nC+bozrqjVw+TUyCkFSA6LLurjGolSLoOK8sk6y29RbaR+SisGd99nZpcAm4EJ4AZ3f9TMrgZm3X0T8GfAXwTO5Z10lAfBcRvoOKr3AR9z9+Rav0J0yfNDb/GPsGqqrnM0MqVc2pj13pLaSP2Ye1xvpvYyMzPjs7OzTYshmua6N8b80JeNTd2jOKqc4Z+27u7RqLTa1KqxRe1jzWyLu8+kHTc0zmchFtBgOF/bqTIseiTCXLuD8975jl/J9w+mmu6Qto8du1pJYoSoOpyv7lDGISUunHVowlx7EsnoKIXuBKLuwXkQDu8a0IpBDDdV2WiHdGY3CIa+N0OTvb2HtH2sVgxCwNDO7AbB0JdyqXpwzrOy7ObZ9NPyhEqtGISAoZ3ZDYqhLuVSZTRSnpXltg3w0o8Wfsaiydb7wbRiEAIaLVgmaqbKRLI8K8u7rob9Ly3c/rJXtt48KcUgBLQ6C1WUpMoghTwry7hj51/If94BI1OSEDBWCXBjSVVBCnnMUm1MqMuIFIMQXVqahSpaRJ7cmSHOs5EpSYgqUS7EaJPHLNVg2eyyqCSGGF0GXYpgjAr1ieFEJTFKMhLVJMeZJhLWmkykEqJCZEqKIK4D1satka0iRBvJElZYtdlHuRDtpg4z34iaDrViiCCpabpWDUNC2iBdZEWRZpoa4iiUkaeqFWT4Hpha0klg6+YqjFAZFa0YIhiJapLjTlrCWt4SGD2F2CK6xYFyIdpM0ZIn4RXBNSfAxo8eugfmdy5MYBuRMipSDBEMfTVJkT5I5zX7ZBlYhjgKZeQpYubrnwzM74QDe4ufa4iQKSmCoa8mKdIT1vKafbIOLMqFaCdFzHxRk4Gs5xpySq0YzOwoM7vTzJ4IHheUEjSzU8zsb83sUTPbZmYfCO37nJk9bWYPBn+nlJGnKoa+mqTocPL5nU5uV+3qPIYH7LxmH9VSGm6KmPmKzPxHxHRYdsWwFrjL3deZ2drg9eV9x+wGftndnzCzY4EtZrbZ3XcF+y9z91tKylE5Q11NUqRz8vnwzP2w5XOdxi02AW/+UPxsf4izWAXFSp7ErTLCLJrsFMWbf2GkyqiUVQyrgHcEzz8P3EufYnD3fwg9/yczex5YCuxCiEHRH1G04mx46EsdpQCdx4e+BMvfHp/FCqqlNMzkNfNFTQYmDofDXzFyiqCfUpnPZrbL3RcHzw14ofs65vhT6SiQn3T3A2b2OeCngD3AXcBad9+Tdl5lPotcRGUkY0DEvX/kso7Zqez5pEBGg4P/y2cH2yu6JirLfDazrwOvidj18fALd3czi9UyZnYM8BfAhe5+INh8BfDPwOHAejqrjchYLzNbA6wBWL58eZrYQhwi0okYc6uWjSipM+NaCmfwdK/vmLV9TXU+u/tZ7v7GiL+vAN8PBvzuwP981GeY2b8DvgZ83N3vD332c95hD/DnwKkJcqx39xl3n1m6dGm+bynGmzyDvS0ql71aV4vQLHkUw0YVWcODyDwew7avZfMYNgEXBs8vBL7Sf4CZHQ58GfhCv5M5pFQMWA2UXMMLEUFs5JAt3OSTd9OnAAAMvklEQVT7yw24dZXFGLXBqQpFNyhlOYalTsoqhnXAO83sCeCs4DVmNmNm1wfHnA/8LPDhiLDUL5rZw8DDwNHA75WUR4iFxIUqznykYzfup8yAW1dY66gNTlUoukEpyzEMVS6lGNz9B+5+pruvCExOO4Pts+5+cfD8L9190t1PCf09GOw7w93fFJimftHdIzpnC1GSuIzkn/sUHHR39VF0wK2rLMaoDU5VKLpBKcsxLHWizGcxHsSFKlZd+K6usNY25lGUcYbHXfepBTmy+T+jamU5hqHKatQjxpu2NteJGnShPYNT0esWDv+MYtEkrP5stu/V1v8dtDaCLGu4qhSDEG37Ebd5wOty3RtjZusJeSCR+SQR5MklSfrfNZWD0OL/nxSDEMNKkUF30Fy1mOhcEOvUpooi7nvl+YwsbNsAd1zeqYYaRd2DdIv/f2rtKcSwMgwRSEXs+1nlz+sj6G+es+eHyeWx6263Ogz/vxTUj0GINrFtQyfJLoqkAXPQLSaLROpkGfDzOtTb2DNhBCLIpBiEaJqDg/qRcNuaQ4X9wiQNmE1kRRdpShSlTCYOh6mjsn9GP23smTAC4a0yJQlRJXkd2QsclRF2e5tIHjCTEr2K9q/OQt5qpVGlzlf+UiefpChFZv6LJusdpEcgvFWKQYiqKFJA747L02e8fiB5UMlr066z0F8S2zbkK3WehSw9E/p52SvrH6SHvJOfTElCVEXeEg3bNsRHzoRJM3vktWk3VXepjvMmmqdimH+h+PnGBCkGIaoi78w9y4CYxTad16bdVNRMHeeN8nWs+gxc/nSwLYIhcgI3hRSDEFWRd+aeNiBmdcbmdQTHyVO25HgadUXrxPX2HgEncFPIxyBEVeStZxRbL+iozow3D3ls2mdeCV/5GOx/qXd7t+R49/PiKOq4HnS9pxFwAjeFFIMQVZF3IIobKN91Tf2yxlU8yBLNVNRx3cRAPeRO4KZQSQwhmqTuOk1Rn59UxA4oVNaiBeUectP2QoU1oJIYQrSZ/kHpveurH4DiZvepRewKlLWoolf2IAfkqGuz8aNgdsjENga9neOQYhCiLGWT2uoagOLCQ5PIUtaiqh4IPSW4jYPJfYMYkKOuTVQpjbrrKrWUUlFJZnaUmd1pZk8Ej5FdNsxsf6it56bQ9hPM7Ftm9qSZ3Rz0hxZieChSjiIpnr/Kmkd5Z/FTRxUra1HEgdxz3WBBxnfdeRVVdIobYcqGq64F7nL3FcBdweso5kNtPc8Lbb8GuM7dfwJ4AbiopDxCDJYiSVux5phnq615lDaLtwkOhre+9087kVBZQ2PDCWSHTcUfH0eWGkdNFLore+yIUFYxrAI+Hzz/PLA66xvNzIAzgFuKvF+I2sgzay9ic4/NI5ioNjM4anYfxg8sjP3Pyr6QnPM7Owrsr3+7/HULM+hCd4smO1nTYcY076GsYni1uz8XPP9n4NUxxx1hZrNmdr+ZdQf/VwG73H1f8Ho7MF1SHiHKkdc0VCRpK84cE1VVFYrPnLuze5vIL2MScauk2Rt6r9ttvwLXnBB97dLOXfeAHJUUuPqznazpPBVjR5RU57OZfR14TcSuj4dfuLubWVzs62vdfc7MXgfcbWYPAy/mEdTM1gBrAJYvX57nrUJkJ2+l0iJJW3Hx/HFhpGVmzt1zVZlYFquoIn7+3dVEWBaIvm5dB/Qg2m925Yk6xxgqgn5SFYO7nxW3z8y+b2bHuPtzZnYM8HzMZ8wFj0+Z2b3ASuBWYLGZHRasGo4D5hLkWA+sh04eQ5rcQhQir2moaNJW3KBUR2Zw1YlleSuaRilWZSW3mrLhqpuAC4F1weNX+g8IIpV2u/seMzsaOA34ZLDCuAd4P3BT3PuFGChFwjGryq6tc7CsMgM4abYfR5RiVVZyaymrGNYBG8zsIuB7wPkAZjYD/Jq7Xwy8HvgTMztAx6exzt0fC95/OXCTmf0esBX4s5LyCFGOQdfz6WcYBssoBbbi7E5vhbhIozGM7BlmVBJDiH4GnYXb1DmrZtuGTuOh/h4Tk1ODc+KOwnWsEZXEEKIog561V50J3eTgePjLO4rBJjpRVoNyJENznelGEPVjEKJpquxsViQTuwr6M5l9/yET3KAG5aY6040gUgxCNE2VhelGqW1nXprqTDeCSDEI0TRVdjYbpbadeamrQ9wYIsUgRNNU2YKyqcGxDYOyWnlWhhSDEE2Tt2dzEk0Njm0YlKu8jmOOwlWFGDWaikpSqGjryRquKsUghBBjQlbFIFOSEONOlc2BxEigBDchxhklhYkItGIQYpxpQ/6BaB1SDEKMM23IPxCtQ4pBiHGmDfkHonVIMQgxzrQh/yCMHOGtQM5nIcaZNnVSkyO8NUgxCDHutKU5UN5+26I2ZEoSQrQDOcJbgxSDEKIdyBHeGkopBjM7yszuNLMngsclEcecbmYPhv7+zcxWB/s+Z2ZPh/adUkYeIcQQ0zZH+BhTdsWwFrjL3VcAdwWve3D3e9z9FHc/BTgD2A38TeiQy7r73f3BkvIIIYYVVUdtDWWdz6uAdwTPPw/cC1yecPz7gTvcfXfJ8wohRpG2OMLHnLIrhle7+3PB838GXp1y/AXAjX3bft/MtpnZdWb2srg3mtkaM5s1s9kdO3aUEFkIIUQSqYrBzL5uZo9E/K0KH+ed+t2xNbzN7BjgTcDm0OYrgJOA/wAcRcJqw93Xu/uMu88sXbo0TWwhhBAFSTUluftZcfvM7Ptmdoy7PxcM/M8nfNT5wJfdfW/os7urjT1m9ufAf80otxBCiJooa0raBFwYPL8Q+ErCsR+kz4wUKBPMzIDVwCMl5RFCCFGSsophHfBOM3sCOCt4jZnNmNn13YPM7HhgGfB/+97/RTN7GHgYOBr4vZLyCCGEKEmpqCR3/wFwZsT2WeDi0Ot/BKYjjjujzPmFEEJUjzKfhRBC9GCdYKLhwsx2AN9r6PRHA//S0LnLMqyyD6vcINmbYFjlhvplf627p4Z1DqViaBIzm3X3mablKMKwyj6scoNkb4JhlRvaI7tMSUIIIXqQYhBCCNGDFEN+1jctQAmGVfZhlRskexMMq9zQEtnlYxBCCNGDVgxCCCF6kGJIwcz+k5k9amYHzCw2WsDMzjWzx83sSTNb0JeiCbI0UgqO2x9qlrRp0HKG5Ei8hmb2MjO7Odj/rSCjvhVkkP3DZrYjdJ0vjvqcQWNmN5jZ82YWWY7GOnw6+F7bzOwtg5Yxigxyv8PMXgxd71Z0+zGzZWZ2j5k9Fowr/yXimOavubvrL+EPeD1wIp1eEzMxx0wA3wVeBxwOPAS8oQWyfxJYGzxfC1wTc9yPWiBr6jUEPgr8cfD8AuDmpuXOIfuHgT9qWtYI2X8WeAvwSMz+dwN3AAa8HfhW0zJnlPsdwF83LWeEXMcAbwmevxL4h4h7pfFrrhVDCu7+bXd/POWwU4En3f0pd38JuIlOE6OmWUWngRLB4+oGZUkjyzUMf59bgDODAoxN09b/fyru/g1gZ8Ihq4AveIf7gcXd4pdNkkHuVuLuz7n73wfPfwh8m4Xlghq/5lIM1TANPBt6vZ2I2lANkLWR0hFBE6T7u/24GyDLNTx4jLvvA14EXjUQ6ZLJ+v9/X2AauMXMlg1GtNK09d7Owk+Z2UNmdoeZ/WTTwvQTmEJXAt/q29X4NS/b2nMkMLOvA6+J2PVxd08qJd44SbKHX7i7m1lcCNpr3X3OzF4H3G1mD7v7d6uWdcz5KnCju+8xs1+ls/JREcn6+Hs69/WPzOzdwEZgRcMyHcTMXgHcCvyWu/+/puXpR4qB5GZEGZmjU1a8y3HBttpJkj1rIyV3nwsenzKze+nMYgatGLJcw+4x283sMOBI4AeDES+RVNm9U4m4y/V0/D/DQGP3dhnCg627325mnzWzo9298RpKZjZJRyl80d1vizik8WsuU1I1PACsMLMTzOxwOo7RxqJ7QqQ2UjKzJRb02jazo4HTgMcGJuEhslzD8Pd5P3C3B966hkmVvc9GfB4d2/IwsAn45SBS5u3AiyHzZGsxs9d0/U9mdiqdsa7xSUQg058B33b3T8Uc1vw1b9pL3/Y/4D/SsfHtAb4PbA62HwvcHjru3XQiDL5LxwTVBtlfBdwFPAF8HTgq2D4DXB88/2k6jZIeCh4valDeBdcQuBo4L3h+BPBXwJPA3wGva/oa55D9E8CjwXW+BzipaZkDuW4EngP2Bvf5RcCvAb8W7DfgM8H3epiYyLwWyn1J6HrfD/x00zIHcv0M4MA24MHg791tu+bKfBZCCNGDTElCCCF6kGIQQgjRgxSDEEKIHqQYhBBC9CDFIIQQogcpBiGEED1IMQghhOhBikEIIUQP/x+1q9ZrmzPLUgAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.scatter(X[y==0,0], X[y==0,1])\n", - "plt.scatter(X[y==1,0], X[y==1,1])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Training Models" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "# Our basic classifier will be a SVM with rbf kernel\n", - "base_clf = SVC(probability=True)\n", - "\n", - "# size of the initial labeled set\n", - "init_L_size = 5\n", - "\n", - "# Make 30 queries\n", - "n_queries = 30\n", - "\n", - "# set random state for consistency in training data\n", - "random_state = 123" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Random Sampling" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 30/30 [00:00<00:00, 650.20it/s]\n" - ] - } - ], - "source": [ - "random_experiment_data = perform_experiment(\n", - " X, y, \n", - " base_estimator=clone(base_clf), \n", - " query_strat=random_sampling,\n", - " n_queries=n_queries,\n", - " init_L_size=init_L_size,\n", - " random_state=random_state\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Uncertainty Sampling" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 30/30 [00:00<00:00, 506.46it/s]\n" - ] - } - ], - "source": [ - "uncertainty_experiment_data = perform_experiment(\n", - " X, y,\n", - " base_estimator=clone(base_clf),\n", - " query_strat=uncertainty_sampling,\n", - " n_queries=n_queries,\n", - " init_L_size=init_L_size,\n", - " random_state=random_state\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Active Search" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 30/30 [00:10<00:00, 3.00it/s]\n" - ] - } - ], - "source": [ - "as_experiment_data = perform_experiment(\n", - " X, y,\n", - " base_estimator=clone(base_clf),\n", - " query_strat=active_search,\n", - " n_queries=n_queries,\n", - " init_L_size=init_L_size,\n", - " random_state=random_state\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Compare" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd4VGX2wPHvSU8gJPTepAjSpSqo2MUVETtib+va2y6uYkNdXdv+1tVVcUGKSrEBKuCq4CqCQpDeUQmETkIaIf38/rg3MYSUSTKTySTn8zx5yNy55b0zYc687byiqhhjjDEAQf4ugDHGmJrDgoIxxphCFhSMMcYUsqBgjDGmkAUFY4wxhSwoGGOMKWRBwRhTa4hIVxFJ9nc5ApkFhRpIRL4VkcMiEu7vstRkIjJWRNLdn6Mikl/kcXoVzttNRHLL2aexiEwTkf0ikioiW0TkQQ/PP1NExle2fJ5yr5MvIn2KbOspIpm+vnaR6/UWkS/c1yhNRL4SkQG+up6qblXVWF+dvy6woFDDiEgH4DRAgYur+doh1Xm9qlLV91W1vqrWB0YAewoeu9t86XVAgK5ALDAa+M3H16yMw8Az/riwiHQDvgeWA+2B1sBCYLGI9PXB9QLq77fGUlX7qUE/wBPAD8CrwOfFnosEXgHigRRgCRDpPjcMWAokA7uAG93t3wK3FjnHjcCSIo8VuAvYBvzmbvune45UYCVwWpH9g4FHgV+ANPf5tsAbwCvFyjsPeKCU+zwVWOHexwrg1CLPfYvzQfaDe43/Ak3Ked2GAwklbG8LzAUOAb8CdxR5biiwyr3PfcDz7vYD7uuS7v70K+G824ELyihPT2ARzofyJuASd/u9QA6Q5Z77wxKOfRd4tti2L4E73d8fB/a65d5U9P0pdsxM4EX33gcXKVdmkX32AcOKPH4B+I/7ezcgF7gF2A0kAje7791692/t1TJegw+BT0q5vwXu7xcA24s9X1gm9+/tcfe9OwS8D8QWK99t7t/rfwu2FTlXI2Cae85dwJNAUJHjl7h/gweBaf7+/18TfvxeAPsp9oY4HzZ3Av3dD4/mRZ57A+cDs7X7n+VUIBznW1gaMAYIBRoDfd1jvqX8oPCV+5+nIMBc654jBHjI/Q8V4T73Z2AdcCLON+U+7r6DgD1F/sM1ATKKlr/INRvhfFhe515jjPu4cZEy/4LzLTzSffxCOa/bcIoFBfc1WgeMA8Lc8+0EznCfXwVc4f4eze8fnMd8sJRyvfeANcANQOdizzXA+dAe65ZhIJBUsB/Oh/X4Ms59HkU+KIFmwFH3Ne2D8wHZ3H39TwA6lnKemcB44C/A1+62igYFxfmSEI5Tcz0CfOyWpZ37vg0u5frJwJgSto/ACYohlB8UxuHUNloBEcAU4N1i5fsPEOX+rRQPCguAf7nPt3Tf8xvc5z4FHnZfx0hgqL///9eEH2s+qkFEZBjOB/xsVV2J88F4jftcEM63tPtUdbeq5qnqUlXNcvf5WlVnqGqOqiaq6uoKXPp5VU1S1aMAqvqee45cVX0F5wPhRHffW3E+0LaoY42773Kcb1xnu/tdDXyrqvtLuN4fgG2qOt29xgxgMzCyyD7vqtM+fBSYDVSmuWEYTjD7u6pmq+pWnG+pV7vP5wBdRaSxqqap6k8VOPcfcT4cHwA2u30K57jPjQbWq9O8laeqK4DPgMs8PPc3QH0RGeQ+vgpYrKqHcL4ZRwInAcGq+quqltds9TrQQ0SGe3pzxUxQ1SxVnec+nqaqh1R1J07ttF/xA9ymnBic4FjcXpwg3cCDa98BPKKqe1Q1E3gauEpEpMg+T6hqRsHfb5EytAdOBx50n98LvMax738HoIWqHlXVHzwoT61nQaFmuQH4r/ufH+ADdxs438wicAJFcW1L2e6pXUUfiMjDIrJJRFLckRwx7vXLu9ZUnFoG7r/TS9mvFU4TWFHxODWgAvuK/J4BVKaPoD3QQUSSC36AB4EW7vM3AL2BrSLyk4ic7+mJVfWIqk5Q1b44NaXPgI9FJNq97unFrnsZzjdVT86dhxMIx7ibrsFpNkFVNwCPAM8BB0TkfRFpXs75MoC/ucdUVJ6qJhZ5fBTYX+zxce+NqubifEko6Z5b4gS3MkcJuR/8bYH5RV7HVTifW43d3fJVdU8pp2iP83/mYJHj/4lTywInoEcBq0RkrYhcW8p56hQLCjWEiEQCVwJniMg+EdmH80fbxx09cgjIBDqVcPiuUraDU92PKvK4RQn7FKbKFZHTcJobrgQaqjOSIwWnil3etd4DRrnl7Q7MKWW/PTj/YYtqh9Nu7U27gM2qGlvkJ1pVRwOo6iZVvQqneeY14BMRCaPI6+EJVU3BaXZpgHMfu3CCe9Hr1lfV+wsO8eC0M4ArRaQz0Isir6WqTlXVU3GajiKAZz0439tAG+DCYts9+fuorK+BK0rYfiXwvarmF7++iITiNC+iqorzN3FWsdcyosgXp7Jey104/TYNixzbQFVPds+/W1VvxglS9wKTRaRdle64FrCgUHNcAuThNAv0dX+647SnXu/+B5oMvCoirUQkWEROcYetvg+cIyJXikiIO1yyoLllNXCpiES5HzC3lFOOaJxvcQeBEBF5gmOr+f8BnhGRLuLoLSKNAVQ1AafTeDrwcfHqfBHzcZptrnHLe5V73597+mJ5aAmAiNwvIhHutXqLyMnu9uvdpqM8nMCn7s8BILisDwgReUpEThaRUDeg34sTuLfjfID3E5Gr3OfDRGSIiHR1D9+P84FeKlVdhtPu/ibwmaoeca97koic4b7vR92f/PJeCFXNBibgtNEXtRoY4742Q4BR5Z2rAp7A+bt8UkRiRaSBiDyEUwN6yt1nE9BIRM52A8LTHPu59Bbwgoi0BRCRZiJStJmxVG6z2o/AiyISLSJB7t/tMPdcV4lIKzf4FNRa8qp2y4HPgkLNcQNOO/pOVd1X8IPTHjzWbaN9GKfjdAVOx+XfcTp2d+J8A3zI3b4ap0MS4B9ANs4H0VTcZogyfIkzbHArTpNOJsc2L72K07TxX5zRL5Nw2rgLTMX5Zlta0xFuc8RFbnkTcWomFxX59ucVqpqD87qcinMvB3E+ZAuaOy4CtohIGvA8cKXbJ3MYZ9TOSrfZoaT+jCCcmlESkIAzkulCt+39MHA+cBNO+/kenG/zoe6xE4GB7rlnlnELM4BzcJoRCxSMQDvknrs+zugcT0zFeb2LehTn/UoG/orTOe0VqroRp01/CM7fULJ7vZGq+p27zyHgPpy/ywScZsOifwcv4tQ4Frnv01Lg5AoUYwzOkOHNOO/VLH5vPjoF5z1Oxxkpdbuqeru2GnDECZLGeIeInI7zYdle7Y/LFOHOwVkGPKyq5X05MX5iNQXjNW71/z6cIY0WEMwxVHUHTs2tvdvkZmogqykYrxCR7kAcztj9C1Q11c9FMsZUggUFY4wxhaz5yBhjTKGASyDVpEkT7dChg7+LYYwxAWXlypWHVLVpefsFXFDo0KEDcXFx/i6GMcYEFBEpnkWgRNZ8ZIwxppAFBWOMMYUsKBhjjClkQcEYY0whCwrGGGMK+SwoiMhkETkgIutLeV5E5DUR2e7mMq9IkitjjDE+4MuawhScpfZKMwLo4v7cjpO90hhjjB/5LCi4qXGTythlFM6yfqqqPwKxIuLRylTGmDooLwfi3oXDO/xdklrNn5PXWnNsnv4Ed9txa7qKyO04tQnatavzCyMZU/dkJMHs62HH9xDZEK6YCiec4e9S1UoB0dGsqhNVdYCqDmjatNxZ2saY2mT/Rpg4HHYth/Oeg/rNYfpo+GkiWEJPr/NnUNiNsyh3gTZ4f41eY0wg2/wFTDoXcrPgpvlw6t1wy1fQ5TxY8Gf47D7IzfZ3KWsVfwaFecD17iikIUCKqh7XdGSMqYNU4buXYOY10KQr3L4Y2gxwnotoAFd/AKc9BD9PhWkXQ/pB/5a3FvFZn4KIzACGA01EJAF4EneNWlV9C2fx9gtxFjrPwFnP1hhT12Ufgbl3wYZPofdVMPKfEFpsobagIDj7CWh2Esy9G9450wkULXv7p8y1SMAtsjNgwAC1LKnG1FLJu5zawb51cO7TcOq9IFL2MXtWwcyxcPQwXPJv6DG6esoaYERkpaoOKG+/gOhoNsbUAfHLnG/8h3fANbNh6H3lBwSAVv3gtsXQohd8eCMseg7y831d2lrLgoIxxv9+ngZTR0J4A7j1G+h6XsWOj24ON3wG/a6F716E2ddBVrpvylrLBdwiO8aYALBnldPWn76//H1VIeMQdDobLp/kzEOojJBwuPh1aN4LvnwUXjnx+L6IqmrcGUa/DQ3be/e8NYgFBWOMd637yOkojmoC3Ud6dkyjTjD4Dgiu4keSCAy5A1r0hPWfAF7sM1WFDZ84TVxXToMOw7x37hrEgoIxxjvy82Hxs/D9K9B2CFz1HtT302TTDsN886F9yt0w42qYNgpGvAgDb/H+NfzM+hSMMVWXmeqMGvr+FTj5eqd9318BwZeadIbbvoFOZ8EXD8LnDzo5mWoRCwrGmKpJ+tWZdbztvzDiJRj5GoSE+btUvhMRA2NmOqOj4iY5KTeOJPq7VF5jQcEYU3m/fgsTz3Q6lK/7FAbf7tkw0kAXFAznToDRE52cTO8Mh30lLh0TcCwoGGMqThV+fAumXwrRLeG2RXUza2mfq+DmBU4T0qTzYNNn/i5RlVlQMMZUTG4WzLsHFo6DrufDrV9BoxP8XSr/ad0fbv8WmnWHWdfCt38P6MlzNvrIGOO59AMw6zrY9SOc9jCc+ZiTh6iui24BN34Bn98P3/4N9q+H3ld6/zotekHDDt4/bxEWFIwxntm7BmZcAxmJcPlk6HmZv0tUs4RGwCVvQvOe8NXjsGme96/xh1d9PgzWgoIxpnzrP4E5d0JUI7h5IbTq6+8S1UwizpoPPUbD0bJWI66k6FbeP2cxFhSMMaXLz4fFz8H3L0Pbwe6EtGb+LlXNF9Pa+QlAFhSMMSXLSoNP/ghbvoB+18EfXnHyC5lazYKCMeZ4Sb/BjDFwaKuTzmFQHZl/YCwoGGOK+e07mH29Mxfh2o+h05n+LpGpRjaWzBjjUIXl78C0S6BeM2dCmgWEOsdqCsYYyM2GBX+GlVOg6wVw6TsQ0cDfpTJ+YEHBGEBVefHLLfRrG8t5PVr4uzjVKzcb3rsUdnwPwx6Es8Y7uX3qgMycPJ7+bCMJhzO8et6QIOHmYR05rUvgZYq1oGAMsCvpKG9++wsA95/ThXvP6kJQUB3pWF32uhMQRr3hLGdZh7z1v1+YsXwnfdrG4s23e39KJjdMXs6jF3bnlmEdkQDqpLegYAywdncyAKec0Jj/+3obW/al8fIVfagXXsv/iyTvhP+9CN0uqnMBYcehI/z721+4qHdLXr/mZK+e+0hWLg/NXsOzX2xi0940nhvdk4jQwKh9WUezMcC6hBTCgoOYevMgxv+hO19u2Mdlby5lV5J3mxVqnAWPOENNL3jB3yWpVqrKE/M2EBYcxOMXneT189cLD+HfY0/mvrO78PHPCYx550cOpGZ6/Tq+YEHBGGBtQgrdW0YTFhLEraedwLs3DWJ38lFGvfEDP/1aexZQOcaWBc7EtDPGQWxbf5emWi1Yv4/vth7kwXO70rxBhE+uERQkPHBuV94cezKb96Zx8es/sDYh2SfX8iYLCqbOy89X1u9OoVebmMJtZ3Rtyty7hhIbFcrY//zEBz/t9GMJfSA7A+b/BZp2gyF3+rs01So9K5cJn23kpJYNuP6U9j6/3oheLfn4T6cSHCRc8dYy5q7e7fNrVoUFBVPn7Ug8QlpWLr1bxx6z/YSm9Zlz11CGdWnCo5+u4/E568nJC9w8+cf4/mVI2elk3azNS2eW4J9fb2VfaibPju5JSHD1fASe1KoB8+4eSp+2sdw3czUvLNhMXr5Wy7UryoKCqfPW7U4BOKamUKBBRCiTbhjIH08/gek/xnPdpJ9IOpJd3UX0roNb4IfXoM8Y6DDU36WpVpv2pjL5hx2MGdSWk9s1rNZrN64fznu3DOaawe1463+/cNu0ONIyc6q1DJ6o5UMrTJ2nCim7QEv/hh+/fTudQg/RJTQRDh+f7jgY+OspkZzcoDF/X7iFP762g+fGnk7Xdl7MgulBOQtJMMS0qVwuIlX44iEIi4Jzn6n48QEsP18ZP2c9MZGh/OX8bn4pQ1hIEH8b3YvuLRvw1LwNjP73Ul65og+N6nlWW4uNCiU6ItSnZRTVmlmFKc2AAQM0Li7O38UwgSAjCT66yVlc3suOaARbh75Cv/O8MIzzSCJ8eIMzV8BTXUfApRMrPut47Yfwya3VslhLTTM7bhd/+WgtL17emysH+L9jfekvh7jr/Z85nOF5beHZS3py7ZDK9YOIyEpVHVDeflZTMLXTgc0w42pI3Q1nPQ4NSl6cJF+VRz9dz+COjRjdz7Nv/ilHs9n/zRv0W3oXy3avY8gNzyOVXZJy/wannGn74ewnnWUdy3M4Hr57CSadC2NmeL4+8tFk+PJRaHUy9L+xcuUNUIePZPP8/E0MaN+Qy09u4+/iAHBqpybMv+80lm5PxNOv5v3axZa/UxVZUDC1z5YF8PFtThPJjV9A20Gl7rp9fxozs2MZ1KcP9PXswyIGCO97BSveupFT4t/i51c30e2O6UTVP75PokybPodPbne+7d+0ANr09/zY9qc6tYuJZ8KVU+GE4eUfs/g5yDgEY2fXmTQWBV78cjOpmbk8c0nPGjVTvWVMJJf1rxlBqoB1NJvaQxW+f8VZB6BxJ7htcZkBAZz5CQC9S+hkLktEZD0G3DeLZZ0foE/ad+z9x3D2xm/xvJz/exFmjYVm3ZxyViQgAJxwhnNcg1Yw/VL48S3nvKXZswpW/AcG3gqt+lXsWgHu552HmbF8FzcP7UD3lpbkrzwWFEztkJ0BH98C30xwFpS/eaFHyyGuS0imXlgwHZvUr/AlJSiIU659ivXD36Fp7j7C3z2HjT9+WU45j8CHNzrf2ntfDTfOhwYtK3xtABp1hFv+62Q1XTgO5t0DuVnH75efB58/CFFN4MzHKnetAJWbl89jn66nRYMI7junq7+LExAsKJjAl7Ib3h3hLC5/zlNw2X8gNNKjQ9fuTqFH6xiCq9Ck0OfMKzh8zQKOSH06LxjD8o9eLXnH5J0w+XzYNA/OexZGvwWhVZxNGx7trJt8+p9h1XSYejGkHzh2n5VTYM/PcP5zEOn7NumaZNqyeDbtTeWJkSdRv7bnsfISnwYFEblARLaIyHYReaSE59uLyDcislZEvhWRmtW4Zmq+nT/BxOGQ+AuMmQnDHvB4qGZOXj4b96TSu3UF+wJK0P7EvjS45zs2R/Zj0Pqn+en1m8nJLvKtPX6p0/5/OB6umQ2n3uO95S2Dgpx015e/C3vXONfZs9p5Lv0gfPM0dDgNel3hnesFiP2pmbz61VbO6NqUET3rWDr0KvBZUBCRYOANYARwEjBGRIpnnnoZmKaqvYEJwPO+Ko+phVa9B1MvgrB6cOvXcOIFFTp82/50snLzS5y0VhkxjZpy0kML+LH5GAYf+pitL59L8qF9zjf1qRc739Jv/Qa6nOuV6x2n56Vwi9t8NfkCp+b01RNO09ofXq1zayw/8/lGsvPyefriHgGVutrffFmfGgRsV9VfAURkJjAK2Fhkn5OAB93fFwNzfFgev5q7fS6/pPzi0b4dGnRgdOfR9odcGlX46nFY+i9n1M3l70JUowqfZp2bLrt3G+81qYSEhjHkT2+xYk5P+qx6kszX+wPp/BozhLkdJpAZlwdsqtI1gkW49OQ2dG5WQj9Iyz5w+2KYdZ0zRwPgtIegac1tT1+bkMz8dftQjwdmlu9IVi6fr93L/ed0oUOTel47b13gy6DQGthV5HECMLjYPmuAS4F/AqOBaBFprKrHpKUUkduB2wHatWvnswL7Sr7m8+TSJwEICSr7Jc/XfHLyc4gOi+bc9j76RhnoNn/hBIQBN8OIlyC4cn/GaxJSiI4IoUPjKC8XEAZecjeb23Qn6ou7+TBvOK8kjiEvMQk4fsZ0ReXmK9OWxfN/V/XlnJOaH79D/WZwwzxY+FenOem0h6t8TV/5MG4Xj326nnzVKvXrlGRgh4bccUYnr56zLvB3z8vDwOsiciPwHbAbyCu+k6pOBCaCM6O5OgvoDek56eRpHg8PeJgbetxQ5r65+bmM+WIMLyx/gVNbnUq9UPuWc4ysdFgwDpr1gBEvVjoggLOGQu82MT6rkXUbcDYM2MQtgDfnDu9NOcrt01Zy2/Q4Hj7vRO4c3un4ewgJh4tK6fCuAXLz8nl+wWYmLfmNoZ0b88Y1JxMbVbcS89VUvuxo3g0UnUvext1WSFX3qOqlqtoPeMzdVvMTjldQalYqADHh5bddhwSFMH7IeA5kHODN1W/6umiB57sXITXB+cALrnwOmKzcPDbvS6VX68AbjdMyJpIP7ziFkb1b8dKXW7h35mqOZh/3XarGSsnI4aYpK5i05DduPLUDU28aZAGhBvFlUFgBdBGRjiISBlwNzCu6g4g0EZGCMvwVmOzD8vhNSrYzQapBmGcTZ/o07cNlXS7jvU3vsfXwVl8WLbAc2ATL3HWE2w2p0qm27EsjJ08rPGmtpogIDeafV/dl3AXd+HztHq58exl7U476u1jl2n4gjUv+/QM//prI3y/rxVMX96i29NXGMz57N1Q1F7gb+BKnZ222qm4QkQkicrG723Bgi4hsBZoDz/mqPP5UkZpCgftPvp8GYQ149sdnyfckc2ZtV5DdMzwazplQ5dMVzGTu5YXhqP4iIvxpeCf+c/0Afjt0hJH/+oGV8VXvs/CVxZsPMPqNpaRl5jDjtiFcNTDw+gfrAp+GaFWdr6pdVbWTqj7nbntCVee5v3+kql3cfW5V1RKmYwa+1GwnKHhaUwCIjYjlgf4PsOrAKuZun+urogWONTMh/gdnclq9xlU+3bqEFBpGhdKmoWeT3Gqys7s359M7T6V+eDBjJv7E7BW7yj+oGqkqb377CzdPXUG7xlHMvXsYAzpUfLSYqR5Wb6sGKVkVaz4qMKrzKPo168erK18lObPWdbV47uhh+O94aDMQ+l3vlVOu3Z1CrzaxtWbYb5fm0cy5ayiDOjbiLx+vZcJnG8mtAavEZebkcf+s1fx94WYu7NWSj+44ldaxgR+IazMLCtWgoKZQkeYjgCAJ4rHBj5GWncY/V/3TF0ULDN9MgKNJzgSsyqaoLiIzJ4+t+9O8MpO5JomNCmPKTQO5aWgHJv/wGzdNWUFyhv9WiduXksmVby9j7uo9/Pn8E3l9TD8iw+pWdtZA5O8hqXVCalYqYUFhRIRUPM/NiY1O5Nru1zJ141Qu6XwJfZr2AWDzvlTun7ma3YdrdudiKDm8I89Sv/OpnHjNSxVP2ZywEuLehcF3QMveXinTxr2p5OWr12Yy1yQhwUE8ObIH3Vs04LE56xj0t28I91NHbmZuHmHBQUy8rj/n9ah6mok1B9cwfsl4Dh095IXSBaa/DPwLo7uM9uk1LChUg9TsVBqEVz5l75/6/okFOxbw7I/PMuMPM/hm0yEemLWa+uEhXD6gDULNbQLpkfQV/X/bBNs3oR/EI5dN8jwpW34efPEA1G8OZz7qtTKtq2S67EBy5cC2dG0RzWdr9pSZUduXQoKFK/q3oUvz6Cqfa872OUxYNoHmUc25pPMlXihdYOoQ08Hn17CgUA1Ss1OJCav8B1C90HqMGziOh/73EHfNe50vf+xCnzYxTLx+AM0bVDHLpq+9+2fSo9rwt5TzefbXach/znYS1zXpUv6xKyY5M3Ivn1zxZSfLsDYhhSb1w2lR01+7KurbNpa+bQNvHkZRufm5vLryVaZvnM7gloN55YxXKtwMayrG+hSqQUpWSpVqCgBDW55JQ+nFD0nvcWHfKGb98ZSaHxAOboH4JYQPvoXPQ8/ntdavOEtCvnM2bPuq7GPT9sOiZ5zcRj0u9Wqx1iYk08eHM5mNd6RkpXDXN3cxfeN0xnYfy1vnvGUBoRpYUKgGVa0p7E4+yhVv/UjC9vMJCcmnXsv5RIQGQIdd3GQICiW0/3Vc3r8tb/zalMSxX0JsO/jgSvjhtdJXC/vveMjNhAtf8Wp2zyNZuWw/mF4r+xNqk1+Tf2Xs/LEs37ecp099mkcGPVJu3jDjHRYUqkFqVuX7FFbsSOLify1hV1IGk64ZwR9738bCHQtZtmeZl0vpZdkZsHoGnDQK6jflmsHtyMlTZm3DSe/cfaST6fTTOyAn89hjf/sO1s2GofdDk85eLdaGPamo1u7+hED3XcJ3jJ0/lrTsNCafP5lLu3i3pmjKZkGhGqRkp1R4jgLAzOU7ueadH4mJDOXTu4ZyZrdm3NzrZtpGt+VvP/2N7Dz/DTcs14ZPICvFyWQKdG5WnyEnNOKDn3aSHxIFV0x1loZcOxOmXAipe53jcrOdmcux7eG0B8u4QOWsTXDme/SsZcNRawNVZfL6ydz9zd20jW7LzD/MpF+zurWedE1gQcHHcvJzOJJzpEI1hZy8fJ6at4FHPlnHKZ2a8OmdQwtz54cHh/PY4MfYkbqDd9e/66tiV13cZGhyIrQ/tXDTtUPak3D4KN9tO+g0CZ3xF2cpyQObndXTElbCsn/Boa1w4cseL6lZEet2p9AyJoJm0TW8P6aOyczN5K9L/so/Vv6D8zqcx9QRU2lZv5JrV5sqsUY6H0vLTgM8n82cnJHNne//zNJfErnttI6Mu6DbcQnDhrYeynntz+Odde/QvF5zwoPDvV7uKjm8A5I3QZ9rYMfCws0alU/Dphv45487OBp2grMxIgT+8LTTvzDrEkCg6zAIzYPfFni9aCsObqRlqwgW/FaDa1l1jKoyfeN01ieu595+93Jrr1ttEIAfifprEHMlDRgwQOPi4vxdDI/tSNnByDkj+duwvzGy08gy9926P41bp8axLyWTv13ai8v7l75k9f4j+7nss8sKU2gYE8iiQqJ4/rTnOavdWf4uSq0lIitVdUB5+1lNwcc8TXHx1cb93D9zFVHhIcz84xBObtewzP2b12vOgksXcPDoQa+V1Suy0uHdC511iM9+/LhFK2B0AAAgAElEQVSn9yVncu3kn7h2cHtuHNrh2CcVyDnirLnsA6t3JvPwh2t4/rLeDOxQ9utrqlfjiMY23LSGsKDgY+Ulw1NV/v3tL7z83y30ah3DxOsG0CLGs/bu6LBoosOqPlvUq5a/A0fTYPDdEHPCcU+fEAOnd0hn4apUHj+/A6HVmILhq6RfyM9uxrmde9Goni3qYkxJrKPZx8qqKRzNzuOeGat46cstXNynFbP/eIrHAaFGUnXyFLXsA61OLnW3awe350BaFt9sOlCNhXMyo7ZpGGkBwZgyWFDwsdLWUtiTfJQr3l7KF+v28siIbvzfVX0DY0JaWXYthwMbnGGoZXQUntmtGa1iInj/p/hqLFzBTObATvtgjK9ZUPCxwuajIkNSV8YncfHrS9hxKINJNwzgjjNKWHg9EMVNgvAG0PPyMncLDhKuHtSO77cdYsehI9VStMNHstmVdNRmMhtTDgsKPpaanUpUSBShQc4i87NX7OLqiT9SPzyEOXedylndmvu5hF5yJBE2zIHeV0F4/XJ3v2pgW4KDhBnLd1ZD4Zz5CUCtW0PBGG+zoOBjBcnwct0JaX/5eC1DTmjM3LuG0blZDeskroo1H0BeFgy4yaPdmzeI4NzuzZkdt4us3DwfF+73oNDDgoIxZbKg4GOp2ak0CGvAn97/mSlLd3DLsI68e+NAYqJC/V0078nPdzqY2w6B5j08PuzaIe05nJHDwvX7fFg4x9qEZDo2qUdMZC163Y3xAQsKPpaalUpkcDRfbdzPHWd04vGLTjpuhnLA2/EdJP1SmOfIU6d2akyHxlG8/6Pvm5DWJaTQy2oJxpSrln061Typ2alonpPD56xuzfxcGh9ZMQkiGzkZUSsgKEi4ZnA7lu9IYsu+NB8VDg6mZbEnJdMyoxrjAQsKPpaalcrRrDBEoEcr760eVmOk7oXNX0C/sRBa8TkWl/dvS1hwEB/4cHjqerc/wWoKxpTPgoKPpWanknYkjM5N61MvvBZOIF/1Hmge9Pesg7m4RvXCuLBXCz75eTcZ2bleLpxjbUKKE5QtKBhTLgsKPpSVl0VmXiYHU4LoXRsnTeXnwcopzpKZjTtV+jTXDmlPWlYun63Z462SHWPd7mQ6Na1P/doYlI3xMgsKPpSa5cxmPnI0rHa2Z2/7ClITYMAtVTpN//YNObF5NO//5JsO57UJKbXz9TfGBywo+FBBigvNi6ydM2njJkH9FnDiiCqdRkQYO6QdaxNSCldG85Z9KZkcSMuySWvGeKju1KczkuDIoWq9ZErSJgBaaQY9QvfBwRqW5roqMhKdmsLpf4bgqo/9v6Rfa56fv5lJS37jnrO6eKGAjp9+SwSgV21svjPGB+pOUFg1Hb56olovmRoZCS2a8mbQ24S//a9qvXa1kGA4+XqvnKpBRCiX9GvFjOW7mLvau30LYcFBnNSyFo78MsYH6k5Q6HoBNGhdrZdMTlwN8XP5ttld9Bjcs1qvXS1i2kJsW6+d7pER3RnauQn5Xl4MsG3DSCLDAjwDrTHVxKOgICKfAJOABaqa79si+UjTE52farQrzhkfH9b9cuhVC4OCl8VEhnJR71b+LoYxdZqnHc3/Bq4BtonICyJSvZ+uAeqXxEOoCgPbtfR3UYwxxiMeBQVV/VpVxwInAzuAr0VkqYjcJCKWYawUu1IOQX4E3VtaJ6cxJjB4PCRVRBoDNwK3AquAf+IEia98UrJa4ED6YUKlHuEh1p5tjAkMHgUFEfkU+B6IAkaq6sWqOktV7wFKXVFFRC4QkS0isl1EHinh+XYislhEVonIWhG5sLI3UtPk5yvJWSnUC6lFayYYY2o9T0cfvaaqi0t6QlUHlLRdRIKBN4BzgQRghYjMU9WNRXYbD8xW1TdF5CRgPtDB08LXZPFJGeSRQaPIRv4uijHGeMzT5qOTRKSwYVxEGorIneUcMwjYrqq/qmo2MBMonltZgYIB5DGAb5Lf+MHahGQIPkqL+hYUjDGBw9OgcJuqFuYfUNXDwG3lHNMa2FXkcYK7raingGtFJAGnlnBPSScSkdtFJE5E4g4GyKzgdQkpBAVn0KqBBQVjTODwNCgEi4gUPHCbhsK8cP0xwBRVbQNcCEwXkePKpKoTVXWAqg5o2rSpFy7re2t2JyPBR4kNt5w7xpjA4WlQWAjMEpGzReRsYIa7rSy7gaLTXdu424q6BZgNoKrLgAigiYdlqrHy8pWNew6C5BNjQcEYE0A8DQrjgMXAn9yfb4C/lHPMCqCLiHQUkTDgamBesX12AmcDiEh3nKAQGO1DZfjtUDoZeekANAiznDvGmMDh0egjN7XFm+6PR1Q1V0TuBr4EgoHJqrpBRCYAcao6D3gIeEdEHsDpdL5RVb2c+ab6rdmVggRnANAg3IKCMSZweJr7qAvwPHASzrd5AFT1hLKOU9X5OB3IRbc9UeT3jcDQCpQ3IKzbnUJEeDYAMWHWfGSMCRyeNh+9i1NLyAXOBKYB7/mqUIFubUIy7dyeEaspGGMCiadBIVJVvwFEVeNV9SngD74rVuDKzctnw55UWjZyWsGspmCMCSSezmjOcoeKbnP7CXZTRnqLumzbgXSycvNp3CAXjlhNwRgTWDytKdyHk/foXqA/cC1wg68KFcjWJThrKERH5RAswUSFRPm5RMYY47lyawruRLWrVPVhIB24yeelCmBrdycTHR6CBB8lJjyGInP+jDGmxiu3pqCqecCwaihLrbAuIYWerWNIzU61OQrGmIDjaZ/CKhGZB3wIHCnYqKqf+KRUASo7N59Ne9O4aWgHfsuyoGCMCTyeBoUIIBE4q8g2BSwoFLF1fxrZefn0ahPDmp2pNIxo6O8iGWNMhXg6o9n6ETywJsFJJNu7dSwp21Jo36C9n0tkjDEV4+mM5ndxagbHUNWbvV6iALYuIYXYqFDaNookNTvVkuEZYwKOp81Hnxf5PQIYTS1aEMdb1iak0Kt1DIqSlp1mfQrGmIDjafPRx0Ufi8gMYIlPShSgMnPy2Lo/jT92O4G07DQUtaBgjAk4nk5eK64L0MybBQl0m/amkpuv9GodS2p2KoA1HxljAo6nfQppHNunsA9njQXjWrfbmcncu00Mh7N+BWwtBWNM4PG0+Sja1wUJdGsTUmhSP4yWMRHs2OsECMt7ZIwJNB41H4nIaBGJKfI4VkQu8V2xAs86t5NZRH5vPrIMqcaYAONpn8KTqppS8EBVk4EnfVOkwJORncu2A2n0ahMLQGqWExSspmCMCTSeBoWS9vN0OGutt3FPKvkKvVs7NYOCmoL1KRhjAo2nQSFORF4VkU7uz6vASl8WLJCsSfi9kxmcmkJ4cDgRIRFlHWaMMTWOp0HhHiAbmAXMBDKBu3xVqECzLiGZFg0iaNbACQIp2SlWSzDGBCRPRx8dAR7xcVkC1trdKfRq83uncmqWpbgwxgQmT0cffSUisUUeNxSRL31XrMCRlpnDrwePFPYngNUUjDGBy9PmoybuiCMAVPUwNqMZgPW7nU7l4jUFCwrGmEDkaVDIF5F2BQ9EpAMlZE2ti9btdmJlryI1hdTsVBuOaowJSJ4OK30MWCIi/wMEOA243WelCiBrE1JoHRtJ4/rhhdtSsqz5yBgTmDztaF4oIgNwAsEqYA5w1JcFCxTrdqcUDkUFyMnPISM3w2oKxpiA5GlCvFuB+4A2wGpgCLCMY5fnrHNSMnKIT8zgqoFtC7elZacBluLCGBOYPO1TuA8YCMSr6plAPyC57ENqv7W7f19+s0BKliXDM8YELk+DQqaqZgKISLiqbgZO9F2xAsOGPc7Io56tfw8AlgzPGBPIPO1oTnDnKcwBvhKRw0C874oVGHYcOkLjemHERoUVbrNkeMaYQOZpR/No99enRGQxEAMs9FmpAsTOpAzaNY46ZltKttt8ZKOPjDEBqMKZTlX1f74oSCCKT8xgYIeGx2wrqClYmgtjTCCq7BrNdV52bj57U47SrlHJNYXoMFuszhgTeCwoVFLC4QzyFdo1rnfM9tSsVKJCoggNCvVTyYwxpvJ8GhRE5AIR2SIi20XkuCyrIvIPEVnt/mwVkYAZ5rozKQOA9sX6FFKzLUOqMSZw+Wz1NBEJBt4AzgUSgBUiMk9VNxbso6oPFNn/Hpz5DwGhMCgUaz6yZHjGmEDmy5rCIGC7qv6qqtk4i/OMKmP/McAMH5bHq+ITM4gIDaJpdPgx262mYIwJZL4MCq2BXUUeJ7jbjiMi7YGOwKJSnr9dROJEJO7gwYNeL2hlxCdm0K5RFCJyzPbUbKspGGMCV03paL4a+EhV80p6UlUnquoAVR3QtGnTai5ayXYlZdCuUb3jtqdkpdjENWNMwPJlUNgNtC3yuI27rSRXE0BNR6rKzqSM4zqZwW0+shQXxpgA5cugsALoIiIdRSQM54N/XvGdRKQb0BAn62pAOJiWxdGcvOPmKGTmZpKVl2U1BWNMwPJZUFDVXOBu4EtgEzBbVTeIyAQRubjIrlcDM1U1YFZyKxh5VDzFRUEyPOtTMMYEKp8NSQVQ1fnA/GLbnij2+ClflsEX4hNLH44KlgzPGBO4akpHc0CJT8pABNo0tGR4xpjaxYJCJexMPEKrmEjCQo59+QqT4VlHszEmQFlQqISdSRnHdTJDkT4Faz4yxgQoCwqVUNpw1MKlOK35yBgToCwoVFB6Vi6H0rNpW0pNQRBLm22MCVgWFCpoZ2LJ2VHBqSlEh0UTJPayGmMCk316VdDv2VGPT3FheY+MMYHOgkIF7Uw6Ahw/cQ0sQ6oxJvBZUKig+MQMYiJDiYk8fmU1W0vBGBPoLChUUGkjj8BtPrLhqMaYAGZBoYJKm6MAliHVGBP4LChUQG5ePrsPHy2xpqCqtpaCMSbgWVCogD3JmeTma4k1hYzcDPI0z2oKxpiAZkGhAgpTZpc0HNUypBpjagELChUQ7w5HLXHimmVINcbUAhYUKmBnYgZhwUG0aBBx3HOFGVJtnoIxJoBZUKiA+MQM2jSKJChIjnvOagrGmNrAgkIF7EzKOG61tQKFfQoWFIwxAcyCgodU1Z24dnwnM/y+loI1HxljApkFBQ8lHckmPSu31IlrKVkphEgIkSGR1VwyY4zxnhB/FyBQxBcORy07xYXI8f0NxtRFOTk5JCQkkJmZ6e+i1CkRERG0adOG0NDj87N5woKCh3Yllb6OAljabGOKS0hIIDo6mg4dOtiXpWqiqiQmJpKQkEDHjh0rdQ5rPvJQvLu4TkkrrgGW4sKYYjIzM2ncuLEFhGokIjRu3LhKtTMLCh6KT8ygeYNwIkKDS3zekuEZczwLCNWvqq+5BQUP7Uw6UuJqawWspmCMqQ0sKHhoZ1JGiautFbA+BWNqnuDgYPr27UvPnj0ZOXIkycnJXjnvjh076Nmzp1fOVdNYUPBAZk4e+1OzSp24lpefR3p2us1RMKaGiYyMZPXq1axfv55GjRrxxhtv+LtINZ6NPvJAYXbUUmoK6TnpKGo1BWNK8fRnG9i4J9Wr5zypVQOeHNnD4/1POeUU1q5dC0B6ejqjRo3i8OHD5OTk8OyzzzJq1Ch27NjBiBEjGDZsGEuXLqV169bMnTuXyMhIVq5cyc033wzAeeedV3jezMxM/vSnPxEXF0dISAivvvoqZ555JlOmTGHOnDkcOXKEbdu28fDDD5Odnc306dMJDw9n/vz5NGrUyKuviTdYTcEDOxPLmaNgKS6MqdHy8vL45ptvuPjiiwFnLP+nn37Kzz//zOLFi3nooYdQVQC2bdvGXXfdxYYNG4iNjeXjjz8G4KabbuJf//oXa9asOebcb7zxBiLCunXrmDFjBjfccEPh6J/169fzySefsGLFCh577DGioqJYtWoVp5xyCtOmTavGV8BzVlPwQHzhHAVLcWFMZVTkG703HT16lL59+7J79266d+/OueeeCzjj+R999FG+++47goKC2L17N/v37wegY8eO9O3bF4D+/fuzY8cOkpOTSU5O5vTTTwfguuuuY8GCBQAsWbKEe+65B4Bu3brRvn17tm7dCsCZZ55JdHQ00dHRxMTEMHLkSAB69epVWGupaaym4IGdiUeIDg+hYVTJMwRTsixDqjE1UUGfQnx8PKpa2Kfw/vvvc/DgQVauXMnq1atp3rx54bf78PDwwuODg4PJzc2t9PWLnisoKKjwcVBQUJXO60sWFDwQn5RB20ZRpY7/tZqCMTVbVFQUr732Gq+88gq5ubmkpKTQrFkzQkNDWbx4MfHx8WUeHxsbS2xsLEuWLAGcoFLgtNNOK3y8detWdu7cyYknnui7m/ExCwoecLKjlj0cFaymYExN1q9fP3r37s2MGTMYO3YscXFx9OrVi2nTptGtW7dyj3/33Xe566676Nu3b2H/A8Cdd95Jfn4+vXr14qqrrmLKlCnH1BACjRS9uUAwYMAAjYuLq7br5eUr3R9fyE3DOvDXEd1L3Oedte/w2qrXiLs2jvDgwP1jMMabNm3aRPfuJf+fMb5V0msvIitVdUB5x/q0piAiF4jIFhHZLiKPlLLPlSKyUUQ2iMgHvixPZexLzSQ7L7/M2cyp2alEBEdYQDDGBDyfjT4SkWDgDeBcIAFYISLzVHVjkX26AH8FhqrqYRFp5qvyVFZ84hGg9OGo4Ka4sKYjY0wt4MuawiBgu6r+qqrZwExgVLF9bgPeUNXDAKp6wIflqZTyUmbD72spGGNMoPNlUGgN7CryOMHdVlRXoKuI/CAiP4rIBSWdSERuF5E4EYk7ePCgj4pbsvjEDEKChJYxEaXuY3mPjDG1hb9HH4UAXYDhwBjgHRGJLb6Tqk5U1QGqOqBp06bVWsD4pAzaNIwkJLj0l8oypBpjagtfBoXdQNsij9u424pKAOapao6q/gZsxQkSNcbOxIxSF9YpYGspGGNqC18GhRVAFxHpKCJhwNXAvGL7zMGpJSAiTXCak371YZkqrLw5CuDkPrKagjE1T0kprp966ilefvlln11zzpw5bNy4sdz93nrrrXLzH61evZr58+d7q2ge8VlQUNVc4G7gS2ATMFtVN4jIBBG52N3tSyBRRDYCi4E/q2qir8pUUSkZOaQczSlzOGpOfg4ZuRnWp2CMITc31+OgcMcdd3D99deXuY8/goJPE+Kp6nxgfrFtTxT5XYEH3Z8aJz7JGY5aVvNRQYZUS3FhTBkWPAL71nn3nC16wYgXKn348OHDGTx4MIsXLyY5OZlJkyZx2mmnkZeXx7hx41i4cCFBQUHcdttt3HPPPaxcuZIHH3yQ9PR0mjRpwpQpU2jZsiXDhw+nb9++LFmyhNGjRzNv3jz+97//8eyzz/Lxxx+zaNEiJk6cSHZ2Np07d2b69OlERUXx1FNPUb9+fR5++OESyzJ48GCeeOIJjh49ypIlS/jrX//K+PHjWbp0KU2bNiU/P5+uXbuybNkyvNnXallSy7DTg+GoKdmWDM+YQJWbm8vy5cuZP38+Tz/9NF9//TUTJ05kx44drF69mpCQEJKSksjJyeGee+5h7ty5NG3alFmzZvHYY48xefJkALKzsynItLBt2zYuuugiLr/8csDJm3TbbbcBMH78eCZNmlSYVbW8skyYMIG4uDhef/11ADZv3sz777/P/fffz9dff02fPn28GhCgjgWF/Uf207xec4/3jy9nHQWwtRSM8UgVvtFXRWlJLAu2X3rppcDvKbIBvv76a+644w5CQpyPx0aNGrF+/XrWr19fmHo7Ly+Pli1bFp7vqquuKrUM69evZ/z48SQnJ5Oens75559f4n4llaW4m2++mVGjRnH//fczefJkbrrpplKvW1l1Jij8Z91/mLRuEp+N/owmkU08OmZnYgZN6odTL7z0l8kypBpTczVu3JjDhw8fsy0pKYmOHTsCv6e2Li9FtqrSo0cPli1bVuLz9eqV3u944403MmfOHPr06cOUKVP49ttvS9zPk7K0bduW5s2bs2jRIpYvX35MtlZv8fc8hWpzTrtzyMrL4qUVL3l8THzSEdo1iixzH1tLwZiaq379+rRs2ZJFixYBTkBYuHAhw4YNK/WYc889l7fffrvwgzkpKYkTTzyRgwcPFgaFnJwcNmzYUOLx0dHRpKWlFT5OS0ujZcuW5OTkVPhDvPi5AG699VauvfZarrjiCoKDgyt0Pk/UmaDQIaYDt/S6hfm/zeenvT95dMyupKOlrrZWoDBttg1JNaZGmjZtGs888wx9+/blrLPO4sknn6RTp06l7n/rrbfSrl07evfuTZ8+ffjggw8ICwvjo48+Yty4cfTp04e+ffuydOnSEo+/+uqreemll+jXrx+//PILzzzzDIMHD2bo0KEepegu6swzz2Tjxo307duXWbNmAXDxxReTnp7uk6YjqGOpszNzMxk9dzQhQSF8cvEnhAaXvJIaQFZuHt0eX8i9Z3XhgXO7lrrfm2ve5N+r/82q61YRElRnWuOMKZelzvaNuLg4HnjgAb7//vtS96mxqbNrmoiQCB4d/Cg7UncwZcOUMvdNOHwU1bJHHoHT0VwvtJ4FBGOMz73wwgtcdtllPP/88z67Rp0KCgCntTmNc9qdw9tr3yYhLaHU/XZ6MPIILMWFMab6PPLII8THx5fZJ1JVdS4oAIwbNI4gCeLvy/9e6j4FcxTalVNTsGR4xpjapE4GhRb1WnBnnzv5NuFbFu1cVOI+8YkZRIUF07R+2aupWdpsY0xtUieDAsDYk8bSObYzLyx/gYycjOOe35l0hHaNokqd/FIgNSvV5igYY2qNOhsUQoNCeXzI4+w9speJayce93y8BymzwUlzYTUFY0xtUWeDAsDJzU9mVKdRTN0wlV+SfyncrqpOymwPgkJqljUfGVOTzZkzBxFh8+bNAOTn53PvvffSs2dPevXqxcCBA/ntt9/8XMqao04HBYAHBzxIVGgUz/30HAVzNg6kZZGVm1/ucNTM3Eyy87Oto9mYGmzGjBkMGzaMGTNmADBr1iz27NnD2rVrWbduHZ9++imxscct+Fhn1fnB9Y0iGnF///uZsGwCn//6OSM7jfw9EV45s5ktxYUxnvn78r+zOWmzV8/ZrVE3xg0aV+Y+6enpLFmyhMWLFzNy5Eiefvpp9u7dS8uWLQkKcr4Tt2nTxqvlCnR1vqYAcFmXy+jVpBcvx71Manbq78NRPZijAJYMz5iaau7cuVxwwQV07dqVxo0bs3LlSq688ko+++wz+vbty0MPPcSqVav8Xcwapc7XFACCJIjxQ8Yz5osxvPbza0SlXkGQQOvYspPhFeY9spqCMWUq7xu9r8yYMYP77rsPcHISzZgxg5dffpktW7awaNEiFi1axNlnn82HH37I2Wef7Zcy1jQWFFwnNT6Jq0+8mhmbZ9A/tDutYhsSFlJ2Raqw+cj6FIypcZKSkli0aBHr1q1DRMjLy0NEeOmllwgPD2fEiBGMGDGC5s2bM2fOHAsKLms+KuLufnfTOLIx6zLfpW2jiHL3L2w+sjQXxtQ4H330Eddddx3x8fHs2LGDXbt20bFjR77//nv27NkDOCOR1q5dS/v27f1c2pqjztQUZq/YxTvf/1rufjlhF5EVPYVfQ57gkjn1y9y3cClOqykYU+PMmDGDceOObba67LLLuOGGG2jUqBFZWVkADBo0iLvvvtsfRayR6kxQiI0KpUvzsj/kAVTPYJem0qxxErFRpafWLtA2ui3RodHeKKIxxosWL1583LZ7772Xe++91w+lCRx1Jiic16MF5/Vo4eHe5aYcN8aYWsn6FIwxxhSyoGCM8ZlAW9mxNqjqa25BwRjjExERESQmJlpgqEaqSmJiIhER5Y+eLE2d6VMwxlSvNm3akJCQwMGDB/1dlDolIiKiSqk7LCgYY3wiNDSUjh07+rsYpoKs+cgYY0whCwrGGGMKWVAwxhhTSAJtZICIHATiK3l4E+CQF4tTE9S2e6pt9wO1755q2/1A7bunku6nvao2Le/AgAsKVSEicapaq6Yr17Z7qm33A7Xvnmrb/UDtu6eq3I81HxljjClkQcEYY0yhuhYUJvq7AD5Q2+6ptt0P1L57qm33A7Xvnip9P3WqT8EYY0zZ6lpNwRhjTBksKBhjjClUZ4KCiFwgIltEZLuIPOLv8lSViOwQkXUislpE4vxdnsoQkckickBE1hfZ1khEvhKRbe6/Df1Zxooo5X6eEpHd7vu0WkQu9GcZK0pE2orIYhHZKCIbROQ+d3tAvk9l3E/Avk8iEiEiy0VkjXtPT7vbO4rIT+5n3iwRCfPofHWhT0FEgoGtwLlAArACGKOqG/1asCoQkR3AAFUN2Ak3InI6kA5MU9We7rYXgSRVfcEN3g1VdVxZ56kpSrmfp4B0VX3Zn2WrLBFpCbRU1Z9FJBpYCVwC3EgAvk9l3M+VBOj7JCIC1FPVdBEJBZYA9wEPAp+o6kwReQtYo6pvlne+ulJTGARsV9VfVTUbmAmM8nOZ6jxV/Q5IKrZ5FDDV/X0qzn/YgFDK/QQ0Vd2rqj+7v6cBm4DWBOj7VMb9BCx1pLsPQ90fBc4CPnK3e/we1ZWg0BrYVeRxAgH+h4Dzpv9XRFaKyO3+LowXNVfVve7v+4Dm/iyMl9wtImvd5qWAaGYpiYh0APoBP1EL3qdi9wMB/D6JSLCIrAYOAF8BvwDJqprr7uLxZ15dCQq10TBVPRkYAdzlNl3UKuq0bQZ6++abQCegL7AXeMW/xakcEakPfAzcr6qpRZ8LxPephPsJ6PdJVfNUtS/QBqdlpFtlz1VXgsJuoG2Rx23cbQFLVXe7/x4APsX5Q6gN9rvtvgXtvwf8XJ4qUdX97n/YfOAdAvB9ctupPwbeV9VP3M0B+z6VdD+14X0CUNVkYDFwChArIgULqXn8mVdXgsIKoIvbGx8GXA3M83OZKk1E6rmdZIhIPeA8YH3ZRwWMecAN7u83AHP9WJYqK/jgdI0mwN4ntxNzErBJVV8t8lRAvk+l3U8gv4tjhccAAAM4SURBVE8i0lREYt3fI3EG1GzCCQ6Xu7t5/B7VidFHAO4Qs/8DgoHJqvqcn4tUaSJyAk7tAJwlVT8IxPsRkRnAcJw0v/uBJ4E5wGygHU6K9CtVNSA6b0u5n+E4TRIK7AD+WKQtvsYTkWHA98A6IN/d/ChOO3zAvU9l3M8YAvR9EpHeOB3JwThf9Ger6gT3c2Im0AhYBVyrqlnlnq+uBAVjjDHlqyvNR8YYYzxgQcEYY0whCwrGGGMKWVAwxhhTyIKCMcaYQiHl72JM7SMizwP/BWKA7qr6fDVf/w4gQ1WnVed1jSmPDUk1dZKILAL+APwN+EhVf6jGa4cUyUljTI1izUemThGRl0RkLTAQWAbcCrwpIk+UsG9HEVnmrlvxrIiku9uHi8jnRfZ7XURudH/vLyL/cxMVflkkFcS3IvJ/4qx9cZ+bv/9h97lOIrLQPeZ7Eenmbr9CRNa7efK/8+0rY4zDgoKpU1T1z8AtwBScwLBWVXur6oQSdv8n8Kaq9sJJklYmN6fOv4DLVbU/MBkoOtM8TFUHqGrxZGsTgXvcYx4G/u1ufwI4X1X7ABd7eo/GVIX1KZi66GRgDU4myU1l7DcUuMz9fTrw93LOeyLQE/jKSbFDMMcGk1nFD3CzdZ4KfOgeAxDu/vsDMEVEZgOfFD/WGF+woGDqDBHpi1NDaAMcAqKczbIaOEVVj5ZwWEmdbrkcW8uOKLgEsEFVTymlCEdK2BaEk/e+73EXVr1DRAbj9H2sFJH+qppYyrmN8QprPjJ1hqqudj98twInAYtwmmf6lhIQfsDJqAswtsj2eOAkEQl3s1Oe7W7fAjQVkVPAaU4Skf9v7w5VEAiiKAyf060+gUUw+DQmow/gG9jF9xJMKq7ZYDQIYrJcw1xGTWtZDP5f2WVgYNrhzi73jlrOdJN0sj3JPbY9zvdBRGwiYiHpos/270AnCAX8Fdt9Sdfsmz9smdM9VxlgdNDb1KqIOKt0CG3yuc31h0qr4qXtvaSdytVQm6mkWe456jUqdpUfuRtJa5UrL6BT/JIKfMn2PSJ6vz4H0CUqBQBARaUAAKioFAAAFaEAAKgIBQBARSgAACpCAQBQPQF1l/ohCxZx9wAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "xx = np.arange(n_queries)\n", - "\n", - "plt.plot(xx, random_experiment_data[\"accuracy\"], label=\"Random\")\n", - "plt.plot(xx, uncertainty_experiment_data[\"accuracy\"], label=\"Uncertainty\")\n", - "plt.plot(xx, as_experiment_data[\"accuracy\"], label=\"AS\")\n", - "\n", - "plt.title(\"Accuracy on Test Set vs Num Queries\")\n", - "plt.ylabel(\"accuracy\")\n", - "plt.xlabel(\"# queries\")\n", - "plt.legend()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xl8TNf/x/HXSSQiYo1Yg9jXSBD7Umpta2vR6hdtabW68NVaSyyxFKVVSqtail9VFS2qqi2lJLbGliB2iZ1IhET2mfP7Y4ZvSjCSzEyWz/Px8JDcucs7E+5nzrn3nqO01gghhMi7HOwdQAghhH1JIRBCiDxOCoEQQuRxUgiEECKPk0IghBB5nBQCIYTI46QQiGxDKbVUKTXVTsdWSqlvlVI3lVL77JHBmpRSzyulLiil4pRS9W14XC+llFZK5bPVMcWTk0IgHkopFa6Uuq6UKphm2RtKqe12jGUtLYEOgKfWuvH9LyqlXlNKBdo+1r3jZ/aEOht4T2vtprU+mJXZRM4nhUA8jiPwX3uHeFJKKccn3KQiEK61vmOlPPb+RFwROGrnDCKbkkIgHmcWMEIpVfT+F9L7lKqU2q6UesP89WtKqSCl1BylVIxS6qxSqrl5+QVza+PV+3ZbQin1p1IqVin1t1KqYpp91zS/Fq2UOqGUejHNa0uVUl8qpTYppe4AbdPJW1YptcG8/Wml1CDz8teBb4Bm5q6TgPu2qwUsTPN6jHn5c0qpg0qp2+afZ1I6783rSqnzwF/m5a8opSKUUlFKqfHmVld782sOSqkxSqkz5td/VEoVN+9yh/nvGHOGZkqpqub36JZS6oZSalU6P3N+pVQcpoJ+WCl15u7PZP5dxSiljiqluqX3O0zzewxM871WSg1WSp0yb79AKaXMrzkqpWab85wFnrs/k8h+pBCIxwkGtgMjMrh9EyAEcAe+B34AGgFVgX7AfKWUW5r1+wJTgBLAIWAFgLl76k/zPkoCfYAvlFK102z7H2AaUAhIrxvnB+AiUBboBXyklHpaa70YGAzsNnedTEy7kdY67L7X7xbFO8ArQFFMJ7y3lVI97jvmU0AtoJM56xfmn7EMUAQol2bdIUAP8zZlgZvAAvNrrc1/FzVn2G1+n/4AigGewOf3/8Ba6ySt9d3310drXUUp5QT8Yt62pPm4K5RSNdJ5zx6mC6bfYz3gRaCTefkg82v1AT9M77PI5qQQCEtMAIYopTwysO05rfW3WmsDsAooD0w2n6D+AJIxFYW7ftVa79BaJwHjMH0KL4/p5BJu3lequZ97LdA7zbbrtdZBWmuj1joxbQjzPloAo7XWiVrrQ5haAa9k4GcCQGu9XWsdaj5eCLAS00k8rUla6zta6wRMJ8VftNaBWutkTO9r2sG+BgPjtNYXzT//JKDXI7qVUjB1+ZQ1/0yWXsNoCrgBM7TWyVrrv4CNwMsWbo952xit9XlgG+BrXv4i8JnW+oLWOhqY/gT7FHYihUA8ltb6CKYTxZgMbH4tzdcJ5v3dvyxti+BCmuPGAdGYPh1XBJqYuyJizN0zfYHS6W2bjrJAtNY6Ns2yCP79ifyJKKWaKKW2KaUilVK3MJ3IS9y3WtpMZfn3zxcPRKV5vSLwc5qfLwwwAKUeEmEUoIB95u6dgRZGLwtc0Fob0yx70vfiapqv4/nf7/BfP6N5vyKbk0IgLDURU7M/7cni7oVV1zTL0p6YM6L83S/MXUbFgcuYTi5/a62LpvnjprV+O822jxpK9zJQXClVKM2yCsAlC3Olt+/vgQ1Aea11EUzXEdQjtruCqQsHAKVUAUxdZnddAJ6572d00VpfSu/4WuurWutBWuuywFuYusqq3r9eOi4D5ZVSaf//p30v7pDx3+kV0vwOzfsV2ZwUAmERrfVpTF07Q9Msi8R08uhnvkg4EKiSyUM9q5RqqZRyxtQHvkdrfQFTi6S6Uqq/UsrJ/KeR+UKuJfkvALuA6UopF6VUPeB14DsLc10DPM257iqEqZWRqJRqjOkaxaOsAbqaL5g7Y+r6SVs4FgLT7l4gV0p5KKW6m1+LBIxA5bsrK6V6K6XuFpabmIpF2k/5D7MX06f4Ueb3sQ3QFdM1FDBdm3lBKeVqLiyvW7DPu34EhiqlPJVSxchYK1LYmBQC8SQmAwXvWzYIGImpi6MOppNtZnyPqfURDTTEdEEZc5dOR0wXiS9j6pqYCeR/gn2/DHiZt/8ZmKi13mLhtn9huv3yqlLqhnnZO8BkpVQspv7+Hx+1A631UUwXZn/A9Mk5DrgOJJlXmYuphfGHeZ97MF1sv9uNNA0IMncdNcV0sXav+a6gDcB/tdZnH/eDmK9PdAWeAW5guoD9itb6uHmVOZiu3VwDlmG+YG+hr4HfgcPAAeCnJ9hW2ImSiWmEsA9z11cMUE1rfc7eeUTeJS0CIWxIKdXV3OVSENPTvqFAuH1TibxOCoEQttUdU9fUZaAa0EdLs1zYmXQNCSFEHictAiGEyOOsNhCWUsoF0/go+c3HWaO1nqiUqoTprgl3YD/Q33wXw0OVKFFCe3l5WSuqEELkSvv377+htX7siADWHBExCXhaax1nHtskUCn1G/ABMEdr/YNSaiGme5S/fNSOvLy8CA4OtmJUIYTIfZRSFj3ZbbWuIW0SZ/7WyfxHA09jerAGTPco3z9IlxBCCBuy6jUC89OmhzA9NPMncAaI0Vqnmle5yEPGN1FKvamUClZKBUdGRlozphBC5GlWLQRaa4PW2hfT+CqNgZpPsO0irbWf1trPwyMjg14KIYSwhE1mTdJaxyiltgHNgKJKqXzmVoEnlg/69S8pKSlcvHiRxMTEx68ssoSLiwuenp44OTnZO4oQIgtZ864hDyDFXAQKYJoPdiamsct7Ybpz6FVgfUb2f/HiRQoVKoSXlxfmyZGEFWmtiYqK4uLFi1SqVMnecYQQWciaXUNlgG1KqRDgH+BPrfVGYDTwgVLqNKZbSBdnZOeJiYm4u7tLEbARpRTu7u7SAhMiF7Jai8A8Y1P9dJafxXS9INOkCNiWvN9C5E7yZLEQQmRDt5JuMWPfDOKS4x6/ciZJIcgER0dHfH19qVu3Ll27diUmJiZL9hseHk7dunWzZF9CiJzn7wt/02N9D1YdX0XwNes/TCuFIBMKFCjAoUOHOHLkCMWLF2fBggX2jiSEyMFuJ99mXOA43vvrPYq5FOP7576nTfk2Vj+uFIIs0qxZMy5dMt0JGxcXR7t27WjQoAHe3t6sX2+6MSo8PJxatWoxaNAg6tSpQ8eOHUlISABg//79+Pj44OPj86+CkpiYyIABA/D29qZ+/fps27YNgKVLl9KjRw86dOiAl5cX8+fP59NPP6V+/fo0bdqU6OhoG78DQojM2HlxJ8+vf55fz/7KIO9BrHpuFbXcLZqJNdNs8hyBtQX8cpRjl29n6T5rly3MxK51LFrXYDCwdetWXn/dNLWri4sLP//8M4ULF+bGjRs0bdqUbt26AXDq1ClWrlzJ119/zYsvvsjatWvp168fAwYMYP78+bRu3ZqRI0fe2/eCBQtQShEaGsrx48fp2LEjJ0+eBODIkSMcPHiQxMREqlatysyZMzl48CDvv/8+y5cvZ9iwYVn6ngghsl5cchyzgmfx06mfqFKkCnPbzqVuCdt2DUuLIBMSEhLw9fWldOnSXLt2jQ4dOgCme+7Hjh1LvXr1aN++PZcuXeLatWsAVKpUCV9fXwAaNmxIeHg4MTExxMTE0Lp1awD69+9/7xiBgYH069cPgJo1a1KxYsV7haBt27YUKlQIDw8PihQpQteuXQHw9vYmPDzcJu+BECLjdl/ezfMbnmfd6XUMrDuQVV1X2bwIQC5pEVj6yT2r3b1GEB8fT6dOnViwYAFDhw5lxYoVREZGsn//fpycnPDy8rp3/33+/P+ba93R0fFe11BGpN2Xg4PDve8dHBxITU192GZCCDu7k3KHT4M/5ceTP+JV2IvlzyzHx8PHbnmkRZAFXF1dmTdvHp988gmpqancunWLkiVL4uTkxLZt24iIePRIsEWLFqVo0aIEBgYCsGLFinuvtWrV6t73J0+e5Pz589SoUcN6P4wQwqr+ufoPPTf0ZPXJ1bxS+xVWd11t1yIAuaRFkB3Ur1+fevXqsXLlSvr27UvXrl3x9vbGz8+PmjUfP9bet99+y8CBA1FK0bFjx3vL33nnHd5++228vb3Jly8fS5cu/VdLQAiRM8SnxDP3wFy+P/495QuVZ2nnpTQo1cDesYAcMmexn5+fvn9imrCwMGrVss0VdfE/8r4L8eQOXDuAf5A/F2Iv0LdWX4bWH4qrk6vVj6uU2q+19nvcetIiEEIIK0lMTWTewXl8d+w7yrqVZUmnJTQq3cjesR4ghUAIIazgcORh/AP9Cb8dzks1XuKDhh/YpBWQEVIIhBAiCyUZklhwaAHLji6jlGspFnVYRLOyzewd65GkEAghRBY5cuMI4wLHcfbWWXpW68kIvxG4ObvZO9ZjSSEQQohMSjYks/DwQpYcWYJ7AXcWtl9Ii3It7B3LYlIIhBAiE8KiwhgXNI5TN0/RvUp3RjUeRWHnwvaO9UTkgbJMSG+46EmTJjF79myrHXPdunUcO3bssestXLiQ5cuXP3KdQ4cOsWnTpqyKJkSekmJM4ctDX/KfX/9DTGIM85+ez9SWU3NcEQBpEeQoqamprFu3ji5dulC7du1Hrjt48ODH7u/QoUMEBwfz7LPPZlVEIfKEE9En8A/y53j0cbpU7sKYxmMokr+IvWNlmLQIrKRNmzaMHj2axo0bU716dXbu3AmYRiodMWIEdevWpV69enz++eeAaRjqp556ioYNG9KpUyeuXLlybz/Dhg3Dz8+PmTNnsmHDBkaOHImvry9nzpzh66+/plGjRvj4+NCzZ0/i4+OBf7dM0suSnJzMhAkTWLVqFb6+vqxatYpq1aoRGRkJgNFopGrVqve+F0JAqjGVRSGL6PNrH67HX+eztp8xvdX0HF0EILe0CH4bA1dDs3afpb3hmRmZ2kVqair79u1j06ZNBAQEsGXLFhYtWkR4eDiHDh0iX758REdHk5KSwpAhQ1i/fj0eHh6sWrWKcePGsWTJEgCSk5O5+2T1qVOn6NKlC7169QJM4xQNGjQIAH9/fxYvXsyQIUMsyjJ58mSCg4OZP38+AMePH2fFihUMGzaMLVu24OPjg4eHR6beAyFyizMxZxgXOI6jUUfp7NWZsU3GUsylmL1jZYncUQjs5GGTud9d/sILLwD/G24aYMuWLQwePJh8+UxvffHixTly5AhHjhy5N4y1wWCgTJky9/b30ksvPTTDkSNH8Pf3JyYmhri4ODp16pTueullud/AgQPp3r07w4YNY8mSJQwYMOChxxUirzAYDSw7toz5B+fj5uTG7Kdm08kr/f9nOVXuKASZ/OSeUe7u7ty8efNfy6Kjo6lUqRLwv2GiHR0dHzkstNaaOnXqsHv37nRfL1iw4EO3fe2111i3bh0+Pj4sXbqU7du3p7ueJVnKly9PqVKl+Ouvv9i3b9+/RkEVIi86d+sc/kH+hESG0L5Ce/yb+uNewN3esbKcXCPIBDc3N8qUKcNff/0FmIrA5s2badmy5UO36dChA1999dW9k3F0dDQ1atQgMjLyXiFISUnh6NGj6W5fqFAhYmNj730fGxtLmTJlSElJeeIT9/37AnjjjTfo168fvXv3xtHR8Yn2J0RuYTAaWH50Ob1/6U34rXBmtprJp20+zZVFAKQQZNry5cuZMmUKvr6+PP3000ycOJEqVao8dP033niDChUqUK9ePXx8fPj+++9xdnZmzZo1jB49Gh8fH3x9fdm1a1e62/fp04dZs2ZRv359zpw5w5QpU2jSpAktWrSwaLjrtNq2bcuxY8fuXSwG6NatG3FxcdItJPKs87fPM/D3gcwKnkWzMs1Y130dz1Z+9qFdwbmBDEMt/iU4OJj333//3l1O95P3XeRWRm1k5fGVfLb/M5wcnBjTZAxdK3fN0QVAhqEWT2zGjBl8+eWXcm1A5DkXYi8wIWgCwdeCaVmuJZOaTaJUwVL2jmUzVusaUkqVV0ptU0odU0odVUr917x8klLqklLqkPmPPM2UTYwZM4aIiIhHXuMQIjcxaiOrjq+i54aehEWHMbn5ZL5o90WeKgJg3RZBKjBca31AKVUI2K+U+tP82hyttfXGYRBCiMe4HHeZCbsmsPfKXpqVaUZA8wDKuJV5/Ia5kNUKgdb6CnDF/HWsUioMKGet4wkhhCW01vx06idmBc9Ca82EZhPoVa1Xjr4WkFk2uWtIKeUF1Af2mhe9p5QKUUotUUql+2ieUupNpVSwUipYhjkQQmSFq3eu8vaWt5m0exJ13OvwU/ef6F29d54uAmCDQqCUcgPWAsO01reBL4EqgC+mFsMn6W2ntV6ktfbTWvvJMAdCiMzQWrP+9HpeWP8CB64f4MPGH/J1x68p5yadFGDlQqCUcsJUBFZorX8C0Fpf01obtNZG4GugsTUzWNu6detQSnH8+HHANFjb0KFDqVu3Lt7e3jRq1Ihz587ZOaUQeVdkfCRD/hqCf5A/1YpVY23Xtfyn1n9wUPIY1V1Wu0agTG2txUCY1vrTNMvLmK8fADwPHLFWBltYuXIlLVu2ZOXKlQQEBLBq1SouX75MSEgIDg4OXLx48ZFDRAghrENrza/nfmX63ukkGZIY1WgUfWv1lQKQDmveNdQC6A+EKqUOmZeNBV5WSvkCGggH3rJiBquKi4sjMDCQbdu20bVrVwICArhy5QplypTBwcH0j83T09POKYXIe24k3GDqnqlsPb8VHw8fpraYilcRL3vHyraseddQIJDeFZgsnxJr5r6ZHI8+nqX7rFm8JqMbj37kOuvXr6dz585Ur14dd3d39u/fz4svvkjLli3ZuXMn7dq1o1+/ftSvXz9LswkhHm5z+Gam7ZlGfEo8wxsOp3/t/jg6yLhZjyJtpExYuXIlffr0AUxjAK1cuRJPT09OnDjB9OnTcXBwoF27dmzdutXOSYXI/W4m3mT49uGM/Hsknm6erO66mtfqviZFwAIy1lAGRUdH4+npiYeHB0opDAYDSikiIiL+dSva7NmziYiIuDcTWU5n7/ddiPRsjdjK5D2TiU2O5R3fd3itzmvkc5ARdCwda0haBBm0Zs0a+vfvT0REBOHh4Vy4cIFKlSqxc+dOLl++DJjuIAoJCaFixYp2TitE7hSTGMPoHaMZtn0YpVxL8UOXH3jD+w0pAk9I3q0MWrlyJaNH//saQs+ePXn11VcpXrw4SUlJADRu3Jj33nvPHhGFyNW2X9hOwO4AYhJjeNf3XV73fh0nByd7x8qRpBBk0LZt2x5YNnToUIYOHWqHNELkHbeTbzNz30w2nNlAjWI1+LL9l9Qs/mRzcYh/k0IghMgxdl7cyaRdk4hKjOKtem/xVr23cHKUVkBmSSEQQmR7scmxzA6ezU+nfqJq0arMazePOu517B0r18jRhUBrnecHi7KlnHCHmch9dl3excRdE7kef503vN/gbZ+3cXZ0tnesXCXHFgIXFxeioqJwd3eXYmADWmuioqJwcXGxdxSRR9xJucMnwZ+w+uRqKhWpxP8983/U86hn71i5Uo4tBJ6enly8eBEZotp2XFxcZMgMYRP7ruxjwq4JXI67zKu1X+W9+u/hkk8+hFhLji0ETk5OVKpUyd4xhBBZKD4lns8OfMbK4yupWLgiy55ZRv2SMkSLteXYQiCEyF32X9uPf6A/F+Mu0q9WP4Y2GEqBfAXsHStPkEIghLCrhNQEPj/4Od8d+45ybuVY0mkJjUo3snesPEUKgRDCbg5dP8T4oPGE3w7npRov8UHDD3B1crV3rDxHCoEQwuaSDEksOLiAZceWUcq1FF93/JqmZZraO1aeJYVACGFToZGh+Af5c/bWWXpV78XwhsNxc3azd6w8TQqBEMImkg3JLDy8kMVHFuNRwIOv2n9F83LN7R1LIIVACGEDx6KOMS5wHKdjTvN81ecZ2WgkhZwL2TuWMJNCIISwmhRDCotCF/F1yNcUdynOgnYLaO3Z2t6xxH2kEAghrOJE9An8g/w5Hn2crpW7MrrxaIrkL2LvWCIdUgiEEFkqxZjCktAlLAxZSGHnwsxtO5enKzxt71jiEaQQCCGyzOmbpxkXNI5jUcd4xusZPmzyIcVcitk7lngMKQRCiExLNaay7OgyFhxagJuTG5889QkdvTraO5awkBQCIUSmnL11lvGB4wm5EUKHih0Y12Qc7gXc7R1LPAEpBEKIDDEYDXwX9h3zDsyjgFMBPm79MZ29Osv8IDmQFAIhxBOLuB3B+KDxHLx+kDbl2zCx2URKFChh71gig6QQCCEsZtRGvg/7nrkH5uLk6MRHLT+iS+Uu0grI4axWCJRS5YHlQClAA4u01nOVUsWBVYAXEA68qLW+aa0cQoiscSH2AuODxrP/2n5alWvFpOaTKOla0t6xRBawZosgFRiutT6glCoE7FdK/Qm8BmzVWs9QSo0BxgCjrZhDCJEJRm3kxxM/8un+T3FUjkxuPpkeVXtIKyAXsVoh0FpfAa6Yv45VSoUB5YDuQBvzasuA7UghECJbuhx3mQm7JrD3yl6al21OQPMAShcsbe9YIovZ5BqBUsoLqA/sBUqZiwTAVUxdR+lt8ybwJkCFChWsH1IIcY/WmrWn1jLrn1kATGw2kZ7VekorIJeyeiFQSrkBa4FhWuvbaf8haa21Ukqnt53WehGwCMDPzy/ddYQQWe/qnatM3DWRXZd30aR0EwJaBFDOrZy9YwkrsmohUEo5YSoCK7TWP5kXX1NKldFaX1FKlQGuWzODEMIyWmvWnV7Hx/98jEEbGNdkHC/WeBEH5WDvaMLKrHnXkAIWA2Fa60/TvLQBeBWYYf57vbUyCCEscz3+OgG7A9hxcQcNSzVkSvMplC9c3t6xhI1Ys0XQAugPhCqlDpmXjcVUAH5USr0ORAAvWjGDEOIRtNZsPLuR6fumk2JIYVSjUfSt1VdaAXmMNe8aCgQedmWpnbWOK4SwzI2EG0zePZltF7bh6+HLlBZT8CriZe9Ywg7kyWIh8hitNb+H/860vdOIT4lnhN8I+tXqh6ODo72jCTt5bCFQSlUBLmqtk5RSbYB6wHKtdYy1wwkhslZ0YjRT90zlz4g/8S7hzdQWU6lctLK9Ywk7s6QjcC1gUEpVxXQ7Z3nge6umEkJkuT8j/uT59c+z/cJ2/tvgvyx/ZrkUAQFY1jVk1FqnKqWeBz7XWn+ulDpo7WBCiKwRkxjDR/s+4rdzv1GreC2+6fgN1YpVs3cskY1YUghSlFIvY7rVs6t5mZP1Igkhssq289sI2B3AreRbvOf7HgO9B+LkIP99xb9ZUggGAIOBaVrrc0qpSsD/WTeWECIzbiXdYua+mfxy9hdqFKvBVx2+okbxGvaOJbIpSwpBB6310LvfmItBohUzCSEyYcfFHQTsCiAqMYrBPoN50/tNnBylFZAjpSZDPmerH8aSi8WvprPstSzOIYTIpNjkWCYETeDdre9SOH9hVjy3gnd935UikFOFbYR5vnAlxOqHemiLwHxd4D9AJaXUhjQvFQKirR1MCGG5XZd2MWHXBCITInnD+w3e9nkbZ0frf5IUVhAfDb+NhtAfobQ32KCQP6praBem+QRKAJ+kWR4LWL9ECSEe607KHT4J/oTVJ1dTqUglvmvzHd4e3vaOJTLqxGb4ZSjER0GbD6HVcPsWAq11BKaxgJoppSoC1bTWW5RSBYACmAqCEMJO9l7Zy4SgCVy5c4VXa7/Ke/XfwyWfi71jiYxIiIHfx8KhFVCyDvznRyjra7PDW/Jk8SBME8QUB6oAnsBCZLwgIewiPiWeOfvn8MOJH6hYuCLLn1mOb0nbnTREFju1BTYMgbhr0GoEPDUK8uW3aQRL7hp6F2iMaXYxtNanlFIyY7UQdhB8NZjxQeO5FHeJfrX6MbTBUArkK2DvWCIjEm/DH/5wYBl41IQ+30G5hnaJYkkhSNJaJ9+dWUwplQ+QGcOEsKGE1ATmHZjHirAVlHMrx5JOS/Ar7WfvWCKjzm6H9e/B7UvQYpjpeoCT/br1LCkEfyulxgIFlFIdgHeAX6wbSwhx16Hrh/AP8ifidgR9avTh/Ybv4+rkau9YIiOS4uDPCRC8GNyrwsDfoXxje6eyqBCMAV4HQoG3gE3AN9YMJYSAJEMS8w/OZ9nRZZQpWIZvOn5DkzJN7B1LZFR4IKx7B2LOQ9N3od14cMoe3XqPLQRaayPwtfmPEMIGQiNDGRc0jnO3ztGrei9G+I2goFNBe8cSGZF8B7ZOhr0LoVglGPAbVGxm71T/YsldQ6E8eE3gFhAMTNVaR1kjmBB5UbIhmS8Pf8mSI0vwKODBwvYLaVGuhb1jiYyK2A3r34Hos9BkMLSbAM7Zr6Bb0jX0G2Dgf3MQ9AFcgavAUv43IqkQIhOORh3FP9Cf0zGneb7q84xsNJJCzoXsHUtkREoC/DUVdi+AohXg1Y1QqZW9Uz2UJYWgvda6QZrvQ5VSB7TWDZRS/awVTIi8IsWQwqLQRXwd8jXuLu4saLeA1p6t7R1LZNSFf2Dd2xB1Cvxehw6TIb+bvVM9kiWFwFEp1VhrvQ9AKdUIuDu5aarVkgmRB5yIPsG4wHGcuHmCblW6MarRKIrkL2LvWCIjUhJh+3TYNQ8Kl4P+66BKW3unsoglheB14Ful1N2SFgu8rpQqCEy3WjIhcrEUYwqLQxfz1eGvKOpSlHlt59G2Qs44aYh0XDpgagVEHocGr0LHqeBS+IHVklINfLHtDPsjblq865GdauBTvmhWpn3AIwuBUsoBqKy19lZKFQHQWt9Ks8qP1gwnRG506uYp/IP8ORZ1jGcqPcPYxmMp6mLd/+jCSlKT4O+PIXAOuJWCvmuhWvt0Vw29eIvhqw9x8loc9TyL4ORoySwAYNDWf373kYVAa21USo0CfryvAAghnlCqMZWlR5fyxaEvKORciDlt5tC+YvonDZEDXDkMP78N14+Cb1/o9BEUeLCgJ6camf/XKRZsP0MJN2e+HdCItjWy1yg9lnQNbVFKjQBWAXfuLtRay5wEQljobMyuMLeHAAAgAElEQVRZ/IP8Cb0RSoeKHfBv6k9xl+L2jiUywpACOz+BHbPA1R1eXgU1Oqe76tHLtxj+42GOX43lhQblmNilDkVcs99EQZYUgpfMf7+bZpkGKmd9HCFyF4PRwP8d+z8+P/g5rk6uzHpqFp290j9piBzg6hHTtYCrIVDvJeg8A1wfLOgpBiNfbDvD53+dolhBZ75+xY8OtUvZIbBlLHmyuFJGdqyUWgJ0Aa5rreual00CBgGR5tXGaq03ZWT/QmR34bfCGR80nkORh3i6/NOMbzaeEgVK2DuWyAhDKgTNge0zTd0/L30HtdJ/hOrE1ViGrz7EkUu36e5blkld61CsYPaeLc6SFgFKqbpAbeDe8Hha6+WP2WwpMB+4f705WuvZT5BRiBzFqI18H/Y9cw/MxdnRmemtpvNcpee4O4KvyGGuH4d1g+HyQajzAjw7Gwq6P7BaqsHIVzvO8tmWkxR2cWJhvwZ0rlvGDoGfnCVDTEwE2mAqBJuAZ4BAHjzB/4vWeodSyivTCYXIQS7cvsD4XePZf20/rT1bM7HZREq6Zq8Lg8JCRgPs+hy2TQNnN+i9FOo8n+6qp6/HMvzHwxy+eItnvUszpXtd3N1sO7lMZljSIugF+AAHtdYDlFKlgO8yccz3lFKvYBqraLjWOt0bapVSb2KaGY0KFSpk4nBCWJ9RG1l1YhVz9s/BUTkyuflkelTtIa2AnOrGKdO1gIv/QM0u0GUOuD1Y0A1GzTc7z/LJnycp6OzI5y/Xp6tPWTsEzhxLCkGC+TbSVKVUYeA6UD6Dx/sSmILpYvMU4BNgYHoraq0XAYsA/Pz8ZCIckW1dirvEhKAJ7Lu6jxZlWzCp+SRKFyxt71giI4wG0yihWydDPhfouRjq9oR0CvrZyDhGrD7MgfMxdKxdimnPe+NRKOe0AtKypBAEK6WKYhqGej8QB+zOyMG01tfufq2U+hrYmJH9CJEdaK1Zc2oNs/+ZjVKKSc0m8UK1F6QVkFNFnYH178L53VD9Gej6GRR6sKAbjZpvd4Xz8ebjuDg58tlLvnT3LZujf++W3DX0jvnLhUqpzUBhrXVIRg6mlCqjtb5i/vZ54EhG9iOEvV29c5UJQRPYfWU3Tco0YXLzyZR1y3ldAgIwGuGfr+HPieDoDD0Wgk+fdFsBEVF3GLk6hH3h0bSrWZKPXvCmVGH7TTGZVSy5WLxVa90OQGsdfv+yR2y3EtNF5hJKqYvARKCNUsoXU9dQOKYZz4TIMbTWrDu9jo//+RiDNuDfxJ8Xa7yYoz8N5mnR50xzB0cEQtUO0G0eFH6woBuNmu/2RjB903HyOSpm9/ahZ4Nyueb3/tBCoJRywTTvQAmlVDHg7k9cGCj3uB1rrV9OZ/HijIQUIju4ducaAbsD2HlpJ36l/JjcYjLlC2X0cpmwK60heAn8MR6UA3SbD/X7pdsKuBAdz6g1Iew+G0Xr6h7M7OlNmSLZY4rJrPKoFsFbwDCgLKZrA3ffoduYng8QIk/QWrPx7Eam75tOiiGFMY3H8HLNl3FQlg0aJv5Na81PBy6x75x9RqkpmnyVXpdmUu1OMCcLNmJtudHEnC0FZ0MfWNegNb+FXkEpxYwXvHmpUflc0wpI66GFQGs9F5irlBqitf7chpmEyDZuJNwgYHcA2y9sp37J+kxpMYWKhSvaO1aOdeVWAqPXhrLjZCTuBZ0tHoEzS2hNN+MWhhqWotDMcHyLdckdIVzxv8EOHtS0sjsB3evgWczVdlltzJKLxVIERJ6jtea3c7/x0b6PSExNZITfCPrV6oejg+PjNxYP0Fqz9sAlAn45SqpBE9CtDv2bVsTBwUafrm9dgl+Gwukt4NUKus9nTDEvxtjm6NmeRUNMCJGXRCVEMW3vNP6M+JN6JeoxpeUUKheRMRYz6vrtRD78KZStx6/TyKsYs3r54FXCRhO4aw2HV8JvY8CYYhoewu91cJBuvbQedbG4hdY6SCmVX2udZMtQQtjLH+F/MHXPVOJS4ni/4fu8UvsV8jnI56WM0Fqz4fBlJqw/SmKKgfFdavNacy8cbdUKiL0Kv/wXTm6GCs2hxwIoLgU9PY/6Fz4PaIjp4bEGj1hPiBwvJjGGaXunsTl8M7XdazOtxTSqFqtq71g51o24JMb9HMrvR6/RoEJRZvX2oYqHjSZw1xpC18CmEZCaCJ2mQ5PB0gp4hEcVghSl1CKgnFJq3v0vaq2HWi+WELbz1/m/mLx7MreSbzGk/hAG1B2Ak0P2mzwkp/g15Arj1x8hLimVsc/W5PWWlW3XCoi7Dhvfh+MbwbMx9PgSSkhBf5xHFYIuQHugE6bbR4XIVW4l3WLGvhlsPLuRmsVr8lWHr6hRvIa9Y+VY0XeSGb/+CL+GXMHHswize/tQrVQh2wU4+jNs/ACS70CHKdDsXZCL+xZ51O2jN4AflFJhWuvDNswkhNXtuLiDSbsmcTPxJoN9BvOm95s4OUorIKM2H7mK/7pQbiWkMLJTDd5qXZl8tro19E4UbBpuKgRlG5haASVr2ubYuYQlV8GilFI/Ay3M3+8E/qu1vmi9WEJYR2xyLB//8zHrTq+jatGqzG83n9rute0dK8eKiU9m0oajrDt0mTplC/PdG02oWbqw7QKE/WLqCkqIgXYToPl/wVEu7j8pS96xb4Hvgd7m7/uZl3WwVighrCHoUhATd00kMiGSQd6DGOwzGGfH7D2FYHa2NewaY34K5eadZN5vX5132lax3QNi8dHw22gI/RFK14NX1kOpOrY5di5kSSEoqbX+Ns33S5VSw6wVSIisFpccx+zg2aw9tZbKRSrzWdvPqFuirr1j5Vi3ElKY/Msx1h64SM3ShVg6oBF1yhaxXYATv5luC42PgjZjodUHIN16mWJJIbihlOoHrDR//zIQZb1IQmSdPVf2MCFoAtfirzGg7gDe9X2X/I45c/KQ7GD7ieuMWRtKZFwSQ56uypCnq+Gcz0atgIQY2PwhHP4eStWFvmugTD3bHDuXs6QQDAQ+B+ZgGj56FzDAmqGEyKz4lHg+3f8pq06somLhiizrvAzfkr72jpVjxSamMO3XMH745wLVSrqx6JWG1PMsarsAp/6EDUMh7hq0HgmtR0E+6dbLKpaMNRQBdLNBFiGyxD9X/2F80Hgux12mf+3+DKk/hAL5ctewwbYUeOoGo9Yc5urtRN5uU4X/tquGi5ONbstMvA2/j4WD/wceNaHPCignz7dmNbm8LnKNhNQE5h6Yy4qwFXi6efJt529pWKqhvWPlWHeSUvloUxgr9p6nskdB1rzdnAYVitkuwJltpkljYi9Di2HQ5kNwyvmzgWVHUghErnDw+kH8A/05H3uel2u+zLAGw3B1yr3DBlvb7jNRjFxzmEsxCQxqVYnhHWvYrhWQFAt/TjBNHONeDQb+AeUb2ebYeZQUApGjJaYmMv/gfJYfW05Zt7Is7riYxmUa2ztWjhWfnMrHm0+wdFc4Xu6urH6rGX5exW0X4NwO0wTyMReg2XvwtD84SbeetVkyZ7G/1nqq+WsZiVRkGyGRIYwLHEf47XBerP4iH/h9QEEnGw1vnAv9Ex7NiNWHiYiK57XmXozqXANXZxt9Vky+A1sCYN9XphFCB/wGFZvZ5tjikcNQjwZ2AL2AqebFMhKpsLtkQzJfHPqCb49+S0nXknzV4Sual21u71g5VmKKgVm/n2BJ0Dk8ixXghzeb0rSyu+0CROyGdW/DzXOmUULbTQBnKei29KhyfxzT08SVlVI7zd+7K6VqaK1P2CSdEPc5euMo/kH+nI45zQvVXmCE3wgKOdtwYLNcZn/ETUauPszZG3fo37QiY56pScH8NmoFpCTA1imw5wsoWgFe+xW8Wtrm2OJfHvUbjwHGAm3Mf2oBHYEx5mIgH8GEzaQYUlgYspDFoYtxd3Hni3Zf0Mqzlb1j5ViJKQbmbDnJ1zvOUqZIAVa80YQWVUtkbqcpiXBkDSTFPX5dbTBdDI46DY3egPYBkN9G8xWIBzyqEHQCJgBVgE+BEOCO1loeJhM2dTz6OOMCx3Hy5km6VenGqEajKJLfhkMa5DKHL8QwfPVhTl+P4+XG5Rn7bC0KuWRyiIaL+2HdYLhx0vJtilSA/uugStvMHVtk2qOGoR4LoJQ6DPwfpmsDHkqpQOCm1rqrbSKKvCrFmMI3od+w6PAiiroUZV7bebStICeNjEpKNTBv6ykW/n0WD7f8LBvYmKeqe2Rup6lJsH0GBH0GhcqYhn0oZ+GzG/kLy0ih2YQlv4XftdbBQLBS6m2tdUulVCbbkEI82qmbpxgXOI6w6DCerfQsHzb+kKIuNhzSIJc5cukWI1Yf5vjVWHo39MS/S22KFMhkK+DyQfj5bYgMg/r9oNNH4CIttZzIkiEmRqX59jXzshvWCiTytlRjKkuPLmXBoQUUdi7MnDZzaF+xvb1j5VjJqUYWbDvNgm2nKV7QmcWv+tGuVqnM7TQ1GXbMgp2fgFtJ+M9qqN4xawILu3iidtmTzFSmlFqCabrL61rruuZlxYFVgBcQDryotb75JBlE7nU25izjAsdxJOoIHSt2ZFzTcRR3seHDTLlM2JXbDP/xMMeu3Ob5+uWY2LU2RV0zOVDb1VBTK+BaKPi8DJ2nQwEbDjshrMKaHXRLgfnA8jTLxgBbtdYzlFJjzN+PtmIGkQMYjAaWH1vO/IPzcXVyZdZTs+js1dnesXKsVIORhX+fYe7WUxQp4MRX/RvSqU7pzO3UkAKBc+DvmVCgOPRZCTWfzZrAwu6sVgi01juUUl73Le6O6VZUgGXAdqQQ5Gnht8LxD/LncORh2lVoh39Tf0oUkEtQaaUajGwJu05cUupj1zVqzXd7Igi5eIuuPmUJ6FaH4gXTaQUYUuHkb6ZxfR67UwP88w1cOQTeveGZj8FVWmq5ia0v2ZfSWl8xf30VeGhnpVLqTeBNgAoVKtggmrAlozayImwFcw/MJb9jfma0msGzlZ5FKWXvaNnK6euxDF8dwuELMRZvU7ygMwv+04Dn6pVJf4XIE/DzYLh8wPIgriXgxeVQu7vl24gcw273bmmttVJKP+L1RcAiAD8/v4euJ3KeC7cv4B/kz4HrB3jK8ykmNpuIh2smb2PMZQxGzZLAc8z64wSuzo589pIvDSta1hdfwi0/BZzTGSnUaIDd8+GvaaYhHF74xvJRPQuWBGcZzTW3snUhuKaUKqO1vqKUKgNct/HxhR0ZtZFVJ1YxZ/8c8ql8TG0xlW5Vukkr4D5nI+MYuSaE/RE36VC7FNOer0vJQpkch//GKVj3DlzcBzW7QJc5pjt+hMD2hWAD8Coww/z3ehsfX9jJpbhLTAiawL6r+2hRrgWTmk2idMFMXsDMZYxGzdJd4Xz8+3GcHR2Y85IPPXzLZa5QGo2w90vYOhnyuZhaAd69QIqvSMNqhUAptRLTheESSqmLwERMBeBHpdTrQATworWOL7IHrTWrT67mk+BPUEoR0DyA56s+L62A+0RE3WHkmhD2nYumbQ0PZvSsR6nCmWwFRJ0xzfB1fhdU7wxd50IhKb7iQda8a+jlh7zUzlrHFNnLlbgrTNw1kd1XdtOkTBMmN59MWbey9o6VrRiNmu/2RjB903HyOShm9apHr4aemW8F/PMNbJkIDk7Q40vTPf9SfMVDyEAfIstprVl3eh0f//MxBm1gfNPx9K7eO0e3Aq7dTuTQE9y5YwmtNct3R7DrTBStqpVgZs96lC2aydm4bkaYZvgK3wlV2kG3z6FIuawJLHItKQQiS127c41JuycReCmQRqUbMbn5ZDwLedo7VoZprVn1zwWm/hpm0X38T6qgsyPTX/CmT6PymSuUWsP+b+GP8YCCrvOgwSvSChAWkUIgsoTWml/O/sKMvTNIMaYwpvEYXq75Mg7Kwd7RMuzKrQRGrw1lx8lImlV2Z0Sn6lk+gXvZIgUolt4DX08i5gJseA/ObodKT0H3+aaJXoSwkBQCkWk3Em4QsCuA7Re3U79kfaa2mEqFwjn3RKS1Zs3+i0zeeIxUg2Zy9zr0a1IRB4ds9ulaazj4f7B5LGgjPPcp+A2UVoB4YlIIRIZprfnt3G98tO8jElMTGeE3gn61+uHokLWfmm3p2u1Exv4Uytbj12nsVZxZvetR0T0bzp976xL8MhRObwGvVqZWQDEve6cSOZQUApEhUQlRTN0zlS3nt1CvRD2mtJxC5SKV7R0rw7TWrD90mYkbjpKYYmB8l9oMaO6VPVsBh1fCb2PAkGwa96fRIHDIuV1wwv6kEIgn9nv470zbM424lDjeb/g+r9Z+NUe3AiJjkxj3cyh/HLtGgwpFmd3bh8oe2XD+3Nir8Msw02Bx5ZtCjy/AvYq9U4lcQAqBsNjNxJt8tPcjNodvpo57Haa2mErVYlXtHStTNoZcZvy6I9xJNjD22Zq83rIyjtmxFRC6BjaNgNRE00xgTQZDDi6+InuRQiAssjViK5P3TOZ28m2G1B/CwLoDyeeQc//5RMUlMWH9UX4NvYKPZxFm9/ahWqlC9o71oLhI2DgMjm8Ez0amh8NKVLN3KpHL5Nz/ycImbiXdYvq+6fx69ldqFq/Jog6LqFG8hr1jZcrmI1cY9/MRbiemMLJTDd5qXZl8jtmwj/3oz/DrcNOcAe0DoNl7Mtm7sAr5VyUe6u8LfzNp9yRiEmN4x+cd3qj3Bk4OmZzw3I5u3klm4oajbDh8mbrlCrOidxNqli5s71gPuhMFm4abCkHZ+tBjIZSsae9UIheTQiAecDv5Nh/v+5j1Z9ZTrVg1FrRbQG332vaOlSl/HrvG2J9DuXknmffbV+edtlVwyo6tgLCNpq6ghBh4ejy0GCatAGF18i9M/EvgpUAm7ppIVEIUg7wHMdhnMM6OmXzy1Y5uxacQsPEoPx24RM3ShVg6oBF1yhaxd6wHxUfDb6Mh9EcoXQ/6r4PSde2dSuQRUggEAHHJccwOns3aU2upUqQKc9vOpW6JnH0i2nbiOmPWhnAjLpmhT1flvaer4ZwvG7YCTmw2PRwWHwVtxkKrD8Ax53bBiZxHCoFgz5U9TAiawLX4awysO5B3fN8hv2N+e8d6wPmoeG4npjx2Pa3huz0RrAq+QPVSbnzzSiO8PbNhKyAhBn4fC4dWQMk60Hc1lPGxdyqRB0khyMPiU+L5dP+nrDqxCq/CXizrvAzfkr72jvWAuKRUPtoUxvd7z1u8jYOCt9tUYVj7auTPlw3vtz+9BTYMNT0k1noktB4F+XJuF5zI2aQQ5FH/XP2H8UHjuRx3mVdqv8KQ+kNwyZfJGbGsYNfpG4xcE8KVWwkMalWJRl7FLdquUomC2fO5gMTb8Ic/HFgGHjXhpe+gXAN7pxJ5nBSCPCY+JZ55B+exImwF5QuVZ2nnpTQolf1ORHeSUpm5+TjLd0dQqURBVg9uTsOKxewdK3PObIMNQ+D2JdPdQG0+BKfsV3xF3iOFIA85eP0g/oH+nI89T99afRlafyiuTq72jvWAvWejGLkmhAs343m9ZSVGdKxBAeds2L1jqaQ4+HMCBC8G92ow8A8o38jeqYS4RwpBHpCYmsj8g/NZfmw5Zd3KsqTTEhqVzn4nooRkAx//fpxvg8Kp6O7Kqjeb0biSZV1B2da5nbD+HdPkMc3eg6f9wSmT01EKkcWkEORyIZEhjAscR/jtcF6q8RIfNPwgW7YCgsOjGbkmhHM37vBqs4qMfqYmrs45+J9n8h3YEgD7voLilWHgZqjQ1N6phEhXDv6fJh4l2ZDMgkMLWHp0KaVcS7GowyKalW1ms+NHxSVxJ8nw2PWMWrNibwTfBJ6jXNECfD+oCc2rlLBBQrM7N0xj+WSlm+dMYwRFnzWNEtpuAjhnw8lthDCTQpALHb1xlHGB4zhz6ww9q/VkhN8I3JxtM75+YoqBz7acYtGOMxi15dv1bVKBD5+thVt+G/2TTEmAv6bC7gXAEwS1VNGK8Nqv4NUy6/ctRBaTQpCLpBhSWBiykMWhi3Ev4M6X7b+kZTnbnYhCLsYw/MfDnLoex4t+njSp5G7RdpU9ClK/gg3vCLoYDOvehhsnocGrUCGLW0qOTlC9M+TPhpPbCJEOKQS5RFhUGP5B/py8eZLuVbozqvEoCjvbZmTNpFQDn289zZd/n8HDLT9LBzSiTY2SNjn2E0lNgu3TIWguFCoL/X+GKk/bO5UQdieFIIdLMabwTcg3LApZRFGXosx/ej5PlX/KZsc/cukWI1Yf5vjVWHo19GR8l9oUKZANx8m5dADWvQORYVC/P3SaBi7ZcNgJIexACkEOdvLmSfwD/QmLDqNL5S6MaTyGIvltc3JLMRhZsO008/86TbGCzix+1Y92tUrZ5NhPJDUZdnwMOz8Ft5LQdw1U62DvVEJkK3YpBEqpcCAWMACpWms/e+TIqVKNqXx75Fu+OPwFhZ0L81mbz2hXsZ3Njh925TYjVh/m6OXb9PAty6RudSjqmg3HybkSYroWcO0I+LwMnadDgRz+dLIQVmDPFkFbrfUNOx4/RzoTc4ZxgeM4GnWUzl6dGdtkLMVcHjy5aa2JiX/8SJ1PQgPf741g7tZTFCngxMJ+Delct3Qmd6oh4WaW5PvfPo3wz2JTS8DVHfqshJrPZu0xhMhFpGsohzAYDSw7towFBxfg6uTK7Kdm08mrU7rrnr4ey4jVIRy6EGOVLM/VK8OU7nUpXjCTrYDrYaZP7JcPZk2w+3n3hmc+Btcc/nSyEFZmr0KggT+UUhr4Smu96P4VlFJvAm8CVKhQwcbxspdzt87hH+RPSGQI7Sq0w7+pPyUKPPjQlcGoWRx4ltl/nKSgsyMjO9WgYBaP0VPJw42nqntkbieGVNj9OWz7CPIXgnYTs/6BqxLV5I4gISxkr0LQUmt9SSlVEvhTKXVca70j7Qrm4rAIwM/PzwpP/GR/BqOBFWErmHdwHvkd8zOz1UyeqfQMSqkH1j0bGceI1Yc5cD6GjrVLMe15bzwKZb/JZYg8aWoFXAqGWt3guU/BLZOFRQiRKXYpBFrrS+a/ryulfgYaAzsevVXecv72ecYHjefA9QO08WzDhGYT8HB98IRpNGq+3RXOx5uP4+LkyNw+vnTzKZtusbArowH2fAFbp4CzK/RaAnVegOyWU4g8yOaFQClVEHDQWseav+4ITLZ1juzKqI2sPL6Sz/Z/hpODE9NaTqNr5a7pntgjou4wcnUI+8KjaVezJNNf8KZk4Ww4vn3UGdM9/Bf2QI3noMscKJQNbzUVIo+yR4ugFPCz+cSWD/hea73ZDjmynQuxF5gQNIHga8G0LNeSSc0mUarggydMo1Hz3d4Ipm86Tj5HxezePvRsUC4btgKMsG8RbJlkmobx+UVQ70VpBQiRzdi8EGitzwIyQ3caWmtWn1zN7ODZOCgHJjefTI+qPdI9sV+IjmfUmhB2n42idXUPZvb0pkwRG45vb3z8iKIAxETA+iEQEQjVOkLXeVC4jHWzCSEyRG4ftbPLcZeZuGsie67soVmZZgQ0D6CM24MnTK013+87z0e/hqGUYsYL3rzUqLztWgE3I+CXoXB2u+Xb5C8M3b8A3/9IK0CIbEwKgZ1orfnp1E/MCp6FURsZ33Q8vav3TvfEfjkmgdFrQ9h56gYtqrozs2c9PIvZaHIZrWH/t/DHeECZ5tq15FZPh3ymbqAinlaPKITIHCkEdnD1zlUm7Z5E0KUgGpduTEDzADwLPXjC1FqzOvgiUzYew6A1U3vUpW+TCrZrBcRcME22fnYbVHoKus+Honn7mQ4hciMpBDaktWbDmQ3M3DeTVJ3Kh40/pE/NPjgohwfWvXorkQ9/CmHbiUiaVi7OrF4+lC9uw1bAwe/g97GmawLPfQp+A6V7R4hcSgqBjUTGRxKwO4C/L/5Ng5INmNJiChUKP/jpWmvNzwcvMWnDUZINRiZ1rc0rzbxwcLDRSfj2ZdgwFE7/CRVbmloBxSvZ5thCCLuQQmBlWms2ndvER3s/IsmQxKhGo+hbq2+6rYDrsYmM/ekIW8Ku4VexGLN7++BVwkZz3WoNh3+A30aDIdk0Rk+jQeDwYE4hRO4ihcCKbiTcYOqeqWw9vxUfDx+mtJhCpSIPfrrWWrPh8GUmbjhKQrIB/+dqMaBFJRxt1QqIvQYbh8GJTVC+KfT4Atyr2ObYQgi7k0JgJdP/XsXKM3MwqiT0zec4cKY1z+05AZx4YF2tISHFQP0KRZnd24cqHjaa61ZrOLIWNo0wTebecRo0fRscsnagOiFE9iaFIIudjb7GGxvHEqn34UxF2hUfSpGSj7+F0qtEQfo0qmC7VkBcJPz6PoT9AuX8oMeX4FHdNscWQmQrUgiy0Kyda1h+6hO0QwJNivZlwXMf4OKUDWfuOvoz/DockmKhfQA0HyKtACHyMCkEWeB8TCQDfxnHNeNunCjP9BZf0KlafXvHetCdKFM30NGfoGx9UyugZC17pxJC2JkUgkz6bNc6Fh//GO1wh4aF+/Bl1xG4OmXDeQDCNpouCCfEwNP+0OJ9cJRfvxBCCkGGXbwVzeu/+HPZsJN8uiyTm8yla61G9o71oPho0y2hoT9CaW/ovw5K17V3KiFENiKFIAMW7PmFr47NxOgQi0+hnizq8iEF82fDVsCJzaaB4uKj4Kkx0HoEODrZO5UQIpvJ1YVg7tpebLh1PMv2pwEjmihHByoaDEyLMeAT9QPM/yHLjpFltIa4q1CyDvznRyjra+9EQohsKlcXgirutWiVfDtL9hWfbCAyNpFUo6adixsfeFSlYKlsfqdN8Sqm5wLyZcPWihAi28jVhaBLmyl0yeQ+4pJSmb4pjB8Pn6eyR0Fm9/ahQYViWZJPCCGyg1xdCDJr15kbjFoTwqWYBP6/vfuP9aqu4zj+fHHlh6BL0DvmAgXJQWRKciVQayyXaW1ZSx1kTTYL3bLRisr1B5mrTKlms6LRYmirEJSKNSeytDR0BOi9/JCBlsup0f4AAAdvSURBVDAj4ofo7EZF6Ls/zof8crv3ey8X7j33fL6vx8a+537u+XzP530/u+fN+ZzvfZ9Pv2c8X7hyIsMGD/CrADOz4+RE0IlDh49w9yPbWfrUTsadOZwVN8+gZdyosodlZtYnnAg6WL/zIPNXtLHr5UPMuXQcX7pqIsOH+MdkZvnyGS75139eZ+Hq7SxZ+yJjRp7KsrnTmX7emWUPy8ysz2WdCO797fOsavtrj/Z95dBhDrQf5pPTz+W2qycxYmjWPxozs//J+mzXfPpQzh/ds5LOTYMGMeuSsVz2trP6eFRmZgNL1olg1rRzmDXND1s3M6vHzyE0M2twTgRmZg2ulEQg6SpJ2yW9IOm2MsZgZmaFfk8EkpqAHwBXA5OB2ZIm9/c4zMysUMYVwTTghYj4c0QcBpYB15QwDjMzo5xE8FbgpZqv/5LajiFprqQNkjbs37+/3wZnZtZoBuzN4ohYHBEtEdHS3Nxc9nDMzLJVRiLYDYyt+XpMajMzsxIoIvr3gNIpwA7gCooEsB74eERsrdNnP7Crl4c8CzjQy74DVW4x5RYP5BdTbvFAfjF1Fs+5EdHtkkq//2VxRByRdCuwGmgCltRLAqlPr9eGJG2IiJbe9h+Icospt3ggv5hyiwfyi+lE4imlxEREPAw8XMaxzczsWAP2ZrGZmfWPRkgEi8seQB/ILabc4oH8YsotHsgvpl7H0+83i83MbGBphCsCMzOrw4nAzKzBZZ0IcqtyKmmnpM2SWiVtKHs8vSFpiaR9krbUtI2StEbS8+l1ZJljPB5dxHO7pN1pnlolfbDMMR4vSWMlPS7pOUlbJc1L7ZWcpzrxVHaeJA2T9EdJbSmmr6X28ZLWpXPeA5KG9Oj9cr1HkKqc7gDeT1HPaD0wOyKeK3VgJ0DSTqAlIir7RzCS3gu0A/dHxAWp7W7gYER8KyXskRHx5TLH2VNdxHM70B4R3y5zbL0l6Wzg7Ih4RtLpwEbgI8AcKjhPdeK5norOkyQBIyKiXdJg4A/APODzwMqIWCbpR0BbRCzq7v1yviJwldMBKCKeAA52aL4GuC9t30fxS1oJXcRTaRGxJyKeSdt/B7ZRFIas5DzViaeyotCevhyc/gXwPuDB1N7jOco5EfSoymnFBPCopI2S5pY9mJNodETsSdt/A0aXOZiT5FZJm9LSUSWWUDojaRzwLmAdGcxTh3igwvMkqUlSK7APWAP8CXg1Io6kXXp8zss5EeTo8oi4mOKhPp9JyxJZiWKtsurrlYuACcAUYA/wnXKH0zuSTgMeAj4XEa/Vfq+K89RJPJWep4h4PSKmUBTunAZM6u175ZwIsqtyGhG70+s+4JcUk5+DvWkd9+h67r6Sx3NCImJv+iV9A/gxFZyntO78EPCziFiZmis7T53Fk8M8AUTEq8DjwAzgjFTYE47jnJdzIlgPnJ/uog8BZgGrSh5Tr0kakW50IWkEcCWwpX6vylgF3Ji2bwR+XeJYTtjRk2XyUSo2T+lG5E+AbRHx3ZpvVXKeuoqnyvMkqVnSGWn7VIoPxWyjSAjXpt16PEfZfmoIIH0c7B7erHL6jZKH1GuSzqO4CoCiWODPqxiPpF8AMylK5u4Fvgr8ClgOnENRbvz6iKjEDdgu4plJsdwQwE7g5pq19QFP0uXAk8Bm4I3U/BWKdfXKzVOdeGZT0XmSdCHFzeAmiv/QL4+IO9J5YhkwCngW+ERE/Lvb98s5EZiZWfdyXhoyM7MecCIwM2twTgRmZg3OicDMrME5EZiZNbhSnllsVgZJdwKPAm8B3h4Rd/bz8W8BDkXE/f15XLPu+OOj1jAkPQZ8CPgm8GBErO3HY59SUwPGbEDx0pBlT9JCSZuAS4CngU8BiyQt6GTf8ZKeTs99+Lqk9tQ+U9Jvavb7vqQ5aXuqpN+nYoCra8ow/E7SPSqeHTEv1b+fn743QdIjqc+Tkial9uskbUl15p/o25+MWcGJwLIXEV8EbgKWUiSDTRFxYUTc0cnu3wMWRcQ7KQqR1ZVq2NwLXBsRU4ElQO1ffA+JiJaI6FjQbDHw2dRnPvDD1L4A+EBEXAR8uKcxmp0I3yOwRnEx0EZRoXFbnf0uAz6Wtn8K3NXN+04ELgDWFCVtaOLYBPJAxw6pCualwIrUB2Boel0LLJW0HFjZsa9ZX3AisKxJmkJxJTAGOAAML5rVCsyIiH920q2zG2dHOPYKetjRQwBbI2JGF0P4Rydtgyjqxk/5vwNH3CLp3RT3MjZKmhoRL3fx3mYnhZeGLGsR0ZpOuDuAycBjFEsvU7pIAmspKtUC3FDTvguYLGloqvp4RWrfDjRLmgHFUpGkd3QzpteAFyVdl/pI0kVpe0JErIuIBcB+ji2lbtYnnAgse5KagVdS3flJ3Ty3eh7FQ382U/N0p4h4iaLy5pb0+mxqP0xR9vcuSW1AK8WyT3duAG5Kfbby5mNUF6Yb1VuApyiWs8z6lD8+alaHpPaIOK3scZj1JV8RmJk1OF8RmJk1OF8RmJk1OCcCM7MG50RgZtbgnAjMzBqcE4GZWYP7L1C/3upm304GAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(xx, random_experiment_data[\"history\"], label=\"Random\")\n", - "plt.plot(xx, uncertainty_experiment_data[\"history\"], label=\"Uncertainty\")\n", - "plt.plot(xx, as_experiment_data[\"history\"], label=\"AS\")\n", - "\n", - "plt.title(\"Number of targets found\")\n", - "plt.ylabel(\"# of targets\")\n", - "plt.xlabel(\"# queries\")\n", - "plt.legend()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "venv" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/example_live.ipynb b/example_live.ipynb deleted file mode 100644 index aa996b6..0000000 --- a/example_live.ipynb +++ /dev/null @@ -1,285 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Example of Human in the Loop\n", - "Shows how you can use a custom `oracle` that asks the user for labels." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Imports" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "from IPython.display import clear_output" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "from matplotlib import pyplot as plt\n", - "import numpy as np\n", - "\n", - "from sklearn.datasets import load_digits\n", - "from sklearn.neighbors import KNeighborsClassifier" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import active_learning as AL\n", - "import active_learning.query_strats as qs" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "%matplotlib inline" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Fetch data" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "digits = load_digits()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "X = digits.images.reshape(-1, 64)\n", - "y = (digits.target == 2).astype(int)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Model definition" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "base_clf = KNeighborsClassifier()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Interactive oracle" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "def oracle(problem, train_ixs, obs_labels, selected_ixs, **kwargs):\n", - " ix = selected_ixs[0] # we'll only use 1 point per query\n", - " points = problem['points']\n", - " model = problem['model']\n", - " clear_output(wait=True)\n", - " plt.clf()\n", - " plt.gray()\n", - " plt.matshow(points[ix].reshape(8, 8))\n", - " plt.show()\n", - " label = int(input(\"Is this a 2? \"))\n", - " return np.array([label])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Learning!" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAC9JJREFUeJzt3d+LXPUZx/HPp5sEjYYsRCtqxFgoARE6CRIqiqQJkVgl\nzUUvIliotKQXrRhaEO1N9R+Q9KIIS9QIxohGV4q01oAJIrTaJK415kfRsMEEdf3BGhVpSHx6MScl\nDal7drvf787s837BsDO7Z+Z5dpfPnHNmzpzHESEAuXxrphsAUB/BBxIi+EBCBB9IiOADCRF8IKGe\nCL7ttbYP237H9n2Faz1qe8z2/pJ1zqp3le1dtg/Yftv2PYXrXWD7ddtvNvUeLFmvqTlg+w3bL5Su\n1dQbtf2W7RHbewrXGrS9w/Yh2wdt31Cw1tLmdzpzOWF7U5FiETGjF0kDkt6V9B1J8yS9KenagvVu\nlrRc0v5Kv9/lkpY31xdI+mfh38+SLm6uz5X0mqTvF/4dfy3pSUkvVPqbjkq6pFKtxyX9vLk+T9Jg\npboDkj6QdHWJx++FNf4KSe9ExJGIOCnpKUk/KlUsIl6R9Gmpxz9PvfcjYl9z/XNJByVdWbBeRMQX\nzc25zaXYUVq2F0u6TdKWUjVmiu2F6q4oHpGkiDgZEeOVyq+W9G5EHC3x4L0Q/CslvXfW7WMqGIyZ\nZHuJpGXqroVL1hmwPSJpTNLOiChZb7OkeyV9XbDGuULSS7b32t5YsM41kj6S9FizK7PF9kUF651t\ng6TtpR68F4Kfgu2LJT0raVNEnChZKyJOR0RH0mJJK2xfV6KO7dsljUXE3hKP/w1uiojlkm6V9Evb\nNxeqM0fd3cKHI2KZpC8lFX0NSpJsz5O0TtIzpWr0QvCPS7rqrNuLm+/NGrbnqhv6bRHxXK26zWbp\nLklrC5W4UdI626Pq7qKtsv1EoVr/ERHHm69jkobV3V0s4ZikY2dtMe1Q94mgtFsl7YuID0sV6IXg\n/13Sd21f0zzTbZD0xxnuadrYtrr7iAcj4qEK9S61Pdhcv1DSGkmHStSKiPsjYnFELFH3//ZyRNxZ\notYZti+yveDMdUm3SCryDk1EfCDpPdtLm2+tlnSgRK1z3KGCm/lSd1NmRkXEKdu/kvQXdV/JfDQi\n3i5Vz/Z2SSslXWL7mKTfRcQjpeqpu1b8iaS3mv1uSfptRPypUL3LJT1ue0DdJ/anI6LK22yVXCZp\nuPt8qjmSnoyIFwvWu1vStmaldETSXQVrnXkyWyPpF0XrNG8dAEikFzb1AVRG8IGECD6QEMEHEiL4\nQEI9FfzCh1/OWC3qUa/X6vVU8CXV/ONW/UdSj3q9VK/Xgg+ggiIH8Nie1UcFzZs3b9L3OX36tAYG\nBqZUb8mSJZO+z2effaaFCxdOqd78+fMnfZ9PPvlEixYtmlK9w4cPT/o+p06d0pw5Uzvw9KuvvprS\n/fpFRHiiZWb8kN1+dMUVV1StNzQ0VLVep9OpWm/lypVV642MjEy80CzHpj6QEMEHEiL4QEIEH0iI\n4AMJEXwgIYIPJETwgYRaBb/miCsA5U0Y/OakjX9Q95S/10q6w/a1pRsDUE6bNX7VEVcAymsT/DQj\nroAspu1DOs2JA2p/ZhnAFLQJfqsRVxExJGlImv0fywX6XZtN/Vk94grIaMI1fu0RVwDKa7WP38x5\nKzXrDUBlHLkHJETwgYQIPpAQwQcSIvhAQgQfSIjgAwkRfCAhRmhNwfj4+Ey3UNTo6GjVelMZEfb/\nGBwcrFqvtjYjtFjjAwkRfCAhgg8kRPCBhAg+kBDBBxIi+EBCBB9IiOADCRF8IKE2I7QetT1me3+N\nhgCU12aNv1XS2sJ9AKhowuBHxCuSPq3QC4BK2McHEmJ2HpDQtAWf2XlA/2BTH0iozdt52yX9VdJS\n28ds/6x8WwBKajM0844ajQCoh019ICGCDyRE8IGECD6QEMEHEiL4QEIEH0iI4AMJTdux+jOp9uy1\n2rPz1q9fX7Ve7dl5zz//fNV6nU6nar2RkZGq9dpgjQ8kRPCBhAg+kBDBBxIi+EBCBB9IiOADCRF8\nICGCDyRE8IGE2pxs8yrbu2wfsP227XtqNAagnDbH6p+S9JuI2Gd7gaS9tndGxIHCvQEopM3svPcj\nYl9z/XNJByVdWboxAOVMah/f9hJJyyS9VqIZAHW0/liu7YslPStpU0ScOM/PmZ0H9IlWwbc9V93Q\nb4uI5863DLPzgP7R5lV9S3pE0sGIeKh8SwBKa7OPf6Okn0haZXukufywcF8ACmozO+9VSa7QC4BK\nOHIPSIjgAwkRfCAhgg8kRPCBhAg+kBDBBxIi+EBCzM6bgq1bt1at14uz16bT7t27q9arPYuwF/9/\nrPGBhAg+kBDBBxIi+EBCBB9IiOADCRF8ICGCDyRE8IGECD6QUJuz7F5g+3Xbbzaz8x6s0RiActoc\nq/8vSasi4ovm/Pqv2v5zRPytcG8ACmlzlt2Q9EVzc25zYWAG0Mda7ePbHrA9ImlM0s6IYHYe0Mda\nBT8iTkdER9JiSStsX3fuMrY32t5je890Nwlgek3qVf2IGJe0S9La8/xsKCKuj4jrp6s5AGW0eVX/\nUtuDzfULJa2RdKh0YwDKafOq/uWSHrc9oO4TxdMR8ULZtgCU1OZV/X9IWlahFwCVcOQekBDBBxIi\n+EBCBB9IiOADCRF8ICGCDyRE8IGEZsXsvE6nU7Ve7Vl9s93g4GDVerVn9fUi1vhAQgQfSIjgAwkR\nfCAhgg8kRPCBhAg+kBDBBxIi+EBCBB9IqHXwm6Eab9jmRJtAn5vMGv8eSQdLNQKgnrYjtBZLuk3S\nlrLtAKih7Rp/s6R7JX1dsBcAlbSZpHO7pLGI2DvBcszOA/pEmzX+jZLW2R6V9JSkVbafOHchZucB\n/WPC4EfE/RGxOCKWSNog6eWIuLN4ZwCK4X18IKFJnXorInZL2l2kEwDVsMYHEiL4QEIEH0iI4AMJ\nEXwgIYIPJETwgYQIPpCQI2L6H9Se/gf9BrN99toDDzxQtd7o6GjVerX/nrVnLdb+e0aEJ1qGNT6Q\nEMEHEiL4QEIEH0iI4AMJEXwgIYIPJETwgYQIPpAQwQcSanXOvebU2p9LOi3pFKfQBvrbZE62+YOI\n+LhYJwCqYVMfSKht8EPSS7b32t5YsiEA5bXd1L8pIo7b/raknbYPRcQrZy/QPCHwpAD0gVZr/Ig4\n3nwdkzQsacV5lmF2HtAn2kzLvcj2gjPXJd0iaX/pxgCU02ZT/zJJw7bPLP9kRLxYtCsARU0Y/Ig4\nIul7FXoBUAlv5wEJEXwgIYIPJETwgYQIPpAQwQcSIvhAQgQfSGgyn8fvWePj41Xrbd26tWq94eHh\nqvWOHj1atV5ttWfZ9SLW+EBCBB9IiOADCRF8ICGCDyRE8IGECD6QEMEHEiL4QEIEH0ioVfBtD9re\nYfuQ7YO2byjdGIBy2h6r/3tJL0bEj23PkzS/YE8ACpsw+LYXSrpZ0k8lKSJOSjpZti0AJbXZ1L9G\n0keSHrP9hu0tzWCN/2J7o+09tvdMe5cAplWb4M+RtFzSwxGxTNKXku47dyFGaAH9o03wj0k6FhGv\nNbd3qPtEAKBPTRj8iPhA0nu2lzbfWi3pQNGuABTV9lX9uyVta17RPyLprnItASitVfAjYkQS++7A\nLMGRe0BCBB9IiOADCRF8ICGCDyRE8IGECD6QEMEHEpoVs/Nq27x5c9V6nU6nar2VK1dWrbd+/fqq\n9cAaH0iJ4AMJEXwgIYIPJETwgYQIPpAQwQcSIvhAQgQfSGjC4NteanvkrMsJ25tqNAegjAkP2Y2I\nw5I6kmR7QNJxScOF+wJQ0GQ39VdLejcijpZoBkAdkw3+BknbSzQCoJ7WwW/Oqb9O0jP/4+fMzgP6\nxGQ+lnurpH0R8eH5fhgRQ5KGJMl2TENvAAqZzKb+HWIzH5gVWgW/GYu9RtJzZdsBUEPbEVpfSlpU\nuBcAlXDkHpAQwQcSIvhAQgQfSIjgAwkRfCAhgg8kRPCBhAg+kJAjpv/zNLY/kjSVz+xfIunjaW6n\nF2pRj3q16l0dEZdOtFCR4E+V7T0Rcf1sq0U96vVaPTb1gYQIPpBQrwV/aJbWoh71eqpeT+3jA6ij\n19b4ACog+EBCBB9IiOADCRF8IKF/A6FsnmDWZkuEAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Is this a 2? 0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 25/25 [00:44<00:00, 1.77s/it]\n" - ] - } - ], - "source": [ - "np.random.seed(seed=42)\n", - "interactive_exp_data = AL.utils.perform_experiment(\n", - " X, y,\n", - " base_estimator=base_clf,\n", - " init_L_size=5, n_queries=25, batch_size=1, \n", - " oracle=oracle,\n", - " shuffle=False\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 25/25 [00:00<00:00, 383.56it/s]\n" - ] - } - ], - "source": [ - "np.random.seed(seed=42)\n", - "random_exp_data = AL.utils.perform_experiment(\n", - " X, y,\n", - " base_estimator=base_clf,\n", - " init_L_size=5, n_queries=25, batch_size=1,\n", - " query_strat=qs.random_sampling,\n", - " shuffle=False\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Plots" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VeW1+PHvSkISMkJIQJIQEpEpAWQIyKQgOOBQrRNI\n1Up/Wutt1Wtb22tbK2qv1fZyqUMdSp1n0VZKr1pHqMqgBBFkJkACSRgykBEyr98f+yREBHKSnOSc\nnLM+z5MnJ3s6a+fAyrvf/e53iapijDEmcAR5OwBjjDFdyxK/McYEGEv8xhgTYCzxG2NMgLHEb4wx\nAcYSvzHGBBhL/MYYE2As8RtjTICxxG+MMQEmxNsBHCs+Pl5TU1O9HYYxxnQra9euLVLVBHe29bnE\nn5qaSlZWlrfDMMaYbkVEct3d1rp6jDEmwFjiN8aYAGOJ3xhjAozP9fEfT11dHXl5eVRXV3s7FL8V\nHh5OcnIyPXr08HYoxphO1i0Sf15eHtHR0aSmpiIi3g7H76gqxcXF5OXlkZaW5u1wjDGdrFt09VRX\nV9OnTx9L+p1EROjTp49dURkTILpF4gcs6Xcy+/0aEzi6RVePMcZ0VGOjsv1gBWtyDlFY3rarWxFh\nSL9oxqf2pm9MuMfjyi6s5IvdJYjANWcM9Ojxj8cSv5smT57MypUrT7rNQw89xE033URERESnxVFa\nWsorr7zCj3/8YwAKCgq47bbbePPNNzvtPY3pjmrqG9iQV8aanBLW7C5hbe4hyqvrm9e35SK3ZWny\ngX0iGJ8ax/jU3mSmxnFqfGSbrphr6xv5Ov9oXFm5hyg7UgfA2JReXZL4xdeKrWdmZuqxT+5u2bKF\n4cOHeyki9zU9dRwfH+/2Pg0NDQQHB7u9fU5ODhdffDEbN25sT4gn1V1+z8YcT9mROr7MPeQk1JwS\n1ueVUVvfCMBpfaOak/X41DiSe/dsU7Kua2hkc0F587HX5ByipKoWgD6RoWS6jjs+NY6MxBhCgo/2\noldU1/HlnlLW7Hb2/WpvKTWuuE6Nj2R8ahyZqb2ZkBZHSlxEu7tdRWStqma6s621+N0UFRVFZWUl\ny5cv55577iE+Pp6NGzcybtw4XnrpJR599FEKCgo4++yziY+PZ9myZbz//vvMnz+fmpoaBg0axLPP\nPktUVBSpqanMmTOHDz74gF/+8pdUVFSwaNEiamtrOe2003jxxReJiIjgwIED3HzzzezatQuAJ554\ngkceeYSdO3cyevRozj33XH7yk580/yGYOHEiTz/9NBkZGQBMnz6dBQsWMHz4cG699VY2btxIXV0d\n99xzD5deeqk3f53GfMvB8mr+vi6fqpr61jdu4dDhWrJyDrHtQAWqEBIkjEiKZd7kVDIHOq3yuMjQ\nDsXWIziI0wf04vQBvbjxzFNRVXYVVbmSufPH5r1NBwCICA1mTEovUuIi2ZBXypZ95TQqBAcJGYkx\nXDtxYPPVQnxUWIfiaq9ul/jv/ecmNheUe/SY6YkxzP9Ohtvbr1u3jk2bNpGYmMiUKVNYsWIFt912\nGwsXLmTZsmXEx8dTVFTEf//3f/Phhx8SGRnJH/7wBxYuXMjdd98NQJ8+ffjyyy8BKC4u5oc//CEA\nd911F08//TS33nort912G9OmTeOtt96ioaGByspKHnzwQTZu3MhXX30FOFcATebMmcPixYu59957\n2bdvH/v27SMzM5Nf//rXzJgxg2eeeYbS0lImTJjAOeecQ2RkpId+g8a0X25xFX/5ZBdvZuVR29BI\nUBsbvBGhIYxJ6cWFI/uTmdqbMQN60zPU/avo9hARBiVEMSghiqsnpABwoLyaNTklZOUc4ovdJWzY\nW8DI5FhunTGY8alxjEnpRWSYb6Rc34iim5kwYQLJyckAjB49mpycHKZOnfqNbVavXs3mzZuZMmUK\nALW1tUyaNKl5/Zw5c5pfb9y4kbvuuovS0lIqKys5//zzAfj444954YUXAAgODiY2NpZDhw6dMK7Z\ns2dz3nnnce+997J48WKuvPJKAN5//32WLl3KggULAGd47J49e6xbx3jVln3lPLF8J/+3oYCQoCCu\nGJfMj846ldT47tkg6RcTzsWjErl4VKK3Q2lVt0v8bWmZd5awsKOXZ8HBwdTXf/vSVFU599xzefXV\nV497jJat7Xnz5rFkyRJOP/10nnvuOZYvX96uuJKSkujTpw8bNmzg9ddf58knn2yO5W9/+xtDhw5t\n13GN8aQ1OSU8sXwnH289SGRoMDeeeSo3TE2jn4dHy5gT6zbj+LuD6OhoKioqAJg4cSIrVqwgOzsb\ngKqqKrZv337c/SoqKujfvz91dXW8/PLLzctnzpzJE088ATg3gcvKyr7xHsczZ84c/vjHP1JWVsao\nUaMAOP/883n00UdpupG/bt26jp+sMW2gqizbdpDZT67iqidXsW7PIX527hBW3jmTX1843JJ+F7PE\n70E33XQTs2bN4uyzzyYhIYHnnnuOuXPnMmrUKCZNmsTWrVuPu9/vfvc7zjjjDKZMmcKwYcOalz/8\n8MMsW7aMkSNHMm7cODZv3kyfPn2YMmUKI0aM4Be/+MW3jnXllVfy2muvMXv27OZlv/3tb6mrq2PU\nqFFkZGTw29/+1vMnb8xxNDQq/1xfwEWPfMYPnl1D3qHDzP9OOivunMFtMwcTG2FzQ3mDDec0zez3\nbDzp/U37+f07W8gpPsyghEhunjaIS0cnERpi7c3O4PHhnCIyC3gYCAaeUtUHj1k/EHgGSABKgGtV\nNc+17o/ARThXFx8A/6m+9tfGGONRB8urue21daTERfDktWM5L/0Ugto6XMd0mlb/9IpIMPAYcAGQ\nDswVkfRjNlsAvKCqo4D7gAdc+04GpgCjgBHAeGCax6I3xvikRz/Opr5BWXRdJrNG9Lek72Pcueaa\nAGSr6i5VrQVeA459+icd+Nj1elmL9QqEA6FAGNADONDRoI0xviu3uIpXv9jD1RMGdNuhmf7OncSf\nBOxt8XOea1lL64HLXa8vA6JFpI+qrsL5Q7DP9fWeqm459g1E5CYRyRKRrMLCwraegzHGh/zpg+2E\nBAu3zRjs7VDMCXjqLssdwDQRWYfTlZMPNIjIacBwIBnnj8UMETnz2J1VdZGqZqpqZkJCgodCMsZ0\ntS37yvnH+gJ+MCXN47NYGs9x5+ZuPjCgxc/JrmXNVLUAV4tfRKKAK1S1VER+CKxW1UrXuneBScCn\nHojdGONjFry3jeiwEG4+a5C3QzEn4U6Lfw0wWETSRCQUuBpY2nIDEYkXkaZj/QpnhA/AHpwrgRAR\n6YFzNfCtrp5AlJqaSlFRkbfDMMZjsnJK+GjrQW6ePsjG5/u4VhO/qtYDtwDv4STtxaq6SUTuE5FL\nXJtNB7aJyHagH3C/a/mbwE7ga5z7AOtV9Z+ePYWup6o0NjZ6OwxjfIaq8od/bSUhOowfTLa6zb7O\nrT5+VX1HVYeo6iBVvd+17G5VXep6/aaqDnZtc6Oq1riWN6jqj1R1uKqmq+rPOu9UOldOTg5Dhw7l\n+9//PiNGjOCGG24gMzOTjIwM5s+f37xdamoq8+fPZ+zYsYwcObL5ad3i4mLOO+88MjIyuPHGG5un\nTwBYuHAhI0aMYMSIETz00EPN7zds2DDmzZvHkCFDuOaaa/jwww+ZMmUKgwcP5osvvujaX4AxJ7F8\nWyFrcg5x28zBnT4zpum4bjdJG+/eCfu/9uwxTxkJFzzY6mY7duzg+eefZ+LEiZSUlBAXF0dDQwMz\nZ85kw4YNzXPjxMfH8+WXX/L444+zYMECnnrqKe69916mTp3K3Xffzdtvv83TTz8NwNq1a3n22Wf5\n/PPPUVXOOOMMpk2bRu/evcnOzuaNN97gmWeeYfz48bzyyit89tlnLF26lN///vcsWbLEs78HY9qh\nsVH543vbSImLYE7mgNZ3MF5nz063wcCBA5k4cSIAixcvZuzYsYwZM4ZNmzaxefPm5u0uv9wZ2Tpu\n3Ljm+fI/+eQTrr32WgAuuugievfuDcBnn33GZZddRmRkJFFRUVx++eV8+qlz7zstLY2RI0cSFBRE\nRkYGM2fOREQYOXLkN+bhN8ab/rmhgC37yvn5eUNsOoZuovu1+N1omXeWpqmUd+/ezYIFC1izZg29\ne/dm3rx5VFcfLd7cNG3ziaZsdlfL6Z+DgoKafw4KCurQcY3xlLqGRhZ+sJ1hp0TznW4wD71x2J/n\ndigvLycyMpLY2FgOHDjAu+++2+o+Z511Fq+88goA7777bnNBlTPPPJMlS5Zw+PBhqqqqeOuttzjz\nzG896mCMT3p9zV5yiw/zy1lDbVqGbqT7tfh9wOmnn86YMWMYNmwYAwYMaK6ydTLz589n7ty5ZGRk\nMHnyZFJSnHJtY8eOZd68eUyYMAGAG2+8kTFjxlhXjvF5R2obeOSjHWQO7M3ZQ/t6OxzTBjYts2lm\nv2fTFk8s38kf/rWVN26exPjUOG+HE/DaMi2zdfUYY9qs7HAdTyzP5uyhCZb0uyFL/MaYNvvLJzsp\nr67nF+cPa31j43O6TeL3tS4pf2O/X+Oug+XVPLsih0tOTyQ9Mcbb4Zh26BaJPzw8nOLiYktOnURV\nKS4uJjzcZlM0rXv042zqGhr52blDvB2KaaduMaonOTmZvLw8bK7+zhMeHk5ycrK3wzA+bk/xYV79\nYg9zxluRle6sWyT+Hj16kJZmEz8Z420LP9jmFFmZaUVWurNu0dVjjPG+piIr8yan0c+KrHRrlviN\nMW5pKrLyH9OsyEp31y26eowx3tHYqGQXVrJ820E+2nqQX5w/1Iqs+AFL/MaYZrX1jXydX8aanBKy\nckrIyj1E6eE6AIb3j+EHU1K9G6DxCEv8xgSwiuo61uYeIivnEF/klLB+byk19U51uVMTIjk//RTG\np8UxPrU3KXERiNhEbP7AEr8xAWZXYSUvrd7D6l3FbN1fTqNCcJAwIjGGaycOZHxqHJmpvYmPCmv9\nYKZbcivxi8gs4GEgGHhKVR88Zv1AnALrCUAJcK2q5rnWpQBPAQMABS5U1RxPnYAxxj0b88t4fHk2\n727cT4/gIMan9ubWGYOZkBbH6AG9iAyzdmCgaPWTFpFg4DHgXCAPWCMiS1V1c4vNFgAvqOrzIjID\neAC4zrXuBeB+Vf1ARKIAq1JuTBdRVVbvKuHx5dl8uqOI6PAQfjx9ED+YkmYt+gDmzp/4CUC2qu4C\nEJHXgEuBlok/HWgqpL4MWOLaNh0IUdUPAFS10kNxG2NOorFR+WjrQR5fns26PaXER4XxX7OGcc3E\nFGLCbVROoHMn8ScBe1v8nAecccw264HLcbqDLgOiRaQPMAQoFZG/A2nAh8CdqtrQcmcRuQm4CWgu\nUGKMabu6hkb+ub6AJ/+9k+0HKhkQ15PffXcEV41LJrxHsLfDMz7CU516dwB/FpF5wCdAPtDgOv6Z\nwBhgD/A6MA94uuXOqroIWAROIRYPxWRMwKiua2Bx1l4WfbKLvENHGNovmofmjObiUf0JCbbnNM03\nuZP483FuzDZJdi1rpqoFOC1+XP34V6hqqYjkAV+16CZaAkzkmMRvjGmf8uo6XlyVy7MrdlNUWcu4\ngb2595IMzh7a12rgmhNyJ/GvAQaLSBpOwr8a+F7LDUQkHihR1UbgVzgjfJr27SUiCapaCMwAvllX\n0RjTZoUVNTz92W5eXp1LRU0904Yk8OPpg5iQFmdj7U2rWk38qlovIrcA7+EM53xGVTeJyH1Alqou\nBaYDD4iI4nT1/MS1b4OI3AF8JM6/xrXAXzvnVIzxf3tLDvOXT3ayOCuPuoZGLhzZn/+YNogRSbHe\nDs10I92i2LoxgW7b/gqeWJ7NPzfsI0jgirHJ/GjaINJsTnzj0pZi6/bEhjE+bG3uIZ5Yns2HWw4S\nERrMDyancuOZp3JKrE2LbNrPEr8xPkZV+WRHEY8vy+bz3SX0iujB7ecM5vpJqfSODPV2eMYPWOI3\nxods2VfOHW+sZ1NBOafEhHPXRcOZOyHFplMwHmX/mozxIQ+8u5WC0iP84YqRfHdMEmEh9tCV8Tx7\nssMYH5FfeoRPdxTy/UmpzBmfYknfdBpL/Mb4iDez8gC4clyylyMx/s4SvzE+oLFReWPtXqYMimdA\nXIS3wzF+zhK/MT5g1a5i8g4d4apMa+2bzmc3d033pQrFO2HPSijOhozLIHGMt6Nql8VZe4kJD+H8\njFO8HYoJAJb4TffRUA/718Oe1bBnlfO9qtC1UmDFw3Dq2XDmzyD1TOgmc9aUHa7j3Y37uXr8AJs6\n2XQJS/zGd9VUQn6Wk+BzV0JeFtRVOet6p8Jp50DKREiZDNH9IOsZWPU4PP8dSMp0/gAMuQCCfLtH\nc+n6fGrrG5mdOaD1jY3xAEv8xvesegy+fhP2rQdtAAROGQFjroGUSU6yj0n89n5Tfwpn3AxfvQwr\nHoHXvgcJw2DK7TDySgj2zcpTr2ftJb1/jE20ZrqMJX7jWxob4P3fQtypMPV2pzU/YDyEu5kUe/SE\n8TfC2Hmw6e/w2Z9gyc2w7Pcw+VYYe52zjY/YVFDGxvxy7r0kw9uhmADi29fAJvBU7Hda+ZN+DDPv\nhsHnuJ/0WwoOgVGz4eYVMPd1iOkP7/4C/jQCPlkAR0o9H3s7vJGVR2hIEJeOPs4VjDGdxFr8xreU\nu4q7xXhoWGNQEAydBUPOd+4TfLYQPv6dcyN46IUQ0smTngWFwCkjnSuXhKHfuOFcXdfAW+vyOT/j\nFHpF2ORrputY4je+pWyv8z02ybPHFYHUKc7XvvXw2UOw+9+efY/jqTvi3HQG6Bnnuhk9CVIm8WFR\nAmVH6phjN3VNF7PEb3xLmavFH9uJDzL1Px2uerbzjt+SKpTscoaf5q5yvm97B4BzJYy/RwxmzJ5Z\nIJNhwAQIi+6auExAcyvxi8gs4GGc0otPqeqDx6wfiFNnNwEoAa5V1bwW62OAzcASVb3FQ7Ebf1Se\nD6HR7evX90Ui0GeQ8zXmWmdZxQGKt/ybfyz9Gxf1ykU+WwifLgAJcrqF+o1wXrfFiCtg0Nmej9/4\npVYTv4gEA48B5wJ5wBoRWaqqm1tstgB4QVWfF5EZwAPAdS3W/w6nFq8xJ1eW5/luHl8T3Y8Xy0fz\ncEMk591wNkQ0QN6ao88r7FzWtuPVVsGGxTDvbWcElDGtcKfFPwHIVtVdACLyGnApTgu+STrwM9fr\nZcCSphUiMg7oB/wLcKsepAlgZXmd283jAxoblTey8ph6WjzJvV0Tsg2a4Xy1R1UxPDUDXpsLN34E\nvQd6Lljjl9y5nkwC9rb4Oc+1rKX1wOWu15cB0SLSR0SCgP8F7uhooCZAlOdDjH+3+FfuLCa/9AhX\neeqmbmQf+N4bUF8Lr14N1eWeOa7xW54ax38HME1E1gHTgHygAfgx8E7L/v7jEZGbRCRLRLIKCwtP\ntqnxZ3XVztw7ft7ifz1rL7E9e3Beej/PHTRhCMx+Hgq3wZv/z5nXyJgTcCfx5wMtmybJrmXNVLVA\nVS9X1THAb1zLSoFJwC0ikoNzH+D7IvKNG8OubRepaqaqZiYkJLTvTEz3V94FI3q8rPRwLe9t2s93\nRyd6fkK2QWfDRf8L2R/Ae7/27LGNX3Gnj38NMFhE0nAS/tXA91puICLxQImqNgK/whnhg6pe02Kb\neUCmqt7pmdCN32l+eMt/u3r+8VWBMyHb+E4au5/5A2eK6lV/hvjBMOGHnfM+J3O45OgMqoeLu/79\nu7PeqTDtl53+Nq0mflWtF5FbgPdwhnM+o6qbROQ+IEtVlwLTgQdERHFG7/ykE2M2/qorxvB72eKs\nvYxIiiEjsROHq557n1On4N1fQu80Z9qLzqIKpXtciX6l871wq7MuOBQi+3ab6bF9QhdNJeLWOH5V\nfQd455hld7d4/SbwZivHeA54rs0RmsBR7roVdLyZN/3AxvwyNhWUc9+lnTwhW1AwXPEUPDML3pgH\nN7wP/dI9c+zGRji42VUPwVUToelKLSwGBpzhzJGUMgkSx0KPcM+8r/Eoe3LX+I6yPIiI96nZMz1p\ncdZeZ0K207ugKyssCr73Gvx1Jrw6B278GKLaef+soR42vQVfL4Y9n0NNmbM8ur+T4AdOdqai6Jvu\n/NExPs8Sv/EdZfl++/BWdV0DS9blMyvjFGIjuqguQGwyzH0Vnr3QqU1w/T/b1gKvq3ZqG6x8BA7l\nOP3PGd89muh7DbRunG7KEr/xHeX5Tp+0H3pv037Kq+uZ01k3dU8kaSxcvggWXwf/+InTBdRasq4u\nh6ynnWpmVQchaRycd78zm6mPVzMz7rHEb3xHWZ5TK9cPvZGVR3Lvnkw6tU/Xv3n6JTBzPnx0rzPS\nZ/oJBtZVFcHqJ+CLvzrdOadOh6lPQdpZ1rL3M5b4jW+oLoeacr/s6tlbcpjPsov46TlDCAryUgKd\n+lMo2gHLH4A+pzmlKJuU7oGVf4YvX4D6ahj+HWf7pLHeidV0Okv8xjf48Rj+N9fmIQJXZnpxmKoI\nfOdhKM2FJT+G2AHODKgrHoKv33C2GXU1TPlP5ylg49cs8RvfUOYayulnY/gbGpU31zoTsiX18vJo\npZBQmPMSPDUTXvwu1B2GHhEw/ocw+Ra/+92bE7PEb3yDnyb+FdlF5Jce4VcXDvN2KI6IOPjeYljy\nH3Dq2XDGzc4kbyagWOI3vqE83yk+EnWKtyPxqMVZe+kV0YNzPTkhW0fFD4YbP/R2FMaLLPEb31CW\n7zwQFOy7/ySf+nQXG/PL2rTP+5sO8L0zUggLsQebjO/w3f9lJrCU7fXpbp6coiruf2cLfSLDiAxz\nP4mnxUdy3SQrjGJ8iyV+4xvK86H/aG9HcUIvrs4lWIR3bptK3xibf8Z0b/YYnvE+VZ+erqGqpp7F\nWXu5YGR/S/rGL1jiN95XVQQNNc7Ych+05Kt8KqrrmTfZumyMf7DEb7yveTpm32vxqyovrMwlIzGG\nsSm9vR2OMR5hid94X3MBFt9L/Kt3lbDtQAXXT0pFbL4a4ycs8Rvva56uwfdG9bywKodeET24ZLR/\nFocxgckSv/G+sr0QHAaR8d6O5BsKSo/w/uYDzBk/wPOF0Y3xIrcSv4jMEpFtIpItIt+a01VEBorI\nRyKyQUSWi0iya/loEVklIptc6+Z4+gSMH2ga0eNjXSkvf56LqnLtGXZT1/iXVhO/iAQDjwEXAOnA\nXBE5toDnAuAFVR0F3Ac84Fp+GPi+qmYAs4CHRKSXp4I3fqI83+du7FbXNfDqF3uZObwfA+IivB2O\nMR7lTot/ApCtqrtUtRZ4Dbj0mG3SgY9dr5c1rVfV7aq6w/W6ADgItLPwp/FbZXk+N5Tz7Q37KKmq\n5fpJqd4OxRiPcyfxJwF7W/yc51rW0nrgctfry4BoEfnGlH8iMgEIBXa2L1TjlxrqoWKfz43oeWFV\nDoMSIplyms1cafyPp27u3gFME5F1wDQgH2hoWiki/YEXgR+oauOxO4vITSKSJSJZhYWFHgrJdAuV\n+0EbfaqrZ92eQ6zPK+P6yTaE0/gndxJ/PtDyOjzZtayZqhao6uWqOgb4jWtZKYCIxABvA79R1dXH\newNVXaSqmaqamZBgPUEBxQfn4X9hVS5RYSFcPtZ3YjLGk9xJ/GuAwSKSJiKhwNXA0pYbiEi8iDQd\n61fAM67locBbODd+3/Rc2MZv+FjiL6yo4e0N+7hyXDJRYTaHofFPrSZ+Va0HbgHeA7YAi1V1k4jc\nJyKXuDabDmwTke1AP+B+1/LZwFnAPBH5yvXlu1Mwmq7nY7V2X/tiD7UNjTaVsvFrbjVpVPUd4J1j\nlt3d4vWbwLda9Kr6EvBSB2M0/qwsH8JiIDzG25FQ19DIy5/v4czB8QxKiPJ2OMZ0Gnty13hXWZ7P\ndPO8v+kA+8urbQin8XuW+I13lef5TDfP86tyGBDXk7OH9fV2KMZ0Kkv8xrt8pADLln3lfLG7hOsm\nDiQ4yIZwGv9mid94T90ROFzkE109L6zKIbxHELMzfesJYmM6gyV+4z3lBc53L0/HXHa4jrfW5fPd\n0Un0igj1aizGdAVL/MZ7msfwe7erZ3HWXqrrGvm+3dQ1AcISv/EeHxjD39CovLg6lwmpcaQnen9I\nqTFdwRK/8Z4y79faXb7tIHtKDvN9K6RuAoglfuM9ZXkQmQA9wr0WwvOrcukXE8b5Gad4LQZjupol\nfuM9Xi7Asquwkk+2F3LNGQPpEWz/FUzgsH/txnu8/NTuC6ty6REsXD3BhnCawGKJ33hPWb7XEn9l\nTT1/W5vHRSP70zfae11NxniDJX7jHdVlUFvhta6el1fnUlFTz/cnp3rl/Y3xJkv8xju8OA//y5/n\n8uC/tjJtSAJjBvTq8vc3xtus0oTxjjLXGP4uTvx/+fdOHnh3KzOG9eXxa8ZaaUUTkPwm8ZeXFrNr\n0TXeDsNjQoODGNIvipAgH7soC4+BixZCWAfnqy/v2jH8qsrCD7bz6MfZXDSqP3+aPZrQEB/73RrT\nRfwm8WtjI1E1B70dhsdU1zWwv64Hyb16ejuUoxpqoXArDP+O89URZfkgwRDd+ePnGxuV+/5vM8+t\nzOHq8QO4/7KRNgOnCWh+k/hj4xKI/e2X3g7DYxZ+sJ1HPtrBny8cw8WjEr0djqO+Bh4YAHtWeyDx\n50FMIgQFeya2E2hoVO782wbeWJvHDVPTuOui4da9YwKeW9e6IjJLRLaJSLaI3Hmc9QNF5CMR2SAi\ny0UkucW660Vkh+vrek8G789unXEapw/oxW/e2si+siPeDscREgZJ42DPqo4fqwse3qqtb+S2V9fx\nxto8bj9nsCV9Y1xaTfwiEgw8BlwApANzRST9mM0WAC+o6ijgPuAB175xwHzgDGACMF9EensufP/V\nIziIh+eMpq6hkZ8vXk9jo3o7JMfASbBvPdRWdew4ZXmdOivnkdoGbnoxi7e/3sddFw3n9nOGWNI3\nxsWdFv8EIFtVd6lqLfAacOkx26QDH7teL2ux/nzgA1UtUdVDwAfArI6HHRhS4yOZ/510Vu4s5unP\ndns7HEfKJGish7ys9h+jsdFp8XfSiJ6K6jquf/YL/r29kAcuH8mNZ57aKe9jTHflTuJPAva2+DnP\ntayl9cAt6W+9AAAUE0lEQVTlrteXAdEi0sfNfc1JzM4cwHnp/fif97axuaDc2+FA8nhAnH7+9jpc\n5Nwo7oQCLIeqarnmqc/5MvcQD189hrkTUjz+HsZ0d54az3YHME1E1gHTgHygwd2dReQmEckSkazC\nwkIPheQfRIQHrxhFr4ge3P76Oqrr3P61do6evaBfRsf6+TupAMvB8mrmLFrF1v0V/OW6cVxyuo/c\nFDfGx7iT+POBlrNYJbuWNVPVAlW9XFXHAL9xLSt1Z1/XtotUNVNVMxMSEtp4Cv4vLjKU/7nqdLYf\nqOTBd7d6OxynuydvDTTUt2//TijAsrfkMFf9ZRV5h47w3A/GM3N4P48d2xh/407iXwMMFpE0EQkF\nrgaWttxAROJFpOlYvwKecb1+DzhPRHq7buqe51pm2mjakATmTU7luZU5/Hu7l6+KUiZCbSUc2Ni+\n/Ztb/B2bFbOqpp7PdhTxpw+2c9WTqzhUVctLN57B5EHxHTquMf6u1XH8qlovIrfgJOxg4BlV3SQi\n9wFZqroUmA48ICIKfAL8xLVviYj8DuePB8B9qlrSCecREO68YBgrdxZxxxvree/2s4iL9FJh8JRJ\nzvc9qyBxdNv3L8uDkHCIiGvTboUVNazNLeGL3YfIyi1hU0E5DY1KkMDIpFgeuHy8lU80xg2i6iPD\nBF0yMzM1K6sDI0b83OaCcr772AqmD03gL9eN894QxT+NhKQxMPuFtu/7xjzYtwFuO/EDd6pKbvFh\n1uSUsCanhKycQ+wqcoaQhoUEMXpAL8anxjE+LY4xKb2ICe/RzhMxxj+IyFpVzXRnW795cjdQpCfG\n8Ivzh3L/O1tYnLWXOeO9NGpl4CTYtRxUoa1/fE5SgEVV+f07W1jyVQGFFTUAxPbswfjU3sweP4Dx\nqXGMSIohLKRzn/g1xp9Z4u+GbpiaxvLtB7n3n5uZkNaHtPjIrg8iZSJseB0O7Ya4No6TL8uHQWcf\nd9WybQf566e7mTGsLzOG9WVCWhynJUQRZHPrGOMxNj1hNxQUJCy46nR6BAdx++tfUdfQ2PVBNPXz\n57ZxWGdDPVTuP+6InsZG5Y//2sbAPhH85bpxXDtxIEP6RVvSN8bDLPF3U/1je/L7y0ayfm8pj36c\n3fUBxA+F8F5tH89fsQ+08bhdPf/cUMDW/RX87NwhVvzcmE5k/7u6sYtG9eeKscn8+eMdrM3t4sFS\nQUFOd09bn+A9wcNbtfWN/O/72xneP4bv+MpspMb4KUv83dw9l6ST1Lsnt7/+FZU17Xygqr1SJkHx\nDqgqcn+f5oe3vtnifz1rL3tKDvPL84da144xncxu7nZz0eE9+NPs0cz+yyqueHwlfWPC3N43SIRb\nZ5xGZmrbxtM3ax7PvxqGX+zePsdp8R+ureeRj3YwPrU304fak9vGdDZr8fuBzNQ47r9sJBFhwVTW\n1Lv9tXpXMYuz9rb+BieSOBqCw9rWz1+WB+GxEBbdvOi5lTkUVtTwy1nDbOpkY7qAtfj9xNwJKW2e\nifK6pz9nY34HZvxsT2GW8vxvdPOUHa7jyeU7mTmsL+Pbe+VhjGkTa/EHsIzEWHYcrKC2vgPDQdta\nmOWYAixPfrKTipp67jh/aPtjMMa0iSX+AJaRGENdg7L9QEX7D9JUmCV/rXvbt3hq92B5Nc+u2M2l\npycyvL/NsWNMV7HEH8BGJMUCdKzAS1sKs9QehiMlzQ9vPfLxDuoblJ+eO6T972+MaTNL/AFsYFwE\nUWEhbCwoa/9Bmgqz5K5sfdvyAud7bDK5xVW89sVe5k5IYWAfL0w5YUwAs8QfwIKChOH9o9nU0ZKO\nKRPdK8xS7hrKGZPEwg+2ExLsDCc1xnQtS/wBLiMxli37nHnt2y1lknuFWVxj+HfU9GLp+gL+35Q0\n+saEt/99jTHtYok/wGUkxnC4toGcYjdH5RxPywe5TqbMeWr3f1aVEx0Wwo/OGtT+9zTGtJsl/gCX\nkejc4N2Y34F+/tgkiE2BPa3085fnURcez/vbSvmP6acRG2HFU4zxBkv8AW5wvyhCg4M6NrIHjk7Y\ndpKKblqWR25DHH2jw5g3ObVj72eMaTe3Er+IzBKRbSKSLSJ3Hmd9iogsE5F1IrJBRC50Le8hIs+L\nyNciskVEfuXpEzAd0yM4iCGnRHX8Bu/ASVB5wCnMcgKHC3PZUR3LbTMH0zPUKmgZ4y2tJn4RCQYe\nAy4A0oG5IpJ+zGZ3AYtVdQxwNfC4a/lVQJiqjgTGAT8SkVTPhG48ZURiLJsKyuhQ/eVW+vkbGxqR\n8gIqw/oxZ/yA9r+PMabD3GnxTwCyVXWXqtYCrwGXHrONAk2PXsYCBS2WR4pICNATqAU62LQ0npaR\nGMOhw3UUlFW3/yCtFGZ5d+02IjjC8GHpVmTFGC9z539gEtByCsc817KW7gGuFZE84B3gVtfyN4Eq\nYB+wB1igql1cMcS0Jt11g3dTR27wNhVmOU4pxrqGRt5c5lwJpA879mLRGNPVPNX0mgs8p6rJwIXA\niyIShHO10AAkAmnAz0XkW5W5ReQmEckSkazCwkIPhWTcNbx/NCJ44EGu4xdmeX3N3uahnEHHKblo\njOla7iT+fKBlp2yya1lLNwCLAVR1FRAOxAPfA/6lqnWqehBYAWQe+waqukhVM1U1MyHBCnF0tYjQ\nEAYleOAG73H6+Y/UNvDIRzuYFH/EWRD77SLrxpiu5U7iXwMMFpE0EQnFuXm79Jht9gAzAURkOE7i\nL3Qtn+FaHglMBLZ6JnTjSRmJMWzqyJw9cNzCLC+uzuFgRQ0XD2yEoBCI6tfBSI0xHdVq4lfVeuAW\n4D1gC87onU0icp+IXOLa7OfAD0VkPfAqME+dISKPAVEisgnnD8izqrqhM07EdExGYgz7yqopqapt\n/0GaC7McbfG/vWEfY1N6kSjFEJ0IQTaM0xhvc6sCl6q+g3PTtuWyu1u83gxMOc5+lThDOo2Pa3qC\nd1NBGWcO7kB3W8pEWPkI1FZRVh/Khvwy/nPmYNibb908xvgIG1dnAKfFD3SsFCPAwMnNhVlW7SpG\nFaacFg9le5sLsBhjvMtq7hoAekWEktSrZ8f7+VsUZllxKI7I0GBGJ8c4c/HHWIvfGF9gLX7TLCMx\npuNz9jQVZtmzihXZRZxxah96HCmGxjpr8RvjIyzxm2YjkmLZXVxFZU0rBVVakzKRxj2fk1tU7nTz\ntCjAYozxPkv8pllGYgyqsGVfx8fzB9VVMUz2MOW0Ps0FWKzFb4xvsMRvmmV4YuoGaH6Qa3rPbIb2\ni25+atcSvzG+wW7ummb9YsLoExna4Sd4NSaRfSRwTuRuRATK8yGkJ/Ts7aFIjTEdYS1+00xEyEiK\n7XDi336gks8bhjCsdpNTmKVpKKeIhyI1xnSEJX7zDRmJMWw/UEFNfUO7j7Eiu4isxqH0rCl0CrOU\n2cNbxvgSS/zmGzISY6hvVHYcqGz3MVZkF1EQM8b5Yc9qp6snxvr3jfEVlvjNN7ScuqE96hoaWb2r\nmOQhpzuFWXZ/ChX77cauMT7Ebu6abxgYF0FUWEi7+/nX7y2lqraBKYP7wpGJsPVtQK2rxxgfYi1+\n8w1BQUJ6/xg2tnNI54rsYkRg4ql9nAnbalzHsYe3jPEZlvjNt6QnxrBlXwUNjW0vvr4iu4iRSbH0\nigiFlMlHV1hXjzE+wxK/+ZaMxBiO1DWwu6iqTftV1dTz5Z5DzjQNcLQwC1iL3xgfYonffMuIpPbd\n4P1idwn1jcrUpsTfVJglvBeERXk6TGNMO1niN99yWt8oQkOC2nyD97PsIkJDghg3sMUTulN/CtP+\ny8MRGmM6wkb1mG/pERzE0H7RbW7xr8guYnxqb8J7tCivOOQ84DzPBmiM6RC3WvwiMktEtolItojc\neZz1KSKyTETWicgGEbmwxbpRIrJKRDaJyNciEu7JEzCdwym+Xo5TOrl1hRU1bN1fcbR/3xjjs1pN\n/CISjFM0/QIgHZgrIunHbHYXThH2McDVwOOufUOAl4CbVTUDmA7UeSx602kykmIpPVxHfukRt7Zf\nubMI4Gj/vjHGZ7nT4p8AZKvqLlWtBV4DLj1mGwViXK9jgQLX6/OADaq6HkBVi1W1/ZPAmC7TVIPX\n3X7+FdlFxISHND/5a4zxXe4k/iRgb4uf81zLWroHuFZE8oB3gFtdy4cAKiLviciXIvLLDsZrusjw\nU2IIEvcSv6qyIruYyYPiCQ6yGTiN8XWeGtUzF3hOVZOBC4EXRSQI5+bxVOAa1/fLRGTmsTuLyE0i\nkiUiWYWFhR4KyXREz9BgBiVEsdmNG7y5xYfJLz3ClMHWzWNMd+BO4s8HBrT4Odm1rKUbgMUAqroK\nCAfica4OPlHVIlU9jHM1MPbYN1DVRaqaqaqZCQkJbT8L0ykyEmPYmN96i/+zbOvfN6Y7cSfxrwEG\ni0iaiITi3Lxdesw2e4CZACIyHCfxFwLvASNFJMJ1o3casNlTwZvOlZEYy/7yaoora0663YrsIhJj\nw0ntE9FFkRljOqLVxK+q9cAtOEl8C87onU0icp+IXOLa7OfAD0VkPfAqME8dh4CFOH88vgK+VNW3\nO+NEjOe5c4O3oVFZtauYKafFO2UWjTE+z60HuFT1HZxumpbL7m7xejMw5QT7voQzpNN0M0fn5i/n\nrCHH74LbXFBO6eE6plr/vjHdhk3ZYE4oNqIHyb17svEkN3ib+vcnD7LEb0x3YYnfnFRGYgybT9LV\nsyK7iKH9okmIDuvCqIwxHWGJ35xURmIsu4uqqKyp/9a66roG1uSU2DQNxnQzlvjNSY1Icm7wbtn3\n7Vb/l7mHqKlvZOrgPl0dljGmAyzxm5NqusF7vFKMn2UXERIkTEizxG9Md2KJ35xU3+gw4qNCjzuk\nc0V2EaMH9CIqzGb3NqY7scRvTkpESE+M/VbiLztcx4b8MuvfN6YbssRvWjUiMYYdByqoqT86seqq\nXcWoYuP3jemGLPGbVmUkxlLfqGzfX9m8bEV2EZGhwYwe0MuLkRlj2sMSv2nV0akbjt7gXZFdxIS0\nOHoE2z8hY7ob+19rWpUSF0F0WEhzP39B6RF2FVVZ/74x3ZQlftOqoCBheGJM89QNK5qmYbb+fWO6\nJUv8xi0ZiTFs3VdBQ6OyIruI+KhQhvaL9nZYxph2sMRv3JKRGMuRugZ2FVbymavMok3DbEz3ZInf\nuKVp6oa31uVTVFlj1baM6cYs8Ru3DEqIIjQkiJdW5wJYfV1jujFL/MYtPYKDGHZKNOXV9aTFR5LU\nq6e3QzLGtJMlfuO2pvH8kwfZpGzGdGduJX4RmSUi20QkW0TuPM76FBFZJiLrRGSDiFx4nPWVInKH\npwI3Xa9ppk7r3zeme2t1WkURCQYeA84F8oA1IrLUVWe3yV04RdifEJF0nPq8qS3WLwTe9VjUxisu\nHNmfnKIqzh7W19uhGGM6wJ35dCcA2aq6C0BEXgMuBVomfgViXK9jgYKmFSLyXWA3UOWJgI33xEWG\nctfF6d4OwxjTQe509SQBe1v8nOda1tI9wLUikofT2r8VQESigP8C7u1wpMYYYzzCUzd35wLPqWoy\ncCHwoogE4fxB+JOqVp5sZxG5SUSyRCSrsLDQQyEZY4w5Hne6evKBAS1+TnYta+kGYBaAqq4SkXAg\nHjgDuFJE/gj0AhpFpFpV/9xyZ1VdBCwCyMzM1PaciDHGGPe4k/jXAINFJA0n4V8NfO+YbfYAM4Hn\nRGQ4EA4UquqZTRuIyD1A5bFJ3xhjTNdqtatHVeuBW4D3gC04o3c2ich9InKJa7OfAz8UkfXAq8A8\nVbWWuzHG+CDxtfycmZmpWVlZ3g7DGGO6FRFZq6qZ7mxrT+4aY0yAscRvjDEBxue6ekSkEMjtwCHi\ngSIPhdPd2LkHrkA+/0A+dzh6/gNVNcGdHXwu8XeUiGS528/lb+zcA/PcIbDPP5DPHdp3/tbVY4wx\nAcYSvzHGBBh/TPyLvB2AF9m5B65APv9APndox/n7XR+/McaYk/PHFr8xxpiT8JvE31qVMH8nIjki\n8rWIfCUifv3os4g8IyIHRWRji2VxIvKBiOxwfe/tzRg70wnO/x4RyXd9/l8dWwXPX4jIAFe1v80i\nsklE/tO13O8//5Oce5s/e7/o6nFVCdtOiyphwNxjqoT5NRHJATJV1e/HM4vIWUAl8IKqjnAt+yNQ\noqoPuv7w91bV//JmnJ3lBOd/D84kiAu8GVtnE5H+QH9V/VJEooG1wHeBefj553+Sc59NGz97f2nx\nN1cJU9VaoKlKmPFDqvoJUHLM4kuB512vn8f5D+GXTnD+AUFV96nql67XFTgTRyYRAJ//Sc69zfwl\n8btTJczfKfC+iKwVkZu8HYwX9FPVfa7X+4F+3gzGS24RkQ2uriC/6+o4loikAmOAzwmwz/+Yc4c2\nfvb+kvgNTFXVscAFwE9c3QEByTUlePfvw2ybJ4BBwGhgH/C/3g2nc7nKuv4NuF1Vy1uu8/fP/zjn\n3ubP3l8SvztVwvyaqua7vh8E3sLp/gokB1x9oE19oQe9HE+XUtUDqtqgqo3AX/Hjz19EeuAkvpdV\n9e+uxQHx+R/v3Nvz2ftL4m+uEiYioThVwpZ6OaYuIyKRrps9iEgkcB6w8eR7+Z2lwPWu19cD//Bi\nLF2uKem5XIaffv4iIsDTwBZVXdhild9//ic69/Z89n4xqgfANYTpISAYeEZV7/dySF1GRE7FaeWD\nU07zFX8+fxF5FZiOMyvhAWA+sARYDKTgzO46W1X98gboCc5/Os6lvgI5wI9a9Hn7DRGZCnwKfA00\nuhb/Gqev268//5Oc+1za+Nn7TeI3xhjjHn/p6jHGGOMmS/zGGBNgLPEbY0yAscRvjDEBxhK/McYE\nGEv8xhgTYCzxG2NMgLHEb4wxAeb/AxCF2XisH9voAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(interactive_exp_data['accuracy'], label='interactive')\n", - "plt.plot(random_exp_data['accuracy'], label='random')\n", - "plt.legend()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/classification.ipynb b/examples/classification.ipynb new file mode 100644 index 0000000..a61ac22 --- /dev/null +++ b/examples/classification.ipynb @@ -0,0 +1,572 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Classification Query Strategies\n", + "This notebooks demonstrates the query strategies for classification problems built into `active_learning`" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import warnings\n", + "warnings.filterwarnings(action='ignore', category=RuntimeWarning)\n", + "\n", + "from matplotlib import pyplot as plt\n", + "import pandas as pd\n", + "import numpy as np\n", + "\n", + "from sklearn.datasets import make_moons\n", + "from sklearn.gaussian_process import GaussianProcessClassifier\n", + "from sklearn.gaussian_process.kernels import RBF\n", + "\n", + "from active_learning.problem import ActiveLearningProblem\n", + "from active_learning.query_strats.random_sampling import RandomQuery\n", + "from active_learning.query_strats.classification import (GreedySearch, UncertaintySampling,\n", + " ActiveSearch, SequentialSimulatedBatchSearch)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Set the random seed for the experiment" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "np.random.seed(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Make some Toy Data\n", + "Have a little binary classification task that is not linearly separable." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "X, y = make_moons(noise=0.1, n_samples=200)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Turn it into a DataFrame" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "data = dict(zip(['x1', 'x2'], X.T.tolist()))\n", + "data['y'] = y\n", + "data = pd.DataFrame(data)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Pretend like most of the data is unlabeled" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "data['is_labeled'] = False\n", + "data.loc[175:, 'is_labeled'] = True" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plot it all" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "xlim = [-1.25, 2.2]\n", + "ylim = [-0.8, 1.3]" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(-0.8, 1.3)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsvXmUbFld5/vZJ+Ypx4ic8+Zwh7qA\n2Ay3oYEGBaGq1Ncggl1YNl0ovAKFbkHpJchbYIMNhdoNorRSIiKKAgoo+oCiEEqkobBu+ZBiqDvk\nPEROEZkRkTFHnP3+iDznxnBO5BQ5789ad93MM0TsjGH/9v4N35+QUqJQKBQKhYF21ANQKBQKxfFC\nGQaFQqFQ1KAMg0KhUChqUIZBoVAoFDUow6BQKBSKGpRhUCgUCkUNyjAoFAqFogZlGBQKhUJRgzIM\nCoVCoajBedQD2AvhcFiOjo4e9TAUCoXiRPHoo4+uSSkj2113Ig3D6OgoV69ePephKBQKxYlCCDGz\nk+ta4koSQnxECLEihPiuzfmfE0J8Z+vfN4QQ/6bq3LQQ4jEhxLeFEGq2VygUiiOmVTGGjwJ3Njk/\nBfyIlPKHgXcB99edf76U8ilSyistGo9CoVAo9khLXElSyq8JIUabnP9G1a8PA0OteF6FQqFQtJ6j\nyEp6NfCFqt8l8CUhxKNCiHuPYDwKhUKhqOJQg89CiOdTMQz/vurwc6SUi0KIHuBBIcTjUsqvWdx7\nL3AvwLlz5w5lvAqFQnEWObQdgxDih4EPAy+RUsaM41LKxa3/V4DPAs+wul9Keb+U8oqU8koksm22\nlUKhUCj2yKEYBiHEOeAzwCullNerjgeEECHjZ+B2wDKzSaFQKBSHQ0tcSUKIvwR+FAgLIeaBdwAu\nACnlHwJvB7qB/y2EAChtZSD1Ap/dOuYE/kJK+cVWjEmhUCgUe6NVWUk/u8351wCvsTg+CfybxjsU\nCoVCcVScyMpnxdklm82STCYRQtDe3o7H4znqISkUpw5lGBQnhmg0SjweR0oJwOrqKn19fXR3dx/x\nyBSK04UyDIpDo1wus76+TiqVwuVy0d3djc/n29G9mUymxigASClZWlqira0Nl8t1UMNWKM4cyjAo\nTKSUbGxssL6+jhCCzs5O2tvb2UoO2BflcpmJiQmKxaI5uScSCQYGBujs7Nz2/kQiUWMUqkmlUnR1\nde17jAqFooIyDAqgYhRmZmZIp9PmBJzJZEgmky0pKIzFYjVGwXjOaDRKe3s7mtY8c7qZcWqF4VIo\nFLdQjXoUAKTT6RqjAJWJO5VKkc1mLe+RUtqu4utJJpO21+ZyuW3vb7ZzCYVCOxqDQqHYGWrHoABo\nMAoGUkrS6XRNLCCbzbK4uEg2m0UIQUdHB/39/U1X/Q6Hw/K4lNL2XDU+n49IJMLq6mrN8cHBQZxO\n9TFWKFqJ+kYpAHA6nQghGoyDEKJm4i4UCkxNTaHrOnArLlEoFBgbG7N9/HA4TCaTaXh8t9u945TT\nnp4eOjo6zHRVFXRWKA4G5UpSABVXjRXGBGwQj8dNo2AgpSSTyZDP520fPxQKEYlEEEKgaRqapuF2\nuxkZGdnVON1uN+FwmO7ubmUUFIoDQu0YFEBlxzAyMsLc3Jy5qtc0jXPnztXsGOziDUII8vl809V/\nT08PXV1dZLNZnE4nXq/3SAPHpVKJfD6Py+XC7XYf2TgUiuOGMgwKk2AwyOXLl83J3+fzNUzcfr/f\n0iUkpdyRS8jpdB55sNiof4jH46b7LBAIcO7cuW2zoxSKs4D6FihqEELg9/vx+/2Wq/murq6G40II\ngsHgiZGniMfjZrGcrutmgH1xcfGoh6ZQHAvUjkGxK1wuF+fPnycajZJOpxFC0NXVRU9PD1JK8vk8\nUsqWuomKxSLLy8ukUik0TaOrq4twOLznx19bW7Pc8RgFd2rXoDjrKMOg2DUej4fR0dGaY/l8ntnZ\nWQqFgjlhDw0N1QSu94JRMV0qlczfV1ZWyGQyuw5cG9QHz+vPKcOgOOuob8ApQUrJ+vo6N27c4PHH\nH2d+fp5CoXBozz01NWXuFnRdR9d15ubmmmYq7YT19XXK5XLD821ubtoWxkkJ+XzlfysCgYDlcafT\nuaOaCoXitKMMwylheXmZxcVF8vk8pVKJjY0NU5vooEmn05arcCkl8Xh8X49tFeiGSlzDyjD8+Z/D\n0BD4/RCJwPvf32ggent7G3YFQggGBgYQQvDII3DXXfBv/y38t/8G0ei+/oQacrkc8/PzTExMsLi4\neGjGW6HYDcqVdAoolUrEYrGGCVTXdWKxGH19fbt+PGDbiuJiscjq6iqJRMLWPbNfw+TxeCwL74CG\nFNNPfxpe+1rIZCq/x2LwtrdVDMOb3nTrOofDQSQSIZVKUS6X8Xq9hMNhfD4fn/kMvPKVkM1W7vvO\nd+BP/gT+5V9gv5JR6XSa6elp82/JZrNsbGwwPj6O1+vd34MrFC2kJTsGIcRHhBArQgjLfs2iwgeE\nEDeFEN8RQjyt6tw9QogbW//uacV4zhq5XM4yEGsEVI2Jfjvy+Tw3b97k2rVrXLt2jZs3b9q6a4rF\nIjdv3iQejze4egyEEHtOTS0Wi8RiMdsAs8vlapDsftvbbhkFg0wGfvM3b+0aUqkU165dY2VlhWw2\nS6FQwO124/P5KJfhda+r3GNcXyjAxgb8xm80jqFUKrGyssLMzAzLy8vbGsHFxUVL4x1t5ZZEoWgB\nrXIlfRS4s8n5Hwcubv27F/gDACFEF5X+0M8EngG8QwixvQazogaXy2UrUFcsFrl+/TqZ+hmzDl3X\nmZycJJfLmeJ4uVyOqakpy4l/bW3N1iBAxSi43W7biupmxONxrl+/ztLSkqmN5HA4TCMRDAYZGxtr\nMBqzs9aPl0hUdgDlcpnZ2Vnz7zP+ra2tkclkmJ2FdLrx/nIZHnyw9lg+n+fGjRusrq6SSqVYW1vj\nxo0btoZU13XbeMt2741Ccdi0xDBIKb8GNHMmvwT4mKzwMNAhhOgH7gAelFLGpZTrwIM0NzAKCzwe\nj2UxmoGu6+aEWE+5XKZUKtmqn+q6TjKZbDi+ubnZdDw9PT2cP39+1xk+xWKRaDTaMHnrus7Y2BhP\neMITGB0dtXRzXbpk/ZjhMPh8lTHb7azW19fp6KgYAbvHqCYajVIul83XzBjjwsKC5f1CCNv3RwW8\nFceNwwo+DwJzVb/Pbx2zO96AEOJeIcRVIcTVeoVNBYyMjBAMBm3P67pes5otFotMTU3x+OOPc+3a\nNaLRqG0A2cpFYqdTJIRgZGSESCSyp7RPKyNkjGNzc7PpJPre91YMQDV+P7z73SAETSXCpZR0dsId\nd0C9OkYgAG9+c+2xtNXWgkrcwC5Y3tnZaVkcqJoMKY4bh2UYrJZKssnxxoNS3i+lvCKlvBKJRFo6\nuNOAw+FgZGRkR0FMI73UkNqWUtq6hTRNs2y/aVdg5vf7KZfLzM3NcfPmTaLR6IFkRpXLZWKxGHNz\nc6Z//4474DOfgSc/GTweuHABPvxh+IVfqNwTDAZtJ+2Ojg4APvYxeO5zKwamvR28XnjjG+Huu2vv\nSSScfPKTXXz4w2G++11fzWPZ0dfXRzAYNIUEjedVn2fFceOwspLmgeGq34eAxa3jP1p3/KFDGtOp\npKury3TFVKNpmmk00un0jgLSRpzAaicSDAbp7+9naWkJqBgbv99PV1cXk5OT5vPncjnW19e5cOHC\njoTqQqGQ+Zj1YzGK5Uqlkln0JqVECEEsFmN0dJQ77/Rzp40z0ul0MjAwUBMEFkLQ3t5u1ja0t8OX\nvwxTU7CwAE96EtR3Hv2Hf4AXv/giui4pFjU+9CHJC16Q5L77FujosG8opGkaIyMjFAoFisUiHo9H\n9ZJQHEsO61P5OeANQohPUAk0J6SUUSHEA8C7qwLOtwNvPaQxtRTDx2ysBI+Kzs5OUqkUm5ub5qQp\nhODcuXPmuOpbbFbj8XhMl1J7e7splW2QTqdZW1ujWCwSDAa5cOECuq7jcDhwOp1cu3bNMvNmeXmZ\n4eFhtsPtdtPb28vy8nLN5B2JREzDVp8BZOx65ufnuWQXaKh6fQKBABsbG+i6Tltbm2V8Zmys8q+e\nQgFe/nLIZG5ttstlwVe/GuKf/inMa18bbrzJ4m9Uaq6K40xLDIMQ4i+prPzDQoh5KplGLgAp5R8C\nnwd+ArgJZICf3zoXF0K8C3hk66HeKaXcX0XUISOlZHl52exT4HQ66evrM10Th41hBLLZLOl0GqfT\nSVtbW41v3s7dZPi7u7u7Lc+vr6/XrLbz+by5G3C5XBSLRVuXVLNgdT3hcJhQKGQGxNva2mrGnEql\nLO8rFouUSiVzFV4oFFhZWTFfh56eHkKhEG63m56enh2Pp5pvfAOsSjayWQdf+EIvv/RLe3pYheJY\n0RLDIKX82W3OS+D1Nuc+AnykFeM4Cgz5ZmOyLJVKLCws4HA4jkxeuloh1Qqfz0cgEGho5+l0Oums\n95tsYeTb1/eELpfLrK6ubis+t9vMG4/HY+t7b7YjM84VCgVu3rxp7n6KxSKzs7P09fXZGr6d0KzF\n9Q7bXysUxx4libEPdF2vMQoGUkpWVlaOaFQ749y5c0QiEVwuFw6Hg87Ozqbppc2kG4zdgMPhoK2t\nzTLzJlyf77kPrKS/oRL4NgzQysqKZae55eXlpiJ62/Gc51QynOoJBOBVr9rZYxhxF7s+2wrFUaMi\nX/ugWYHXcdfA0TSNnp6eHbtUHA6H7SRWHUAdGBigXC6bktxSSrq6umx3InvB6B9dXZfgdDoZGhoy\nr7FLJ4XKe7NXCQq3G/7qr+ClL624lPJ58HolP/mTOj/1U4Jmay0pJbOzszXjdrlcjI2NqSC04lih\nPo37wOl02ur4HGftG13XyWazZqbSToLlLpcLv9/fMOHW7wYcDgejo6MHmnlj1Erkcjmy2Sxut7uh\nsZAR86hHSrnv8bzoRTA9DZ/4hGR+Ps1TnrLCD/1QlmvXKruZvr4+y9d0dXXVTAqojtPMz883yJgr\nFEeJMgz7QAhBT09PTQaNcby3t/cIR2bPxsZGTaey3dQ/DA8PMzs7SzabNQ1iT0+PZc+Fw8i88Xq9\ntuOORCIN1d6GdlMzw5DP580VfVtbm+214TDcffd6TdxFyoqch6Zplu//+vq65SJic3OTmzdv4vV6\niUQiJ6YTnuL0ogzDPgmHwzidTlZWViiVSni9Xvr6+mwDv63k61+HD34QVlfhp38afv7nGyt/q8nl\nciwsLNRMTrquMz09zW233bbtzsHpdDI+Pk6hUKBUKuHxeI6tnEMoFGqoswiFQjXupnpWVlaorqqP\nRqMMDQ3Z6j2trq5axpdisRg9PT0Nr2ez2EYulyOXy5FIJBgbGzuUz49CYYcyDC2go6Pj0NNTf/d3\n4dd//ZY89De/CX/4h/Ctb9kbB6tAOVQmrHQ63VRSo5qTkoff1dVFR0cHxWLRrLOwI5vNWk708/Pz\nBIPBGgOYyWSIxWK2Fd1GH+l6w9DW1sb6+nrTMUspWVxc5MKFC9v9eQrFgaGykk4gGxvwlrfUykNn\nMjAxAR/9qP19dtXOuq6ztLRENptt/WCPGE3TdhTnsOpnARX3U3XdxPr6OlNTUyQSCdvHcrlclruv\n3t5eMy7VDEPhVqE4KpRhOIE8/HCj0BtUjMNnPmN/n1UqqUEul2NycrJpNs9ppVwuN53ojUnaqpaj\nHiGEbfDZ6XRy8eJF+vr6aG+3l84AzFiOQnEUKMNwiJRKpR03zWlGR4d19a0QlXaWdrS1tZkd0ayQ\nUp66pjHZbHbbmoGNjQ3bc0ZsArDttWDg9/sZGRlp2oPC4XDQ3d3N8PBwg9xINalUisnJyV1VjCsU\nrULFGA4BIyXRmFg8Hg/Dw8N7zj555jMrWTHpdG21rc8Hr7esL6+gaRrj4+PE43FLoTrYfvJrBYlE\nZWcTj8OP/Rg85Smtfw5d15mZmTGb4BiCgFa9HJr9zdWZSc1qOYLB4K5TTiORCIVCgUQiYfm4Rrxh\nO/0nhaLVqB3DAWN0RjN0+o3OaJOTk3uuwBUCvvQlGB2FYBDa2ipG4b77KpW5zdA0jXA4bFvhfNCF\nVl//OgwNwX/5L5Xg+XOeA//5P+9NTqJQKNj641dWVshkMjWNfnK5XE2qroFdLYfRQ8HA4/FYGnMh\nxLYyG0Y/ifX1ddMQCSEYGhritttua/o37qdSW6HYC2rHcMDYdUYz+jHvtSL44sVKsPmRR2B9HZ71\nrIqB2CnhcLghC6fV0hX1lEqViuFq70ihUNk9vPjFFdXSnWDoHhm9roUQDA4O1tRT2NUMGO9HtSHo\n6OhgZWWloZLd7XabctwG586dY2ZmhkKhYNZyRCKRprpYRlMkQyYcKjsMQ/HW6XTidDot3YzNOr8p\nFAeFMgwHjN2KT9f1BunoRCJhpjMaKbDVk0I2myWZTKJpGu3t7bjdbp7xjL2NKxKJUC6Xicfj5gTX\n3d29L4G57fjWtyoSEvWk0/DHf7wzwyClZHp62uyfbEy0c3NznD9/3ix4265bW/Xr6nA4OH/+PAsL\nC6aUR1tbGwMDAw2Tstvt5sKFC+RyOUqlUo0+kx1zc3MNEimbm5usra2ZQoHhcNiyUNJOF0qhOEiU\nYThgfD4fmqY1GIf6zmhzc3OkUimiUSfve18vX/96CL+/zOtf7+DXfk2wthatqUNYWVlhYGBgzzsO\nIQT9/f309PRQKpVwuVx7asW5G8plawE649xOyGaztlIXsViMwcFKZ9hgMGjZJtR4P+pxu92MjY3V\n9ICwQwhh2dXOilKpZJldZPSZNgxDd3c3xWKxxlC3t7cf2wp6xelGxRgOmGAwiNvtrplo6jujZTIZ\nUqkUGxuCu+46zxe/2E4y6WBpycm73w3/8T+WGorTjMDkfrOcHA4HHo/nwI0CwL/7d2D1NIEA3HPP\nzh6j2d9bbTD6+/tragaMdpqG4bCj1a6bZjuX6sWCEMLMGDMK5I666ZPi7KIMwwEjhGB8fJzu7m7T\nl9zd3c3Y2Jj5pTdSKT/96U7SaQ1dr3YfCb74RQdTUy7Lx7ZrWnMccbvhk58Ev7/SSxkqRuEFL4BX\nvGJnj+Hz+WwL0aort10uFxcvXqS3t5f29nZ6enq4dOnSoYsbOp1OXK7G9w6oiYkkk0mi0WiNsVhf\nXz916cOKk4FyJR0CmqbR19dHX1+f5XmHw4EQgm9/O0A+32irnU7J9eteRkdr/dQnsTr29tthchL+\n8i8rGk+33w7Pe569i6kel8tFV1dXzQ7KCODWu9UcDseBBtN3gpF5ND09bWZIGeOtljxfWVmx1F1a\nX1+nr6/vUHZ0CoVBq1p73gn8LuAAPiylvK/u/PuA52/96gd6pJQdW+fKwGNb52allC9uxZhOEu3t\n7SwtLTE+nuPrXw9SLNZOArouGBqy1uU5qi5x+6G3F974xr3f39fXh8/nIxaLUS6XaWtrIxwOH1tB\nP7/fz8WLF4nH4xQKBQKBAB0dHTWTvZ3uElQqs5VhUBwm+/60CSEcwAeBHweeCPysEOKJ1ddIKd8k\npXyKlPIpwO8B1cINWePcWTQKcEv6+u67E7hctatGtxue9CTB854XrPGXGymaZ7HBixCCjo4Ozp8/\nz6VLl+jr6zs2r8N3vwvPfz64XNDZWdG0KhQqO53e3l6Gh4fp6upqmOjtgtnG7kKhOExasQx5BnBT\nSjkppSwAnwBe0uT6nwX+sgXPe6oIBAI8//kX+MIXijzxiWVcLonbDf/hP8ADD0BfXy8XLlwwXVKX\nLl1qiaKrUXCXTqdVIdU+mZ2FZz8bHnqoUrOxsQEf+AD83M9tf29vb69lS1Q73SWF4iBpxVJkEJir\n+n0eeKbVhUKIEWAM+ErVYa8Q4ipQAu6TUv6Nzb33AvdCpcjoOCKlZGNjw3RxhEIhenp6drziE0Lw\nvOd5+d73KrIRHs+tIC3YV97ulXw+z8zMDMVi0Zx8BgYGDl1C/LTw/vdDvbpGNgt///cwMwMjI/b3\n+nw+xsfHWV5eJpvN4nK5bJsgKRQHTSsMg9Vyxi4q+grgr6WU1Vnr56SUi0KIceArQojHpJQTDQ8o\n5f3A/QBXrlw5llHXpaUlYrGY+Xs8HieZTHLhwoVduwOa6LC1BCmlWY1r/A6wsLDQtDOawp5HHwWr\nUIHHA48/3twwQMU47EZvySiebCaMqFDshVa4kuaB4arfh4BGQZoKr6DOjSSlXNz6fxJ4CHhqC8Z0\n6Bg7hXqM6uKDpFQqkU6nmwYw67FzHUkpa8ZbLpdZXV1lcnKS+fl5JQXdhKc+tRJbqCefh1bq4BUK\nBW7evMmNGzeYmJjg2rVrSoVV0VJasWN4BLgohBgDFqhM/nfXXySEuA3oBL5ZdawTyEgp80KIMPAc\n4LdaMKZDJZPJsLCwYHlOSnlgPQ6klCwtLdVUywaDQYaHh7fNYqnXBarGMDClUomJiQlT4yeTyZBI\nJBgcHFTuJgt++Zcr0h7V9tnrraTkjo1Z31Mul4lGo2Y/iGAwSH9/v22HPCklk5OTNYV+pVKJmZkZ\nLl68eCI66ymOP/veMUgpS8AbgAeAHwCfklJ+TwjxTiFEdZbRzwKfkLXJ2k8Argoh/hX4KpUYw/f3\nO6bDxqolZDV2BU77JRaLmfn8RrXs5uampYJoPX6/37ZQzPBrr62t1Qi/wa2KaxWobmRsDP7xHysV\n3kJUivfuvbdS1GeF4c4zZLellGYfBjvDvdOdnkKxH1qSByel/Dzw+bpjb6/7/Tcs7vsG8ORWjOEo\nyVspw1VxUEVWVu0oDTG+gYGBprsGl8tFOBxmbW2tplDM7XabjWZSqZStwcvn8zvWCzpLPO1plf7b\nul4xDs1c/+l0mkKh0PAal8tlNjY2LAUNm7kLd+NKVCiaoRKkW4DX621QzzQYHh4+sECu3arSWH1u\nR29vL36/38yiam9vr8mxtysYk1Ie22Kyo8AQv8vlcvh8Prq6uswVfCaTMY/Vu3ny+bytJLtd86Bm\nO71qSRCFYj8ow9ACenp6GlbXhmRyszaP+yUQCFhqJe1GKTUUCtlWT4fDYebm5homIq/Xq3zZWxhN\nlwxjbMhpG6+ZEWOKx+OMjY3V7LKMbKL611cIYbuY8Hg8tLe313R9q9/pKRT7RdXZtwCv18v4+Dh+\nv9+sVO3t7bXVRmoVVho6RkV0K9IXDakJQ5nUUAA9rnUkR8HCwoIZ3wFq4j3VE76u6w2xn0AgYBl/\n0jStaXB/cHCQgYEB00CHw2HGx8eVbIaiZagdQ4swCpT2yuYmfOUrFVnqF7ygokC6HR6PhwsXLrC2\ntkYmk8Hj8RCJRFrquurt7aW7u5tsNovT6bRtg3kWkVLuKn3XaO9aLW0yPj7O4uKi2VkuGAwyMDDQ\n1FVntBw1RAPL5bKt69AoXlSyGordoD4tR4Su66yvr5NKpfjSl4K86U3dOJ1i6xx86lPw4z++/eO4\n3W4GBgYOdKxOp/NEivUdBlauoGbX1uNwOBgeHt5Rg6B6SqUSc3NzZDIZoOJCHBoawu/3k8vlajrH\neb1ehoaGWlo5rzi9qL3nEaDrOhMTEywtLXHzZo7/+l+7yGQEySQkk5Xdw8tfDhb1copjhBCC9vb2\nHU3mhvCf3bW7bRBkpLoavTyklBQKBaanp824hxHcNnY2U1NTKs1YsSOUYTgCDPllKSUPPNCO3YLz\n058+3HEpdk9/f7/pXjPiMIFAgGAwWHPM7/fT39/fsufNZDK2LU6XlpYsdzG6rp+oxk6Ko0O5ko4A\nw58MsLnpoFhsXCmWSqC+w8cfh8PB+fPnyWazFAoFPB6PGePJ5/Pk8/mWix8CtunRxs7BzjCoWgfF\nTlA7hiOgOrD43Oem8Hobv8QOB9x552GOqjnFYpGlpSVmZmZYWVnZd6/p04bP56O9vb0m8O/xeGhr\nazsQv36zFqd+v98yQ0kIoYoSFTtCGYYjoKury/QnP/nJWW6/PYHPZxSrSQIBePWr4UlPOroxVpPN\nZrlx4waxWIxUKsXq6io3btzYtuJbcXB4vV7TXVWNw+Ggr68Pl8tVc84wCv6dpLspzjziJPYNvnLl\nirx69epRD8OWdDpt6gyFQiG6u7sb0g9XVlZYXV2lVBKsrjq5ds3P177Wj9vt4J574Md+bOd9kA+a\niYkJy7TMUCjEyHZa0ooDQ0rJ2toa8XgcXddpa2ujt7cXp9NpquIa4nydnZ2mNEsqlaJcLhMIBFSW\n0hlDCPGolPLKdtepGEOLicViNcG/XC7H+vo6Fy5cqDEOPT09fOxj3bzznRrlMkgJr3ud4Ld/u+JG\nOi40y9VXUs9HixCCSCRCJBJpOGfsHKqLLI3MJLjVf6Ojo4OBgQFVm3IMKJfLJJNJdF0nGAweqdFW\nhsEGQ8pA13UCgcCOtIF0XW/ICJFSUiqViMVi9PT0mMf//M/hHe9wsJWCDsCHPlRp6vKe97T0T9k3\ndrn6qtL25CClZGZmpiFddWNjg2AwqOQ0jphUKsXs7GzNse7u7gNXT7DjzH+zDanjaDTKysoKhUKB\nXC7HtWvXmJ2dZX5+nscff5y1tbVtHyubzVquvIznqOZd76LGKEDl99/7vUpGElQMTT6fP9Lcc7tc\nfaP6VnEyyGazSq77mKLruqlJVv0vFosdWC+X7TjTOwZjFZXJZMwvzcrKCpqmNXyJlpeX8fv9TYN3\nDofDtgq2XpLArmVCoQCbm5J8fqXGGHV1dR1ZY/j+/n4KhYJp+KSUBAKBmh2Q4njTLJZ4EuOMpwk7\nl6yUkvX1dVwuF6VSCa/Xe2i79DNtGBKJhFk5Wo3dyioWizU1DF6vF4/H0yCZLIRo0NZ/+tMrTV3q\niUSgWIzVKHRCpSjO4XAcyWTscDgYHx8nl8uRz+fNv1NxcrBLUzUqshVHRzPDnEqlSCQS5oKwr6+P\nrq6uAx/TmXYlVUsX74RUKrWtW2dkZKShEra3t7dBK/+9720UyvP74X/9L4jF1hrGZWSgHCVer5f2\n9vaWGwVD1+f69evMzs6qvtK7JJvNEo1GiUajpm5SPZqmMTw8bJnCqgzD9hSnF0n8yWdJfuILlBOt\nrTwNBoO285AhkKjruqnQOzc3d+Du5ZYYBiHEnUKIa0KIm0KIt1icf5UQYlUI8e2tf6+pOnePEOLG\n1r97WjGeXYx7V9frur6tP9aCi8yeAAAgAElEQVTlcnHhwgXOnz/PyMgIly9ftuzg9sxnwte+Bnfc\nAX198Oxnw2c/C3fdhW3xWLW882khk8kwMTFBIpGgUCiQTCaZnJw8Mt/qSWN5eZnJyUlisRixWIyp\nqSmi0ajltaFQiIsXLxKJRPD7/bjdbhwOh3qttyH2nj9i7t+/ktjbP8jaW97HzJN/msxDj7Ts8R0O\nR0NmWLO5KZFIMDc317Lnt2LfriQhhAP4IPAiYB54RAjxOYvezZ+UUr6h7t4u4B3AFUACj27du77f\ncRmUSiVWVlZIJpNmwDQcDqNpGp2dnU3bV1qRSCR21KpzJ9LXT386fPGL1vdadfByu92nLq0wGo1a\n7o4WFxe5ePHiEY3qZJDP5xtcjkYwuaOjw9J95HK5SKfTpgR4Pp9nc3OTcDhMb2/vYQ7/RJB9+Dsk\n/uBTyHytBMnSq97G6Pc/h+ZvjcR9Z2cngUCAjY0NdF3H5XLZal5BJS5hyK0cBK3YMTwDuCmlnJRS\nFoBPAC/Z4b13AA9KKeNbxuBBoGVCEIaKaTwep1QqUSwWWV1dNdPCgsGgWYVsuH40TWuQsf7nfw7w\nyleO8dznXuauuwZ56KFWjdCa/v5+yyygVoqwHRfs3EZ2bS8Vt7ATxJNSkkwmLc8lk0lyuVyDMVlb\nW1M6ShakPvkFZM6iwl8Isv/Yul0DVBZ+PT099PX1beveE0LYtn9tBa0wDINA9b5mfutYPS8TQnxH\nCPHXQojhXd6LEOJeIcRVIcTV1dXVHQ1sY2OjwS1j1CcYGTb9/f1cuHCB/v5+BgYGuHz5Mp2dnWZn\nrX/6pyCvf/0I3/52gI0NJ48+6uUnfgIeeGBHQ9gTgUCAsbExgsEgTqeTQCDA6OjoqeyJYFcfomok\ntqfZ7tHu9TMKqKxQLqVGZKGInfyxLB6cXth2iSZSygNNAGnFt8/q01n/Sv4dMCql/GHgy8Cf7uLe\nykEp75dSXpFSXrGq9LQik8nYrjqrV6oej4euri46OjrMgPHIyAgOh4Pf/u1+cjmt7l74lV9p/rzz\n8/PMzs6ysbGxp5Wv3+9ndHSUy5cvMzY2RiAQ2PVjnAS6u7std0dWxxW1tLW1NRyTEh59NMD739/F\n+98Py8u15+0MsRBiR0WcZ43gS1+I8FtkdJVK+H5kW2WJfRGJRBgcbFwnG0kDrezUWE8rDMM8MFz1\n+xBQk6UvpYxJKY392B8BT9/pvfvBzidvNE9vhtfr5bbbLjM1ZX3d449b37e2tsbU1BQbGxskk0kW\nFxeZmppqahyKxaIZPLSTUz6tRCIR051nGOWOjg5VI7EDXC6X2d+7Ul+i8aY3neOXfmmUd7/bwVvf\nCuPjtbvbagHHaoQQDZlzCvD/2DMJ/Pi/R/i9FfEypwPh9RD+nTfjaD/4HXxnZyfnz58340VGwelB\na5TtW0RPCOEErgM/BiwAjwB3Sym/V3VNv5QyuvXzS4Ffk1L+u63g86PA07Yu/Rfg6VLKpqk/OxXR\nK5VKXL9+vWHr7Ha7uXjx4rYr0kKhwMCARizWGKPv64P65I9SqcS1a9cajIARt7DyG66vrzc0ie/p\n6bHUvznNlMtlCoUCLpdL9SfeJaVSiVQqxac/7eKNbwyQTtd+rtvbJSsrAmMttLGxwcLCgvn51zSN\n0dHRA12BnmSklOS++a+kH/g6WsBP6OW34xofOvRx6Lq+605/9RyaiJ6UsiSEeAPwAOAAPiKl/J4Q\n4p3AVSnl54D/KoR4MVAC4sCrtu6NCyHeRcWYALxzO6OwG5xOJ2NjY8zPz5srcb/fz9DQ0I5e3NnZ\nWV79aj8f+EBfjTvJ75f8+q833p9Opy11hXRdJ5lMNhiGUqnE4uJiw/UrKyuEQqEz9UV1OByqV0AV\nm5ubrK2tUS6XbRV6DZxOJ52dnfz1X4NVmKBU0nnooRK3317xSXd0dNDW1kYmk0HTNHw+n3LbNUEI\nge/ZT8H37Kcc6TgOM+7WkqWZlPLzwOfrjr296ue3Am+1ufcjwEdaMQ4rfD4fFy9epFQq7cqPWigU\nyOfz/Kf/lCOb1fjjP45QLgucTsnrXpfgDW9orD5s9thWb6pd5oiUkkQicaYMg+IWq6urrKysbKvQ\nW4+mSazDdrCxEQf6q67VlOtIYcuZ2bPv1j1RLpe3tm2Se+9d4yUvWeef/znIyEieK1ckFS9YLYFA\nwNZ/a1XGfpb0a6SU5HI5CoUCPp9v2xjPWaVcLtcYBbil0BuPx5u6GO+5p8RDD2lks7XGw+mUPPGJ\nCaoNg0LRjDNjGHaLIWshJXzgA7187GPduN2SchmGh3W++lWoK3fYMgCjXL8+R1dXCSEqX+re3l5L\njaW2tjaWlpYajhsBptNCqVRienqafD5vutra29vNwOl+HjeVSiGEIBQKnYqsmmqhwmoMhd5mhuHl\nL3fwF3+R4EtfaqdcrhgEIeB3f3cWv18ZYsXOUYbBBiEEg4ODfOQjG3z8410UChpGwtDEhMZLXwrf\n+tat62MxuOceePBBH0JcZHBQ8vu/n+FFL/La7lZcLhe9vb0sLy+bE4GRqnma/O3z8/NmMY7xdxqu\nsp1UkVsRj8dN6QdjIh0eHrZM4TxJOJ3OHSv01qNpgve+d5lXvjLGww8HaG8v88IXJgmFdMLhcw3X\nG+qdRrdBn89HX19fzWdP1/WGTnDd3d2qzuSUowxDE9ra2vjUpwINW/NyWfDYYzA9DaOjldzx22+H\nxx6DSvGoYHJS8DM/E+Sxx2BszP45wuEwoVDIFPRra2s7VUahXC5bFk4Z0g17MQz5fL5GSsP4f25u\njttuu+1EZzV5PB7cbndDP20rhd56jMZSly/nuHz5VlWskQpcz+rqKqurq+brl06nmZqaYnx8HK/X\ni5SSycnJmir0lZUVNjc3GR0dVQHrU4wy+9uQSNhlgsDWIop/+Re4ds0wCrcoFCTveU9y2yI3j8dD\nT08Pvb29p8oogLWE+U7ONaPZ62knE3FSEEKYqaPVtR39/f3bFjlW9xWpRkrZoLpq7ASsMuhWVlaA\nymtZKBQa4h2ZTMZWxVWxPzKZDLOzs0xMTLC0tGQrqHnQnNyl1SHx0pfCxATULeBwOuGJT6z8PD1t\n3ae5WBRcuwaLi4vE43HGxsbO3CrL6XTidDotdXj2KvFxEMbmOGEo9Obzecrl8o4btLhcLts2rKur\nq6RSKVMCvlAo2F5rqAI0MzTZbPbUVuMfFUZ9iVU2miHRc1ioHcM2/OqvQn8/GAt5Tav0TfjQh8B4\nr572NLAqWPZ4dK5cqWzvs9msbXrqaadetVMIgdPp3HN1s1WrUYPTpCfl8Xjw+/079ue3tbXZvi7G\nZD4zM2N2DLMzom63m0KhYMYV6tE07US7644jUsoGpWEpJeVymZ1qw7USZRi2obMT/vVfKz2aX/hC\neNWr4P/8H/iZn7l1zdgYvPzltY13nE5JKFTmZS+rKIgbtQlniXQ6zfXr11lYWAAquwe/309PTw8X\nL16sSEA/8H+YfdbdTPQ+j+kf/mkSf/Z32z6uz+drMA5CCCKRyJlOgzU67Xk8nqYGIhqNMj09bfs4\nkUiEqakpW7VVIcSJD/IfNwqFgq2hPooFpTL7O6CtrbJz+NVftb/mox+t7Bw+8AGdZFLnR380yRve\nsEJb2603+zSkU+6UQqHA9PR0zQqoVCqhaRrhcBghBOkvP8zy//0OZLbipytHV4m97QOQL9D+mpfZ\nPraRMdbR0WG2Pezs7Dx18Zm94PV6uXjxIvl8nhs3blhek8/nbXchRr1NuVy2PK9pGuPj4yorqcU0\n6xdfKpXIZrNm3OkwUIahRTgc8KY3wRvfKHj88RsNXyxj8jorrK+vW37QS6USmUyGQCBA/H98yDQK\nBjKbI/5bH6HtF16KaDL5GKJvqnrXGrfbjaZplqtQh8NhO/E7HI6mAc9QKKT6fR8A27nmpqamEEIw\nNDR0KO5SZfZbjJFV4nA4zMY/Rt9nqyK300ozlVjDRVGcnLc8r6cyyLTq+7wfDNealaS5IS9fj6Gb\n5PP5LI26EEIFnFtM7l++z/xPvI6JwRfgfdmv4fjUl8HCmOu6TrlcZnZ29lAUmM/kjiGXy5HP5/F6\nvQey+vH5fFy+fJl0Ok25XCYQCJy5YJ3f7yeZTFpW8LrdbmKxGAz1wvWZhnu1oA8RUG6h/RIOh83u\nbFJKNE2jp6eHrq4u0ul0TX2CkRAQCoXMynujtsY473K5tu0sptg5+R9MsvjSX0ZmKjUnIl7E9Sd/\nh4glKP2itSvVqP/p6+s70LGdqdmqXC4zMzNTIzsQCAQ4d+5cy32mZ13fvrOzs6FdpLHiNGIP4lU/\nifu//xEiX3WNz0vnm1/V1I2k2BlCCFPCXdd1c/cKMDY2xsrKChsbG0Al06u3t9c8Pzg4SCAQIBaL\noes6bW1tRCIRFVtoIev/808b2oaKXAHn3/wj5Xt+EmnTT/owahvOlGGIRqNmE/Tqas+VlZUDt8Bn\nDU3TOH/+PKurqySTSVNIMB6Pm35v+awnU3jbz+P6w8+gLa7h6Omm88330Paqnzri0Z8urFSFHQ4H\n/f39tn3EjZjYWYqLHTb5x66D3uiy0zwu+nERtagzOSxV3DNjGIx0USvXxmFszc4iTqezZvIpFoss\n1/Wa1J/7VPLPfSoup5Pzly8fxTAViiPBfdsopamFxp7ShRLtl8bJpmtVEwx33mGkCp8ZwwD2Utan\nTeL6uNI01e6MVYQfJbkcfPazcOMG/NAPwZ13Fkml4mSzWXw+H11dXYdeaXsW6frVV5F96JGazDzh\n8xB8+e04OkIMtAdNd56hSHxYAoYtMQxCiDuB36XSwe3DUsr76s7/CvAaKh3cVoFfkFLObJ0rA49t\nXTorpXzxXsZgyBIbaZLt7e10dHSYk5HRQNso969GZVocDs3kMc6aVMhRMTcHz3pWRedrcxOCQUl3\nt87HP75Oe3uJdDpNLBYzhfQUB4fn39xG35/fx9pb3kfx5hzC76X91S+j662vBm5lkB1FwH/fhkEI\n4QA+CLwImAceEUJ8Tkr5/arL/j/gipQyI4T4ReC3gLu2zmWllPvumReNRmty59PpNBsbGzUqkIOD\ng0xOTpoxBqN/qp2fVXF4FAoFyuXymSoCPApe+1pYWgKjjGFzU5DLufid3+nlXe9aML8b0WiUsWay\nwIqW4H/eFc594+PIUgkcjmOzQGrFnuQZwE0p5aSUsgB8AnhJ9QVSyq9KKQ05xoeBlnbSzufzDQVV\nhjaMoQsDt6pCw+EwwWCQcDjM4OAgm5ubpNNp5VI6BOzK/oUQ5jld11lfX2dmZobZ2dlt1WkVO6NU\nggcfvGUUbh3X+NKXav3W6vvQOgqFArOzs3z/+9/n8ccfb+jQByCczmNjFKA1rqRBYK7q93ngmU2u\nfzXwharfvUKIq1TcTPdJKf9mtwOw0vuHygSTSqVqKgWN5jilUompqSnTfyeEwO12MzY2platB0gw\nGLTUjHI4HKabaWJioiYlL5lMsry8zPnz589cPchRYeymFfujVCoxMTFhVpobcuf5fJ7h4eEjHp09\nrdgxWH16LJcaQoj/BFwBfrvq8Dkp5RXgbuD9QojzNvfeK4S4KoS4Wq826GiyBbObSKLRKPl8Hl3X\nkVKi67rZAEZxcPT29jYEzwztIyEE0WjUMk+7WCyq92afOJ2VhlL16x6XS+eOO24Z67Mm39IKpK5T\nmJilFK2dm4w6kJprpSSZTB5KBfNeaYVhmAeqTd8QsFh/kRDihcDbgBdLKc0wvJRycev/SeAh4KlW\nTyKlvF9KeUVKeaW+762ddogRvLF4LEvFwrOogHrYuN1uLl68aLYvbW9vZ3x83HwPmzXaUe/N/rn/\n/kqv8lCoIiEfCkmGh0u8+c0rZgGc3+9X6du7IPPVf2bmyS9l/gWvZvbfvoL5O15LaaGSlp3JZBpd\nclLiePBbLL7g1Uw/6SUsvfa/U5xumDKPlFbsyx8BLgohxoAF4BVUVv8mQoinAh8C7pRSrlQd7wQy\nUsq8ECIMPIdKYHpXaJrG6OgoMzMzNW/C0NCQrQyz8p8eHS6XSwX8j4jBQbh5E/72b+H6dXjykwU/\n8RNuSqVR0uk0qVSKbDbLjRs36O7upru7W7mUmlCcWmDpnrchs7daqeb/9RoLL/1lzn3rL/F4PA2u\nbudH/x7np75MOVfZMaT/5itkv/www1/7KM7B2t4lR8W+DYOUsiSEeAPwAJV01Y9IKb8nhHgncFVK\n+TkqrqMg8FdbHzIjLfUJwIeEEDqV3ct9ddlMO8bv93P58mXTQjdrcGJIM1jFJs6yjMVxoL293ZRp\nqOc0NeE5Stzu2n4iFZysrKyYvvByuczy8jK5XI6hoZbmipwqEn/6N5WMomrKZcorcXIPf4fup12u\nTYxJZ3F+8sEaGRh0HT2bY/33/oLIfW+iUCiwsbFBuVwmFAoRCAQO3Ti3JJInpfw88Pm6Y2+v+vmF\nNvd9A3hyK8YAO1d/zOfzlv49o1JXcXT09fWRyWQa3h+Hw8HAwMARjer0Y+cLTyQS9Pb2qoI3G0qz\nUShaaBcJQXlpDZ/Hw9jYGAsLC5U+GLPLCJcT8nW1PMUS2W98m/n5+ZqFUTweJxgMcu7cuUM1Dmcu\nxUNKyfT0tGWRVTPXk+JwcDqdXLx4kVQqZUqYhEIh2tvblYDbAWLpC6ey2Mrlcsow2OB73hUy//At\nUyHVpFjC8/RKU3i/38/Fixcpl8uUOyPMF8uN2TlCkOtuo1C3W5ZSsrm5STKZpL29/eD+kDrO3Dct\nm83aNimxc2EoDhajr+3k5CQzMzNkMhna2toYHh7m3LlzdHZ2IoQglUqxtrbG5uamihG1GDv5eSml\nMgpNCP3MHTh6usB96zUSfi/Bn7kd17la74PD4cA92IvvR64gPHULUI+L4s++yPI5pJSsr6+3fOzN\nOHM7BjujAIcjZ6uopVwuMzExQbFYNCf7zc1N+vr66O7uBirvy+TkJKVSyaw5cblcjI+Pq5qTFtHd\n3d1QJGrIyChpDHu0gI+hBz/Mxu//BZt/9xBawEf7a15G6BU/bntP74fewcqb3kv68/+E0DREyE/2\n9S9HPnHc9p7DjjGIk7jyunLlirx69eqe7i2VSly7dq2x8lCImslIcTisrq5aV4IKweXLl3E4HMzO\nzlqmF3d0dKjAaAtJp9MsLCyYbtZQKMTg4GCN8S0UCuRyOdxutzIY+0TfzKAnUmh9YX7w+OO2u2Ah\nBMPDwy1RVRVCPLpVN9aUM7djcDqd9PT01ExGRtWzKuo5fFKplK1vO5vNEggEbGsbEomEMgwtJBAI\nmL5woy2tgZSShYUFEomE2eTK6/WabWwVu0cL+tGClXa/9R3zquns7Dz0jLwzZxgAIpEIPp+PeDxO\nuVymra2Nzs5OFdw8AuwmFSmlWbV+ELvafD5vfhHb2trw+VQrUbjV4rOeWCxmvl7G+5HNZllYWODc\nuXOHPcxTR39/P/l8nlzuVhDb6XQyNDREKpXiBz/4gZmG39/ff+C7tTNpGKBSr6BqFo6ecDhsGUx2\nu914PB6zRWq1GKLBXldRsViMpaUl8znX1tbo7OxU6bA26Lpu9o2uJ5VKmW1DFXvH4XAwPj5ONps1\n+9H7fD5mZ2drdtXpdJrJyUkuXLhwoBmU6t1UHCmBQIC+vj6EEKYkg8fjYWRkxAy4DQwM4KxSn9Q0\nbc81J8ViscYowK2sj0wm0+TOs0e5XGZ2dpYf/OAHTRMz7BRzFbvDkCPp7OzE5/NRKBQsXa26rhOL\nxQ50LGd2x6A4PnR3d9PR0UE2m8XpdJo7BQO3282lS5dIJBLkcjm8Xu+e6xqsdh5wq5jL7/fv+e84\nbczMzJg90u1wuVwqxnBA5PN5M55Tj1XDsVaiDEMd6XTaVF51Op1EIhEzj15xcDgcjqauPU3T9pUc\noOs6yWTSMrtJ0Ugul9vWKFSr4ipaj8fjsX39VYzhEMlkMkxPT5tvhiH1XC6XqVd0VZwcCoUCk5OT\n6LretFHQUbRQPCqMHhf5fB6Xy0VPT0+N4S0Wi7arVU3T6OjooLu727YwTrF/3G63GV+rfh80TSMc\nDh/oc5/aGIPRY2E3LC8vN3wRpJSsrq6qStsTzPz8PKVSyfLzYDSkMTLVzgKpVIq5uTny+Yr6fbFY\nZHFxkXg8bl7j9Xpt04i7u7sZGBhQRuEQGB4ervFY+Hw+xsbGDly659TtGHRdZ3Fx0Uyt83g8DAwM\n7EhcrzpVrBopJaVSacfSAJubm6yurlIoFPD7/fT09Kgv0RGh67ptUNkoagyFQmdKI6s++A6Vz/jy\n8rI5CblcLsvcek3TVBHoIaJpGgMDA2aixWG57U6dYZidna3pV5vP55menubChQvbTs5ut9s2qLPT\nANv6+jqLi4vm8ycSCVKpFOfPn1fGYRdks1mi0SjZbBZN0+jq6qKnp6fhiyGlNIN0u319z+okZ9c5\nrFwum5IjAIODg3i9XlN5NRgM0tvbq9qrHgH1n/tisYiu67jd7gMxFqfqHS4UCpZNzA130HZVsr29\nvQ3Nfoyt804yYKSUlqsxXddZXl5WhUA7pFAoMDU1Zbp+yuUya2trpNNp+vv7TZdPOp1mbm7O1L9y\nu92cO3euxkBomobf72/YNQghDlWt8jjhdrtNN1I19S1yhRCEw+ED92crdk6pVGJubo5MJmO6QQcG\nBlr+WT5VMYZCoWBrPa2+CPUEg0GGhoZMl5GmaUQiEXp7d9ZVybDiVqgc+Z2ztrZm2Rsgk8kwMTHB\nxMQEuVyOmZkZU1jP2DkYQeZSqUQikSCZTDbsJAwJlJ2+r6eJUqlkmdFixFlUhlHryefzLC8vE41G\n960MPD09bS5+dV2nXC4zPz/f8vTVU7VjaJbetdP89Pb2dtrb29F13bTIO6WZu0ltv3fOdh/ybDbL\n7Oys5XstpTRjTEIIS0MtpaSzs/PM5d/ncjkmJycbXjdN0+jt7aWrq+uIRnZ6qXctr6+vEwwGGR4e\ntpxbMpkMGxsb6LpOe3s7wWDQvC6Xy1kucKWUxGKxluqGtWTHIIS4UwhxTQhxUwjxFovzHiHEJ7fO\nf0sIMVp17q1bx68JIe7YzziMgFn9C74XX7JRhbsbHA6H5fMbqzHFzthJrKBQKFgaBl3X2djYuJWV\nVtbRvvkYjr/6B7RHfwBbhmJpaenM7eIWFxfRdb3hdQsEAqq38wFQLpdrjAJUPp+pVMqy0HJlZYWp\nqSni8TgbGxvMzc0xPz9v3l8qlWzfI7u40V7Z9zJWCOEAPgi8CJgHHhFCfK6ud/OrgXUp5QUhxCuA\n9wJ3CSGeCLwCeBIwAHxZCHFJSmnfNGEbBgcH8Xg8ZsDMEJ06rKyTgYEBpJQkk0nzTYxEImfWn70X\nIpGIrdLkrogn8fyX30FsJCvtF51O5GCE/Pt+BYI+lpeXGRsba82gjxlSSrLZLKVSCZ/Ph9PptDWE\ndtXgiv2xublpWQsipWRjY6NG66tQKDSkxRtGJJ1OEwwGm6YQt1r3rRX+jWcAN6WUkwBCiE8ALwGq\nDcNLgN/Y+vmvgd8XlVnzJcAnpJR5YEoIcXPr8b6518EYq/OjWqFrmsbw8DClUolSqYTb7VYCY7vE\ns9Und3Fx0TaF2OVyoWlazc5BCIHD4TB1fdz/8+OI5RiivOVOKpZhZgnX/Z+l+Ct37yjudBIpFApM\nT0+br4OUsqmb6KzuFIxq+GKxiM/nIxAItPS1aPZY9XOCnXE2xhgMBnE6nXR3dxOLxWoMhMPhaLkb\nsBWGYRCYq/p9Hnim3TVSypIQIgF0bx1/uO7ewRaM6chxOp0qrrAP/H4/Fy5coFAocPPmzRq5ZyGE\nWaVr+F2NYw6Hg4WFBWSpjPat794yCluIYgnHV65S/JW7T2368OzsbINrIR6P4/f7G2Quql/Ls4SR\nqGC4HDVNM/tLtGohZ7eKt3rNDde1XaW5QW9vr5lCXC6XCYVCRCKRls81rXg0K7NY/9fZXbOTeysP\nIMS9wL2ASvs8QxgCevF4nHQ6jdvtJpPJmJlLxhdqfHzcTD5YX18nk0zZfJKAciWx4DRmJeXzedsA\nJVR2Y9XnfT7fqXwdtqM6zRkqK/NsNsva2ho9PT0teQ5N0zh37hyzs7PArfcgHA43JMOEQiFbN1G1\nETGkWw5avqUVhmEeGK76fQhYtLlmXgjhBNqB+A7vBUBKeT9wP1Rae7Zg3IoTgtF1DypB42r3kZF1\nND8/z/nz5xFCMDo6SjKZJPb0JyAf/T7oVRLbDg2e91RGRkZOpZKqkU1nF5g/f/58g+b/Tsnn86yv\nr5vNraozZk4SxWLR1niur6+3zDBAZddw2223mX0rgsGgZbzT4XAwMjJiGhFjPP39/Ueys22FYXgE\nuCiEGAMWqAST76675nPAPVRiBy8HviKllEKIzwF/IYT4X1SCzxeBf27BmBSnFCPjqJ5cLke5XDaL\ntNrb2/H/wduZv/O1yEwemckiAj6cnW0Mvu+tOE9pkya7SUQIQVtbm6n5v1ujWJ92ubGxQSAQqOmb\nobDG4XDsaIUfDAa5fPmy2YPBiCscBft+1q2YwRuABwAH8BEp5feEEO8ErkopPwf8MfBnW8HlOBXj\nwdZ1n6ISqC4Br99PRpLibFNvMFwjA4xc/RSbn/0HCten8fzwJQL/14+geU9nbAFuaessLCzUxGRc\nLtee5T+s0i6NgsNkMnniMu5cLpdl9fdhKuzaSVpomnYsXk9xElVDr1y5Iq9evXrUw1AcAdFolHg8\n3mAEfD4f58+fP6JRHT9yuRyxWIxisUgoFNpXT/NkMsn8/LxlsWBbW9uJjPlVF/tJKdE0zcyGO8gs\nwkKhwOzsrGmUHA4Hw8PDOxL5bAVCiEellFe2u06lzShOFD09PWxubporLqMl6H6qPtPpdI0abiQS\nOfEZS16vl8HB1iT47Sbt8rizubnJ0tISuVwOh8NhKuv6/f4Dj5lIKZmcnKxpk1oqlZienubSpUs7\nVm8+DJRhUJwoHA4HF9nQjdYAAB+YSURBVC5cIJVKkc1mcbvde27zCRVfebXbpVAokEwmlRpuFXar\n2ZOW6prJZGpEMsvlMslkkkgkUlNsdlBsbm7aaqnF4/FjlR12ssy9QsGtQGpvb+++XCRSSqLRqK0a\nrqKCpmmMjIygaZqZHmworx6WC6QV2DXishJtPAgKhYKtdlexWDzw598NasegOLPYdXWDintJcYtA\nIGBmzJTLZdu0y+OMXaW70YhrL39PPp8nGo2STqfNHVRvb2/DYsUwQFYIIY6dgVWGQXFmUWq4u+O4\nZMzsFbfbXePfr2Yv73epVGJyctIslJNSEo/HyeVyDRpcyWSypqCu/rmP2+uqXEmKM4sx0Sk13LNB\nb2+v5Xu900Zc9ayvr9v2DanX+Mrlcra70/24Qw+K4zUaheKQGRgYIBQKmdlNhlE4bis4xf4JBAKc\nO3fOdBntthFXPfW6UwZCiAbDYNeC00iTPW6o/bLiTGPo2ZRKJYrFIh6P59it3hStIxQKmbpE+01N\n9Xg8trLa9ZN9e3s7S0tLDe4kTdMOJSNqt6hvgEJBxc/r8/n2bBSMBkHLy8skEolDyXJR7J1W1CtY\nNTcSQlhqUGmaxvj4eI0USSAQYHx8/FguRNSOQXGqKJVKptZMKBQ6lKKhYrFoBiENxVeHw8H4+Pix\nKlpStBan08n4+DiLi4tkMhlTo6u/v9/yeo/Hw/j4uLlrOM6tZc+sYTC6rCWTSbPRhVWTdMXJwShW\nM4hGo/T19e1ZI2inLC4u1uSh67qOrutEo9ETKReh2Dler5fx8fEaXartOM4GweBMGgZd15menq4J\nHq2vrzMwMHCiKjkVtyiVSjUVzAZLS0sEg0HT51soFFhaWmJzc9NcEITD4T27FqSUpFIpy3N2xxVH\nT+Yr32LjDz5JeSWO/0XPouMX78LRvXcBPePzUygUyGQyOJ3OlneEO0zOpGFIJBINGQVSShYXF/cl\nr6A4OpLJpOVxKSWJRIKenh5KpRITExPmVl7XdVZWVsjn87ZaS0b6YSKRMNU36/3Hdv0PFMeTjT/4\nJPH3fBiZrWQOFW7OkvrkFxn+x4/i6Np5Npqhu5TP53E4HLjdbrLZrGkMNE1jbGzsWGYdbceZNQx2\naWbpdPpYZgkomtNsYjbOxWIxy7zzRCJBb2+vZTwgGo2yvr5uPkY8HicSiZjNXIQQhEIhS8PU1ta2\n579HcTDomxni7/kjZLaqCrpQpLyeIHH/X9H1ltdY36frrK6uEo/H0XUdr9dLNps1zxs93uHW503X\ndWZnZ7lw4cKJ2zmcyaVxsx2B2i2cTOyMuaGrBBURtZ3mnUMlT73aKEDlS7+6ulojrzAwMIDL5TI/\nO5qm4Xa7bYOQiqMj/72bYFXlnC+S/vLDjce3mJ+fZ21tjXK5jJSyxig0o1AoNPTfPgmcyR1DV1eX\nmblSjaZpp7Ld41nA7XbT09PDyspKTSCwq6vLdP14PB5LDSQppaVOTjKZtN2JbG5umi4Cp9PJpUuX\nSKVS5PN5PB6PWTSnOF44wp1QtJHF6Leudi8UCpbzxU4QQpzI1OUzaRiCwSDhcJi1tTXzyyuEUG0K\nTziGfLLR/rO9vb3G0Hd3dzfsAIxWl1Z+4GafBav8deU6Ov64zw/jftJ58v96Hap0k4TPS8cv3mV5\nTy6X23McyahrOGnsy28ihOgSQjwohLix9X9DSo8Q4ilCiG8KIb4nhPiOEOKuqnMfFUJMCSG+vfXv\nKfsZz27o7e3l0qVLDAwMMDw8zOXLl3fVGF1xPPF6vfT19dHf39+w+/N4PIyMjNSkC3o8HtvAs5WO\nkoEyAieXvj+7D+/TLiO8HkQogAj46P7NN+B7tvX04/F49mwUBgcHT+Ric1+tPYUQvwXEpZT3CSHe\nAnRKKX+t7ppLgJRS3hBCDACPAk+QUm4IIT4K/L2U8q9387yqtadipxSLRaSUuFwuhBDMzMywublZ\n427y+XyMjY1ZfoHj8TjRaNQ8J6VkaGhIaSmdAoqzUcrxBO7LY9v2AZ+amrKMUTkcDsrlMg6Hg+7u\nblwuF5ubm7hcLjo7O49dRtJhtfZ8CfCjWz//KfAQUGMYpJTXq35eFEKsABFgY5/PrVDYUigUmJub\nM4PKTqeTSCRSYxSgMtHncjk2NzctA9hdXV20tbWRSqXMDKRmBUpSSgqFApqmqarnY47rXD+ucztL\nEBgZGSEajZpuSsO1pOs6HR0dDAwMmMkHp6EWar+GoVdKGQWQUkaFED3NLhZCPANwAxNVh/+HEOLt\nwD8Ab5FSWnfTUCh2iJSSqampmmrkYrFo2a0NKmmFzdKUnU7njr7syWSShYUFM9jo8/kYHh5uiYEw\nCulyuZwZ3FYZdIeHpmkMDg4SDoeZmJgw32Mj3blYLDb0YDjJbGsYhBBfBvosTr1tN08khOgH/gy4\nR0pphOnfCixRMRb3U9ltvNPm/nuBewElM6BoyubmpmVTlGZuU8PltFd/cC6XY25uruY5MpkM09PT\n+85jL5fLTE5OUiwWTS0mQ5TtpHVRqyeTybC8vEw+nzczy4LB4K4fJ5fLEYvFKBaLBINBOjs7D0R6\nYrseDCcx0GzFtoZBSvlCu3NCiGUhRP/WbqEfWLG5rg34f4H/R0ppJgsbuw0gL4T4E+DNTcZxPxXj\nwZUrV1SZqcKWUqlkawTsskuMyma7QPR2xGIxy8ctFArkcrl9JTYY1bUGhhbTwsLCiV6lptNppqen\nzdetVCoxMzOz6xhOIpFgfn7efJx0Ok08Huf8+fMtNw529QtCCAqFwqkxDPvdi34OuGfr53uAv62/\nQAjhBj4LfExK+Vd15/q3/hfATwHf3ed4FArbSdhoYG83WSQSCctCt51g18xdCLHvRu+JRMLyeDqd\nPpE58gZLS0uWvQysjtthSNnUx42KxaJtj+X94Pf7LXd/Vj0YTjL7NQz3AS8SQtwAXrT1O0KIK0KI\nD29d8x+B5wGvskhL/bgQ4jHgMSAM/OY+x6NQ4PV6GwrMhBC4/v/2zj1Isruq459zu6ef89ie6Xnt\nY3Z2N8sGKDEJCwIBTQIoRiGRl2gpQRNjpKSwECSKpVVW1AAFVKm8AohQUEl4CQGJQF5GLRNYQ0iA\nkOwm+3B25907u/Pod//8o/t2umfu7e7pnn7tnE/VVvfc+7t9z73d+zv39/ud8z09PQwPD1cMNV1b\nW6vrnG6CacYYDYN2wc0J29N6tX6GU1tbPXmrGRwcdMxhKRVqvBBoaPHZGLMIvNJh+xHghsL7LwBf\ncDn+qkbOryhu7Nmzh8XFxeKc8MDAAMPDw8VoIacpJRGpe+phcHCQWCxWNjqwM68bXXweGBjg7Nmz\nG7aHw+G2L0A3Uv/C4/EU9YVKEZGa12Q8Ho+rE2nGGkNPTw8HDhxgenqa1dVVLMsiEokUtbMuFLZl\n5rNy4WNPG0Wj0Q37IpEI8/PzjsfUK6Do8Xg4cOAA8/PzLC8vY1kW0Wi0rnyHVCrF3Nwc8Xgcn89H\nJBJhdXWVTCZTtvi8a9euumzdKuz6F7aTrbX+RTweZ2pqytEpAEUtolrkaXw+H36/f8PoQ0SaVofD\n7/czOTlZc/tcIsnKV+9h9dsP4olG6P+9awlccnFTbNsqGkpwaxea4KY0ysrKSjGKyBiD1+tl7969\nbV88TCaTZeGQ8GwGrWVZJBIJfD4f/f39bR0tZDIZnnzyScdR10UXXeQ6rZLJZHjqqaeqro2Ew+Ga\nF9ZTqRQnTpwoUzcdHBxkbCwfTJnL5UilUsWppWAwWCzl2uys5Fwiyemr30766VOYtQRYgvh9DN3y\nDgbeek1Tz+1EqxLcFKUr6e3t5eKLLy7q4NiF3dvN7OysYzjkzMwMhw4d6hgpjkr1L5aWlhgdHXXc\nbyeIVaM0CqsaPp+PgwcPEo/HyWQyBINBPB4Pp0+friixb1kWu3fvbqrM/vLtd5M+dqpY+4GcwcST\nLP7lP9L3+ldj9XamaKdmyCjbFlsOIxAIdIRTABzVXyGfy+A29WJjZ+K2glrqXziRSqVqcgybXci1\nxRAD80uk//MRTj3ymKtTsG3MZrOcOnWqqbLYK9984FmnUIrXS+LIT5p23kbREYOidADZbJa5uTnH\nxDwbt8XUXC7HzMxMUTnW7/ezc+dOwuFws8ylr6+PmZmZDdtFpOK6SigUYmlpqaIDE5FNL+bmVtaY\nfutfkPzBj6HHSy6ZxPuqXyD9rt8Gj/vzrz3CadbisSfiMhrJ5bD6OnO0ADpiUJQtw34K3UwMvi2j\n8dRTTxGLxRzb2Z2t25rC1NRUmZx4MpnkxIkTdedk1IKdpbw+JDgSiVQMz+3v78frVCingK2Au1mn\nNveuD5D4/mOYRBKzvIqkMnju/QGer91X9dhG80wqMfD7r0dC69atRPBE+vFf9rymnbdRdMSgKA1i\njGFxcZH5+fmi0ubIyIhjzLtNLpcrdt7Vpn/6+vrYuXOn4750Ou1YRMYYw8LCQsVM7uziEvH//iFW\nb4jgK16I9GyuO6hW/8IJy7KYmJgom8IJhUJMTExUdBiVyCWSrP7bg5Aq7+AlmcL7tfvJvslVvAHL\nspq6xhC8/FIi77qOsx/8LPi8YMDqCzP+pQ91zPSlE+oYFKVBYrEYs7Ozxc45m80yMzNTzGNwYnZ2\ntqZkumg0WoyucSKVSrnKfFRawF36xJ3EbrkNCs5Aerzs/PKH8f/8oao2lWLXv6iVbDZbFkEEz4av\nbiYEtBSTSIGbBMpK5VGTnQzZTCLv/B36f/e1JB5+DGugj8BLXoB0uABiZ1unKF3A/Py84xP73Jyj\ndBjJZJLFxcWqn1tL9a9KRWTcpnQSj/yU2N99CpNMYVbWMCtr5M6e58yb/xTjUvZyq1haWtqwjmKM\nYXV1te6pL2ugF+9uB+dkWZgXPdf1OBFhcnKyJU/unsEBwr/6CoIvu6TjnQKoY1CUhjDGuEYLuW13\nSq5zwrKsquGpXq+XHTt2bOjc7AQ7J85//puY5MZIHJNKE/+vR2qyrV6cit3YuAnUVUNEGPnIn+Xn\n8u2FZn8PVn+YiVvf7drxBwKBtmeOdyo6laQoDSAi+Hw+x5BHN0nsWjrAUCjE7t27a+q4du7cic/n\nY3FxkWw2SygUYnx83PX8ufMrkHOpS7EQo5mxMna+iFNiXCMS4sHLL2X3vZ9h6eN3kj56ksCLX8DA\nH7wB7+gQg9MeYrHYhlrfF5qMxVaijkFRGmR0dLRM9hnyHY/b3Lvf73ed/5+YmCAcDm9K50dEGB4e\nZnh4uKb2va+9grV7H8pn4paSyTK3e5Dw2lrFReR0Os3s7GxR+mNoaIihoaGapmQikQgLCwsbHENP\nT09NEhiV8F00wciH3rNh+9jYGCJSnL6zLIuxsbGmry10M+oYFKVB7FBSu+CM3+9ndHTUteMZGRnZ\nEEkkIvT397cks9n7yy8lc2gv1hPHkUQKYwn09JC+8VpMX4j5+Xn27t3reGwmk+HYsWPFdYJsNsvs\n7CyJRKKmWhY9PT3s27ePqamponPs7e1l9+7dTZvrt5306OhoMWqskyOCOgF1DIqyBfT19dX8BBoI\nBIo1hJPJJJZlMTg46Coj0QjxeJylpXx59R07dhAMBlk4GyP1wXfgefBRrAcfgd4QmV9/OeZQ3hlU\nimaKxWKOkh3nzp1jZGSkpumgYDDIwYMHyWazRWmKUty2N4qIuIbE5nI5lpeXSSaTxUileDzO/Px8\ncdvIyEjbtbRahToGRWkhy8vLTE9Pk0qligvEIyMjTVkEnZ2dLZu2icViRKPR/BqHx0P2yheSvfKF\nG46rNKWzurrqqj1kC/zVij1dlslkSKVSxTDfZDJZTOobHx9vinx2Kel0uihcWKpeW5qsmEqlWF5e\nZv/+/duivoY6BkWpEztz+dy5c8Wn/mqd6qlTp4qdTS6XK5YEHR8fByguYjdayzmRSGyYy7eT3ip1\nbPZ6hRt+v99Rz8kYs+m6E8aYotCdiJSNROxRSDqdbnr50tOnT5dFkNkOwsne6elp9u/f31R7OgF1\nDIpSB8YYTpw4URZ+aU+nuHWsc3NzjvkOsViMgYEBTp8+XeYY9uzZU/fUhVM2tH0+t8Q6r9fL5ORk\nRQG7oaGhMvkNm0AgsOkn6bm5uaLQXSVb7XWbZmCMYWVlpeb29YbUdhsNjV9FZFBEviciRwuvEZd2\n2ZKynneVbN8nIg8Xjr+zUB9aUTqe8+fPE4/HNzyRz83NueYvVJq7P3HiBMlksthJJpNJjh8/Xrda\nai1V0OwnfDuy6NChQzUl1E1OThar4NnHBwKBolPL5XIVxQBt7NFStetopvrpZqlXtqPbaPQqbwbu\nNcbcKiI3F/5+r0O7uDHmEoft7wc+Yoy5Q0Q+AVwPfLxBmxRl0ySTSebn50kkEgQCAYaHhys+pZ4/\nf96x0xYRVlZW2LFjx4Z9gUDA8em0kjT0uXPniEQcn7cq0t/fz+zsbMU2dq6EbXethMNh9uzZw/Hj\nx4G8Izh79ixLS0sEg8HiiMTn8zE0NMSOHTs2rBPUKhFuq8U2C7tecy2jBrsq4Hag0RWva4DPFd5/\nDri21gMl/0u8CvhKPccrylYRj8d5+umnWVpaIpFIsLS0xLFjxypqGVVaEHXbNzo66lhIPhwOOzqH\nXC5Xt/Knz+dzFd6zsaN/6gndnJ6edpwWK71nqVSK6elpfvazn22QB6lF7sMO4W10vaUau3btwuv1\nFgMA7LrgAwMDxegou1Som/bVhUajI4ZRY8w0gDFmWkTcUgkDInIEyAC3GmO+DgwBS8YYe9w9BbS3\niK2yLZmennYMwTxz5gwXXXSR4zGRSMRxrt3u6J0IBoNMTk4yMzNDIpHA4/EwPDyMz+cjHo9vsMGy\nrIaSviKRCKFQiKNHj27YZ3e69WDXZN5M+/n5eYLBYFlI786dOzl+/LjjPfR4PAwNDbXkCb2np4fn\nPOc5G8JVRYRsNks6naanp6fp0VGdRFXHICL3AE4pnO/bxHkmjDFnRGQ/cJ+IPA441QZ0nXAUkRuB\nGyGfHaooW4XbyCCRSGCMcXyiDgaDjI2NFVVUId+h7d27t2LoaTgc5sCBA2Xb7OkS+3z2ZwUCgYaL\n7fj9fsbHx5mZmSn7bL/f7zjdVStuiq5u2NLkpY4hFApx4MABFhYWiMfjBINBotFoW3IFLMtyLDDk\n8Xi2lUOwqeoYjDGuYuYiMisi44XRwjjgKCdpjDlTeH1GRB4ALgW+CuwQEW9h1LAbOFPBjtuA2wAO\nHz5c+y9SUarg8XgcF0vtKQQ3hoaGGBgYYHV1Fcuy6O3trWtaRkTYt28fCwsLZclo0Wh0SzJ0h4aG\nCAaDxGIxMpkMAwMDFQv/1GKv24ipEk73OBAI1JQxrbSWRqeS7gKuA24tvH5jfYNCpNKaMSYpIlHg\ncuADxhgjIvcDbwTucDteUZrN4ODghph/u/OrhtfrrVjKslbs80Wj0aYku4VCoYa1iEoZGxsjnU6z\nsrJSzEFwc7DQ2NSV0noadQy3Al8SkeuBU8CbAETkMHCTMeYG4LnAJ0UkR36x+1ZjzE8Lx78XuENE\nbgF+CHymQXsUZdOMjIyQTqeLiVbGGPr7+5siUeHEwsJCWY7D4OBgUfitU7Esi71795JKpYp5Bl6v\nl7m5uQ1hqCJCT09PSxduM5kMsViMRCJBMBgkEolsm1DTrUA2MxTsFA4fPmyOHDnSbjOUC4xMJkMy\nmcTn8206i7dezp0756jMOjQ0tKnKaKWYTIa1ex4idewUvov3E7ryRUgL5snj8TgnT54kl8sV8zH8\nfj+RSIRIJNKyufpEIsEzzzxTtMGOLNq/f39TQ1+7ARH5X2PM4Wrt1IUqSgGv19vyp8rSkqA29kKt\nU3hrNTJzMU7/2h+RXVjCJFJIwId31wi7vvUxPDuaJzOdy+UcE/JSqRT9/f0tXcA9c+bMBnmNbDbL\n9PR03eVDtxtavkhR2ohbljQ4L9ZWY/7dHyQzNYtZWYNMBrOyRvqZKRb/+qONmFmV5eVlx+3GmOKC\neiuoJPnhpPGkOKOOQVHaiFtopmVZm37KNrkca9/9H8iscyjpDCvfuK9eE2uiVIl0PZWcXzNwG2V1\n8ppNp6GOQVHaiNMis11Ypr6OzGXNsE7NpVpxy7ewLKulldJsuW6ne9pI3sZ2Qx2DorSRUCjEvn37\niuU8g8EgExMTdekjiWUR/KUXgWfdf2uvh/DVr9gii52xE+ZKO2QRIRgM0tvb29Rzr2d8fJxgMFgm\nZxEKhepezN+OaFSSolxApKdmOf0rf0hudQ2zGkfCQTyDA+z690/iHWluuKhdn8JOfBsYGCASibRt\nCicejxdDabdDcZ1a0KgkRdmG9OweZeLInazedT+poyfxPf8AvVf/IuJvvqK9PY2zFQl/W0EwGFSH\nUCfqGBTlAsMK+un7zdfU1NYuFLS4uEg2myUcDjM2NtZ0RVOls1HHoCjbmJmZGWKxWDGi6Pz586ys\nrHDw4MGWJfkpnYcuPivKNsWWjVi/zpjL5VhYWGiTVUonoI5BUbYpyWTSdWG4UpEi5cJHHYOibFN6\nenpck9LavcaQzWZZWVkhkUi01Y7tiq4xKMo2xefzEQ6HWV1d3SDi187axvPz88zNzRWVbv1+P5OT\nk6qO2kJ0xKAo25g9e/YUy1ja8tgTExNtC/NcXl4uSpDbKq2JRIKTJ0+2xZ7tirpgRdnGeDweJiYm\nyOVyZLNZvF5vWzWF1hdMskkkEqRSqbZPcW0X1DEoioJlWU2pHLdZ3AT3RKQutVmlPtr/S1AURSlg\nT2s5sd2L7LSShhyDiAyKyPdE5GjhdYPyl4hcKSKPlvxLiMi1hX3/IiLHS/Zd0og9iqJ0N9FoFI/H\ns0GMb3x8vCNGNNuFRu/0zcC9xpiDwL2Fv8swxtxvjLnEGHMJcBWwBny3pMl77P3GmEcbtEdRlC7G\n6/Vy8OBBotEowWCQ/v5+9u3bV5farFI/ja4xXANcUXj/OeAB4L0V2r8RuNsYo9kziqI44vF4GB0d\nZXR0tN2mbFsaHTGMGmOmAQqvI1XavwW4fd22vxWRx0TkIyKik4iKoihtpuqIQUTuAZwqXLxvMycS\nkXHg54DvlGz+c2AG8AG3kR9t/I3L8TcCNwJMTExs5tSKoijKJqjqGIwxr3LbJyKzIjJujJkudPxz\nFT7qzcC/GmPSJZ89XXibFJHPAu+uYMdt5J0Hhw8f7r7qQoqiKF1Co1NJdwHXFd5fB3yjQtvfYt00\nUsGZIPkQhGuBHzdoj6IoitIgjTqGW4FXi8hR4NWFvxGRwyLyabuRiEwCe4D/WHf8F0XkceBxIArc\n0qA9iqIoSoM0FJVkjFkEXumw/QhwQ8nfJ4BdDu2uauT8iqIoytYjbrK7nYyIzAPNVNWKAt1eqaTb\nr6Hb7Yfuv4Zutx+6/xq22v69xpjhao260jE0GxE5Yow53G47GqHbr6Hb7Yfuv4Zutx+6/xraZb/m\nmCuKoihlqGNQFEVRylDH4Mxt7TZgC+j2a+h2+6H7r6Hb7Yfuv4a22K9rDIqiKEoZOmJQFEVRylDH\nAIjIm0TkJyKSExHXCAAReY2IPCkix0Rkg8R4O6mlNkahXbak/sVdrbbTwZ6K91RE/CJyZ2H/w4Vk\nyY6hBvvfJiLzJff8BqfPaRci8s8iMicijqoDkucfCtf3mIhc1mobq1HDNVwhIudKvoO/arWNlRCR\nPSJyv4g8UeiH3unQprXfgzFm2/8DngscIi8bftiljQd4GthPXvTvR8Dz2m17iX0fAG4uvL8ZeL9L\nu5V227qZewq8HfhE4f1bgDvbbfcm7X8b8E/ttrXCNfwicBnwY5f9VwN3AwK8BHi43TbXcQ1XAN9q\nt50V7B8HLiu87wOecvgdtfR70BEDYIx5whjzZJVmLwaOGWOeMcakgDvI16PoFK4hXxODwuu1bbSl\nVmq5p6XX9RXgldLOavXldPpvoirGmAeBWIUm1wCfN3keAnbYGmedQg3X0NEYY6aNMY8U3i8DT7BR\nKaKl34M6htrZBfxfyd9TOMh8tJFaa2MEROSIiDxkl1htI7Xc02IbY0wGOAcMtcS66tT6m3hDYfj/\nFRHZ0xrTtoxO/93XyktF5EcicreIPL/dxrhRmCq9FHh43a6Wfg+NVnDrGirVlTDGVFKFLX6Ew7aW\nhnRtUW2MCWPMGRHZD9wnIo8bY57eGgs3TS33tO33vQK12PZN4HZjTFJEbiI/+ukmjbBOvv+18gh5\nKYgVEbka+DpwsM02bUBEeoGvAn9ijDm/frfDIU37HraNYzAV6krUyBR5hVib3cCZBj9zU1S6hlpr\nYxhjzhRenxGRB8g/nbTLMdRyT+02UyLiBQbonGmDqvabvNCkzaeA97fArq2k7b/7RintZI0x3xaR\nj4lI1BjTMRpKItJD3il80RjzNYcmLf0edCqpdn4AHBSRfSLiI78Q2vaonhKq1sYQkYhdPlVEosDl\nwE9bZuFGarmnpdf1RuA+U1iN6wCq2r9uHvh15OePu4m7gLcWomJeApwzzxbY6gpEZMxelxKRF5Pv\n9xYrH9U6CrZ9BnjCGPNhl2at/R7avSLfCf+A3yDvkZPALPCdwvadwLdL2l1NPmLgafJTUG23vcS2\nIeBe4GjhdbCw/TDw6cL7l5GvffGjwuv1HWD3hntKvrzr6wrvA8CXgWPA94H97bZ5k/b/PfCTwj2/\nH7i43Tavs/92YBpIF/4PXA/cBNxU2C/ARwvX9zguUXsdfg1/XPIdPAS8rN02r7P/5eSnhR4DHi38\nu7qd34NmPiuKoihl6FSSoiiKUoY6BkVRFKUMdQyKoihKGeoYFEVRlDLUMSiKoihlqGNQFEVRylDH\noCiKopShjkFRFEUp4/8BtQdQhZfMjhgAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots()\n", + "\n", + "# We are going to be reusing this data a few times\n", + "data['color'] = data['y'].apply(lambda x: 'crimson' if x else 'blue')\n", + " \n", + "ax.scatter(data['x1'], data['x2'],\n", + " color=data[['color', 'is_labeled']].T.apply(lambda x: x['color'] if x['is_labeled'] else 'lightgray'))\n", + "\n", + "ax.set_xlim(xlim)\n", + "ax.set_ylim(ylim)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We have a few examples of each class, are going going to try more of the red ones" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training Model\n", + "Train a basic SVC to use for use during the search procedure" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Our basic classifier will be a SVM with rbf kernel\n", + "base_clf = GaussianProcessClassifier(RBF(1.0))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "labeled_subset = data.query('is_labeled == True')" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "GaussianProcessClassifier(copy_X_train=True, kernel=RBF(length_scale=1),\n", + " max_iter_predict=100, multi_class='one_vs_rest', n_jobs=None,\n", + " n_restarts_optimizer=0, optimizer='fmin_l_bfgs_b',\n", + " random_state=None, warm_start=False)" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "base_clf.fit(labeled_subset[['x1', 'x2']], labeled_subset['y'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plot the decision surface" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "xx, yy = np.meshgrid(np.linspace(*xlim, 8), np.linspace(*ylim, 8))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "prob = base_clf.predict_proba(list(zip(xx.flatten(), yy.flatten())))[:, 0].reshape(xx.shape, order='C')" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAADuCAYAAAA9UKBmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsvXeUJNd13/951XmmZ3pynp3ZgEVc\nhF1kMIAZkmxSOhR/FC1blE39INuSaAVbR7It/mjJR6IpWzq0bNoHImlKogkqkRIpkqZggkgkFsQi\nLIDdRdg8OXdP90xPp3q/P6q7p7q6Ynf1bMB8z5kzVe/dd9+r6qr7rXtfElJKdrGLXexiF7uoQLnU\nDdjFLnaxi11cXtglhl3sYhe72EUNdolhF7vYxS52UYNdYtjFLnaxi13UYJcYdrGLXexiFzXYJYZd\n7GIXu9hFDXaJYRe72MUudlGDXWLYxS52sYtd1GCXGHaxi13sYhc1CF7qBjSCnp4+OTY22ZSOVk74\nvlJ1X471+gkhrqx6vOpxK+9GzkvdTrJ+1tf0vfXrQb5CX4jnXnhhWUrZ7yR3RRLD2Ngk3/jGMct8\nN7+ZmYxTOWO+/tzquBF5N+XdlmvkOt3U7ZesX/Db2JnJ251bHTvJ25UzypiVMabb6TWT08tY5XtJ\nt6rfjQ63bTPqr0mn/PBZvRxmL57XF9bNuR28Gho/dJYhOjsvuJHzJZQkhPiCEGJRCPGKRf5PCyFe\nKv/9QAhxiy7vvBDiZSHEi0IIa2vvI65Qst/FLnaxix2BX30MXwQesMk/B7xdSnkz8DvAQ4b8d0gp\nb5VS3t5sQxo1+l7LtYpc/NTrl7fQbJ2XCy7ntl1N2KmQnSUaDRnsBPz2FqRsybX4QgxSyieAVZv8\nH0gp18qnR4ExP+qtraO1z4Nduavd4Li9vkt5Hy71b+AmlHipcLm1xwrNEko1jLTTuFT9Fi38YS/F\nqKSPAd/WnUvg74UQzwkhHrQqJIR4UAhxTAhxbHV1abtwawizYexUW5oJcTZTT7NyrcTl0IZLjavt\nHrSi78gVWvEl2OpwgI/Y0c5nIcQ70IjhLbrk+6SUs0KIAeARIcSrZQ+kBlLKhyiHoG6++faGtpFw\n26nbqB63Zbx2WjdS506Gka4kYyRl44akmbJW5ZvV6WdbWgE3nfItR7Mv1k484JfZF9iOeQxCiJuB\nzwEfkFKuVNKllLPl/4vA14A7W1F/M6RwuYWRGvEWWjXQ4XIjhZ38KLvcrv3NgoYJxo8frFFX3Q9y\n2sEHbkeIQQixB/gq8E+klK/r0tuFEB2VY+C9gOnIpmbgl6fgVKaRUW/N1umHfCM6LrcQnh5Xm0E3\njqq80tp/2cHLA9Lql+ky/fryJZQkhHgYuB/oE0JMA/8fEAKQUv5P4BNAL/BZodF9sTwCaRD4Wjkt\nCHxZSvl//GiTVndjeY3o8DO85KU9bj5g/CaWZp9Tr+Ub+UK8FCEaL3W6kb1UYabLFb7fi2ZfjFaH\nGy5hn4QvxCCl/IhD/s8BP2eSfha4pb5Es+1pLt9K7lL3LVyppOAXkahqkVTqNOn0BYRQ6OzcRyKx\nFyHMHV87w+rW6BrlGi2307jU9XtBM/0QDU1sc4tWfem3ihB8JJIrcuazEV7uRyu+zC9Fp67fHqjf\nz6r1OyFR1QKKEkCIgAd9KjMz3yOfX0fKEgDLyy+SzS4wPHyvbTuaJQf7dnnT0er2XI3w9Z40+qD7\n9wJ4L+OXvAdckcTQSHy7GSPZKCk06y3YtcEpv9HrlVJSKGQAQSjUDrh/K53q3NxcYHHxGMXiJkII\nOjom6Ou7DUVxfgw3NmbJ59NVUtDqK7GxMUculyQS6bJtl5+jifw24G8mQmh2DSbf7pOTe+3FVXej\n26mcX4bAJ1yRxOAFfocRG/n9djqE1OgztrW1yuzs05RKWwAEgzGGh+91NLpukMslmZt7qmrYpZSs\nr1+gVMozPHyfY/lsdgkpi2YtYHNzybaNlXaaGRW/vQar41bicglvuVkjyk16w/AzjOS2Djf5O+n6\n+4SrctntikfRalJw82HQSlIwu8ZGSaFUyjM9/RjF4gZSlpCyRKGQYXr6MVS13iB7vb9ra6dqvvY1\nqGxuzlEsbjqWDwajFn0JCsFgzFV7mokSNOOReW3PmxGNrNBa17/gBDui8MO990IKTg9so0bMJ1zx\nxKC/f43eRzcG1k9ScGqL1bGTPrcGzkwunb6I2axBKVUymZmasqqqkk5fZH7+aRYXj5HLrdWVMyKf\nT1vkBCgUnImho2MSs7CWEAHa24d17bXXc6UZYy8fvZfi2lrtETU9Z6GZr8NWkIKdkbrEZKDHFRtK\n8uveufk9/SYFr+W9ts9tOX16oZA1+aIHKUsUi1nde6YyM/MYudxaWV6QTl+gr+9WEon9lnVHo73k\n8ymo+7orEQ53OLY9GIwxMvI25uefRlUL5bQ2hofvrevErrTVSyevmzCLnYyXcJJdmlM7dio8tVPw\nMiLJ1GuwGo1khFtvwa6s2/xWhJN2GFcsMTQLLwRvl9+MnJvyjZKU1zpjsT6SySDGOL4QCtFoX/U8\nnb6oIwUAiZQllpdfpKNjD4oSMq2ru/s60ukLNfqFCNDZuY9AIOJ8EUAs1s/k5D8kn19HCIVQKI6w\nsSZuDbkbea/6rzY0cp1+DEO1KtuSTuhmXXyrcs3EMC8RrvhQkldYeWpmv2+zpKDX4eRBOh27OXfS\nbdauCtrahohEEjVf30IEiEb7iEZ7q2mZzJSpZwGCbHbJJF1DKNTO+Pi7aWsbRoggwWAbvb2H6Ou7\n1fki9LUIQSSSIBzusCWFCuzuu6qqFItb1I50ctbnJq9ZL66ZcpfS3rg12C0lVKN34Cas5JUgzAyE\nkxFxSm8E+vCTmz+XeFN4DF5fIi8G2M3z5Mfz6NXoePeYBSMj95NKvcH6+nmEEHR27iWROFBjgK08\nAi3P/nEKhzsZGXmrQ8tbA+PXfSp1huXll8sejKCr6wB9fYcQQvHkCTQagnozeRsVNDNqqakwkhXM\nCMMtKdide0nzgh1k/KuWGBr58vP6Rb4TpNBMm7ymK0qA7u7r6O6+zlJ/IrGfjY2ZOq9BUQI1ISe/\nnuFWzBfIZKZZWnqx5hqSydMA9Pc7T8RvtVG/0kjDrq3NhqA857v1FpwMQKOk4Cch+EkEHnVd8cTg\n9d75Re7NkIJbb8APUvBKFE6Ixfrp7r6BtbUTVCKRQgQYHn4bUgpyuVVUtUQk0u1q4poTpIRiMcvq\n6stsbMyhKAE6O/fT3X2t5VIYTvpWVk7UEZuUJZLJ0/T13YQQAUfjrM93c+zUJruOZ6f8KxVu+iAc\nr9fuofcSQvLzxXXSYwcvZVroQVyxxODH/WuWEIznXp8tv0JHO0EIenR3X09Hx162tpZQlBCx2ACF\nQpqLF79JqZSvyvX3H6GjY6KpulS1wPT0I5RKOUCiqrC2dpJcbtXVxDgzWM+bkKhqkUBA62fx0xC3\nWtdOk4axrlZ6DWbHpnMX3D7cdsTh9cV1c+62Pc3K+Igrlhjs4He4xSy/EUPeSlKQUmVl5QTJ5GlU\ntUg02kt//21Eo92u9LipQ49gMEo8Pl6te3b28eqM6QqWlo4RiXQRDicarnt9/Vx5eOp2I6Qssbk5\nTy63TjjcWWd41tdhehqGh6G7u15nJNJl2lEuRBBFCbtum1+ewpWCZpezMMvzQjB18OoJ2KXb6fPy\n9eeVnBrN97ucAVcsMfjppfnlJbitw4/QkTFvfv5ZMpnpaohka2uZ6envsWfPewmF4taKXOq3w+bm\nnMXs6BLr62fo67vNY+3b1mFra7ku7FORyeWShMOdNe/6H/4h/NVfQSgEhQK8+93wiU9o5xX09t7M\nzMxjNXqFCNDXdzNSCr72Nfj852F1Fa69Fn7t1+Dmmz1eAtttqu30nmZx8WXy+U0ikU6Gh28mHh9s\nTPkVhEY8Cst0O2+h0a91P7/m3NTnNa8RuSZwVQ9XlbL2zyrPSYfVufHYL1Jwapcxr1DImg4jlbLE\n2tpr1opM9BaLW6RS51hfP0uxuGUnzebmPBcufIv5+e9jvo4RDjqsdVf+tMlv5o9pKNRWc/7lL8NX\nvwr5PGxsaP+/+134zGd0mqWkVMoRDicQIogQAcLhBENDd5NI7OMLX4Df/32YnYWtLTh+HB58EE6d\nctFqh2dpdfUc09PPkMtpiwFuba1x/vxTZDILrnW0Gs14BX6Fl5zCSID917pbb8EtKRhfSC8vq17G\nqs1OxsPOkLUIvhCDEOILQohFIYTp7mtCw38VQpwWQrwkhDisy/uoEOKN8t9HG22D2/vn9t6aPQte\nPEpjGbflnQjBrE35fLpu9m9ZgnT6vOVyF3odUsL6+nkuXPg7lpdfYHn5BS5e/DvW189Wden/crk1\n5ue/T7GYsWyvtlzFiPUFWV6nSjL5OlNT3yGdnirXWaOZUKidSKS3JvV//2/NmOuRy8HXvgaqqp2v\nrBxnfv5pcrnVKplVltXI5eALXzDX8dnP1rcznZ7hzJlv8eqrf8mZM39HKnVBdw3Ga5IsLr5sSt5z\nc8cb+gC91CTSLOyIpCFvwS7NCykYX1yzdKuyVnrM0p3yvBCO2z+X8Mtj+CLwgE3+jwDXlP8eBP4H\ngBCiB223t7vQ9nr+/4QQJlHhejR6/9zqtjp3+2y4/cDwQgpW56FQvM7gbMuVWFx8ltXVk3Vt0Oso\nFjdZXj6GlCpSFtEW0lNZXn6+vAx3LdbWXrWsEzRjGwp1Vvsh3EJKydzcU6yuvkI+v14mHlEegSQA\nhba2QUZG3l43wS2VMteZz2t/hcIGqdRpjEt35/Op8hBWqzbBawbHK52eYW7uaPneSAqFTebnj5FM\nnjUtry0tkjPVn8tZrSNVq8MN/CaLpoaOmuQ36lV49hbs8ty+aI16CWb5zXgNRpkd8Bz82sHtCSHE\npI3IB4A/ldpn61EhRJcQYhhtO9BHpJSrAEKIR9AI5uHm2uNPOSeCcKPHDy/B6TwYbKO9fZSNjVlT\nY60NxXyVrq6DNRPUcrkk6fT5Mglg6lVICRsb03R1XVuTns+vW7RYEA4n6OzcS0eH9e5qVtjaWjHp\nV1CBIAMDR2hvH9ENg5Xo+yMOHYJjx+p1jo9DNArr60to30Kq4RpLZDJz9PbuqXoWZjr0WF5+yfTr\nf2npFbq69tWVFyKAogSr6zzpYQyJ7TT87gBvVYjJFGYvkl9fgE7HTvq8pHnJ3wHsVB/DKDClO58u\np1ml10EI8aAQ4pgQ4tjq6pIf3lIVZl/xjXqQrSIFq4+QCgYH77RdxE4IUWPMU6k3mJn5LqnU66yv\nnyGdPgemyxdLpKy3ltFoj2U9IyNvJxrtI5udd7Vyqh5bWyum9UlZJJ9fr5kboapFcrlVisUsAL/6\nqxCLQXm0KUJohPAbv6GdW6/JJAgGI8Ri8MEPamVqrxX++T+vTcvnN8jnFTY2ameCa0tsqOU262oQ\ngr6+6+tCfkIEGBy80aJdlwZeDbvfk9hsw0heDa2Vt+CGFPQvnZ1xsfMQnNL06TvgCbjFTo1KMvup\npU16faKUDwEPAdx00+1N3z03ISDjeSOEYJfnh74KtFE1t1IoZNjYmDXRpxIMxgBtwtjKynGDATb/\nVBZCoa2tvp+gq+u68iio2kXx4vEJ5uaeKJOQAFTi8XH6++9wtbaRtu9CgPrF/ALV9kspWVs7RTL5\narWOtrYRDhy4ky99Kcj/+l9w8iTs2wc/+7NwXXkid1vbIIoSoFSqXyiws3MfUsKv/IpGLg8/rPU1\nDA7Cv/k3cPvt2n3XZk7DH/3RvRw9OoiUgoGBDD//88e4/vplAoGIpZfU16d5XcvLp8rzJcIMDh6i\nq2uP432poNKGyw1uyMKpT8FVp7MeTt5Cs6Rglm/XBjfnbnQ2Ch917hQxTAN6Z3wMmC2n329If6yV\nDWmEEOzKGfPckoDfxFRBV9d1bG4uGMIcCtFof9mwSrLZBcw5uRZCBEgkDhCJbM9DUNUSGxszlEpb\n9PcfZn39HLncKoFAmETiINnsIrlcEj2/ZzLThMPddHVd41hne/soy8svmlyrqPZXZDIXSSZr+zg2\nN2dZWnqOiYm7+OQn6/VqBlVhZOR+5uaerJmINzBwB+FwJ6B5G7/wC/Av/oXW6RyNgmKw87/8y3D8\n+BDFopYxN9fJ7/3e2/j0p7/LLbccsLw2IQT9/dfR338tqlpEUYKuyNILWkUajXgJjayK6rrT2a0B\nbxUpNGIwmjXcO+hN7BQxfB34RSHEV9A6mlNSyjkhxHeA39V1OL8X+E2/K/dC2l4IoZK/sbHI0tJJ\n8vkN2tr66Ou7oWaPgVZ5CWblo9E++vvvYHn5ebS+A0l7+zADA3dU5bY7cuvR3j5e/TKPx8drQka5\nXIrZ2cfQOqhVhBDEYgPs3fvjCCFQ1QIrKy9jdPqkLJFKveGKGBQlyMjI/Sws/KAaIgoEogwO3k0g\noE0+M+v4llJlY2MKVT1c7UfR+g5myGYXCYXa6OzcSySSYGLix6pLh0ejPTXhncoXuaJonoMR58/D\nyy9DoVDLFoWCwqOP3s3995tP5tN/6QshCASsFyPcCbQqHNSs52A8t+10tjL2xjQ/ScErITRqzHe6\nnAG+EIMQ4mG0L/8+IcQ02kijEICU8n8C3wJ+FDgNbAL/tJy3KoT4HeDZsqrfrnRENwM3xtxNuhsj\nnkxeZHb22aqhSqU2SadnmJx8N5FIp+NzZpbXCDnp0dGxh3h8jGJxA0UJVw1qBW1tw5hBiADd3dcS\nidQPDJNSsrDwA1Q1r0uDbHaR9fUzJBIHUFXrUUraBDirC6m1DJFIgvHxBygWNwAIBttrvqy15TGs\n6imUO3mLzMw8SqGQKf82CmtrpxgZeRuxWL9lH4kTpqa0yXI5QxNUVWF2ViOFyzXc0wo04hWYydj2\nOVh5C/pjN0TgVocbA+E3IbiV3yGvwa9RSR9xyJfAL1jkfQH4gvc6/ZNv1BBLqTI//4Lh61Vbb2dx\n8WXGxu6rK+OlbrdegkkqQgjLGc+KEmRo6F7m57+PZpQlIOnpuVFHCrWKC4V09Qu+tv4S6+vnSSQO\nEAhECAZjVYO+DVGz/aZZe/WygG37Y7E+NjZmTK4rRCCg9Rwnk69RKKTZ7kdRkRLm548yOfkPGg7h\nXHONNvTViHAYbnW5tUShkGVrK0U43E4k4rx7nR7NEE6zQ0ubSTP2IVjV79lbMMvTp7k19m69D6e6\nvRgmL0bJC3wgjyt2SQw3aOS+e/k6LxRypktBAGSzy57qatZLKKfaF9KhrW2Qycn3l5ezKNHWNkgw\nGLXRYadbM75CCAYGbmdu7qmyQZYIoaAoIXp63I68qdRjbUV6eg6V+1FqO757e2+rGvxM5iJmo5tU\nNU+hkLHdTtTui39wEN73Pnjkke2JcIoiiUbhwx9WAbOJhhW9krm550gmz5c72FVisR4mJ99yyUNL\nVmjUI3Bb1okw6kYiuSEEt/0KXknBj1hvI3mNyDWJq4IY/CBpr4ZYSsovs3lBzcj6TwrNEgJo4ZZs\ndhlFCdDePurq6zkU6kBRwpRKtV5DZSRSBbFYP+Pj7yGVOk0+nyYW6y9v3+l+cToNEityCIc7GB9/\nD2trr7K1tUwo1E5X1/XEYtv7QZjPBK/keRulbSSKT34SDhyAhx+WZDIqhw7N8dM//TKrq5tIeYD+\n/ptN7+nq6hskkxeo9NEAZLMrTE8/y8TEvZ7apF2H5yINDT31kuZXp7StHjfGeqdIoVFCaDTe7QZv\nZo9hJz02q+dAUYJ0du5hfb3261SIAD0913smhVZ7CQCp1FlWVl6gMoVFUQIMD7/FtF9BDyEEQ0N3\nMzv7JFpYRqWyzpA2f2LbkIdCcc9bdprDmhxCoTgDA7dbluzs3MfKSv0ktFCog1Co3bw2KcnlVslk\nZlAUhY6OCVPPIhCAj34UPvCBcywubocSpYS1tdMoSpD+/noPaWXljbr2SKmyvj7Nq6/+HdFogsHB\nG2hv760r6yUM5LVjudHwlNvF7yqE4Xl+hJm3oDfkbvsarGTckIKfL+uux3Bp4RcZu+lfGho6gpQq\n6fQ02raQkt7eG+nsNB+b7tdzJqXkm9/U1gfKZOCtb4WPfQx6621KFbncGisrL5ZJTCOyUqnI7OyT\n5bi78Uu6ttJotIeJiQdIp6colbJEo320tQ3pvo7tLqLR4Lg1Odihs3M/2ewSm5tz1forfSumtUjJ\n0tJzpNMXysZbsLb2Gn19t9LVZT5xcGXlpImhL7G6+hp9fTfUeQ1ms54rKBQ2KRQ2yWQW2bv3LXR0\ntG7F1WZGE7klHa+kUZdm9yy56S9olBS8ko1T2/wKUewwrlhiaOS+NeNlWHsNAUZG7qZUylEsbhEM\ntpvuXOZWn5u2gOQP/gD+5m8gm9Xeqq9+VfLoo/Dnfw6JRK1sBevr5zCfVayyublAe/uQfUPQZg93\ndVmP1beGc9+Bn9A8nHvI5VLkcisEArEyiZmHkba2lnWkoLVXyhLLyy/Q0TGqCw1KkskzrK29Zrnh\nj6qWkLKEENvPgRAQjw+RSl20bbeUJWZmnue6637E5tqc01s5O9lLGMkLQdTBylswyzM7NqZ57Xtw\no9upTqc0u/RLiKt62W3Yfp68eAlun0f9eSAQIRxO+EoK9e2WgGR5Gf76r7dJAaBYFKTTWnqt/DYq\nu6DV11NkYeEoyeTrWK3C6h+kaRucyzSGSKSLzs79tLeP2PYtrK29Vvf1r0FhY2O+era09CJLS8cp\nFIwjr7YRCsVM+zgGBg4RCIQd+zhyubStd9EqeDXsXrwKu/S6NKfhqVZf9XaGvRlSsDMgZm1zSrNL\nbxR6fXZ/LnHVEUOD96FattHzRsJT3kKO2wmvvaYNjzQilxM8+2xFtl55PD5q2SmrhUBOsr5+3rpR\nvuLy+UoqFjfZ3KxfRqSCSkioVMqRSp2xIJCKbICBgVsRQtQZvXC4nWuueYDe3mtpa+uz7SB/5ZWv\nMzv7cg1Ru1kywnaEj8sRQc10LDvpsWpHHdyGZIx5fpCCFckYy1gRhxNBNBru8MHgu8WbKpTkRZcX\nEnDzzOZyKdLpKaSUdHSMEw53eWhPbcLAABRNRskGApJR0yUINbS3jxKJnCaXS5oaN21jn1MkEnut\nlVxByOfT5f2h1wiFOujuvoFYrH5i2/r6Obbnc9RCSrU6B6Oy74VZOA4EsVgPfX032e7IFgxGGRo6\nBMDy8mssLLxi+VssLb1eHhxwg6vrNYOfo47sZL14EFbpDXkLdsbZ7r/VsRtCMDu2KtMoCbRS3gWu\nWGLwA14Jwc25Wd7KyklWV09VDUoy+TpdXdfR21s/csWJFECbZDU5CadPS4rF7bcrFIKf+imrBmmT\n3kZG3kY6PcXSksn61FC3b7PfKBbhz/5M8Od/LtjY0Ban+1f/SjI5aVfKe4A8l1vTbd8pKRTSZLML\nDA/fR1tbbV+Ktsuc+X3r7JysDrUNhdosvYWOjjFGR+/xZGR7ew9SLOZMRyuBRg6Li68xNHR91Wtp\nVf+CH2EkN+1r2FtoxF13QwpOaXZtcpPnBDeyLTD8TrjqQklusVOkkM+nWV09WTVQlDs1k8lX6/Y0\ncEMKFfzRH8HhwxAKSaJRSW+v5FOfkhww7RfWhyMUOjsnCAbNh2yGQt5m4nrFf/yPgs9/XrC8LMhm\nBU89BT/7s4LFRW96NjbmmJt7kunp77K29lrdRMPl5eNoE+C2r137Cn++Tldb22BNR/E2FLq7D1bP\nQqE20w5sbXLddYa0Wk3J5HneeOPbnDz5Vc6efZTNzRUURTA8fDM33PB+rMhPVYtoQ4NNs23rtMvz\nK4zUKHlYegtOHoIeViEfs3Qn78AtKbgJITnBLgTkNUzkFGJqMOT0pvIY3HiHVmmNkAJAJjNjoV9l\nY2OmuqqnlNpwxs1Nbf/ftraBmk11NBlJsbiBEArd3W189rOwtibZ2ICRkfpVQO2Ipbf3EIuL22s8\ngWbg+voa3PXeBRYX4e//XpDPb1sFKQW5nOTLXxb88i+btbfeuqyunqxZXTWfT5FOn2Ns7N3Vzv9c\nznzJrUJho7qyaQXt7SOEw53k86mqTm3i3hiRSGdN+ZGRu1lYeL68XarW0Tw0dIRotNvSqC4vv8bi\n4nbIaHNzmXPnHmP//ncQi/UQCISIxbrIZtfqymsd2UYiMj82q9vuvJLmhTSaCSV58mTchpDMZI3H\nXkNMbvVandvp8ZLnRcZnvCmIwevv4hcpaMcKQpiVEeiN3sbGDAsLR1FVwfPPD/HDHxbp6+vhJ3+y\ngwMHtCU2FhePlpeLloRCnQwN3U13d5xu07lp9g9TPD6KogRYWTlBsbhBKNRBb++NxGIDtuWawdmz\nWqe5ca2hQkHwyivuHv5SKUcyuR2WA80TKBY3Sae1NZu0OQv1s7RB85jqDa3C6Og7yhsWXUCIAF1d\n++no2KOT0f4rSpDh4TsZHDyClMXyKCNh0Ld9LKXK0tKJulCRlCXm519h7963ATA8fAvnzj1ZR9Sj\no7f4Hkby0olsN3ehpd4CuAshWRl9v0jBb0JoliSakfeAq5YYGiVxv0ihgnh8lJWVl+rShdjeX6BQ\n2GJh4Silksp/+k9v4dVX+9jaCqEoKt/4BvzKr+Q5fPhJ9GsD5fNJZmYeY2LiR+sMnRMpVNDWNlQX\nb28lxsagYDIKMxCQ7KvfDRMzb2FrawWr7Tk3NuZIJLSlvbu6rmV19eU6Q9vZuc/kfkEgEKC7+2A1\ndOT0Va4ogZpRRVYGOJ/PYjUEeGtrrVquo2OAsbEjzM6+SKmUR1ECDAxcT3f3uC+jkRr1HpzyGul4\n1tdv2cfgNoRkV7ZVpNDsl6YbPV5kWoCrqo/BS1jOqqydnJfnoXIeCrXT13db+Us1UP5T6Ou7hVCo\nHSm1PZUBfvjDUU6d0kgBtKWccznBH/xBiEymfmijqharoSddzdaNvMQYG9P6RcLh2jaGQvDTP21s\nt7lFsd6ec3t9KoBE4hoSiQPl+x5ECIV4fJy+vlvqytkZzWaHaAaD1u3Vrx6bzSaZmXm+uoGQqpZY\nWDjFyso523bYEYVT2730M7jJc9JpRSK2W3Z6CSG5DSlZkYIZqRjT3BCS/twqzapuO5kdxBXpMTR6\nz7wQtx+kUEEioU2wqiwV3d6//40BAAAgAElEQVQ+QjDYppPXOhiffnqcXK5+lc1gUOXkyV7uuMM4\n1l4awiWXLyloEHz605L//J8F3/62pFjURlf9238rmZjYlrFDJNJDIBChWKzf+rPiLWjngr6+W+jp\nuYFCIUMw2GZKKo2M4DHK2RnXQCBIT88+VlfP1nkvQ0M3VmXn51+p60DXZkEfp7d3wtd+Bq99DU55\nVvW6IhGz4amNhJCcjLcbUjCrx0zWql1uzp3SW4EG6vJro54HgM+grTn8OSnlpwz5fwi8o3zaBgxI\nKbvKeSXg5XLeRSnl+/1okx7Nen5e7qtV2WAwVo5/1+e1tQ2zunqCSKSIECpSGh05xXRCG0AkUlkc\n6fInBdB2Rfut35L85m9qYaXaXdKcrXJlyO3c3FPlJSm0Mn19h2sWAtzuEwhZLhDoRAqNfKWblRsa\nugUhFFZWTqPtvR1hePhWOjqGqrIbGyumdalqiWJxi3C4zbVn00zYqBFvQX/dXgkGcDbW+nM3noFZ\nGTf1OIWQrNrmNnzkBxnsEKE0TQxCC7T+d+A9aHs4PyuE+LqU8mRFRkr5Kzr5XwJu06nISin9WIqz\nDo2GlJrV4TUvHE7Q0bGPd73rAkePjpPL1RJDICC49dYNQKHS6SpEgLa2ofJ+zJczKZhbk2BQ+3OS\nM0MoFGd8/H3k89rSEZFIV80oIzdf/a0ihTfegN/9XXjxRY30PvQh+KVfUhgauoXBwUNIWURRQihK\nbQXhcIxs1mxnOkkwGHbVHmP6TngLDfUhVHTYdTjbeQj6dGO+nSdhlm+my5hnVqeZjJ9k0AwB+EAe\nfngMdwKnpZRnAcr7On8AOGkh/xG0rT9bAjf3xK1355UUGv89JH19t/COdyxx+vQcDz88SjAIiqIt\nq/CZz8Dk5P0kk6+TyUwhhEJHx77yctfNobKAnpRFotH+mjh983Bj7D0G8SulhFImRX2a27LW526/\nmM2O5+bgn/wT2Cgvo5ROa6vfTk/DH/4hKIoChOuMphAwOHgDFy8+U7M9qjY/YqJu/S2jQd5Jb8GJ\niJz6Fyx1OMXejcdW+XbpbonAycA3Gj6yglv5HQw/+UEMo8CU7nwauMtMUAgxAewFHtUlR4UQx4Ai\n8Ckp5d9YlH0QeBBgeHiP70Tc7D3XnsMSKysnSKXOoKolYrE++vtvIxxO1MkaIYQgFhvg4x+Hj3wE\nnnkG4nG45x6IRAC0XdDqd0JrvOFbW6vl3dYqOlR6em6gq+vahnVqcGvsGyEF8zKNeAnGtGZIAeBL\nX6rfCzqXg8cfh9lZGB2tN46V/11dYxQKWebmXgYkUkp6evYwPn6baxLQp5uRRyPeglv9xny7OsHC\nW6jAKSRkdW6W54UUmiGEZuLNXvP9LmcCP4jBlP8tZH8K+CtZO6h7j5RyVgixD3hUCPGylPJMnUIp\nHwIeArjxxts934Fm+xnc5M/Ofp/NzUUqQymz2UWmp7/Lnj0P1HQ2O6G/H/7BP3At3hCkVJmbe6pu\nFc/V1VNEo31EozYbO1iilYRgXa6VoSNjnt3xiRPma1iFw3DunDYqy66ugYFr6OvbR7GYJRiMEAya\nb/cpBBSLeVKpOVS1SFfXEJFIu60h1pe1I0i78JMbonAin2oISQ+9t2Bm4L14ETtBCl4JodF4cyNy\nPsEPYpgGxnXnY4DVUpU/BfyCPkFKOVv+f1YI8Rha/0MdMTQDv0nBCFWVLC4+x+bmvEmeSjL5RnWY\nZL1uN5VZy0ipksnMkMutEQ7HicfH62ZMmyGbXcRsfL2UJdbXz1WJYWtrleXl4+RyayhKiK6ua+jq\nupbaSV2XLyGYybWCFACuvx6OH68nh3xeG31lpVubBb+MlCodHX1EInFTY1w5TibnOX36+1Be+O/8\necno6PWMj2+PcmrUW9Cne82zkrMMHVX+u/UQ9P/tZOzk7f7b6bE7NkMjoYnLwFOowA9ieBa4Rgix\nF5hBM/7/yCgkhLgW6Aae1qV1A5tSypwQog+4D/i0D20CGvvtGvkAyGSmSafPW0ir5HL1Sx34gVIp\nz8zMdykWc0hZRAhtJvPo6P22m90DNbHs+jzNsuXz68zOPkHFwVPVPGtrr1IsZunvv403CyEYz82O\nhYCf+Rn46ldriSESgfvug/HxenmAjY1lzp59iu2Z3JKJiTvp6TGf2KaqRU6f/n7d7zc7+yrd3UN0\ndtZ6em4JwuyajXlOYSdXnoWbOQt2xt8uzUmP3X8rObtjs2twk+Ymz4tMC9D0BDepTcf9ReA7wCng\nL6SUJ4QQvy2E0A89/QjwFVn7mXo9cEwIcRz4Hlofg1WntYc2tS6MZ6ZH2+DGbClmDcY+Br+wsvIK\nhUKWyoxoKUuoap7FRfOVU/WIxfoxzh4GqKwRBLC2tr0eUQVSlkinz1MqudlIRtB4P0J9OTdfqlZy\nzXgJZjFzM0M4Ogp/+qdwyy1aWmVU0h/8Qa2c3sifOfMkpVIeVS2W/0qcP/9D8vmMabtSqXnM7o2q\nllhcPG95rW77ERy/9DG/B2YyDYeQjHlWaWb/3ZKCE7mY6bPzAryEwNzkuTViXv9cwpd5DFLKbwHf\nMqR9wnD+SZNyPwAO+dOG5mUbJfDKbFVzBOjqOuhavxdoM6brjXsut4aqFmxDSoFAmN7em1lZeblM\nahIhAkSjvbS3j5T1JC1KKxQKGQKB+r0NNFweHoJZmp9egv64VCpQKGS59to2Hn44WH4P8+TzGwQC\nbQgRqSubSs2C6R4QkuXl84yO3lQtow87WcGYZxZSMrtOvaxTnqs+BDsCdxtCcvISvIZ/rEjDKGOX\nZnUtVud25S9jbwGu0JnP0Pg9a+Z3skJ7+zDJ5Ab1RlowNnY/oZD5EtdmRsEb7CymszVNJPYTjfay\nvn4OVS3Q3j5a3gJTKxsOd1IopE1KqhbXdGUSgjHfC0FonfgvsrZ2FiEUpJT09l6DlEVWVytpKt3d\nk4yPH0bvpKtqHvN1lFSKxbxpfYnEoCk5KEqAvr5xU+Nv5xlYXa+TV+AmhFTnLbgJITkZZ6uvc6fy\nbknBTdjISaYVIaUdxhVLDI2g2ftv9Uz19FxPJjNFqZRHv2zzwMCdDY7ucYeOjgnW10/XGYpYrM90\n72kzRCJd5f4CIwTd3dezuTmPcSmHeHxct7xEo2RgXdYtIZjJNkMIxnM3x4uLJ1hbO4eUavV3WF5+\nrZwrq2lraxcIBsOMjNxc1dHRMYjZx4GiBOnqGjY1zuFwhMnJ27hw4UVUVfP0FCVAT88o3d21O8c5\nEYJZuhevwAyu+hucwhteQ0tmZZohhUYIwW9v4RLjTUMMrSTsQCDCnj3vI5k8QzY7TzDYTlfXNZZL\nMVghm9VepGjdHDOBmQHp6bmBra0l8vk02mYuCoFAiIGB2xu8ktq3OBLpYnj4LSwvv0g+n0KIIInE\n/vJciktHCH57CMZzt3lSSovd18zCQyWWl08zPHyoOus5Gu2gr28/Kytnq53JihIgHu8jkRiyrHtw\ncD+JRD9LSxdQ1SK9vaMkEv3ol+d2CiG5Cf34FUKq8Ra8xt/dhJYaJQW3XoJTmMpK1i6tUewQqbwp\niGEn7mUgEKan53q0/nR3yOVSZLOLzM+38elPj3DihPZW3XYbfPKTMOSwIraiBBkdfSdbW0vkcilC\nofbyDmNujbazXCzWz/j4e6ohD7Mlq5upqxnvwCytGUIwntuRhQZZt/CdHVS1iBASEFV94+O3kkgM\nsbx8FilVenr20Ns7jhCizujq2xKLdbJnz43Mz5/mzJnnKJWK9PWNsWfPDYRCEYrFAhcvnmBpaQoQ\nDA1NMjFxPcFgwPFr3/p6nclCX7YuhGT3xe6lX8EptGRMt9Nh5yW4JQS/yeAy8CjeFMRgh0vxG0gp\nWVp6lkxmimxW4Zd+6UdJp0FK7Y16/nnJP/2n8PWva0tS26EyY7qVG+xohuzS9B9YyV5aQqgYRoVw\nOG46gsgM0Wgn2jLget2Crq5hurqGa+qzI4XK/9dff4bV1dmqtzE3d5qVlVkOH34Px48/SjabqYay\npqZeI5lc5MiRd1D5TYxG3irs1Gh/Q90oJKgnASeyMDs202NX1izf7L9VO4xpbs+d0Gr5JnBV7cdg\nBr/upZ+/ycbGNJnMFFKWOHp0lFxOQb+iqqoKNja0pRS20UzophEI3Z9/Ze2+NM3kjEbRKs1Jj5tz\nL0awcjw6ehj9hj1aemWXOKFLCzA+ftjWGJuFZazys9n1GlIA7YOjUNji3LmXyOU20fc9qWqJTCZJ\nKrVsW4dVntM9s/wdzAy5U/+BnUdgJm9Wj5keq/aY6bbTaXduB7t7YJbvJO+2ngZ0XNUew2XgkZli\nfX17bf75+bjpHgy5HMzMGFMFtHQl1WbJ5/LwDpzKeMkzk9PLdHYOsX///SwsnCCXWycW62Jw8EaE\nUFhYOEk2myQa7WR4+Aba2rpNdVjptiONTGYVs/utqiVSqSVKpfoQl6qqrK+v0t3db1u/G8/A7Bpq\n5JrpV7AqYyerz9enucm3kzfLNzs3g52MV0O/w7iqieFygxCVZ3P7S27v3jWi0UJ117YKIhG41nQt\nu8rb6MfD4t5iFwobpFKnyedTRCLdJBLXlFdi3RkycKuzlYSgP9anxeO9xONvqyuzd+89jmWN6XYE\noc+PRuv3Z9BkFGKxdnK5zbrZ0Yqi5en12hl6N16BZQjJypg1ShZWXoUbUnAKHTkRgR1BWF2f23S3\n+TuMqz6UdDmio2OyGn44cmSWnp4sweD2SxwOS8bH4c477bQI3Z8bCJM/d8jl1pia+ntSqTfIZhdI\nJl9naur/WM7OdUsKdobGi05T42RybqbPTrddmMlNW92EhtyEbcz0dnb2EwrVE7MQCvv23VJe4rsm\nh0AgSF/fsHdD7yBXRwp6mBl6pzCQXXjHKG/Mb5YUrNppF4pxIjIn8rvMSAF2icE13H6tSilJpc5y\n4cK3OXfub5mfP0qhkKmR6eiYIBrtQ4ggq6sxPvjBk9x11wydnSpdXZIPfhAeegjq3m3rml38NY6l\npefQlt2oPMAqqlpgZeXF6jW5JQQnA2mXZqXLzbkTWVjpdSIHYzkrObuvcSs5e32Cm29+B52dvQih\noCgBotF2br75rcTjCW677R3E4wkq/R2dnT3cccc7yeWynD79MidPPsvCwnR5LoQ1edldn6WsmaF3\nY7jdhH/sSMMLKdi1zSshmOk0k2s1CZgRToN174aSfMbKynFSqTNU+hAymYtsbs6Vl97W9rEUQqG/\n/2381m9t8fjjEUIhSamkcMMN2oYu8bhdDTsLKa0XAcxmFz15B05pbonFz3M3clbHdvluCaTy3444\nrEgjGo1x663vpFjMoaolIpEYlWGuHR1d3HXX+8jntxBCEA5HWFqa4eWXj6JNxpPMzU2V5d6OogQc\nPRqrtlbT3fQrmOU5GXRjeiM6nDwOqzyzttud25V1wFI6zZmlJQqlEuM9Pezp7kZx+4L5jF2PwQJb\nW0kWFp5ndvYZ0ukpzJYhMP5mpVKOVOo0xglPqloimXytptyXviR48skYhYLC5maAXE7wyivwe7/X\nkstpABVPozLCxkTCMBqnPt+bd+BGVyPndl+3br0KK+Nt9xVtJW/2Zydn1GXMD4cj5T4HUVcuHI4S\nDkeQssSJE9oOcZU5KaVSkfX1Naanzzfchuqx1/kKdl/VVmWcZPV5dmWt/tuRglUbrNrkBVJycnaW\nJ954g4urq8ylUjx34QKPv/46qhd9ux6DfygUNlhaeomNjfnyUgQHUJQQS0vHdV/900SjvYyNvc3S\nSIK2TLUQARMSUclml2pS/uIv6nf7KhQEjz4K+bwkFCqRycyQz6cIhztpbx9DUewNcfOot5xCCOLx\nCTKZCzXXJUTAcmvRy907MOZ58Q6sZOzKufUo3HoSbgjDmJZKrZraBVUtMTt7gcnJ/Z6I1JIUKjAz\n3l7DLH6QglGfWZ4TIXg5t4OFbLZQ4OTcXA0JlFSV1c1NZtbWGO/2toKCH7iqiUEI+9+tWNzi/PlH\nUFVtwTJVLbC8fAKQ5T8NUpbY2lohk5mio2PCso5gsK3OW6ggFKrdHyGbNW+TqkI2m2Nu7v+W114q\nIkSQlZWXGRt7p6ed4Jxhb50rL39//20Uixtsba0ghEBKlba2IXp6bqiTNSvvlGaX75Vk3JKBXTm/\nCMFMzs6bMP63qs8rKQiB7UdFIBCo02/VHjuPCDA39G48CCcd+nSjHmOenU5j/Wbts9JvdmwHN3JS\nsrS+jiJEnXdQUlXOLS+zkE6zVSgwkkgw0dNDwKrzscEQlhmuamIAe3JIJk9T2ctgG+bLGktZYm3t\ndB0x6BEKtRON9rO1tVT3dd3dXTv29O674dFHNSLQY3ISstkXKBazVMhJyiKlUpGlpecZHn6LZf3u\n4I4M9NCW3riffH6dQiFDONxJKBS3NYROaUYUCmlyuTThcJxIpNOTN+CFbNx+2TuVb5QQnPQ5eRJW\ncsViDiEgFIrUyFbyE4luQqFQ3fyGQCDAxMT+Ov1m9Zp6MFb9Cl49CLdkYTx2k+fmv7Eup2Mr2MlY\n5IUC1qQ9n04j0mkkMLe+zonZWd517bW0hcPObWkCvhCDEOIB4DNAAPiclPJThvyfBX4fbYc3gP8m\npfxcOe+jwL8vp/9HKeWf+NGm2vrNf5PNzRWTsI81trZWyOWSRCJdlvpHRu5lYeEYmcxM+UstwsDA\nkboF9T7+cfjhD2FrS9v6MRjUlr/4d/8ONjfN1+nXVjqVuFuewo3MdvvdIBzuJByuN9pmOtzoFKIS\nyniajY2FsjciaWvrY2zsPgKBYJ282zpa4R24Ketk2O1k7TwBK7lsNsPJk0dJp7W9M+LxBDfddBfx\neGeNnBCCw4ffyrPPPoaqqtWVWfv7hxgcHLGt09IbcUMKVh6ElfG2+7K3KusXKViQQGFqnuX/8iU2\nnz2JCAfp+NG30PuL/w9KNGKux0yfTf5APG7bySyr4pJssci3Tp7krfv2MdjZ6b1OlxDm68F7UKD1\nQL4OvAdt/+dngY/od2IrE8PtUspfNJTtAY4Bt6Nd/3PAESml7V6YN954u/zKV5x3Kasgm10rG+oA\nHR17CIe1ST7z88+TTJ7BzAhboaNjgpGRuxw/NFS1gKoWUZSopSFfXYW//Et46SXYtw8+/GFtF7Cz\nZ//aIiQl2LfvJ10SgzUaKe6Xd2DMX1o6zupqbYe9EApdXXsZHj7SEjKw03M5EYLxvzG/VCrx9NN/\nR6FQ21kVCoV561t/jGAwVGfkS6USr732EhcvnqnOdQgGg9x119vo6uqyJIW6ttiNQLIjC7syVsdm\n8nZljOWsdOphQUKlVIaLP/nrqOnNap4Ih4jctJ/Rz/6GuQ4zONjZtc1NnjhzhlI5hKBKiZTS0jIF\nFYUP3HSTdVjJAuLaa5+TUjouv+yHx3AncFpKeRZACPEV4AOAmy063wc8IqVcLZd9BHgAeNiHdiGl\nZGHhBZLJc2XDo7C8fILBwSN0de2lp+cgqVQlrwKFYLCNYnF77sHmZognnpjg/PkuDhzI8TM/Ax0d\n9b+13nNQlFB1BzWrZ6KnB37+5+vT4/Ex0ukpasNaomYTHa/wiwzM0t16B2bna2tnqd8+VCWZPM/w\n8GHQrUTqRrcXcnAjZ2Xc7fLtyjgZfKs8sy/6lZUZ0727VbXE/PwU4+P76spnsxmmprSVXEsl7fkq\nlYocPfoY73vf+6tkYRtWatUIJH2elS4rOTsPxJhm1l4LmfW/fRyZy9ekyXyB3Mmz5F6/QOSaPZjC\nY9ipOxbjH954I8uZDEVVJaQoPHHuXJUozLCYyTBs5jX4AD+Gq44CU7rz6XKaER8UQrwkhPgrIcS4\nx7INIZtd1pECgLaZysLCc5RKOcLhOOPjbycc7kAzQAodHaOMjb2NSihmcbGdj3/8R/jyl2/me9/b\nx5/+6UHe/35tHSO/vqKN6Ou7lVCoHSGC5XYFCQbb6Os74ljW6PqbGR63Oqz0mp07tcXq3KqzXsoS\nQkhbg2ym25hn104ruUbKWqW5qcuNXmM+wNbWBqVS/f0rlUpsbW2YktDU1NnqpDY9VFVlaWnBtr46\nUqjA6uvfTs4tWewEKdh4E7lT55A5k/3NFYX8Wd1iZm6vzUoGUIRgoKODkUSCvnicWND+u73ZaI8d\n/PAYzF5dY4u/ATwspcwJIf458CfAO12W1SoR4kHgQYDhYQuWNiCVumhheBQymXkSiQna2vrYt+9H\nKJXyCBFAUQJICV1d+0mlzvH5zx8mkwlTWf00lwtQKMCnPgV/9Efm9ZZKW6TTU6hqkfb2ISKR7cXT\n3PyWlY1/NjfnyedThEKdtLcP0/heCM7wk+ScvtL1aW1t/WxsLNTJxGK9ddfrxRtw4+24OfZS1q2H\nYNdeK0/CLL2zs4dAIFDXoZxMdvD446OcPw/vfKfWd1Upl8/nMHvFpIRCIW9L4nWkYBUqchtWsks3\n06VPMx6b1WVMM+q2yy8jcnAPm99/sZ4cVJXwxLB9WKoJCCF46759fPeNN8ibkL9E65toFfwghmlg\nXHc+BszqBaSUK7rTPwb+k67s/Yayj5lVIqV8CHgItD4Gs5ewPrRj/oZrD3xtXiAQrskfHLyNYDDO\nSy8Nol8SG7SRRD/4wbasvu5MZpa5uacBLUa4unqSjo4JBgaOUJmAZGynqhbY3FygMgw0EAgjhEJ7\n+wjt7SOm1+AHWkUGXs6Hh2/j7NnvImUJbSCAgqIojIwcNi3rhRzcyHkhBC/lmiEEK/1GsujpGaC9\nvZN0OkllEMXDDx/im988SCSi7fsQicA3vgGHDmllh4ZGWViYrSMTKVX6+wdq6mmYFPQyVrLNpLup\ny0zGKKf/b0Q5veP9byf5pW8j84VtPg0FCV8zTuTaiXq9PqIjGuX9N9zAE+fOsbyxgSolihAI4K7x\ncYKK0pJ6wR9ieBa4RgixF23U0U8B/0gvIIQYllLOlU/fD5wqH38H+F0hRGXIznuB32y0IfqXRkpI\nJCYMoaRKniQeH7L9ihdCkEjsIRCQdUNKoX4DHSG0WO3c3NM19UlZIp2+QDw+Rnt77XaNUsLGxhzz\n8z9g23lS6e8/QmfnXg9X7g5uDacbebP8Rs6j0U6uueYBVlbeIJtdIxbroqfnGiKRNld6Wk0GVsdu\nScQvQjDLE0Jw++33c+7cKWZnz/PCC3185zsHKRQ0rxYgnZb8xE9I3nhDI4rh4VHOnXud9fVkNQwV\nCAQ4cOAgsVisOVIw5lvJNpNuPNbX7QcpGPQGuzsY/eN/z9Lv/ylbL76GCAaIv/du+v7VR5o3yi7K\nK4rC2/ftY2Vzk9n1dcKKwp7u7st/uKqUsiiE+EU0Ix8AviClPCGE+G3gmJTy68DHhRDvB4rAKvCz\n5bKrQojfQSMXgN+udEQ3CyGgra2Xnp6DrK5WlqPQnvaRkTvrPASz32h29gfcd99ennpqD8Xi9ljj\nUEjywAP1lkFbO0iYPI8l1tfPV4mhAlXNMz//gzriWlp6nlisn1CoOVfRKxE4lbGSaTbEEwrFGBq6\n2bWeS0UGVmXdGnM7GaPcysosU1OvUyjk6OsbYWLiIKFQpK5MMBjk4MFDXHvtIf74j80mTgqSySKP\nP77JO98ZRwiF++67n+npC8zMTBEMhti3bx8DA0ONk4JZnv7cCDekYCZvptMqbOSVFMyIpnwcnhhi\n9L/9OrK64KCLl8RKdwMQQtDX1kZfm25ya4s8hQp8mccgpfwW8C1D2id0x7+JhScgpfwC8AU/2mGG\nwcFDdHVNksnMog1XHSvvI1ALo/dQKGywtbXKRz+a4sKFBLOzlQ5qydhYln/9r72NBtC/cNthp2kw\n6WaRUiWdvkBPz42e9Dcq08pQkZP+ZknFS5lWEYJbj8Ktl3D+/EkuXDhVHXG0uZlmfv48d9/9PkKh\nsKkBB807MHuehJC8/vo53vWuQ4A2C3pych+Tk/vqicBICnp4+cp3OnZDKvr/Rj1mes1krOSM+S6O\nXROCW6PdYuPeDK76mc8AkUgHkcj2zGO736PiPWid0QptbQV+93f/L488sp9XXhlkZGSdD31ohnj8\nPXVl29oGMBspIESAzs4J3XmlHSUw7WuXaEthuL1C62tpJM9Ozovh1p+XSgXW16cpFDaJxbrp6Bii\n0rncrDG3K7MT3oH+uFFCqJwXCnkuXDhZM3JISpVCIcf09Gn27bvBlBSEgA98IM8zzwTI5Wpfa1VV\nGB29CByyJwIzUmgkJOSWFNykm+U71aGXMftvzHd7bAUnmVYSQIt0vymIwQjjl7tZfiSieQTFouDT\nn34Lr77aT6GgEAoN8sgjB/nc5+D662vLlUpBLly4n+np09x44yKJxBbaWvh7aGsbqqunvX2Y5eXj\nJvUHaG/3PmrXydj7QQZO52Z1bG2tc+7co0iplif9BQmH4+zf/w4CgZArHcY8KSXp9DxraxcRQqGv\nb5J4vN+2vB9hJrflvISX9OmZzBranNHaji1VVVlZmWP//lpiqJQTAv7xPw7w0ENJzp9PkMuFUBSV\nYFDlwQefpasrYkkE+XyOfH6L9vY4wWDAlBRyuRxCSsL6zjUrUjDmG2UNuh2NeStJoVFCsMpvxFBf\nhp7DFUsMdsbD7X22IwhFCTA4eBtf/GKSU6f6yee1W5XLKeRy8Gu/Bt/85raOEyfgX/5LKBZ7kbKH\nYlHy0Y8u8bGPhYlGa5fDqNQXDsfp7r6WtbXXqfQzaOGu8fJwTXfX4XR9jcg2e65Pn54+SqmUr6ap\napFcbp2lpVcZHj7k2FajEZRScuHCM6RS2xO81tYuMjBwDWNjN9uWdTr2UrbZ8JIZYUQiUayWaals\n52lGClq+4D/8hyc4enSIY8dG6OzM8a53nWVsLM3Bg/fWkUKxWODYsR8yPz9Xndh286FDHDhwoPqQ\nrqdSPPPss6RSKSTQ093NXUeOEG9r8+YBWBltOwO/U6TQLCFcSo+hRbhiicEOxhfQ6XexIoiurr08\n9VShSgp6rK7C+fPaUnS6mFYAACAASURBVBb5PPzCL0AqVdUICP7szwa4554Shw/XltW3r7//EO3t\nw6yvn0dKlY6OCdraBtzHMy30NirfrHdgzCsWt8jl1uvypVRZW7vAyMghx7YZjzOZ5RpSAG2278LC\n6/T17SMajTsadav2N1vOybNwko3HE7S1dZDJpNCHGbV+gYOmZSvHCwszBINwzz3T3HPPtK6sUg1x\nVuSFoEoK22snwfGXXqK9rY3h4WEK+TyPPvYY+cL2GP7V1VUefeIJfuzd79aWY3AbFtLnOR2b6XFT\nxqy8HnbkYTy2Kuck65TnBCkplErkSyViodAl26jnqiQGI/T31u43M5MTImQpW5F5/nkomEyOzOUk\nX/ziFD09bzA0dEed51BBW1sfbW19dpfgG5olAysdVnJ+eS3641Rq1nQpCBCk0/PEYgdaHiqyyvfi\nTVhd7+HDb+X48e+TTqeqHwg33HAbXV29VTkzYtjYSNfNTwAtDDU9fZF4PE53tzbZMp/PVUlBj1Kp\nxKnXXmN4aIipqam6JRkkUCwWmZ2fZ3x4uLaiRjyInSaFZgihmfCRC5miqvLc7CxTqZS2RZaicOvQ\nEHt392NoPbySxI//OHzmM9oqqHokEpq3ANZ7K0ipkM2GyOWSTE19j8nJBwiF/NxPwR3ckIExzS0Z\n2JUTAkKhKNFogmx2zVBGoadn0rG82bG26qoAQ8e9EAJFCZqGW9y2u1mPohlC2A4Jxbj77nezuZmh\nWMzT0ZGobr1ZkTPT1dmZIBAImpLDwsIci4vz9PT08pa3vEXrM7C4MdlNbcG4xaUl8yU3VJWNzc3t\nBL0hb9SDMNNlJucnKTRDCC3wGI7NzDC9vl7dl6FUKvH87CyxUIihHd7v94okBjdfom5+Gzck8eEP\na/smvPKKRgDRKAQC8F/+C1QWNjxyBIr17yKRSIF77tGWglJVlWTyDP395qETP+Hli95OphEy0J9r\nW5peQBt9pJS9LBVFCRKNdjI0dB0in0N891E49SpM7kE88F5thUKb+nt7J5ibe5X65U4kPT0jls9H\no2EifXqjZGAmZxcWEgLa2+M1smZy+vJDQyNEozE2Nzcw9lNUPIOVlWVOnHgFIRRToy+A/v5+Tpw6\nxfTsbF0+QEBR6E4kGvMIWuUpOJGFXZoxz+25U7oH5EulGlKooCQlJxcXd4nBL5i9rF7DSKDNcP78\n5+GZZ+CFF6C3F973PtAvaphIwK//Ovz+72vbcqqqIBIpcODAKnffXYn1quTzqWo9PjxLde12k9dI\nKMkqz+5YSpUzZx5jayupC/sotLf3MzR0PZ2dg4i1JHzs/0Wk1rdZ96HPw+f/J2Kidj0svf5YLM7k\n5BHOn3+u5qv34MF7CYXqZ4Q2EiZqtFwzISUzQjPzDoxtqJwrisLb3/5OTpx4iZmZKYomXyuqqnLu\n3DnThfQAAsEg+ycn+d6TT5rKCCHo7OhgoKdHS7hSSMEpdGSX79VjaABbxSJWi+IljeGKHcBVSwxm\n8BpGqsgJoe24dvfd9bIVPR/6ENx0E/zlX+aZm1vgzjunueOOGQIBWdYZIBbrNa3DTzTqLTRLBsbz\nZHKara1UTV+AlCqbmyvEYp0IIRCf/R+wtAyVL9etLW0j7N/5Pfjc/7Ctd2Bgkp6eEdbXFxBCIZEY\nrNnYp5EwUaPl3HgRVsbcTXjJyVvRH4fDEQ4fvoNbbz3M3/7tX9c3Fq2PwCqMtHdigo2NDRRFMSWG\njvZ27r/7bq28F1Ko4FKSQrOE4DMZ6HW2B4OWey8UVZW/OXWK0Y4ODg0OEnVYddUPvKmIQQ/je2H1\nmzuRiT7/+uvhE5+IMDV1gY2NhZovAEUJ0N29r4kW29ftJs8LGRjzvX5tr6/Poqr1X6xCKGQyS/T2\nTsDjT26TQgVSwslTkNuCaNTWKIZCYXp7x+vSrY53mgzM8rwQQuW/kxdhRjqBQICenl5WV1cwItHZ\nyXo6XZeuKAqxaJSQjeEZ6OvbXrytGY+gGVJw4w04kYJbQmiGDFyULaUypP76e2w+d4q9kQAL77yV\nzb31c57ypRLnk0nmMxkeOHDAdjtQP9C6dZwvYxSLObLZtZqx9ZUXysnY2slU8sbH76W39zoCgQiK\nEqSjY4y9e99DIBCp0dHsn1XbzAyF0ZA4XYOZPjuDZCwfDEbAYnvRYDBEOr2AavX0CRABpaHrNv4p\nivtyetlG6nPbJq9l3Pw2ZvlHjtxOKBSqzlEIBAJEIhGO3H675cav46OjDPT3mw6TDCgK+8bGtJOr\nkRSs6nYD/bW4LFtcW2fqn/0Oa1/5DrkTZ+l+4Q0O/tev0f3D10zlJRpBXEgm3berQbypPAYpVWZm\nniOV0jpEpVTp6dnP0NCt6F1r4zvh5CkY84UIMDBwEwMDN5m0oZkrMG+fmzwvZYzGxovuynFf3z6W\nl+t3ZxNC4cKF5ymV8ozccQ39j7+EUtTJBAKIu+5AiYQt63TTvkbKOV2fW/12uq3utZ2cWZ4ZIRiP\nE4kEDzzwI5w7d5ZUKkVPdzd7JycJh0LccfvtPHvsGEJo85yllBy55RbaYzGQkrfffTdPPPNMdbiq\nVFVuu/FGuipbF14KUtDLGdPdlHMqa1XGDF5fZBP55Je/QymVgfLzLyQE8kX2/MVjrB05oI1yMaAk\nJcvZLAdaEdbS4YolBjvDbIWFhVfKm/doO7kBrK6eJRiM0d9/XcN1uSESK1k/4GQE3ZRzSw5OMpX/\nbW0JJidv58KFY1Q8h0AgSDAYJZvVJm/N/thdtJ+dIza7giJBBEPQ203gE7/REFm1ggycZN2SQKsI\nwZhuzI9FI9xwne7ZLhvOPWNjDPX3Mzc/j5SS4cFBIuFwNb87keAfvuc9LK+uUiwU6O/p2Q5fuCEF\nL2ThpoxVObv2mMkY5a1kzNBsvgGbR1+pkoIeAQkjySzzvXGMvTyKEHS2eMltuIKJQQ83RlBVZd2m\n8wBSllhZec2WGOzqckMUdrJe4HSdXsnAeN4oGVgd9/ZO0NU1ysbGSjmMEef48b+DcjebGgnx2q/9\nJO1n5uhcSDN+z3sRd92O0H0pORnZy4UMzMq48SrsPAkn0nAkDLsls4FwKMTE+LilwVaE0EYfeTXk\nduGYZsNHVvrM5PUyxnQvXoJVfpMvdCARpzBVv3uhoqrcde01/N/1FTL5fE2ntCLEjkx4uyqIwQ2E\nkKadoaCt/Nm43vo0q+fFb2/BrT47MjCeN0oIVuWCwSCJxCBCQD5vMhNQCDYOjJC/PsrEkbtc6b8U\nZODUlka8AytZt16EJWGYrYxa+S8ly0uSL3xR4aVXFO68XeUnfjzN3MLrJFMpujo7ue7AATrj8Voj\nbvfl7oYUnHSZ6TW7Bj2aJYVGCMELGTjIJj70LnKnp5Bb232dBAJErttLqL+Ld3TH+eHsLAsbGwig\nIxLhzuFhYoFA81+ZDrhiicH4QqlqieXl11ldPYuUKonEOAMDN1Q35BFCIRLpNF23JxbrrtHXyi97\nr7obJRM/yECf58cXezgcJRAIUizmTcoEHA1hK9vXrP5mvQMnObM8T15C+fjkScl994fI5SCbFXz1\nbwS/9R/a+dSnVujt3SSVTjM1N8f9d99Nb2USWzOk4DV8ZGXA/fAU3HoJfoeYLBB/663kz72X5Je/\nA6EAFEuE944y9MmfAyAaDPK2PXsoqCqqlERaPBJJD1+IQQjxAPAZtB3cPiel/JQh/1eBn0PbwW0J\n+GdSygvlvBLwcln0opTy/Y204fz5p9jYWK6GilZWTpNOz3HNNe9FUbQbOjp6mHPnntSFkwRCKAwP\n32a4nlrdfpKz316DnW6v52bpfhpcbc9r84c7n9+kVCoQDIZ8qMd7GTNZP8jATt4LcTiRRp2XUDk2\nGNCf/5dBUimQUlOQzSpsbYX5kz+5hV/91aeRUlIqlXjhlVd49733evvqdwoB+UEKdnUYZdzk6+GW\nEBowCKqUrOdyhAIB2nVLl/f8zI+S+In7yZ+eItCbILynfqhqSNn5waNNE4PQ3vT/DrwHmAaeFUJ8\nXUp5Uif2AnC7lHJTCPEvgE8DHy7nZaWUtzbThs3N1RpSANA2N8mSSs3Q3a3Noo3HBzhw4J0sLJwi\nl0sRDsdRlCBzcy8Qi3XT13eQcLh+6rmXcNFOwskAmqW58Q6sjpsNsWgwn3GrbYlaRIgQmcwq584d\nJ5PRdnlNJPrZt+9W2to6PbWtEVJrhgzM5Jr1JFx5EWZeQuW/zrAWCvCDo6JKCtviCi+8ULsg3moq\nhVRVqhPZTPSZpjVKCrUNss6zqsMqzyrNTJ/VuV1ZF3JT6+scK3fyq0B3JMK9Y2PEyvNFAvEYsVsP\nuq+nlV+XZfhBRXcCp6WUZ6WUeeArwAf0AlLK70kpK6tuHQXGmq208tIIAdnsKpjMG1TVIpubSzVp\nsVg3k5P3Mj5+FxsbS6RS02xurrCycoY33vh7traSNbrd1G/8awXs6nGTZtU2u+uwu65Gy3R1DYPJ\nKPpwOEY4HGV9fYmXXnqU9fUlVLWEqpZYW5vn+ef/nnR6pUa3lzkKTveoVfeg0XbZzamoniMRUrX/\nqtflKUKajYAEIBSqJexQMEjd7OZWkoJVWT2cwkd2OqyMvVXb7NKMeWbXWUZya4sfzs1RUFWKUqJK\nyerWFk9cvIjVEhiOMKvX7Z9L+EEMo8CU7ny6nGaFjwHf1p1HhRDHhBBHhRA/blVICPFgWe7Y2lqt\nsQ+H26hsE1lbJkA4HDd9wWZmni93RldultY5PTv7vEHH/9/eewdJdtx3np98r2x3VXvvprvHDwbA\nzHAAkAAJOkCiuBLBk0gupdsQFUEdj9JqYxUbuyHqtKfbU0gnkhshbWxIKxEraUWZFSlShjiKOkog\naAEQwMAMZgbjbXvfXW3LvJf3R5kp83y9agPUN6KiqjJ/ad6rV79v/n6Z+Uv3yt+LwqlGmRmll6eZ\n9dHpZ6tyTssIAUNDx3PnFud/L4GiqBw8+ABCCG7ceBWjg2qk1Ll27RVbpWu3Sc0oz6yMVXmjMkZE\nVZ5X/m7XN0MZpPWKIyOFLSWqIvnIhzVCoVIFEQxmeM97bha+q4rCgaEh65G9FSnYEYmRnFEdVm0Z\nyRi9l382SrNrr7xcWZ7UddafPcvM5/6Mud/7a5LX756FcWVpqSIwngTWUinmXr3E+g/OkllcYbfB\njzkGIzVpSE1CiH8FnAbeXZQ8JKWcFEKMAs8IIc5JKa9XVCjlk8CTAMePn5bFf5amph5UNZiLy3O3\naSFEIbRzWV05K6MS6+sLhT+iFcHakYPXwYDT+s1krMqVKxijPLPyRvlWZczkIpEGTp78ANPT10kk\n5ohG4/T1HSQajSMErK+b/0myx17at+G0L1b5RuWdlrFqz+6+W8mZziPk382UYlH67/9uistXw1y+\nqhSyjxxe46d/+g2CgQAZTaM5HqepsZFMJkMgvwKmGmvALM/uOsrTjfLNZMrvQ7ms0Xcry8AEUtOZ\n+o9/wNbr15FbSVAEq994jvb//SdpfuJRNtLpCmUYWkhw6L/+Patrm6wpCqQzNP/Ue2n7+SfwckBX\nLeAHMYwDg0XfB4CKeL1CiMeAXwPeLaVM5tOllJO59xtCiO8AJ4EKYrCCEAoHDryP27d/WIj7Hwo1\nsm/fQwSDYQP57CjV6LAXs0Bs5bBT/H7/vnYK22lZLwrR7LNbJZp/D4XCDA0dM5QLBkOk00mMEAiU\nng/tF5lVSyZmcm6I2JY4yi0EsFfEBjKtrXDmB0me/6Hg6jXB8aMabzupsrn5CK9dvMjE3Bwrq6u8\n/MYbvHLxIo+ePJldnWRUp1tScNPX8nSjfDOZ4ncjWaN8u+8mWH/udbZev3Z3yakukck0C3/4t8Te\n+zZ6GhtZ2NxEK6pv/xe+TnAhAVIWSGPl779L+OgwsXdmp1u3Mhkyuk5jMLgjZOEHMbwEHBRCjAAT\nwMeBnykWEEKcBL4AfEBKOVuU3gpsSCmTQogO4BGyE9OuEQ43cujQ+8lkkkipEwxGTWWl1IlEWtnY\nmC9JF0KlvX2/LxbD3bbsZdz87k5l3Y5o7fKrGVW7kR8cPMzNm+eo9L8K+vsPULxAo5p+7Jbr9oUQ\nit+N8svyBPDwQzoPP5hX2LC8usrk/HxFRNXvv/YaH3rXu7Kxk9wofqO+mH23SzfKN5Mpb79c1uqz\nWVkLrH33ldJ9CHkEVDZfvczoO+/n2tISW5qWXXI6s0RkbgWl3L20lWLl775D6m2HOTM9zUoyiSIE\nQUXhgd5eevfaeQxSyowQ4peAb5JdrvonUsoLQojfAM5IKZ8C/jMQA76SY7/8stSjwBeEEDrZ+Y7P\nlq1mco1s8DZr3LnzUsWJYgDxeA+9vXfjG/nhLqqG7N2WdUoATqyPWoySncgPDh4mlUoyPn4Fiozw\nnp4hRkbuqUqh+00GdtfjRMY1IeTfzZSwXb6Jor0xNmZ4cI+u6ywsL9PZ0mJYzhEp7KSlUCNCyEOJ\nRrI/mkF5EQkRUlUeHxnh8sICE2trxKRAUVWyK/dLsbq4wrO3bxe+azK7dPi5iQkeGx6mOWyv2/yC\nL/sYpJTfAL5RlvbrRZ8fMyn3HFD7I82KkEqts7w8RvkEpxAK0WgLRpPYZthpd6Ab94+ZfHG6UyJx\nq0ztFKmUkpWVeZaXZwkGQ+zbd4TR0XtYW8se8hOLNRMOR0gmN1ldXSMajRMOV4bk3ikysCvjijzs\n5hHy7z64lYrfjUghD03XjRWvGSmYtfdmIoVcmaYfewdr33oJmSy1GoQiaDhxCKQkrCjc19nJfZ2d\nyIFBbilfqVi0LYMB5u43DsuvS8mVxUUeKD9ju4bYkzufhfCulLe2VshHVi1G9hCZeZNSuwd2Sqv8\nuxvroPhzLUbKRmlS6pw//xxLSzPouoaiqFy//jr33/8u2tq6gOyu9vPnf8jc3Hhhbqi7e4hjx06j\nKErN3T52Zb22YUkGxd/t3Eb5d5dWQnH6vt5e5paWCtFUi9HR1OSaaKomBTtl7wcpuCEEE9nI0WFa\nf/bHWPrTf4CACkIghKD3//kFRChYIS+CATr+3c8w9/k/R6Y10HVEJMRWUwMz777fuGlgPe09bI8X\n7EliqAahUCPG64cFkUhTyZ/Xo3XpG5wodbO03UQIVsp0dnasQApA4f3cued417s+hKIoXL9+nrm5\nCXRdL/jAZ2bGiEYbOHDgeNXkZHetXsp4tg6Kv1dLCHb5RemDXV3cmpxkYWWFjKah5BTcA0eOZA/m\nySGZSpHY2KAxFKIhHHbWpp+kYEcIZnleCMGhXOu/fJz44w+x+epllGiY6OmjKAakkEf83acIDXaz\n8tT30eaWiD54jKdH2tHDxmUUIehuaLjbn21wVexZYvB6b6LRZhoaWtjYWCqxGhRFobPzYNVtOHmW\n3NbrlCCs6nXrLrIr45cvfXr6puHqMCl1VlcXaWnpYGLieoWMrmuMjV3j4MHjlvdht1gTJXl+EUL+\n3U6BOlDAiqLw6MmTTC8sMDk3RzgYZLinh1gkkhORvHbtGtcnJ1EVBU1KupubeceRI3eXtDpp2wuJ\n2d0DI7lqSMGDJRFojRN/3+nKOkz+lOHRfrp++eOF7623b7NgcrZzSFHYn5/jseqfj4SxZ4nBK4SA\nAwfexe3bZ1hZmQQkoVCMffseIBKpnPl3azVU89vYlXViKZjl7QQhOKvb/AKyMhJNM46Km8mkHSvo\n8r4kEsuMj99C1zV6ewfp6OgkG8upsqx1/11aB1Cde6RaK8KiLQH0trXRmw/rXFT2+sQEN6am0KVE\nz81HzCwv8/K1azx06JC3tp302cl1G9VVnmZWxqi813w3ZcoellPd3Xz7zh00KUv2PQzEYoRUla/f\nuIGm67RHo5zq6qIlR9hV988Ee5YYvCpgKUFVQ4yOPoyuZ9B1nUDA/OCLbbDaXLXrljxqRQh+uZf6\n+oZZWZmvsAgURaGlpQ0hBE1NrSQSlavImpvbPY3cr127xOXLFwptjo3dor9/kBMnsruv7fru9FrB\ngXVQ/NkPQiiXd5tvUP/q+joX79ypmH/QpWRsfp7T+/ejFp8DbdS/7SYFvwjBR2VrWq8QtEYi/Mjw\nMFeWllhOJmmNRDjU2sprs7PcTiQK+yDmNzd55s4dfnRkpCQYn9/Ys8TgFcV/WlUNmMaOMYKfz4hT\nwvEi54YMitO3w0Iol+npGWJuboL5+Sl0XUdVFUBw4sQjhbOKjx07xUsvfQdd18nODwlUVeXYsZOu\n+7y1tcHly+dL1utrmsbExBhDQyN0dHQ6uheOrINqFZgTpeiENNzkF+VtbG7y/fPnWdvcNJyUziOj\naahC7D1SMMurFRnY9CMWDHKqu7uQvJ5OM7W+XhFSQ5OSK0tLnOzqqlmX9iwxVDOSt/vdNS3F5uZq\nLrBbgy9t2sFL3X5YB2afa00Id9MF99//MCsrCywtZZer9vQM5uIpZWVaWtp5xzse5+bNS6yurtDU\n1ML+/UdobIy7ngeYm5s2tAo0TWNycpzOzlJicHUfvRCCmawTQjCTNWvHhWKWus53z51jbWPDOL5N\nDpFQdq2+r6Tg9F5ZlTGSNyvrRH67UGyppVKoQhjGWloymY/wC3uWGKqBmRKWUjIxcZ6Zmcvkl7Q2\nNfUwOvr2klAZOwUrIij/7sQV5dT1YibvRtaujpaWdlpb203LxONN3H//g7b1m7WxuLjAlSsXWV5e\nqNjdm5MiEMgeFlQzMjBLM0r3SghmeQYKe31zk5W1NWKRCE2NjSX5S6urbG5tWZKCqiicHh3NzhI5\nUexO+2ZXh10ZI3mrdDtCqJYwPI4oY8FgSSiNQnVAq9Ecg4/YeW23i7CwcIvZ2StIqZNfsZRITHP7\n9hlGR9++7f2xU0pOvhul7yZCcNIHL7LFn2dmJnnppectN3GpqsK+ffsqiMGoPUdkUPzZixvEC3E4\nHInrmsYLb7zBxPx8YUTa3tTEI8ePE8yN/pOplKFlBRBQFPpaWznc309r8TJKJ/3bTaTgxc3kBfm6\nXBJELBikt6GB6Y2NEoJQheBQjc99flMSQzqdZGnpDul0kqambmKxDtOHvBjT05crJkGl1FlaGkfT\n0qiq88mezc0EqdQGDQ0tBIPW7G7XNa9kUJzn1M203YRQS5dV1u0tOXv2FVNSUNUAUuocP34vzflg\ncUb1W60qKk5zYh0Y5flFCFYyue8Xb90qxEXK207zKyu8evUqDx4+DFLSFo8bziuoisLxwUEO5Xfh\n2hGTQfu7mhRq6U7yQBBv7+3l3MICN5aXyUhJRyTCqe7umk48wx4mBrN7m0jMcvXq93PPocbMzGWa\nmro5cOBh7MJdZDLGUT0BNC3jiBgymRRXr36fzc1lQEFKjc7OUYaGTjoiJycjfiu58jy38wdGcjth\nIUgpWVpaYGFhlnA4TG/vIKFQyFDWqh+ZTJqtrU2MoCgKJ0+epKenh0jkbtBFX6wDozJuyMONonSp\ncK9NTBiuMLozO8vpgwdRgHAgwNHBQS6NjxdkFSGIBoOM5Cc9d4oUzO6xEzfRThCCWVsO9IGqKJzo\n7OREZydSSkc6xA/sWWIwQvYwl+dKRv26rpFIzLCwMEZHxz7L8vF4F0tL41DmWQ0EQoRCEUdEf/Pm\nC0Wb57L9mJ+/STTaTFfXfsDZgMGJG8kqvxaE4ETGDwsBdF555Tnm52fQNA1VVXnjjbM89NC7aG+/\nOzmsaRp37txgbOw2iqIwPDzK4OC+wp9HCAjkTiIz2u0ejUYZHh6p7Md2WQdG8tVYEg4VcsbEepJS\nInWdfAjbewYHaW1s5OrkJMlMhoHWVg709BTcTYZ92QlSqMZK2E5CMGrbhaLPP9fT6+tcXVoiqWn0\nxeMcaGnJLgDwEXuSGMz8wGtrixid/qXrGvPzN+nstCaGwcF7SSSmcxuqsg+MoqgMD78NRbH/ATOZ\nFInETEUfdF1jZuYq3d37Ta/HDNWSgVkd200IbspMTNwpkALcDfD23HPfJhpt4PDhexgcHOK5575D\nIrFcyF9ZWWJubobTpx8CsvddCEFfXz/j48WHDIKqqhw6dMQfMjAq40T5WdVfjSVhQxptTU3MLS9X\ndKG5sTG7H6GobF9rK33FkVWt2qo1KRileSUFJ4TgljRM/qxSShKpFBkpaQmHs0t7i9twQQ4XFxZ4\nY3GxMOewnEpxc2WFHxkeJqj4cSBnFnuSGLzAiQkWicQ4fvxHmJq6zOrqHJFIjN7eI8Ri7Y7ayB4V\natyOpqVczyU4lfOLEKzqdkoI+c9e6sy/37lz03ROYHNzg3PnXmFhYY5EYqVELr/kdHKyjytXLrO0\ntGhqLezbN8z+/aPWrqLidCejfTu5WhNCeXqZTEbTeP7CBRZWKk/JCygKbztwwLSsY8XvtK9eSMEJ\nwdrVYVXOLs8O+bJFD/xqKsX3JyfZzGQQZPXQ6a4uBuPx0nK5MpquM7OxgS4lXQ0NJZZAStO4sLhY\nsnxVl5KtTIbry8scaWvz3vcy7FliMFJ68XgbiqJQPmemKCqdnSOOFG8k0sjIyClPfQqFogQCIdLp\ncp+2oKWl5+43D25CuzmGnSIEK1kvVoVZ/4uhaRrj47cNFb6uS1566YXCklQpJcGlNcLzK2x1t5Jp\nyu5LuXXrJvccO0rULrSAW2XklUD8JIT8e5nM2evXmTE4g7g1FuPhI0dozAfFeyuSQjWEYFKXDnxn\nfJzN4kGOlLw4M0NzOExTqDTiwuzGBs9OTmaf69yA5lRXFyO5RRGLW1soonJfgyYlU2trvhKDL7aH\nEOIDQojLQohrQojPGOSHhRBfzuW/IIQYLsr71Vz6ZSHEj1bXD4VDhx5BUQIoigq5g+ZbWvpobx+0\nLV9d26Aogv37TxfazvcpEAgxMHDc1AVmVWdxGaffjeoolzcrYyZTXqeRrFFZL3KDgyOoDnymxlag\nLBCGSGcY/e/f4Ph/+jP2f+Hr3Pt//ilDf/EtyEVpvXDhQllRaf9yK1+u3NzK5j8bvdvUuZFMsrS6\niqZpSF3n1vR0jojsPQAAIABJREFUhVIBWN/aypJCeRt212DUFyf9tZNxWqYcZvfPKN9KzifMbWyQ\nNljZpUvJ9TKrLa1p/GBigrSuk5GSjK6jSckrs7OsprJnPYRV1XAwBBAJ+DvGr7o2IYQK/D7wONnz\nn18SQjxVdhLbJ4ElKeUBIcTHgc8B/1IIcYzsUaD3AH3A00KIQ1JK8wXnhXaN05uaOjl16sdZXBwn\nnU7S3NxNLOYfk9qhpaWX48cfY2rqCltbqzQ1ddHTc8DzklU3loJZuhsLwUzeKL0WFkKWGIaYnh5n\nbm7G1KWUJwWjP0o+beBvn6X5wi2UjIaSydbT9vIVUh1NTH/gAaamp60VUPlnJ3K1kHdpJSTTaZ6/\ncIH5RCJ7JCdw38iIaViLwmS0FRnZ9We3kYJZXhH0nNtmI5OhLRKh1ecT0pJmk/xkz3QuxtT6unEf\npeRWIsG9HR20hMM0BoOsplIly2NUITjo874GP2jmQeCalPIGgBDiS8ATQDExPAH8p9znrwK/J7L/\n7CeAL0kpk8BNIcS1XH3PV9OhQCBEV5fxaUjbgYaGZvbvf8A038pqsHMDuSnvhBCM5KolhHyaV5eV\nEAoPPfQIi4sL3L6dXXVUTACqqtLT00swGCxMKkspCYcj9Pb2cvPmDfSMRsfzF1DSpX9ONZWh6ztn\nmf7AA4SCQffK2kxupwmh6P258+dZSCSykVBzRc7euEEsEmHNIJRCZ/4gnt1OCmawyjfJW0+leGZi\ngrSmFZRsdzTKw729BTKtFh2RSMVJbZBV5L3FO82BtK4b7jKXuTzIDoYe7e/n+xMTrKXT2cGRlJzo\n6qIjan7GvRf4QQz9QPGSj3HgITOZ3BnRK0B7Lv2HZWX7jRoRQnwK+BRAf/+QD92uDdw+U14tBTN5\nMyVcDSFYyVZrJZjLCdrbO+jo6KCnp5dz515ja2sLVVUJBAJMTU0hpY6iKCiKykMPPUhvby+pVIqx\nsTtkUmlExniErG6lUBSFQwdz52/UYrRvV9aJvEtCgKxbaGF1tdIPreuEg0G2Uim0nBJShEBRFE6M\njJi7h8rb8dAnx6RgVa+ZvJP7aZD//PQ0W5lMiTKe2dzkyvIyR3wafTcEgxxobub6ykphFZEqBLFg\nkMFYaYj/noYGQ2JQhaC/SLYhGORHh4dJJJOkdJ2WSKTkICW/4AcxGDt6nck4KZtNlPJJ4EmA++47\nLX0i9W2Bk746sRSs8v0kBCuZ2hNCZfrAwCADAwNoWoaLF9/g2rWrhcllTdPQNJ1Lly7S39dLJBzi\nsfe/n7Nnz7LV10Z0YqGkXgms7e9j/8gIw4ODxsoOdich2MlKSTKZzE5QVvaQjKbxI6dOcWV8nOWN\nDdoaGznU21t6EptR27m02USCa7OzpDWNgZYWhtvb74bbLpO17a8bi8lO3ui7hfxWJsNyMlmhaDQp\nuZFI+EYMAPe3t9MRjXJteZmMrjMYj7O/ufnusuAcGoNBjrS2cnlpqYREehob6TKwBprybq8aKUI/\niGEcKJ7ZHQAmTWTGhRABoBlYdFh218Lrb+LUSnAiUw0hWNW304RQKSMIBoOMjd0xCIInWVpaIpVM\nEgqFiDU28sjDD7P5h/83Ux/798hUCjQdAioiHOTAf/k/iN93uFKRbae7yEjOB/dSU0MDRvMuihB0\nt7QQC4c5NTpqX2dZ2sXJSd6YmirMU8yvrXFjfp73HTpUeRaD27477YuRvNF3G3ldyuxDZSBnNDlf\nDYQQDDQ2MhCrPASsHMc7OuhuaOBm7vyFoXicvsZGjBdZUDNSAH+I4SXgoBBiBJggO5n8M2UyTwGf\nIDt38BHgGSmlFEI8BfxPIcTvkJ18Pgi86KTR3W4xeLUSnMg6JQerfLs63JTZTvIwW5UBZaamlEQf\nOM7AN7/A8n/7EqlLNwmfOEzLpz9GcLDnTUcI+feAonDv8DDnbt0qCWURVFWO9Pcb12dTZzKV4sLk\nZInS1HSdxNYWY0tLDLe1+UsKRnlWsJMpy48GAjSoKmtlE8AKVLh4aoGUpjG5vo4mJT0NDdm4R7mH\nvLOhgc6GBpsaqLkCrJoYcnMGvwR8E1CBP5FSXhBC/AZwRkr5FPDHwJ/nJpcXyZIHObm/JjtRnQH+\ntZMVSdsFP++9l7rcEIBXQjDK20m3kZ3c0NAQ165dq7AampubCYdCFUomdGCQrt/5D7ZKJ51KsZVM\n0hCNlrpHTORrTiBVkMehvj7i0SiXx8fZTKXoaWnhSH8/keLJdodEg5TMra0Zr5/XdcaXlxkud71Y\n9dnuGpyUt7rndt+BTU1jf3Mz5xcXkWSthIAQRAMBjtYiamnRgzy5vs7zU1N3uwcca23lWEeH67pq\nCV8Wv0opvwF8oyzt14s+bwEfNSn7W8BvuWmvfES70/CrL07mGeysAzMZPwghn7aT7qV7jh1jenqa\njY0NMpkMqqqiqioPnT5dOvp1ojykRNN1Xj57ljsTE2TPfBYcP3yYQyMjxuW3mxBs+m9Wpre1lV6z\nUBYuSAEwjcMjyK6tt6zDTXtWsuX5VvWZfJdS8vLcHLdWV1FEdgNZSFXpjkbpybl71BoqlrSm8fzU\nVMUZCxeXluiJxWizO2NhG5Xent35XA12C6k4dR06sQ7K090QglHabrISgELoimAgwI889hhTU1Ms\nLi3R2NDA4ODg3cBubhQq8OrrrzM2MVFigZy7dImGSISBnh7LsjUhBCt5N+4oOzJwUVdHLEZAVcmU\nWWmKEOwvHul6vRY7WaMyHiyF6ysr3M6t2MpbP0lNI6nr7CsOUVEjTG1sGPZLk5LbiYQ1MWyz0tqz\nxLBblLtTWPXXraVgJeuEEIzkdjshFP+hFEWhv6+P/r6+0jyXCjWTyXBrfLzCLaVpGm9cvZolBicu\nHbN8vwjBKM8PQnCYrwjBuw8c4HvXrpHObdqSUnKiv5+2vD/cL4Izg1W+XdkcrhYtGy0UJRuKIqVp\nniOULiWTTK+vE1AUBmOx0l3IRQ/27MYGZn5yo5PajOrYLuxZYtiNcPr7ebUUrOStCMEo38pqcFOm\nWjlLi8YswJ1TpWcjm0qlTEIewubWVqkFYlW3WR9qSQhG8jUghfx7czTKj99zDws5F157Y2PWSnN7\nTV7uWTmc3uMyGIWnAEAIMlISMs616IbklZxrSs8R6OsLCzzU3V2xCmk9nebW6qphPQowZGax7NAI\nuE4MNqjmd3Gj2L2WsZtzcEIEe8FKKPleJSHkEYlEUFXVMFREe0tLKTEY1W322Q8SsZK3uy9eyjjI\nF0LQUb5ixi3RlcvYpZnV7abeHHobG7mVSFTsX4ioKlEP1sLM5ia3VlcLo/38+wszM3Q3NNwlTmBs\nbc20by2RCJ1GO5e9Kh8fyMT/LXPbhPxos9avavphJ2eUZ9eOVZtW7eXTzPKKy1m910Ku0BekMSkY\njdytFFrxy0JWAe4/dqwiYF9AVbn30CHzus0+GxGJnZxZGbtrMZI1KmNXv9M6reSsytrl2ZGIU/J0\ngONtbYRUtRDyQpDdRPZAVxdeTka7XUQKxRBk3UbF0IqOUS2XHYjFKtt30x+3SssB6hZDEfyy2rxY\nCk7k7KwDM5m3hJXgZrRclDbS308kFOKNq1fZ2NykvbWVew4coLl8PbtbpebWQnDbhpPrdWNZeK3L\niazTts3KOM2zqS8aCPCBoSGur6wwt7lJLBjkYEtLRfhrX1D2oPc2NnKpaFdzHooQ9JU/a14VhI/Y\ns8RQw3via9tu+2k3cPCbEIzytpMUbM9UdqMkPSqo3o4Oejs6qlLWUkoWEwlW19dpamykNR4vnb/w\nSiJO+lTNPXIi4xcp2NVhhWpIpAhhVeWYT+cW7IvHGV9bM5zQ7ipzubVFIgzH4yWuJ1UIDhQT0y4g\nhDz2LDH4AT/vbzV1WSlru/rNrAgrcjCTsVP0+c+7zkooSs+k00zPzSF1ne7OzmwEVbM6fHJ7pDMZ\nvvfqqywXTS62xGI8euJE5QStURt27TmR90okTuv1Imsnb1bGKt0ubxvRHY0WlH1+8hngoe5uw2M2\nT3V3M9jUxO1EAiEE+5qa7s4tOFEg2zga3rPEUO09klJnbm6ShYUJgsEwPT2jNDY21aQtJ/U5TTPK\n80oIRvLb6jryY9Ra9D41M8NzL79MPhyxlJLT993Hvv5+Z3WWpzmRB167coWlXJjrPJZWVzl79Sqn\nDx921obTfnmxEpzIVFufF6vAKSn4ZC34DSEEb+vqYrS5+e5y1Xjc+NAcIRBkLYlya8I337OP2LPE\n4BRG91TXdc6e/S6rq0uFc5onJ69z6NBpenr2bUsfzNLdPCNuyMEsr1rZ3WAlICXJVIrnzpypWGF0\n5vXX6WhpobGhAaRkdX2d1y5eZHZxkYCqcmBoiKOjo6Ux+J2ObHNpdwxORtOl5PbMTCUxeLVcjOSc\nWBdeZbyQjBncuo3Mynopb4K1Z8+y/JVn0FbWaHjbUVp/+nEC7c2u65FSEstFRhWKwnIyydjaGhFV\npbexMRsS2+kIz01ejbEniaHYneEFs7N3WF1dRNfz200kuq5x5coZOjv7UVXvt8XL7+zWiqyGEIzk\nttVKAP9Ioeh9YmrK8MKllNyZnOTo/v1sbm3x9HPPkc4FT9M0jUs3brC6tsbb77/fUIHpus74zAy3\npqYQQjDa10dfRweFE+Ryx4QaQTdTiC6Jx/NI3S9ScFu3kzQvLiQ/5IHF//lNlr/0NDKZPTIzMfMc\na997lcEnP0OgxXoHdFrXyeg6EVVlbG2N1+bnSWoaCtAYCrGeTmfPuiA7sfyewUFazE6G26WkAHuU\nGKrFzMztIlK4CyEEKyvztLX1GJQqlnPWTrW/uxMXkNXn8rRazDvsBlKA7FkDRpFXdSkLRHC1KOJo\nHpquMz47y/rGBo3RaEm7Utd57vXXmVlYKJSbXVpiqLubB44cyV2foKu1lZmlpYq2u81iFVldpxM5\nry4nv+o066tRmhsXkp2cD9aCvr7F8l/9MzKVvpuoaejrW6z8zbdp/+SHDMulNI0XZ2aY3tgAIQgq\nCmlNKyxB1YBE7mxmIJsuJc9OTPDBkREcL0fdYULIY8/uY6gGqhq0yFMLFonZywh2ck7qKJc1+u7k\ns1m58vdakkLhWvN7E6SsVBhWLzu5fF4OvZ2dGEFVVfo6O0FKFlZWDEf3iqKQKN6AlGtjbmmJmcXF\nEjLRNI0709Ms5+Wl5NShQ4QCgcLhK6qiEAoEOHXwoLGidqqEqynrVKZW8lafzeCTm8gKyVuTEDTY\nzJbJsPHqFdNy35ucZHpjA53sYCNZRApW2NK0EsKwVSK7BHvWYqjmHvb3j7K4OFVhNSiKSnNzuy9t\nuu2f3YDCjYVQnO7E1bRXrYRiuXhjI4dGRrJWQS6ej6qqDPb0ZHcxA82xGPNLSxWWhdR1YvnVIUV5\n04uLhbpKm5PMLC7S0tiYbTsa5cceeoibU1Msr63REosx0tND2M2Z0mb5btxCdm3UmhTs+uSknFs5\nlwi0NUHaIGKRgGCXccjt5WSSlVTKEREYVHvXpbhdhOBDfXuWGKpBW1sPg4OHGBu7jBDZUZ4QCidO\nPIri8vzUan4Du7mA8u9uCKH4s5O5h71MCvn3+w4fprezk1vj40gpGertpbu9vWDGH9q3j1sTE2SK\nlL2iKHS0tBDPTU4XIxQIGJ5DoAhBKBAokQ8HgxwZGnKv1M3yqxmh+z3yt0pzk19NWZ+IItjbQfjQ\nEFuXbkHm7nMgQkFaPvI+wzIbmYxpTC07qIpCczhcO1KokZVRFTEIIdqALwPDwC3gY1LKpTKZE8Af\nAE1kXXG/JaX8ci7vT4F3Ays58Z+TUr7mrO1qeg77999Lf/9+lpZmCQZDtLV1oyjm8VL8uP9Onw2n\n5OBExk6BG8nsRVLIv3e2ttLZ2mooG2to4NTRo5y5cKGg7AVwdHiYCkjJvp4eLty4UVmXEAwYHaxi\nRwRuFbZZvlc5v+r16kLaBleRE/T8X59k5re/yNa569njXlWFjl/8KSLHRgzlW8Jhx9aCQnZ+QRHZ\n5anv6O01Hmz6PaL0GcLqqETbwkJ8HliUUn5WCPEZoFVK+StlMocAKaW8KoToA14Gjkopl3PE8HUp\n5VfdtHvixGn59NNnPPe7tH++VOO6/mqsBbNyTq2LmrqZdpAUyuVmFxa4MT5OJpNhqLeXnvZ2vvGD\nH5BKp0uqCqgq/+Kd7zR0/UzNz/P8hQuFJEUIHjl+nM78xHJ53/xSpNUSihO5avvlppxVWaM8u+9O\n6rRBZimBntgg2N+JCFgH0XtpdpY7ZbGRBNmd1ClNoyEY5GhrK+FAgJmNDaKBAPuam4ma7GlwDZ8U\nlbjvvpellKft5Kp1JT0BvCf3+YvAd4ASYpBSXin6PCmEmAU6geVqGt5F8zQFOFHeTtLdEkLxZ7uy\nThW9G9ndSArnr1zh8u3bhTmCmcVFGqNRw8lnKSV3pqc5ODBQ0UZveztPvPOdzK+soADtTU2lI8Ci\nNtPpNNcmJxmfnycUCHCwr48+o/ALboig6LOu62xlMoRVtfL40WrIwwhuytSaFGqEQGsTtBpvai2B\nEJzu6qIlFOLK8jKbmQwSkGRXKwkhuLe9ncGmbF19sZh/rqNaTmxaoFpi6JZSTgFIKaeEEF1WwkKI\nB4EQcL0o+beEEL8OfAv4jJQyaVL2U8CnAAYGhqrsdvVw8hv4ZS1YybqZg3Cr6N3ImkZFNXp38rkK\nUtjY3OTSrVslJKBpGmvr6xhZyJqus5qPhmnQF1WIu0tPTfqS0TSefvVVNpLJwiqm+USCw/39HN+3\nz9m1mqRJKbkyPc2FiYlC/w92dXHvwAD5Xd6mfTNL86tMldCkZCKRYDmZJB4KMRiPE/Cg4KSUzG9t\nsaVptEciNBiN1L0g1xchBAdbW4kEArw4M1NwReaXpb44M0NvLGa+oa0WhFDD0bHt3RNCPA0YLez/\nNTcNCSF6gT8HPiGlzP9jfxWYJksWT5K1Nn7DqLyU8smcDCdOnN6WIYWfZO30WaklIRR/dus6Mq0H\nlwreTtG7kTWRm11YyE4al5ZGl9JUkd4YH6eruZmBri7rPpj09eb0dAkpQJZwLo2Pc6Cvj0ixm8rl\nqPrW3Bznx8dL6r46O4uiKBzPn2Jn0z/Pri2bvnmyEHJIZjI8ffs2yUyGjJSoQvD63BzvHxoiFjRf\nUl4CIVhPpfjOxATJnHWoA/ubmjhRtBHRNUzKWYXantvaorex0XFdbtr1VFcVsF2CI6V8TEp53OD1\nNWAmp/Dzin/WqA4hRBPwD8B/lFL+sKjuKZlFEvgfwIN+XJQTCGH/clPWTsYo3SrNqD43n8vTjPrj\nZj7BkhSkdObK2AZSQEoCFqPFhnDYcDJQl5IX3ngjO//gkhSQkqmy/Q55KIrCYiLhnBQM7uUbk5OG\nG/OuTE9XWkB+koJfFoJJPWfn5thIp8nk8jUpSWkaZ6anXVX/7NQUGzlyycjsec43Eons4ThuYfNH\nVMzyhKhUpnZ/aqeyThSEmbxTpVaGaje4PQV8Ivf5E8DXKvsoQsDfAX8mpfxKWV6eVATwYeC804ad\nKPZq74+bslYydulO2i+XN/tsVLfd6N9IxpY8nIbLtlKy5ek+kAJkw2gbjRRVReGREyfoaDaOiSOA\nyfl58/5a9DUSChkvaZQyO6ltVt7qenLYKt4gVQRN10sJw+uo3658DawFgPHV1YrT1CQwt7lZOSo3\n+aOspdOs5sJQFEOTkqsrK4ZlDOHwzzja3IxqICeAzuLgeH5YCU6UVRXK3wrVEsNngceFEFeBx3Pf\nEUKcFkL8UU7mY8CjwM8JIV7LvU7k8v5SCHEOOAd0AL9ZZX8cYzstBifpRjJG381G7+Xlyz/bkYjV\ne4WcHSmUf7YjDytZty4psgTw6KlThIJBAqpKIDdZe+rIEVpiscpDePJVkJ3g9XJdBw2WJQqyhNEW\ni9lbBxbKt7k8GmcOkWCwsNvas5Ku0irI6DrjiQRjiQQpg82AXtt2qt4yum4qa3rGc6ERh8q0KL+7\noYHR5mYUIVCFICAEAUXhnf39WWvCDyvB6wjUR1Q1QyOlXADeb5B+Bvj53Oe/AP7CpLzxjhIb1Pie\nlLTjh5zXeqy+Wz03ZrJWJOKUFAqwcx1ZKRwnLiWjdAekkP/c3tzMhx59lLmlJTRNo7O1lWDOxTTY\n1cWNiYkK94wEeq1WEVko39ZYjNMHDvDy9eu5JEljJMK7jh61P7Cn7HrWk0kuTU4yv7pKLBJhpLOT\nlc3Nkv6qisKJwcFKy8gPgnBoLUytrvLc2FihD1JK3tbby7CJRZZHIpnkxampggupomqycYcqgs8J\nUdGfplAoq5DL0hUhGLRaHeQERpaBEJzs7uZAaysz6+sEVZW+WCx7/oJJWzKVZvXbL7P+7FnUljjN\nP/EuwocMFtDY/amr6LdbvCV3PoPvz0tVdftJCMWfnVoWbkjB8TkK5elmctWSgkUZRQi6yxW9lLQ3\nNzPc28utqSm03IhTURTuHR0lWq6MXPjk93V1MdDRwfLaGkFVpal8N7UDUljd2uLp8+ezQQGBlc1N\npldWuKevj+mVFVY2N2kMhzne10dPXgF7tQqqsBZSmsZzY2NZl09RPS9PTdERjRLLn0pW1kZK0/jW\n7du2o/nz8/O8M3+OhgUUIXigu5sf5kKfS7KryBoCAQ61tJDUNFKZDOuZDBNrayhCEA+HaQgE6IxG\nDQ/UsTXngXgoRNzByWt6Ks3Ev/0d0nemkVspUARrT79Ixy99jKZ/8Yh1eSeKpEYj5D1LDHvJYnBa\nxgshFOeZyTup1zEpGMFvZe+GFDzICuBthw8z3NPDeG51z76uLprKV5S4VOqQVUrt8bgjWaO0c2Nj\nBVLIQ9N1rszM8BP33be9FoJFHeOJhEm1kjsrKxwzCWp4a2WlIsSIERa3towzDKyD/liMx4eGuLa8\nzEYmQ09DA32NjTw/Pc3sxkZhz0Ex8juTT3d1sS+3/8DRn8xpeg6r3/zhXVIA0CUymWb+975C7P2n\nUaIRd3Vuh+JjDxODH/B6j/22NtxYDGbpbl1HRnKOSKHWyr4KC8GVLNnNau1NTc6tAjeK3qPCnl1Z\nMaTfVCbDVjpNtPjg+qK6pK6zsrVFKpOhNRq1PlK0WkiJpuuG/dTB1EUEsJpKGS73LEej1XLV/INZ\nVE9TKMSJllbWn32dzPQCZ5sjzO7vRjeJfZYnpzOzs7RFo3dH/2ZtOU0vw9r3Xr1LCsUIKGxduEnD\n6aO+teUn9iwx7CaLwUs9XgjCLO9NQwpWsKvTiayX8g5H+o7KW9Sb2Nzkh9euWU7gFpR9GTZSKb5/\n9SprqVQ2Vo+UHO/t5XCXyX5Tr1ZGEXpiMc7OzFSkq0Jkd/6aoC0S4ZYQluSgCsE9HR2G1kEJiggi\nPbXAxC//LvpWErmVoj0UINbRzOV/91PoEROlT/Ze3UokuNco9pXTP65F/9RY1KxhlIaIdZ07QAh5\nvCXPYyhGfiLb6OVXnUZ5VmlW7RfnWX02KmcmZ0lAtTiK0wxu63TbjltCsrMaikV1nblEguvT08wn\nEsgy37vR57WtLZ6/epW/P3OG/+/111nO774ugyIEA62tBIqJoai+71+7RmJrC03XSes6mpScn55m\nZnXV2XUaXpD1PYqHwxxqby9ZuqkKwUA8TnvURBkCg01NhFXVcCWRIBt76HR3Nz15t55DP/vsf/4L\ntJVV5GYSpERNponMLNL39Rcsi0ooJWMrBeDGt5yTbfrQuxHlxCRAiTcQPjpsrgxqQQou6tyzFoMZ\ndopk3boFvTx3TuYQ3LZlmeeWFJzAT6vCyxyEUZ6dvM31pdJpvnv+PInNTSCr4OLRKO85diy7Esqg\nro1Uin86d46MzSSsAHqamjhdHFajCInNTdaSycp1/LrOlbk5uuNlR1Xm2te3kiS+9j3Wvv0SamMD\nTT/5Xhrfca9lX8pxX3c3ffE4t5aW0KVkqLmZ7oaG0nmQslF/QFG4p72dM0XWRkAIHh0cJBYMZknD\nSFla/Ab6ZpKtizdBL5VRMjptZy4z/pF3mZYNCEG/3eolN4RQhoa3HaHlf/0Ay3/+DQgEAIkSjdD3\n+X+DKHZz+e2frhJ7lhh20Mpy1L4b69ArIdjlFae5dTc5JgWjPDPZWlgVTmSdlvfoVnrt5k1WNjZK\nJlVXNjZ47dYtHjhwwKCI5HuXLtmSAsCh7m7uHxw07VPK4qyAZO5I03LoyTQTn/5t0mMzyGQ20uzm\n2Ss0f/Qx2v+3D9v2qRgdDQ10FFsINr/LairFK7OzJUSWkdkjMH9i//5KUsjDYF7BUZs2j0lnQwPd\nRmEsitu0g82fuu1f/RjNP/5ONs9dR403ELnvIEJV3LXhpV9V4C3vSnICJ24mJ+4jJ24lq3wzd5Od\n5esbKRjBrQXgBLvJhWSVlsOdubmKlTa6lNxZWDAkljsLCwXrwgoBRaG7yTr6Z0s0aqj7FCHoMym7\n9vQLpMdnC6QAILdSrHz5n8gsuNgtbAQbpXV9edlwVZImJTMmrrSK+svaUBoihI/sq2w7oLL20FHM\nEBCCR/r7jS0Un0gh/11tbSL26EmiJw9nScGtu8iNi8lIYblsr04MRXB7L50ShdMy5XLlZczyrMr7\nSgrbMYlczVyElYxVH8zk7foiJTL3MoJuUtc1g0nbcihC0BSNVhJDWZ0BVeX+/v4KX380GOSAyZLR\n9WfPGq6U0VWV9bPm5x57RlHfkrmQ1RWQ2XOUvSq/7l/5WZTmRkQ0uw9FRMOEBro4+W8+Tthk0r67\nsfFu7CO3ytPJSM/LaNBI1upPXoXyt8KedSV5hdf75vRZrbZdN64iu8GKZ1Iwgh++fycWgFmeXy4k\nL7AoK0Q2JPfM8nLJ3RNwdwNaGYyC7eWhCEEkGGS4vZ0jPT3mrpUiHOjspDkS4ersLJuZDH1NTezv\n6CBUrhBz1xFobwZFVPjkdV3nlbUVHtE0QhZH3EopmV1fZ2ptjZCqsq+piUaz5Z5l6I3FGDeIUKpj\nEmvI4e+H8oU0AAAYcElEQVQW7Otk31/8Buvff4309ALh/f00PHQcoSqc6urixenpQpuC7M7xezs7\n3SsEJxaCUVo1FojbOnwotyeJwUdiNK3fD9lq6nFDEHYyrkmhFvMKVvCjjJWM0zSX1kIep0ZHefr1\n1wtB7VRFIaAonBoeNqxrsL2dxOam4VnSHzpxolKhG/WprGxnLEZnfpmoTd8bfuJRlv/hByipu3MQ\nUoAWDbEw3M2VuTmOd3ebdEHy7NgYM2traFKiABfn5niwr4/BPBEKUdmHXNpAPM7lxUUSyWRBUatC\nsL+lxXjvgguCUCIh4o9XBmgebGoiEghwaXGRtVSK9oYGjrW3392d7QQ7RQhuXAs+Yk8Sgx+oFflW\nOwBx+90o3S0plKAav36tLYBaTDh7qa8sPxaJ8MFTp7g9O8vS+jqtjY3s6+ggZBL++2B3N3fm51lP\nJgtB4BRF4eH9+7Nl3PbTQj6laYVlqz2xGEFV5XZzmMmfeT9Df/UMUgiElKSbGrj2Cz+BLrK7ms2I\nYTyRKJACFB1UMzlJbzyePagGTMlBAd43NMSN5WXurK4SUBQOtLRY7n0o1OfiusvLdDY20mk2yZxD\nIplkNZWiORwmFgqRzGS4sbJCIpmkLRpluLk5u5ek1oRQC6vBJd40xFCL++WH9ee0jBu3kFM5p6Tg\naLLZD1eSFbbDWnAq58JayCMUCHCwt9e2jpWNDc6NjbGZShEKBGgJheiIxxnt6CAWMQiP4BZF7Y4t\nLfHinTsUB7l7cGiIseVlVh44xNKJURpvz6CFQ2wOdBQeCrONdAC3l5eND6oRgrn1dXrLl8dWCqIq\nCgfb2jiYi2Ol6TqbmQwZXefC/DzT6+sEFYUDra0camuzPAPBD2R0nR+Mj7OwuYkispvvOqJRlra2\n0GT2fIfx1VUuzs/z2OgoDcWWzXYQQrV+bA/Ys8Tg533ww81YTTmvVkJ5nu+ksJOK3YusVXkvcjZl\npZTMLi8zsbhIQFUZ7uykKRo1dfmsbGzwrQsXCstUU5pGMpNhoK2tQArJ3EFBYacnmJlgM53mxTt3\nKoLcvXDnDk25QIEyGGDtQGmgOgU41N5uWq+pkjbKM7IaitKllLyxsMClhQWAEsJJ50hiJZnkIaNT\n6nzEqzMzzOdce/k+zJatkNKkRNc0zs7M8I6BAfcjfSuZXWg17FlicINq72Otyru1GqzynJJCSXkn\nk81GabuBKPzIq8JakFLy/OXLTC0tFSK0Xp2a4sTwMPtNQlGcHx+v2Lug6Trnx8fpisd56eZNVnLL\nWJujUd4+OkrcoxUxVjYZnocuJctbW9mjTw2u9WBHBwMWYbNHWluZMpg8FkCH2bGWJuRwZXGRSwsL\npuExNCkZW13l3nS6dJTuI6SU3E4kHAX2k8DU2po3K8HPP/s2oKrlqkKINiHEPwshrubeW03ktKJD\nep4qSh8RQryQK//l3GlvDtt2/nJ3TdWVN6rLSbrRdytC8UoKhXyryeZieHEdWcHNyN9PEvAZU0tL\nBVKArNLQdJ1Xb9403Vi2aHLUpC4l3750iaXcJjldSpY2NvjWpUtk8uEaXF6TpuuYLaPNt6mI7CEz\nqhCEVJX3jY5yf29v1vVk8gD1xGKMtLaiClGIUArQ3tDARjoNQrCeTrOaTN5t36Q+K1LIQxWClWTS\n0TW7Qq5P0oQgzaCU/9lsR18Gf1anf3Y/lJBHVGsxfAb4lpTys0KIz+S+/4qB3KaU8oRB+ueA35VS\nfkkI8YfAJ4E/qLJPjlCre+2W9N0OPqzmHZySQgncWgZ+zwH4NV/gta85uflEgvO3brGyvk5TNMrx\n4WE6LTaXjc3Pm57xPLuywqCBO6YhHGYzna5IlzkyKIeu64wvLzNs4doxQ29TE28ULdE0Qk88zmBz\nM5FAgM5YzNEoUQjBqb4+2qNRXpyYKAwvZtbW+Kdr12gIhVjPucNURWG4pYXRlhaaI5ES60HK3L4F\nG+hSWkdadQODP4AiBK2RCEtmYb7LZIdbWmprIfilmKqsp9oNbk8AX8x9/iLZc5sdIXfO8/uAr3op\nb123v1aEm/ac5jmxGozqMpN1Vdbp0ZxWaeV5flsLtShvgtnlZb77+uvMLi+TTKeZSyT43vnzTC0u\nmpZRzdb550bi+c/FONbfX1FOFYLmhgZDksnoOhtuR8u5tluiUUba2y3nBDRdZ19rK93xuLGcRdlL\nCwslrioJaGRDXuhSopOdJ7i6uMg/37zJCxMTWQsi9+ALIcxDXeegCEFbJFKYE3EFp396ITjd20tA\nUQrKUBHZIzvjoVDBolKFoD0aNV2tVdGu2Xc3aU5QIwVXrcXQLaWcApBSTgkhTOL8EhFCnAEywGel\nlH8PtAPLUsq83T0O2B/ZlMMOWFeu2nczanc7ALEiBbv6LecVirGd1oLT9r3m2aS9ev16hWLWdJ1X\nb9wwPuoTGO7q4vbcnKFC725pMSzT29LCqeFhzt65U3D17OvooLe5mRdu3KiYfwgoCm02SyytcLK/\nn+54nGdv3qzIU4Wgv3wuoWhEbwVdSlYcjLCL5ccTCXpjMYaK9jqc6Om5ewpcGRSRDW53ure3NiO5\nIrRGo/zo/v1cW1xkeWuLtmiUA21tRAIB5jc2WEunaY5EaLOIGuvZHeD22rzeC5flbIlBCPE00GOQ\n9Wsu2hmSUk4KIUaBZ4QQ5wCjI6BMn0ohxKeATwEMDhqcl7oN8GJBek03y3dLCiWydvMKXt1KtbQE\ntmG+IGESo2dtawspJYXdx0V96YjHOdLfz8XxcYTI+tol8Mjhw3fX8htgpLOTfe3tbKXThFSVgKqi\n6zrxSISVok1vihDEIxHbWElWEDnlf19vLxdmZgokpgpBYyjEsAnplVVS8RsIshaT1Q7ucmhScn1p\n6S4xkN0F/ejQEOfn5kgkkzSFwxzv7KQ5HM6O0i3uo2vY+HgbQyHu7+mpSO+MxTAOLGJRr5+EUI3S\nqQK2xCClfMwsTwgxI4TozVkLvcCsSR2TufcbQojvACeBvwFahBCBnNUwAExa9ONJ4EmAU6dO115b\n4M9v55UQjGS8kILjeQWjfLdKuZYK3g8rwQThYJCtVGXsoFAgcJcUDHDP4CAjXV1MLy0RUFV6W1uN\nzxAugyIEDUVuFEUI3nPkCBenprg9Pw/AcHs7R/MTweB4NG+EI93dtDY0cG1+Prs8trmZ0fb2uwRW\nXK+DdoQQ7G9t5frioqPT2PIwmkfpbGzkvWZWkddnpto/rldlvF2EsA3ukmpdSU8BnwA+m3v/WrlA\nbqXShpQyKYToAB4BPi+llEKIbwMfAb5kVn674LdFV80zZyTnZX7BlQvJq+L1OiexWyAlRwYHOXfz\nZskIWFUUDjs4jL4hHGa02O/s4foymsbY4iKpTIZD3d0Md3QQNtk1bQgHyrw7Hr97NoNdH8vrM6j/\n3u5ukpkMY4kEqshuCosEAmyl0xjZEaoQ7GtudkdwVSpATdfZymSIBAJ3rQ8/FK6bEZubuv1WGlWg\nWmL4LPDXQohPAneAjwIIIU4Dn5ZS/jxwFPiCEEInO9n9WSnlG7nyvwJ8SQjxm8CrwB9X2R9HqKWb\nrha/ra+kYKfIa2EtWMnvAmI52NdHOpPh8thY4U4d6uvjyMCA722VYyud5ukLF0jldv6qQvDG1BTv\nO3KEZiufthWEQOo6pDMQNLB6HCh+ozqLZVRF4aHBQe5Lp1lPp4mFQgQVhZfGxxlLJEqGIQEhaI1G\nGWltvVtXHjX4PaWUnJub42pu45wQgsPt7Rzr7Kw8u8KNMthOQvA68vQJwmqt827FqVOn5Q9+cMYw\nz8/7td1W346QQnG6U4KwkysvYyXv1BrxKu+ijKZpbKVSREKhkjDWtnW6abcs76WbN7k1N1dhz7U2\nNPD4sWPm7Zu0JaUk8bfPsPjHX0NfWUdta6LtUz9J04+/03ldbtNyeP7OHSaLNr8JIKSqnO7ro9ds\n5ZOL+h1BCC7OzfHG3FyJm0sVgvu6uzmYX/rrJyE4lbErY1WuWkWSz9q//2Up5Wm7KvbseQxCGL/8\nrNOpvNs8M3mr8l6eO0eoxj+/C0b7tnDRpqooNEYi/k562mBicdHQybe8uUnawTr/AnIPROJvn2Hh\n97+CvrwGUqItrDD/u3/J6j+/YChv+t1NGrC8tVVCCkBh058ku7/D8XVU8wIuG2yc06Tk4vy8uz+m\nkayTP6qT0b5dPU7r81sJ5rBnicEPeLmfdvJefhu3FqpTC8Xx7maz/GoIYiflt7u+KmClML24PRb/\n+KmKQ3jkVorFJ//OvjNVkMP8+rphlRkpmcnn+ay8jCClJGVCqGY70ivgRHE7IQ0v9RanWRFFje/j\nW4IYqrUunJTxSgg7SgpeFOReI4FdjuGOjgoXiyA7WRywiHJqBJnR0FdWDfMyswab9apRLGVlIyYr\nuBQhjOMc1Ui5CUUxPWehyS7ulBvFbSfjtV47Mtgm7Eli8GBdVtWGEzkv9duluR2QeIZTv7hZGbdl\nvcrvNTj4we7p76c9FkPN7bANKAqN4TAPjIy4rl8EA6gdxpvrgv1me0+t6zRNy6fn8nrjccN5GQHZ\nMBJ2bfr1Ak729lb0RRWCk+X7FAyuw1WaUwVRTVvbSAbFeEtEV3UCL6N9P9uplhR8txZ8mIz0JFfr\nOnYZVEXhPUeOsLi2xvLGBo3hMN3xuOHo2wnaf+GjzH3+iyXuJBEO0v6vP2pcQIjK++o0rShPVRTe\nMzLCs3fusJlOI3Ik9/aBAaI1ioxqht54nEeHhzk/M8NqKpXdONfdTUfx0aG5fhvCyZ/PCyE4qcMv\nIqiynrcsMfg1yvejfLWWgSkpOIEXa2E7sJ2EUy2KlaaVAjUtLmiPxWh3cjSnTf3xD7wDEQqy+IW/\nIT29QHCgi/Zf/AiND9/vqk9eyKE5EuHHDh5kLRcvqSkc9kxw1aKzsZH3jo4aZzr9IxqlObkeL6M8\nv0eaVeJNTwx+3LftIgQ3aU7yfLUW7PL9tjCqwXa04YEAHJd3U3e5rBDE3nea2PuKViRa1CWl5PLc\nHFfn5khrGp2Njdzf15f1x5uRg1mdIhccz0vQu+1ALf+ITsv4QQrbQLZ7co4BXLkcq66/2n46Ta/2\nmXFtLRRjN4y8dzt2aPRbS7w8Ps6FqSk202kyus7U6ipPX73KukGIkBLYjVZ2y72y+iNbpTuRc1PG\n6dyCVf1O5X1QiHuWGGoBv+d7dmKAUoATa8ENGezU6L+OSvjh3ya76/qWQbwjTde5PDdn35aTfuwU\nQdi17ZQoauE68kIITmR8vN9veleSHWrx3Hp5Ht3UYVhvNdZCrVENgbxVyKca15THsqvJJKqoPMFM\nAgvF+xKqdXtZuZ/8hBclbpVeC9eRG0LwkucT3pLEUMv7ul2/pydrwUzGjRXh15/7rWKBVDsXUcN+\nNIZChtFRBQZr/u3IAZwThI2sputMJBJMr60RDQYZbW2l0Whvgis/q4+EYCTjdX7BaZ/clPcqX4Q3\nPTFslyXr1e3jx0ClqsN3/Ci3GxSfG+wWZW2FavvooHxDKERPPM5MWSgLRQiOdBnse7Cr002fTR7y\njK7zzI0brCaTaFKiAFfm53lk3z568tFh3cAP891OxgtJOO1PtWU94k0xx+DDXEvV7XvJ98OFVIHd\novB2Sz92M6r5sf3whQNvHx5mqLUVRWQPG4qHw7xrdDR7RrPDOiryq7iuqwsLBVIA0MnGOXphbMzw\nPAdP/TDL222k4GSepEaKbs9aDDs1p1WO7SYFX6wFr24ks/rqKMVesEhyCCgKDwwN8bbBQTRdJ1gc\nhqPaeQUP92BsednQvaXJ7HGirXahyP38Q9rJ1JoU3Mj7jDeFxbBTqOUz6Am7XRnt9v5VAz9+VL8f\nDBcKRxGilBTs+uNU4bm8JrPItlJK66i3TtrableN36SwjW6QOjF4hNdnsBp5z9aCX3gzK/Y3A7bb\njPZjhU0Z9re1GcZcagiFiJtNQO+E/96L9eClXjf1+YiqiEEI0SaE+GchxNXce6uBzHuFEK8VvbaE\nEB/O5f2pEOJmUd6JavqzXaiGFHaLC8wx3uxksJt/kN3QN7/64FCB72tpYailBUWIQlDBSCDAI/v2\nURJew83o2W9l+yYnBah+juEzwLeklJ8VQnwm9/1XigWklN8GTkCWSIBrwD8VifwHKeVXq+zHtqEW\nv1HNf/edVO5vdmLZSXj19fs5B+K2rvzDblJGCMEDAwMc6exkfn2dSCBAd/70Nz/dOTWwdhxjl5MC\nVE8MTwDvyX3+IvAdyoihDB8B/lFKuVFluzuCal2Rfi5CqYCXoHP1iKfVYQ9NMlcFu+v0ch+KH2iD\nsvFwOBtzqaZ/Gh/q9GIteG1rG1HtHEO3lHIKIPduF/T948BflaX9lhDidSHE7wohTKNvCSE+JYQ4\nI4Q4Mz8/V12vPaBW81Ne4fv8wltBwdWxO1G87LL8VU2dXvK8yLmB15VL2wxbYhBCPC2EOG/wesJN\nQ0KIXuBe4JtFyb8KHAEeANqwsDaklE9KKU9LKU93dHS6abpq1HLByS58Juqowz1204Ncq77U0jqw\na2ubYetKklI+ZpYnhJgRQvRKKadyin/WoqqPAX8npUwX1T2V+5gUQvwP4N877HcddewdWLla/HZH\nvVXcW17xJlbmfqJaV9JTwCdynz8BfM1C9qcpcyPlyASRXW7wYeB8lf2pYztQa8VTV2y7D3tF6fm5\nimo7sEvva7XE8FngcSHEVeDx3HeEEKeFEH+UFxJCDAODwHfLyv+lEOIccA7oAH6zyv7sGHbp71tH\nHXXsFuzk3IZLCLkHR2dCiDngdg2b6ADma1j/dmCvX8Ne7z/s/WvY6/2HvX8Nfvd/n5TSdpJ2TxJD\nrSGEOCOlPG0vuXux169hr/cf9v417PX+w96/hp3qfz0kRh111FFHHSWoE0MdddRRRx0lqBODMZ7c\n6Q74gL1+DXu9/7D3r2Gv9x/2/jXsSP/rcwx11FFHHXWUoG4x1FFHHXXUUYI6MQBCiI8KIS4IIXQh\nhOkKACHEB4QQl4UQ13LRZHcNnIRAz8lpRWHOn9rufhr0x/KeCiHCQogv5/JfyO2J2TVw0P+fE0LM\nFd3zn9+JfppBCPEnQohZIYTh5lKRxX/NXd/rQohT291HOzi4hvcIIVaKfoNf3+4+WkEIMSiE+LYQ\n4mJOD/1bA5nt/R2klG/5F3AUOEw2OuxpExkVuA6MAiHgLHBsp/te1L/PA5/Jff4M8DkTubWd7qub\newr8IvCHuc8fB7680/122f+fA35vp/tqcQ2PAqeA8yb5HwT+ERDA24EXdrrPHq7hPcDXd7qfFv3v\nBU7lPseBKwbP0bb+DnWLAZBSXpRSXrYRexC4JqW8IaVMAV8iG3Z8t+AJsqHPyb1/eAf74hRO7mnx\ndX0VeL8Qu2BraBa7/ZmwhZTye8CihcgTwJ/JLH4ItORD2ewWOLiGXQ0p5ZSU8pXc51XgItBfJrat\nv0OdGJyjHxgr+j5O5Y+3k3AaAj2SC1/+w/xJejsIJ/e0ICOlzAArQPu29M4eTp+Jn8qZ/18VQgxu\nT9d8w25/7p3iHUKIs0KIfxRC3LPTnTFDzlV6EnihLGtbf4dqD+rZMxBCPA30GGT9mpTSKvhfoQqD\ntG1d0mV1DS6qGZJSTgohRoFnhBDnpJTX/emhazi5pzt+3y3gpG//L/BXUsqkEOLTZK2f99W8Z/5h\nN99/p3iFbCiINSHEB4G/Bw7ucJ8qIISIAX8D/LKUMlGebVCkZr/DW4YYpEX4cIcYJxsIMI8BYLLK\nOl3B6hqchkCXUk7m3m8IIb5DdnSyU8Tg5J7mZcaFEAGgmd3jNrDtv5Ryoejrfwc+tw398hM7/txX\ni2IlK6X8hhDivwkhOqSUuyaGkhAiSJYU/lJK+bcGItv6O9RdSc7xEnBQCDEihAiRnQjd8VU9RbAN\ngS6EaBW5U/KEEB3AI8Ab29bDSji5p8XX9RHgGZmbjdsFsO1/mR/4Q2T9x3sJTwE/m1sV83ZgRd49\nR2VPQAjRk5+XEkI8SFbvLViX2j7k+vbHwEUp5e+YiG3v77DTM/K74QX8L2QZOQnMAN/MpfcB3yiS\n+yDZFQPXybqgdrzvRX1rB74FXM29t+XSTwN/lPv8MNkQ52dz75/cBf2uuKfAbwAfyn2OAF8BrgEv\nAqM73WeX/f9t4ELunn8bOLLTfS7r/18BU0A69x/4JPBp4NO5fAH8fu76zmGyam+XX8MvFf0GPwQe\n3uk+l/X/nWTdQq8Dr+VeH9zJ36G+87mOOuqoo44S1F1JddRRRx11lKBODHXUUUcddZSgTgx11FFH\nHXWUoE4MddRRRx11lKBODHXUUUcddZSgTgx11FFHHXWUoE4MddRRRx11lKBODHXUUUcddZTg/wdQ\ndhUe0yCr9QAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots()\n", + "\n", + "ax.imshow(prob, cmap='seismic_r', origin=(0,0), interpolation='bicubic', alpha=0.2,\n", + " extent=xlim+ylim, vmin=0, vmax=1)\n", + "\n", + "ax.scatter(data['x1'], data['x2'],\n", + " color=data[['color', 'is_labeled']].T.apply(lambda x: x['color'] if x['is_labeled'] else 'darkgray'))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define the Active Learning Problem\n", + "`active_learning` uses a class, `ActiveLearningProblem`, that defines the space of entries to be searched" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "problem = ActiveLearningProblem(\n", + " points=data[['x1', 'x2']].values,\n", + " labeled_ixs=data.query('is_labeled').index.tolist(),\n", + " labels=data.query('is_labeled')['y'].tolist(),\n", + " budget=50,\n", + " target_label=1\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run Different Querying Strategies\n", + "The main capability of `active_learning` is that it implements many strategies for how to select which points to label next. These are known as \"querying strategies.\" In this section, we demonstrate using several of the methods available for classification" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "num_to_select = 5" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Random Sampling\n", + "Just pick points at random" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Wall time: 674 µs\n" + ] + } + ], + "source": [ + "%%time\n", + "random_ixs = RandomQuery().select_points(problem, num_to_select)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Greedy Sampling\n", + "Just pick the points most likely to be red. Uses the model we trained earlier" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Wall time: 25.2 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "greedy_ixs = GreedySearch(model=base_clf).select_points(problem, num_to_select)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Uncertainty Sampling\n", + "Pick the points with the highest uncertainty in the ML model" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Wall time: 28.1 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "uncertainty_ix = UncertaintySampling(model=base_clf).select_points(problem, num_to_select)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Active Search\n", + "Active search is a more developed technique. It uses both the probably an entry will be a target class (here, we define the target class as `y == 1`), and the probability that - if labeled - a point will improve the model's ability to find new target classes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Wall time: 46.7 s\n" + ] + } + ], + "source": [ + "%%time\n", + "active_ix = ActiveSearch(model=base_clf).select_points(problem, num_to_select)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Batch Active Search\n", + "Batch Active Search is different from regular active search in that it picks a group of points that collectively has the best utility. This is more expensive, but prevents the algorithm from picking a homogeneous batch of points." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "%%time\n", + "batch_active_ix = SequentialSimulatedBatchSearch(base_clf, ActiveSearch(base_clf), 'optimistic')\\\n", + " .select_points(problem, num_to_select)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compare the Selections\n", + "To demonstrate the effect of each labeling, we show the results of each pick" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def plot_selection(ax, data, indices): \n", + " \"\"\"Plot the selections of a certain query strategy\n", + " \n", + " Args:\n", + " ax: Axis to plot on\n", + " data: Data to plot\n", + " indices: List of selected entries\n", + " \"\"\"\n", + "\n", + " # Plot the classifier\n", + " ax.imshow(prob, cmap='seismic_r', origin=(0,0), interpolation='bicubic', alpha=0.35,\n", + " extent=xlim+ylim, aspect='auto', vmin=0, vmax=1)\n", + " \n", + " # Plot the original data\n", + " original_data = data[[x not in indices for x in data.index]]\n", + " ax.scatter(original_data['x1'], original_data['x2'], label=None,\n", + " color=original_data[['color', 'is_labeled']].T.apply(lambda x: x['color'] if x['is_labeled'] else 'darkgray'))\n", + " \n", + " # Plot the selections\n", + " new_selections = data.loc[indices]\n", + " ax.scatter(new_selections['x1'], new_selections['x2'], marker='x', color=new_selections['color'], label='Selected')\n", + " \n", + " ax.set_xlim(xlim)\n", + " ax.set_ylim(ylim)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "fig, axs = plt.subplots(1, 5, sharey=True)\n", + "\n", + "plot_selection(axs[0], data, random_ixs)\n", + "plot_selection(axs[1], data, uncertainty_ix)\n", + "plot_selection(axs[2], data, greedy_ixs)\n", + "plot_selection(axs[3], data, active_ix)\n", + "plot_selection(axs[4], data, batch_active_ix)\n", + "\n", + "for ax, t in zip(axs, ['Random', 'Uncertainty', 'Greedy', 'Active Search', 'Batch Active Search']):\n", + " ax.text(0.5, 1, t, transform=ax.transAxes, ha='center', va='center',\n", + " bbox={'facecolor': 'w', 'edgecolor': 'k'})\n", + "\n", + "axs[0].legend()\n", + "\n", + "fig.set_size_inches(12, 3)\n", + "fig.tight_layout()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that random picks points clos eto where the boundaries are. Uncertainty sampling picks points close where the classifier has a probability near 50%. Greedy picks points near farthest from the decision boundary. Regular active search picks points that are somewhere in between, but all clustered. Batch active search picks a larger diversity of points" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6191f4f --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +scikit-learn>=0.20.2 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..f0412d2 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,6 @@ +[bdist_wheel] +universal = 1 + +[flake8] +exclude = .git,*.egg*,src/* +max-line-length = 100 diff --git a/setup.py b/setup.py index 9d738cc..3d5d944 100644 --- a/setup.py +++ b/setup.py @@ -7,17 +7,21 @@ setuptools.setup( name="active_learning", - version="0.0.1", - author="Theodore Ando", - author_email="tando2@icloud.com", + version="0.1.0", + author="Logan Ward", + author_email="lward@anl.gov", description="Active learning library for python", long_description=long_description, long_description_content_type="text/markdown", - url="https://github.com/theodore-ando/active-learning", + url="https://github.com/globus-labs/active-learning", packages=setuptools.find_packages(), + install_requires=[ + 'scikit-learn' + ], + python_requires='>3.5', classifiers=( "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", - ), -) \ No newline at end of file + ) +) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..7d0b1d0 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,2 @@ +flake8 +coveralls