diff --git a/README.rst b/README.rst index 3c6e4c0..39a78d6 100644 --- a/README.rst +++ b/README.rst @@ -25,9 +25,10 @@ Sklearn-genetic-opt ################### -scikit-learn models hyperparameters tuning, using evolutionary algorithms. +scikit-learn models hyperparameters tuning and feature selection, using evolutionary algorithms. -This is meant to be an alternative from popular methods inside scikit-learn such as Grid Search and Randomized Grid Search. +This is meant to be an alternative from popular methods inside scikit-learn such as Grid Search and Randomized Grid Search +for hyperparameteres tuning, and from RFE, Select From Model for feature selection. Sklearn-genetic-opt uses evolutionary algorithms from the DEAP package to choose the set of hyperparameters that optimizes (max or min) the cross-validation scores, it can be used for both regression and classification problems. @@ -37,7 +38,8 @@ Documentation is available `here `_ Main Features: ############## -* **GASearchCV**: Principal class of the package, holds the evolutionary cross-validation optimization routine. +* **GASearchCV**: Main class of the package for hyperparameters tuning, holds the evolutionary cross-validation optimization routine. +* **GAFeatureSelectionCV**: Main class of the package for feature selection. * **Algorithms**: Set of different evolutionary algorithms to use as an optimization procedure. * **Callbacks**: Custom evaluation strategies to generate early stopping rules, logging (into TensorBoard, .pkl files, etc) or your custom logic. @@ -82,8 +84,8 @@ The only optional dependency that the last command does not install, it's Tensor it is usually advised to look further which distribution works better for you. -Example -####### +Example: Hyperparameters Tuning +############################### .. code-block:: python @@ -134,6 +136,49 @@ Example print("Best k solutions: ", evolved_estimator.hof) +Example: Feature Selection +########################## + +.. code:: python3 + + import matplotlib.pyplot as plt + from sklearn_genetic import GAFeatureSelectionCV + from sklearn.model_selection import train_test_split, StratifiedKFold + from sklearn.svm import SVC + from sklearn.datasets import load_iris + from sklearn.metrics import accuracy_score + import numpy as np + + data = load_iris() + X, y = data["data"], data["target"] + + # Add random non-important features + noise = np.random.uniform(0, 10, size=(X.shape[0], 5)) + X = np.hstack((X, noise)) + + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=0) + + clf = SVC(gamma='auto') + + evolved_estimator = GAFeatureSelectionCV( + estimator=clf, + scoring="accuracy", + population_size=30, + generations=20, + n_jobs=-1) + + # Train and select the features + evolved_estimator.fit(X_train, y_train) + + # Features selected by the algorithm + features = evolved_estimator.best_features_ + print(features) + + # Predict only with the subset of selected features + y_predict_ga = evolved_estimator.predict(X_test[:, features]) + print(accuracy_score(y_test, y_predict_ga)) + + Changelog ######### diff --git a/docs/api/featureselectioncv.rst b/docs/api/featureselectioncv.rst new file mode 100644 index 0000000..e263f27 --- /dev/null +++ b/docs/api/featureselectioncv.rst @@ -0,0 +1,23 @@ + +FeatureSelectionCV +------------------ + +.. currentmodule:: sklearn_genetic + +.. autosummary:: GAFeatureSelectionCV + GASearchCV.decision_function + GASearchCV.fit + GASearchCV.get_params + GASearchCV.inverse_transform + GASearchCV.predict + GASearchCV.predict_proba + GASearchCV.score + GASearchCV.score_samples + GASearchCV.set_params + GASearchCV.transform + +.. autoclass:: sklearn_genetic.GAFeatureSelectionCV + :members: + :inherited-members: + :exclude-members: evaluate, mutate, n_features_in_, classes_ + :undoc-members: True \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index d123101..e7126d5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -55,7 +55,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints"] # -- Options for HTML output ------------------------------------------------- diff --git a/docs/images/basic_usage_accuracy_6.PNG b/docs/images/basic_usage_accuracy_6.PNG new file mode 100644 index 0000000..cf71c5b Binary files /dev/null and b/docs/images/basic_usage_accuracy_6.PNG differ diff --git a/docs/images/basic_usage_fitness_plot_7.PNG b/docs/images/basic_usage_fitness_plot_7.PNG new file mode 100644 index 0000000..7afb3b8 Binary files /dev/null and b/docs/images/basic_usage_fitness_plot_7.PNG differ diff --git a/docs/images/basic_usage_train_log_5.PNG b/docs/images/basic_usage_train_log_5.PNG new file mode 100644 index 0000000..f442f60 Binary files /dev/null and b/docs/images/basic_usage_train_log_5.PNG differ diff --git a/docs/index.rst b/docs/index.rst index a12c6a4..d7c9b0c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,10 +5,13 @@ sklean-genetic-opt ================== -scikit-learn models hyperparameters tuning, using evolutionary algorithms. -########################################################################## +scikit-learn models hyperparameters tuning and feature selection, +using evolutionary algorithms. -This is meant to be an alternative from popular methods inside scikit-learn such as Grid Search and Randomized Grid Search. +################################################################# + +This is meant to be an alternative from popular methods inside scikit-learn such as Grid Search and Randomized Grid Search +for hyperparameteres tuning, and from RFE, Select From Model for feature selection. Sklearn-genetic-opt uses evolutionary algorithms from the deap package to choose a set of hyperparameters that optimizes (max or min) the cross-validation scores, it can be used for both regression and classification problems. @@ -73,6 +76,7 @@ as it is usually advised to look further which distribution works better for you notebooks/sklearn_comparison.ipynb notebooks/Boston_Houses_decision_tree.ipynb + notebooks/Iris_feature_selection.ipynb notebooks/Digits_decision_tree.ipynb notebooks/MLflow_logger.ipynb @@ -87,6 +91,7 @@ as it is usually advised to look further which distribution works better for you :caption: API Reference: api/gasearchcv + api/featureselectioncv api/callbacks api/plots api/mlflow diff --git a/docs/notebooks/Iris_feature_selection.ipynb b/docs/notebooks/Iris_feature_selection.ipynb new file mode 100644 index 0000000..c2d8dab --- /dev/null +++ b/docs/notebooks/Iris_feature_selection.ipynb @@ -0,0 +1,260 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Iris Feature Selection" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 1, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "from sklearn_genetic import GAFeatureSelectionCV\n", + "from sklearn_genetic.plots import plot_fitness_evolution\n", + "from sklearn.model_selection import train_test_split, StratifiedKFold\n", + "from sklearn.svm import SVC\n", + "from sklearn.datasets import load_iris\n", + "from sklearn.metrics import accuracy_score\n", + "import numpy as np\n" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "### Import the data and split it in train and test sets\n", + "Random noise is added to simulate useless variables\n" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 2, + "outputs": [ + { + "data": { + "text/plain": "(150, 14)" + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data = load_iris()\n", + "X, y = data[\"data\"], data[\"target\"]\n", + "\n", + "noise = np.random.uniform(0, 10, size=(X.shape[0], 10))\n", + "\n", + "X = np.hstack((X, noise))\n", + "X.shape" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "### Split the training and test data" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 3, + "outputs": [], + "source": [ + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=0)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "### Define the GAFeatureSelectionCV options\n" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 4, + "outputs": [], + "source": [ + "clf = SVC(gamma='auto')\n", + "\n", + "evolved_estimator = GAFeatureSelectionCV(\n", + " estimator=clf,\n", + " cv=3,\n", + " scoring=\"accuracy\",\n", + " population_size=30,\n", + " generations=20,\n", + " n_jobs=-1,\n", + " verbose=True,\n", + " keep_top_k=2,\n", + " elitism=True,\n", + ")" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "### Fit the model and see some results" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 5, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INSTANCE\n", + "True\n", + "gen\tnevals\tfitness \tfitness_std\tfitness_max\tfitness_min\n", + "0 \t30 \t0.558444\t0.155441 \t0.893333 \t0.253333 \n", + "1 \t54 \t0.659333\t0.132948 \t0.893333 \t0.333333 \n", + "2 \t54 \t0.742667\t0.0867111 \t0.893333 \t0.586667 \n", + "3 \t55 \t0.805778\t0.0740117 \t0.893333 \t0.653333 \n", + "4 \t52 \t0.873333\t0.0435125 \t0.906667 \t0.746667 \n", + "5 \t53 \t0.896222\t0.00659592 \t0.913333 \t0.893333 \n", + "6 \t55 \t0.901111\t0.0131186 \t0.953333 \t0.893333 \n", + "7 \t54 \t0.911778\t0.0206332 \t0.953333 \t0.893333 \n", + "8 \t50 \t0.926444\t0.0210455 \t0.953333 \t0.893333 \n", + "9 \t51 \t0.941333\t0.020177 \t0.966667 \t0.913333 \n", + "10 \t49 \t0.955556\t0.00978787 \t0.966667 \t0.913333 \n", + "11 \t55 \t0.959111\t0.00660714 \t0.966667 \t0.953333 \n", + "12 \t57 \t0.965333\t0.004 \t0.966667 \t0.953333 \n", + "13 \t55 \t0.966444\t0.00271257 \t0.973333 \t0.953333 \n", + "14 \t58 \t0.966667\t6.66134e-16\t0.966667 \t0.966667 \n", + "15 \t53 \t0.966889\t0.0011967 \t0.973333 \t0.966667 \n", + "16 \t56 \t0.967556\t0.00226623 \t0.973333 \t0.966667 \n", + "17 \t53 \t0.969556\t0.00330357 \t0.973333 \t0.966667 \n", + "18 \t51 \t0.971111\t0.0031427 \t0.973333 \t0.966667 \n", + "19 \t58 \t0.972889\t0.00166296 \t0.973333 \t0.966667 \n", + "20 \t54 \t0.973333\t3.33067e-16\t0.973333 \t0.973333 \n" + ] + } + ], + "source": [ + "evolved_estimator.fit(X, y)\n", + "features = evolved_estimator.best_features_\n", + "\n", + "# Predict only with the subset of selected features\n", + "y_predict_ga = evolved_estimator.predict(X_test[:, features])\n", + "accuracy = accuracy_score(y_test, y_predict_ga)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 6, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[ True True True True False False False False False False False False\n", + " False False]\n", + "accuracy score: 0.98\n" + ] + } + ], + "source": [ + "print(evolved_estimator.best_features_)\n", + "print(\"accuracy score: \", \"{:.2f}\".format(accuracy))" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 7, + "outputs": [ + { + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmMAAAJdCAYAAAB+uHCgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABWFUlEQVR4nO3dd3xUVf7/8fdMeg+EJPTekaaggK6ICIhS7OKq6FrXtayoq9hWxLLKuquurH5FV92fYmWxgLuI4uoqoKIgIAYCJISSkJ5MeiYz5/dHYCS0UDJzp7yej0cezNw75XNyk8ybc84912aMMQIAAIAl7FYXAAAAEMoIYwAAABYijAEAAFiIMAYAAGAhwhgAAICFCGMAAAAWIowBLaRPnz6aPHmypk6d6vm6//77JUlTp06Vw+FQRUWFpk+fbnGl2Llzp4YOHdrs47744gs9++yzkqRly5bp0Ucf9XZpIWPu3Ln67LPPJEnPPvusPvjgA2sLAiwUbnUBQDD55z//qdatWx+w/cMPP5TUGALWr1/v67JwjNavX6/y8nJJ0tixYzV27FiLKwoe3377rXr27ClJ+v3vf29xNYC1CGOAD/Tp00crV67Uvffeq9raWk2dOlULFy7UkCFDdMMNN2j58uUqKCjQ9OnTdfXVV0uS3nvvPb311ltyu91KTk7Wgw8+qB49euj777/XE088IbfbLUm68cYbNWHChENu35fb7dbjjz+utWvXqqqqSsYYPfroo+rdu7dGjx6tTz75RKmpqZKkSy65RDfffLNGjhypp556SqtWrZLL5VL//v31wAMPKD4+XmeeeaYGDRqkTZs26Y477lB4eLhefPFF1dfXq6SkROedd55uv/12SdK8efO0YMECxcXFadiwYVq2bJk+//xz1dfXH/L19/fCCy9o6dKlcrvd6tChgx566CFVV1dr2rRp+uqrrxQZGSmXy6UxY8bolVdeUXx8vGbNmqVdu3bJGKPzzjtP1113XZPXfO6551RaWqo//vGPTe5PnTpVb7/9tlwulxISEtSlSxd98sknevHFF7V79+6Dvu7OnTt19dVXa/To0Vq7dq3Ky8s1Y8YMnXPOOQe05bPPPtPcuXPlcrkUHx+ve++9VyeccILGjBmjuXPnauDAgZKkGTNmaPjw4fr1r3990Panp6fryiuvVFJSkrKysnTZZZfpyiuv9LyPy+XSnDlz9PnnnyshIUGDBg3S1q1b9frrr6uiokKPPfaYMjMz5XQ6NXLkSN19990KDw/XwIEDj/pnc+bMmSorK9OOHTt0xhln6KKLLtLs2bNVXV2tgoIC9e3bV88884wWLFign376SXPmzFFYWJiWLVumXr166dprr9X333+vOXPmqKamRhEREbr99tt1+umna+HChfr0009lt9uVk5OjiIgIPfnkk+rdu7eWLl2qF154QTabTWFhYbr77rs1fPjwo/kVBaxlALSI3r17m0mTJpkpU6Z4voqKijz7iouLzY4dO8yQIUOaPOf11183xhizfv16c8IJJ5ja2lrz7bffml//+temurraGGPMV199ZSZOnGiMMWb69Olm8eLFxhhjMjIyzKxZsw67fV+rV682t956q3G5XMYYY1588UVz4403GmOMufvuu83LL79sjDFmy5Yt5owzzjAul8s899xz5oknnjBut9sYY8xf/vIX89BDDxljjBkzZoyZO3euMcYYt9ttrrjiCpOdnW2MMWb37t2mX79+pri42Pzvf/8zEyZMMOXl5cbtdpt7773XjBkzxhhjDvv6+3r//ffN7bffbpxOpzHGmLfffttcd911xhhjLr/8cvOf//zHGGPMF198YaZNm+bZ/sorrxhjjHE4HGby5Mlm8eLFTY7D3/72N/Pwww973mff+/ve/te//mVuuOGGZl+3d+/e5vPPPzfGGLNkyRJzxhlnHNCWLVu2mFGjRpnt27cbY4xZsWKFOfXUU01FRYV59tlnPe9ZVlZmTj75ZONwOA7b/iuuuMLce++9B7yPMca89dZb5vLLLze1tbWmrq7OXHPNNeaKK64wxhgzc+ZM8//+3/8zxhjT0NBg7rrrLjNv3jxjzLH9bN5zzz3mqquu8rz3E088YT744ANjjDH19fVm0qRJZsmSJZ6a9x6ze+65x7z88sumpKTEjBw50vz444/GGGMyMzPNySefbLZv327+9a9/mZNOOsnk5eUZY4yZPXu2ufvuu40xxowdO9asWbPGU89zzz130O8F4K/oGQNa0KGGKQ9n79DXgAEDVF9fr+rqan3xxRfKycnRtGnTPI8rLy9XWVmZJk6cqNmzZ+vzzz/XqFGjdMcdd0jSIbfva+jQoUpKStLbb7+tHTt26Ntvv1VcXJwk6eKLL9bDDz+sa6+9Vv/61790wQUXyG6364svvlBFRYVWrFghSXI6nUpJSfG85rBhwyRJNptN//d//6cvvvhCixcv1tatW2WMUU1Njb788kudffbZSkxMlCRdfvnl+uabbySp2dff67///a/Wr1+vCy+8UFJjL19NTY2n9vfff19nn322Fi5cqIsvvljV1dVavXq1XnnlFUlSQkKCLrjgAv3vf//T4MGDj+oY7au5142IiNDo0aMlSf3791dZWdkBr/HNN99oxIgR6tSpkyRp5MiRat26tX766SddeOGFuuiiizRz5kwtXrxYY8aMUUJCwmHbv+9x2N+XX36pqVOnKioqSpJ06aWX6vXXX5fU+L1fv369FixYIEmqra1t8tyj/dmUpJNOOsmz/Q9/+IOWL1+ul156Sdu2bVNBQYGqq6sP+b1dt26dOnfu7Dk+vXr10oknnqjvvvtONptNAwYMUNu2bT3f208//VSSdO655+qWW27R6NGjdeqpp+r6668/5HsA/ogwBlhs74ekzWaTJBlj5Ha7NXXqVP3hD3+Q1PjBW1BQoKSkJE2bNk1jxozR8uXL9dVXX2nu3Ln66KOPDrk9ISHB815ffPGFHnvsMf3mN7/R2LFj1b17d3300UeSGj/MGxoatG7dOi1evFhvv/22573vu+8+T8CoqqpSXV2d5zVjY2MlNYaU888/X2eddZaGDRumCy+8UJ999pmMMQoPD5fZ5zK4YWFhntvNvf6+j7vuuuv061//WpJUX1/vmc919tln609/+pO2bt2qVatW6YknnpDL5Wrynntfo6Ghock2m83W5HFOp/Nwh0tut/uwrxsRESG73e557YPZ//l7tzU0NKhDhw7q37+/vvjiCy1cuFD33Xdfs+2XfjkO+wsPb/pnfm9te1/z2WefVY8ePSRJDoejSc1H+7O5fx133HGHXC6XJk6cqDPOOEN5eXkHbfu+9Rzq+xIREaHo6GjP9n2P24wZM3TRRRfp66+/1sKFCzVv3jwtXLiwSVsBf8ZPKuBD4eHhBw0J+zv11FP18ccfq6CgQJL01ltv6aqrrpIkTZs2TRkZGbrgggv0yCOPyOFwqLy8/JDb97V8+XKNGTNGv/71rzVw4EB99tlncrlcnv0XX3yxHnnkEfXp00ft27eXJJ122mmaP3++6uvr5Xa79eCDD+qvf/3rATXn5OSosrJSt99+u84880x99913nueMHj1aS5cuVUVFhSR5emKO5vVPO+00LViwQJWVlZIaz8C7++67JTWGhnPPPVczZ87U+PHjFRMTo/j4eA0ePFjz58+XJFVUVOiDDz7QqFGjmrxuq1attGHDBhljVF1dra+//tqzLyws7IDwdqSvezgjRozQ8uXLtWPHDknSypUrlZeX5+kRuuSSS/TSSy+ptrbW09N0uPYfzujRo/XRRx+pvr5eDQ0Nev/99z37TjvtNL322msyxqi+vl433XST3njjjcO+3uF+Nvf39ddf6+abb9Y555wjm82mtWvXen7eDva9HTx4sLKzs7Vu3TpJ0ubNm7Vq1SqdfPLJh6ynoaFBZ555pqqrq3XZZZfpoYce0tatWw94bcCf0TMG+FBqaqr69++viRMn6q233jrk4371q1/p+uuv1zXXXCObzab4+HjNnTtXNptNd911lx5//HE988wzstvtuuWWW9SxY8dDbt/XtGnTdNddd2ny5MkKCwvTsGHDPBPC7Xa7zjvvPP31r39tEoZ+97vf6cknn9T5558vl8ulfv36aebMmQfU3KdPH51xxhmaOHGiEhMT1blzZ/Xs2VM5OTn61a9+pUsuuUSXXnqpoqOj1atXL8XExBzV61988cXKz8/XJZdcIpvNpnbt2umJJ55osv+NN97QrFmzPNueeuopzZ49WwsXLlR9fb0mT56sCy64QLt27fI8ZsqUKfrqq680fvx4paena+jQoZ6wPHLkSN16662KiIjQgAEDjup1D6dnz5566KGHdMstt8jlcik6Olr/93//5+nFPPPMM/Xwww83GW5rrv2HcsEFFyg7O1vnnXeeYmNj1bFjR8/3/v7779djjz2myZMny+l0atSoUQec4LC/w/1s7m/GjBm6+eablZSUpJiYGA0fPlzbt2+XJI0ZM0ZPPvlkk57I1q1b69lnn9Ujjzyi2tpa2Ww2/elPf1K3bt20Zs2ag9YTHh6u++67T3fddZfCw8Nls9n0+OOPKzIystnvDeAvbKa5/6IDwHFav3691qxZ41lj7dVXX9XatWv1zDPPWFtYCPj6669VXFysqVOnSpIeffRRRUVFeYYZAViPMAbA6yorK3XfffcpKyvL06vzyCOPKD093erSgl5+fr5mzpyp4uJiuVwu9e3bV7NmzWoylxCAtQhjAAAAFmICPwAAgIUIYwAAABYijAEAAFiIMAYAAGChgF5nrLS0Sm63d88/SEmJV3FxpVffw1/R9tBsuxTa7Q/ltkuh3X7aHpptl7zffrvdplat4g65P6DDmNttvB7G9r5PqKLtoSuU2x/KbZdCu/20PXRZ2X6GKQEAACxEGAMAALAQYQwAAMBChDEAAAALEcYAAAAsRBgDAACwEGEMAADAQoQxAAAACxHGAAAALEQYAwAAsBBhDAAAwEKEMQAAAAsRxgAAACxEGAMAALAQYQwAAMBChDEAAAALEcYAAAAsRBgDAACwEGEMAADAQoQxAAAAC4VbXQAAAMDRchsjl8uoweWWy20av1xuNez51+Vq3Nbg3nN7z+MaXEYu997bjf+OGd7F0rYQxgAAgE85G1yqqHbu+aqXo7peFdVOz7+Ve25X1jjV4HI3Bqj9wpQxLVePSzadMahdy73gUSKMAQCA4+JscKtiT5Cq2C9Y7bt977baetdBXyfMblNCbIQSYiOVGBuhNknRigi3KzzMrjC7TWF2u8LCbAoP23PbbvtlX9g+t/fbHhZmV/g+z2/cbld4mE3hdrv69GijoqJKH3/XfkEYAwAAkiRjjOqdblXVOlVZ0/hVVdvwy+09X5U1TlXWOlVR5VRFTb1q6g4druJjI5QYG6mE2Ah1T05qErYS9mzfuz8mKlw2m83HrZYl77kvwhgAAEGoweVuDE61DZ4AVbUnRDXe/mV7XYNbZRW1qqxpUIPLfcjXjIywKz4mQvHREYqLiVC39jFKiIloDFhxkUqIiVRi3C8hK9aicBVoCGMAAASIBpdbjqp6lVXWq7yqTuVV9XJU1qu8au9XncorG+daHWooUGrssYqLidgTrMLVNiVWHVPjGu/HRCguOnyf2xF7HhuuiPAwH7Y2dBDGAACwkNsYVdY49wlVjSGrvLJejqp9glZlnapqGw76GvExEUqKi1RiXKR6dkhSfGxEkzC1b8iKi4lQdGRYkx6r1NQEFRZW+KrJ2A9hDACAI+R2Gzldbjkb9nztud2w936Dq+n+fR7jbHCrzun6JWDt6d1yVDnlPsipgZERdiXHRSkxPlLtUmLVt3OykuIilRQfpcS4yMbbewJYeBjLhgYywhgA4IgZY1Tf4FZtXYNq612qc7rkNkZud+O+xttGxjT2+Oz9d99tbreRUWOwaXxM4/M9t432PL7x9r6v67nvbrrP7Hm+57aMzJ7Xdxt5bh+uHnuYXdXV9QcNW3u/XO7jW0/BbrMpMS5CSXFRSoqPVKf0eE+oSoqPahKw9u+9QvAijAFAkHMbo7p6l2rrXaqtbwxRe8NUzd779S7V1DXIFmZXaXmNaut+eWzNnufV7NnWkus7HQ+bJLvdJpvNJrtNstltsu+9bbPt2aeDbGu8b7fbZJNNdnvjY6KiwmW32xQfGamIcHvjV5j9l9v73A8/7P4wRYTbFRm+z2P3PC5sz/sD+yKMAUCQMMZoV1GV1m8t1rqtxSooq1FNXYPq6l06kvxks0mxUeGKigxTdGS4YiLDFB0ZpuT4KEVH7dm259/oyDDFRIYrMsIu+94Q1CT87Llt33v7l9DTJAztve15TNPg5Lm95/m2fV67pUMN86ZgFcIYAASwOqdLG3NKtW5PACt21EqSOqXFa0DX1ocMUdGRYYqJavw3OjJM0VHhigy3Ky0tkUAC+BhhDAACTFFZjdbuCV8bt5fK2eBWVESY+ndtpUmjumhg9xS1Toy2ukwAR4gwBgB+rsHl1pad5Y29X1nFyi2qkiSlJcdo9OD2GtQzRX06tVJEOGfUAYGIMAYAfshRVa/1WcVau7VYG7JLVFPXoDC7Tb07Jev0Qe00qGcbtW0da3WZAFoAYQwA/IDbGOXsrtgz96tI2/IqZCQlxUdqWJ9UDerRRv27tlJMFH+2gWDDbzUAWKS6tkE/byvxDD86quplk9S9faKm/qqbBvdoo07p8bKzFAIQ1AhjAOADxhiVVtQpt6hKOwoqtT6rWJt3lsvlNoqNCtcJ3VtrUI8UndA9RYmxkVaXC8CHCGMA0II8oau4SrmFVdpVVNV4u6haNXW/XFewQ2qcxp/cSYN7tFGPDokKszP5HghVhDEAOAbGGJVV1mtXUaVyCxsD166iA0NXfEyEOrSJ04gB6erQJk7tU+LUPjWO3i8AHoQxADiMvaErt2hv2KpsNnS1T4lrDF6ELgBHgDAGAHs4G9z6MbNAG7YUKbeoyhPA9g9d7dvEaUT/dLVvsyd0tYlTYhyhC8CxIYwBgKStueV65eMM5RVXSzowdO0NXoQuAC2NMAYgpDkbXPrgq2wt+W67WiVEaeZVw9U2MUoJsREtfiFqADgYwhiAkJWV69A/Pv5ZecXVOn1wO10yppe6dGrFhbIB+BRhDEDIcTa49OHX2/Sfb3OUHB+lOy4ZrBO6p1hdFoAQRRgDEFKy8xz6x8cZyi2q0q8GtdOlZ/ZSbDR/CgFYh79AAEKCs8GtD7/O9vSGzbhksAbSGwbADxDGAAS9fXvDThvUTtPoDQPgR/hrBCBoORvc+mh5tv7zzXYlxUfq9osHa1APesMA+BfCGICglJ3n0CsfZ2hXUZVOG9hO08b2VGx0hNVlAcABCGMAgsqBvWGDNKhHG6vLAoBDIowBCBrbdjfODdtVSG8YgMBBGAMQ8JwNbi1aka1/r9yuxLgIesMABBTCGICAlrO7Qi9//LN2FVbp1BPaatpZvRRHbxiAAEIYAxCQGlxufbR8m/69MkeJcRH6/UWDNLgnvWEAAg9hDEDAydldoX98/LN20hsGIAgQxgAEjAaXW4uWb9PHK3OUEBeh2y4apCH0hgEIcIQxAAGhsTcsQzsLKzXqhLa6jN4wAEGCMAbAr7ndRv/5NkcffJWt+NgI3XbhIA3pRW8YgOBBGAPgtwrLavTy4p+1eWe5hvdN05UT+ig+ht4wAMGFMAbA7xhjtHz9br35WaZsNun6Sf01YkC6bDab1aUBQIsjjAHwKxXV9fp/Szbph8xC9emUrGsn9VObpBirywIAryGMAfAb67OK9crHGaqsceriMT00YXhn2e30hgEIboQxAJarc7r03n+36PPVu9ShTZxmXDJYndMTrC4LAHyCMAbAUtt2OzTvo5+1u6Ra44d30oWjuysiPMzqsgDAZwhjACzhcrv172+266Ovs5UYF6k/TBuifl1bW10WAPgcYQyAzxWU1ejlRT9ry65yndI/XVeM780CrgBCFmEMgM8YY/T1ujy9uWyz7DabbpjcXyMGtLW6LACwFGEMgE84quv1z/9s1JrNRerbOVnXnttfKUnRVpcFAJYjjAHwunVbi/TKvzequtapS8b01PiTO8nOAq4AIIkwBsCL6pwuvfv5Fv13zS51TI3TXZcOUce0eKvLAgC/QhgD4BXZeQ7NW/SzCkqqdfbJnXX+6d0VEW63uiwA8DuEMQAtyuV26+OVOVq0fJuS4iP1h8uGqm+XVlaXBQB+izAGoMUUlFbrpcU/a+suh0YMSNcV43orliUrAOCwCGMAjpsxRl+ty9Nbn21WmN2m304doJP7pVtdFgAEBMIYgOPiqKrXP5c0LlnRr0srXXtuP7VOZMkKADhShDEAx2xNZqFeW7JRNXUuTTuzp84azpIVAHC0CGMAjlp1bYPeWpap5et3q3N6vO6+rL86pLJkBQAcC8IYgKOyMadU//j4Z5VW1GvyqK6afGpXhYexZAUAHCvCGIAj4mxw6V9fZmnpqh1KbxWje688UT3aJ1ldFgAEPMIYgGZt2+3Qy4szlFtUpTNP7KCLz+ipqMgwq8sCgKBAGANwSPsu4JoYF6k7Lh2sE7qlWF0WAAQVwhiAg8orrtLLizOUnefQiP7punx8b8WxgCsAtDjCGIAm3MZo8ddZenXRBkWE21nAFQC8jDAGwKPEUatX/p2hn7eVamD3FP3mnL5Kjo+yuiwACGqEMQAyxuibn/P1xtJMud1GN180WCf2aC0bC7gCgNcRxoAQV1Fdr9c/2aTvNxWqZ4ckXTepnwb0TldhYYXVpQFASCCMASFs7ZYivfafjaqsceqiM3ro7JM7y26nNwwAfIkwBoSgmroGvfP5Fv1vba46psZpxiWD1Tk9weqyACAkEcaAEJO5o0wvL/5ZxeW1mjiis847rbsiwrmcEQBYhTAGhAhng1sffJWlJd9uV0pStO65/ET17pRsdVkAEPIIY0AI2J5foZcX/6ydhVUaPaS9LhnTUzFR/PoDgD/grzEQxNxuo/98m6MPvspWXEyEfn/RIA3u2cbqsgAA+yCMAUHIbYxWbyrUh8uztauwSsP6pOrKCX2UEBtpdWkAgP0QxoAgsn8Ia5cSq9+dd4JO6pPKAq4A4KcIY0AQOFgIu2FKf53cN511wwDAzxHGgAC2N4R9tDxbOwlhABCQCGNAANo/hLVtHasbJvfXyf0IYQAQaAhjQABxG6M1mYX68Ott2llYSQgDgCBAGAMCwP4hLL11rK6f3F+nEMIAIOARxgA/1hjCivTR8mztKCCEAUAwIowBfuiAENYqRtdP6q+T+6cpzM51JAEgmBDGAD9CCAOA0EMYA/yAMUZrNhfpo6+ztX1PCLtuUj+d0j+dEAYAQc6rYWzRokV64YUX5HQ6dfXVV+vyyy9vsv/LL7/UU089JUnq3bu3Zs+erbi4OG+WBPiV/UNYGiEMAEKO18JYfn6+nn76aS1cuFCRkZGaNm2aTjnlFPXs2VOS5HA4NHPmTL3++uvq2bOnXnrpJT399NN64IEHvFUS4DfKq+qVkVOiJd9s94Swa8/tpxEDCGEAEGq8FsZWrFihESNGKDk5WZI0YcIELVmyRLfccoskadu2bWrfvr0nnI0ZM0bXXXcdYQxBx22McouqtGVnuTbvLNfWXeUqKKuRJEIYAMB7YaygoECpqame+2lpaVq3bp3nfteuXbV7925t3LhRffv21X/+8x8VFRV5qxzAZ+rqXcrKc2jLrnJt2RO+qusaJEmJsRHq2TFZZwztoJ4dk9StXQIhDABCnNfCmDHmgG022y/rIiUmJurJJ5/Ugw8+KLfbrUsuuUQRERFH9R4pKfHHXeeRSE1N8Mn7+CPa3rzi8hr9nF2ijdtK9PO2EmXtKpfb3fjz37ltgn41tIP6dW2tft1aq11KXJPfA3/GsQ9dodx+2h66rGy/18JYenq6vv/+e8/9goICpaWlee67XC61bdtW7733niRpw4YN6tSp01G9R3FxpedDz1tSUxNUWFjh1ffwV7T9wLa73UY7Cys9vV6bd5ar2FErSYoMt6tbu0RNPKWzenVMUo8OSYqL3uc/GMaoqKjSV004Lhz70Gy7FNrtp+2h2XbJ++23222H7UDyWhgbNWqUnnvuOZWUlCgmJkZLly7VI4884tlvs9l0zTXX6L333lNaWppeeeUVnXPOOd4qBzgmNXUNjUOOO8u1ZWeZtuY6VFvvkiQlxUeqV4ckjRveSb06JqlTWrzCwxhyBAAcHa/2jM2YMUPTp0+X0+nURRddpEGDBun666/XbbfdpoEDB2r27Nm67rrrVF9fr5EjR+raa6/1VjnAEXNU1ev95eu1dlOBdhRWyhjJJqlDarxGDmirnh2T1KtDklKSogNmyBEA4L9s5mCTuwIEw5TeFYptr651as6ba5RbXK1eHZPUq2OSenZMUo/2SYqJCp01kkPx2O8Vym2XQrv9tD002y4F8TAlEGjqnC49u2CddhVV6Y/XjlCnlBirSwIAhAAmuACSGlxuPf/+T9qys1w3TBmgE/umNf8kAABaAGEMIc/tNnp58c9an1Ws6Wf30XCCGADAhwhjCGnGGL3xaaa+yyjQxWf00OghHawuCQAQYghjCGkL/5elL9bs0sQRnTVxRBerywEAhCDCGELWkm+36+OVORo9pL0uGt3D6nIAACGKMIaQ9L+1uXr3v1s0vG+arhzfh/XCAACWIYwh5Hy/sUD/XLJRJ3Rvresn95fdThADAFiHMIaQsiG7RPMWbVCP9km6+byBXL4IAGA5PokQMrbsKtdzC9epbes4/f7iQYqKDLO6JAAACGMIDTsLKvXse2uVHBelOy8drLjoCKtLAgBAEmEMIaCgrEZ/eedHRYTbdde0IUqKj7K6JAAAPAhjCGqlFXV66q01anC5dee0oWqTzPUmAQD+hTCGoFVZ49Rf3/1RFTVOzbhkiDq0ibO6JAAADkAYQ1CqrW/Qs++tVX5JtW67YKC6t0+0uiQAAA6KMIag42xw6+8L1ysrz6HfTj1B/bq2trokAAAOiTCGoOJyuzVv0QZt2Faq30zspxN7p1pdEgAAh0UYQ9AwxuifSzbph02Fmja2l04b1M7qkgAAaBZhDEHBGKN3/7tFX6/L05RTu2r88E5WlwQAwBEhjCEo/PubHH3y3Q6NPbGjpp7WzepyAAA4YoQxBLz/rtmlf32ZpRED0nXZuF6y2bjwNwAgcBDGENC+/Tlfb3yySYN7pOiac/rJThADAAQYwhgC1rqtRXp58c/q1SlZN513gsLD+HEGAAQePr0QkDJ3lOn5939Sx9R43XbhIEVGhFldEgAAx4QwhoCTs7tCzy5Yq9aJ0Zpx6WDFRodbXRIAAMeMMIaAkl9Sraff/VExUeG6a9oQJcZGWl0SAADHhTCGgPLGp5lyG+nOS4eodWK01eUAAHDcCGMIGFW1Tm3MKdXpg9urXUqc1eUAANAiCGMIGOu2FMvlNlxvEgAQVAhjCBirMwuVHB+pru0SrC4FAIAWQxhDQKh3urQ+u1hDe6eysCsAIKgQxhAQNmSXqN7pZogSABB0CGMICKs3Fyo2Klx9OiVbXQoAAC2KMAa/53K7tXZLsQb3TOGSRwCAoMMnG/xe5o5yVdY4GaIEAAQlwhj83prMQkWE23VCtxSrSwEAoMURxuDXjDFas7lQA7q2VlQkFwMHAAQfwhj8Wk5+hYoddQxRAgCCFmEMfm11ZpFsNmlIrzZWlwIAgFcQxuDX1mQWqk+nZMXHRFhdCgAAXkEYg9/KL6nWrqIqDWWIEgAQxAhj8FurMwslSSf2IowBAIIXYQx+a/XmQnVJT1BKUrTVpQAA4DWEMfilsso6bd3l0Im9mbgPAAhuhDH4pTWbiySJJS0AAEGPMAa/tCazUGmtYtS+TZzVpQAA4FWEMfid6toGZeSU6sTeqbLZbFaXAwCAVxHG4HfWbS2Sy20YogQAhATCGPzO6s1FSoqLVPf2iVaXAgCA1xHG4FecDS6tzyrW0F5tZGeIEgAQAghj8CsbtpWqrt7FECUAIGQQxuBXVmcWKiYqTH27tLK6FAAAfIIwBr/hdhv9uLlIg3q0UXgYP5oAgNDAJx78xuadZaqscTJECQAIKYQx+I3VmUUKD7PrhG6trS4FAACfIYzBLxhjtGZzofp3baWYqHCrywEAwGcIY/ALOwoqVVReyxAlACDkEMbgF1ZnFspmk4b0amN1KQAA+BRhDH5hdWaRenVIUmJspNWlAADgU4QxWK6grEY7CysZogQAhCTCGCy3elOhJBHGAAAhiTAGy63eXKjOafFqkxxjdSkAAPgcYQyWKq+q19ad5RpKrxgAIEQRxmCpHzcXyoghSgBA6CKMwVKrM4uUmhytjqlxVpcCAIAlCGOwTE1dgzJySjS0V6psNpvV5QAAYAnCGCyzPqtYDS7DECUAIKQRxmCZ1ZmFSoyNUM8OSVaXAgCAZQhjsISzwa11W4s1pFcb2e0MUQIAQhdhDJbIyClVbb2LIUoAQMgjjMESqzMLFR0Zpn5dWltdCgAAliKMwefcbqMfNxdqUI8URYTzIwgACG18EsLntuaWy1Ht1NBeDFECAEAYg8+tzixUeJhNg3qkWF0KAACWI4zBp4wxWp1ZqH5dWismKtzqcgAAsBxhDD61q7BKhWW1Gtq7jdWlAADgFwhj8KnVmYWyScwXAwBgD8IYfGp1ZqF6dExSUlyk1aUAAOAXCGPwmcKyGm0vqNSJ9IoBAOBBGIPPrNlcJEk6kfliAAB4EMbgM6szC9UxNU5prWKtLgUAAL9BGINPOKrrtXlnGdeiBABgP4Qx+MTazUUyhrMoAQDYH2EMPrE6s1ApidHqnB5vdSkAAPgVwhi8rqauQRu2lerE3qmy2WxWlwMAgF8hjMHrNmSXqMHl5ixKAAAOgjAGr1udWaj4mAj16phsdSkAAPgdwhi8qsHl1tqtxRrSq43sdoYoAQDYH2EMXrUxp1Q1dQ2sug8AwCEQxuBVqzcXKSoiTAO6tbK6FAAA/BJhDF7jNkZrNhdqYPfWiggPs7ocAAD8EmEMXpOV61B5Zb2Gsuo+AACHRBiD16zJLFSY3abBPVKsLgUAAL9FGINXGGO0OrNQfbu0Umx0hNXlAADgtwhj8Ircoirll9ZwYXAAAJpBGINXrN5cJEka0pNV9wEAOBzCGLxidWaherRPVKuEKKtLAQDArxHG0OKKy2uVs7uCIUoAAI4AYQwtbvXmQkliSQsAAI4AYQwtbk1modq3iVPb1rFWlwIAgN8jjKFFVdY4lbmjXCf2ZuI+AABHgjCGFvXj5iK5jdFQLgwOAMARIYyhRa3ZXKjWiVHq2jbB6lIAAAgIhDG0mLp6l37KLtHQXqmy2WxWlwMAQEAgjKHF/JRdLGeDmyUtAAA4CoQxtJjVmUWKiw5X705JVpcCAEDAIIyhRTS43Fq3tUhDerZRmJ0fKwAAjhSfmmgRGTmlqqpt0Il9GKIEAOBoEMbQIr7LyFdMVJhO6JZidSkAAASU8OYesGnTJn366afKzs6W3W5X9+7dNWHCBHXv3t0X9SEANLjcWp1ZpKG9UhURTr4HAOBoHPKTs6SkRLfddpvuvPNOVVdXa/jw4RoyZIgcDod+//vf6/bbb1dRUdFhX3zRokU655xzNG7cOM2fP/+A/Rs2bNCFF16oKVOm6MYbb5TD4Tj+FsHnNmSXqKauQSf3S7O6FAAAAs4he8buu+8+XXfddRo2bNgB++655x59++23uv/++/Xiiy8e9Pn5+fl6+umntXDhQkVGRmratGk65ZRT1LNnT89jHnvsMd12220aPXq0nnjiCf3jH//QjBkzWqBZ8KXvMgoUGxWu/l1bW10KAAAB55A9Y88///xBg9hep5xyil544YVD7l+xYoVGjBih5ORkxcbGasKECVqyZEmTx7jdblVVVUmSampqFB0dfbT1w2LOBpd+3FKoE3unKjyMIUoAAI7WIT897XuWJzj//PP13nvvqaam5pCPOZiCggKlpv5yZl1aWpry8/ObPGbmzJm6//77ddppp2nFihWaNm3aUTcA1vopu0Q1dS6GKAEAOEbNTuB/8MEH9c477+jZZ5/V+PHjddlll6lXr17NvrAx5oBt+14ip7a2Vvfff7/++c9/atCgQXr11Vd1zz33aN68eUdcfEpK/BE/9nikpobudRaba/u6TzKVEBupXw3rHHQ9Y6F83KXQbn8ot10K7fbT9tBlZfubDWMnnniiTjzxRDkcDi1atEg33XST0tLSdOWVV2rixImHfF56erq+//57z/2CggKlpf3Se5KZmamoqCgNGjRIknTppZfq2WefParii4sr5XYfGPpaUmpqggoLK7z6Hv6qubbXO136ZkOeTumXrtKSKh9W5n2hfNyl0G5/KLddCu320/bQbLvk/fbb7bbDdiAdUVeGw+HQhx9+qHfffVcJCQmaOHGiPvzwQ919992HfM6oUaO0cuVKlZSUqKamRkuXLtXpp5/u2d+lSxft3r1bWVlZkqRly5Zp4MCBR9ou+IH1WcWqq3dpOEOUAAAcs2Z7xu688059+eWXGjNmjGbNmqWhQ4dKki677DKNGjXqkM9LT0/XjBkzNH36dDmdTl100UUaNGiQrr/+et12220aOHCg/vSnP+n222+XMUYpKSl6/PHHW65l8LpVGwuUEBuhvp2TrS4FAICA1WwY69Wrl+6//361bt102YLw8HC99dZbh33u5MmTNXny5CbbXnrpJc/t0aNHa/To0UdTL/xEndOlH7cUadQJ7bgWJQAAx6HZT9EzzjhDM2fOlNS4Gv/UqVM9Q4s9evTwbnXwW+u2Fqve6dbwvgxRAgBwPJoNY7NmzdLFF18sSerTp49uvfVWPfTQQ14vDP5tVUa+EuMi1adTstWlAAAQ0JoNYzU1NRo3bpzn/llnnaXKykqvFgX/VlvfoHVbizWsT6rsdlvzTwAAAIfUbBiz2WzatGmT5/7WrVsPu9grgt/aLcWqb2CIEgCAltDsBP7f//73uuKKK9S7d29JUlZWlp566imvFwb/tWpjgZLiI9WLIUoAAI5bs2FszJgxWrJkiVavXq2wsDANHjxYKSkpvqgNfqimrnGI8owh7WW3MUQJAMDxOqLxxvz8fLVq1UoJCQnavHmz3n33XW/XBT/145YiNbjcLPQKAEALabZn7IEHHtCyZctUW1ur9PR0bd++XSeddJIuueQSX9QHP7Mqo0CtEqLUo0OS1aUAABAUmu0ZW7FihZYtW6bx48dr3rx5eu211xQdHe2L2uBnqmud+im7WMP7pjFECQBAC2k2jKWmpio2Nlbdu3dXZmamTj75ZJWWlvqiNviZNZuL1OAynEUJAEALajaMRUREaNWqVerRo4f+97//qaKigjAWolZtLFBKYpS6t0+0uhQAAIJGs2HsD3/4g95++22NHj1aGRkZGjFihKZMmeKL2uBHqmqd2pBdouF902VjiBIAgBbT7AT+n376SX/5y18kSe+9954cDocSE+kZCTWrMwvlchvOogQAoIU12zP21ltvNblPEAtNqzIK1CYpWl3bJlhdCgAAQaXZnrFu3brpgQce0LBhwxQbG+vZPn78eK8WBv9RWePUz9tKNeGUTgxRAgDQwpoNY2VlZSorK1NOTo5nm81mI4yFkNWZhXIbo5P7pltdCgAAQafZMPb666/7og74se8y8pXWKkad0+OtLgUAgKDTbBh79NFHD7r9gQceaPFi4H8c1fXKyCnVOSO6MEQJAIAXNDuBPzk52fMVFxenNWvW+KIu+InVmwpljHRyP4YoAQDwhmZ7xm655ZYm92+88UbdeOONXisI/uW7jHy1bR2rjqlxVpcCAEBQarZnbH+xsbEqKCjwRi3wM6WOWm3aUabhfdMYogQAwEuOas6YMUYbNmxQ9+7dvVoU/MOKdbl7hihZ6BUAAG9pNowlJyc3uT9lyhQuhxQivlqbq/Zt4tQhlbMoAQDwlmaHKW+88UZ17txZt9xyiy699FLV19c3WfwVwam0ok4/ZxdreF96xQAA8KZmw9js2bP1xRdfND7YbtcPP/ygxx9/3Nt1wWLfbyqQMSKMAQDgZc0OU65Zs0aLFy+WJKWkpOjZZ5/V1KlTvV4YrLVqY4G6tktU+zacRQkAgDc12zPmdDpVX1/vud/Q0ODVgmC9Ekettuws12lD2ltdCgAAQa/ZnrEzzjhD1157raZOnSqbzabFixdr9OjRvqgNFvl+Y+PSJacN7iDJWFsMAABBrtkwdvfdd+vNN9/UsmXLFB4ervHjx+vSSy/1RW2wyKqNBeqcFq8OqfEqLKywuhwAAIJas2HMGKOkpCS98MILKiws1Mcff+yLumCRovIabc116MLRrCUHAIAvNDtnbNasWZxNGUK+31goibMoAQDwlWZ7xn788UfOpgwhqzbmq0vbBKW1Yi05AAB8gbMp4VFQVqPsvAoufwQAgA9xNiU89p5FObwPYQwAAF85orMp58+f7zmbcty4cZo2bZovaoOPrcooUPf2iWqTHGN1KQAAhIxmw1hYWJimT5+u6dOne7ZVV1dzfcogk19arZz8Cl16Zk+rSwEAIKQ0G8Y+++wz/e1vf1N1dbWMMXK73SorK9OaNWt8UR98ZFXGniFKzqIEAMCnmg1jc+bM0e2336633npL119/vT777DPFxXG9wmDzXUaBenZIUuvEaKtLAQAgpDR7NmVMTIzOOeccDRkyRFFRUZo1a5a++eYbX9QGH8krrtLOwkp6xQAAsECzYSwyMlL19fXq3LmzMjIyZLfbmyx1gcC3amOBbJKGEcYAAPC5Zocpx44dqxtuuEFPPPGEpk2bph9++EHJyck+KA2+siqjQL06JqlVQpTVpQAAEHKaDWO//e1vNWXKFLVt21bPP/+8Vq1apUmTJvmiNvjArsJK7Sqq0uXjeltdCgAAIemQw5Tvvfee53b79u0lSf3799dVV12llJQUSdI777zj5fLgbXuHKE/qk2p1KQAAhKRDhrGGhgZdeumlmj9/vnJzcz3bd+3apbfeeksXXnihnE6nT4qEdxhjtGpjgfp0TlZyPEOUAABY4ZDDlJdddplGjx6tF198UXPnzlVFRYUkKTExUePHj9ezzz6rjh07+qxQtLxdhVXKK67WWSdxHAEAsMph54y1b99eDz/8sB5++GGVlpbKbrcrKSnJV7XBy77bmC+bTTqJa1ECAGCZZifw79WqVStv1gEfM8ZoVUaB+nZupcS4SKvLAQAgZDW7zhiC046CSuWX1mh4P3rFAACwEmEsRH2XUSC7zaaTenMWJQAAVjqqMLZjxw6tWrXKW7XARxrPosxXv66tlBDLECUAAFZqNoy9+eabuvPOO1VSUqJp06bpgQce0F/+8hdf1AYvycmvUGFZLdeiBADADzQbxhYsWKB7771XS5Ys0dixY/Xxxx9r+fLlvqgNXvJdRoHC7DadyBAlAACWazaM2Ww2tWnTRitXrtSIESMUHh4ut9vti9rgBXvPouzftbXiYyKsLgcAgJDXbBiLjIzUSy+9pO+++06nnnqq3nzzTcXExPiiNnhBVp5DxQ6GKAEA8BfNhrHHHntM27Zt05NPPqmkpCT98MMPevTRR31RG7xglWeIso3VpQAAAB3Boq/du3fXY489JqnxbMpp06apR48eXi8MLc9tjL7fVKATurVWbDRDlAAA+APOpgwhWbscKnHU6eR+6VaXAgAA9uBsyhDy3cZ8hYfZNaQXQ5QAAPgLzqYMEW5j9P3GAg3s3loxUUd8SVIAAOBlnE0ZIrbsLFdZZT3XogQAwM9wNmWIWJVRoIhwuwb3YIgSAAB/ckRnUz744IPKycmRMUaPPfaYoqOjfVEbWojb3XgW5aDuKQxRAgDgZ5rtGfvxxx911lln6cYbb1R+fr5Gjx6t1atX+6I2tJDMHWUqr2KIEgAAf9RsGJszZ45ee+01JScnq23btpozZ45n3TEEhnVZxQqz2zSoR4rVpQAAgP00G8Zqa2vVs2dPz/3Ro0fL5XJ5tSi0rI05perRPlHRkQxRAgDgb5oNY+Hh4SovL5fNZpMkZWVleb0otJzqWqdy8ivUt0srq0sBAAAH0WxXyU033aQrrrhCRUVFuuOOO7R8+XLNnj3bF7WhBWzaUSZjpH6EMQAA/FKzYWzMmDHq3r27li9fLrfbrd/97ndNhi3h3zJyShURblf39klWlwIAAA7iiCYRhYeHa8iQITLGqK6uThs2bNCAAQO8XRtawMacMvXskKSI8GZHpAEAgAWaDWN//vOf9cYbbygl5Zcz8Ww2m5YtW+bVwnD8KqrrtbOwUuef3t3qUgAAwCE0G8b+85//aOnSpUpPT/dFPWhBm7aXSZL6dWa+GAAA/qrZsat27doRxAJUxvZSRUWEqWu7BKtLAQAAh9Bsz9jIkSM1Z84cjR07tsllkJgz5v825pSqV6ckhYcxXwwAAH/VbBhbuHChJGnJkiWebcwZ839llXXKK67WaYPaWV0KAAA4jGbD2Jtvvqm2bds22bZ582avFYSWsXF7qSSpL/PFAADwa4ccvyorK1NZWZluuOEGlZeXq6ysTOXl5SoqKtLNN9/syxpxDDbmlComKlxd0pkvBgCAPztkz9idd96p5cuXS5JOOeUUz/awsDCNGzfO+5XhuGzMKVOfTsmy221WlwIAAA7jkGHsH//4hyTp3nvv1Z/+9CefFYTjV1xeq4KyGp15UkerSwEAAM04ZBjbunWrevTooSuuuEIbNmw4YD9nU/qvvfPFuB4lAAD+75BhbM6cOXrxxRd16623HrCPsyn9W0ZOqeJjItQhNc7qUgAAQDMOGcb69esnqTGUDRs2zGcF4fgYY7Rxe6n6dk6W3cZ8MQAA/N0hz6ZcvHix8vPzNXv2bM/ZlPt+wT8VltWoxFGnvgxRAgAQEA7ZM3bqqafqjDPOkDGmydmUUuMwZUZGhteLw9HLyGG+GAAAgeSQPWMPP/ywMjIydNJJJ2njxo1Nvghi/mvj9jIlxUWqbetYq0sBAABHoNmLFs6fP98XdaAFGGOUkVOqvl1aycZ8MQAAAgJXkA4iecXVclTVM0QJAEAAIYwFkV+uR5lsbSEAAOCIEcaCSEZOqVISo5SaHGN1KQAA4AgRxoKE2xht2l6mvp2ZLwYAQCAhjAWJnQWVqqxxsr4YAAABhjAWJDZuL5Mk9e1MGAMAIJAQxoLExpxSpSXHKCUp2upSAADAUSCMBQG322jTjjKGKAEACECEsSCQk1+hmroG9e2SbHUpAADgKBHGgsDGvdejZL4YAAABhzAWBDK2l6pdSqyS4qOsLgUAABwlwliAa3C5tXlHOfPFAAAIUISxALctr0J1ThdDlAAABCjCWIDL2HM9yj5cjxIAgIBEGAtwG3NK1SktXgmxkVaXAgAAjgFhLIA5G1zasqucVfcBAAhghLEAtnWXQ84Gt/oxeR8AgIBFGAtgG7eXymaTendKtroUAABwjAhjASwjp1Rd0hMUGx1udSkAAOAYEcYCVJ3TpaxcB0OUAAAEOMJYgNqys1wut2GxVwAAAhxhLEBl5JQqzG5Tr45JVpcCAACOA2EsQG3cXqpu7RIVHcl8MQAAAhlhLADV1DVoW14FQ5QAAAQBwlgAytxRJrcx6sclkAAACHiEsQCUkVOq8DCbenRgvhgAAIHOqxOOFi1apBdeeEFOp1NXX321Lr/8cs++jIwMzZw503O/pKRESUlJWrx4sTdLCgobt5eqZ4ckRUaEWV0KAAA4Tl4LY/n5+Xr66ae1cOFCRUZGatq0aTrllFPUs2dPSVK/fv304YcfSpJqamp08cUXa9asWd4qJ2hU1ji1I79SU0/rZnUpAACgBXhtmHLFihUaMWKEkpOTFRsbqwkTJmjJkiUHfeyLL76o4cOHa9iwYd4qJ2hs2l4mIzF5HwCAIOG1nrGCggKlpqZ67qelpWndunUHPM7hcOjdd9/VokWLjvo9UlLij6vGI5WamuCT9zkSOV9nKyoyTCcP6qCIcO9P+fOntvtaKLddCu32h3LbpdBuP20PXVa232thzBhzwDabzXbAtkWLFumss85SSkrKUb9HcXGl3O4D36clpaYmqLCwwqvvcTTWbCxQz/aJKiut8vp7+VvbfSmU2y6FdvtDue1SaLeftodm2yXvt99utx22A8lrXSvp6ekqKiry3C8oKFBaWtoBj/vss890zjnneKuMoOKoqteuoiqGKAEACCJeC2OjRo3SypUrVVJSopqaGi1dulSnn356k8cYY7RhwwYNHTrUW2UElY3bSyUxXwwAgGDi1Z6xGTNmaPr06TrvvPM0adIkDRo0SNdff73Wr18vqXE5i4iICEVFRXmrjKCyMadU0ZFh6to2tMf1AQAIJl5dZ2zy5MmaPHlyk20vvfSS53ZKSoqWL1/uzRKCSsb2MvXulKwwO2v1AgAQLPhUDxClFXXKL6lWP4YoAQAIKoSxALExZ898sc6EMQAAgglhLEBk5JQqLjpcndJ9s7YaAADwDcJYgNi4vVR9OreS/SBrtQEAgMBFGAsAhWU1KiqvVd/OyVaXAgAAWhhhLADsnS/G5H0AAIIPYSwAbNxeqsTYCLVvE2d1KQAAoIURxvycMUYZOaXq26XVQa/tCQAAAhthzM/ll9aorLKeJS0AAAhShDE/51lfjPliAAAEJcKYn8vIKVWrhCilt4qxuhQAAOAFhDE/ZozRxu2l6ts5mfliAAAEKcKYH9tVVKWKaidDlAAABDHCmB/zrC/G5H0AAIIWYcyPZeSUqk1StNokM18MAIBgRRjzU25jlLmjjCFKAACCHGHMT+3Ir1RVbQNDlAAABDnCmJ/KYH0xAABCAmHMT23cXqr01rFqlRBldSkAAMCLCGN+yOV2K3NHmfrRKwYAQNAjjPmhbbsrVFvvUt/OyVaXAgAAvIww5oc816Nk8j4AAEGPMOaHNuaUqkNqnBLjIq0uBQAAeBlhzM80uNzavLOcXjEAAEIEYczPZOU6VN/gZvI+AAAhgjDmZzbmlMomqQ+T9wEACAmEMT+TkVOqzukJiouOsLoUAADgA4QxP1LvdGlrbrn6dkm2uhQAAOAjhDE/snVXuRpchsn7AACEEMKYH8nYXiq7zabenZKtLgUAAPgIYcyPbMwpU9d2CYqJCre6FAAA4COEMT9RW9+g7DwHS1oAABBiCGN+YvPOcrnczBcDACDUEMb8xMacUoXZberZMcnqUgAAgA8RxvxERk6perRPVFREmNWlAAAAHyKM+YHqWqdy8ivUl/liAACEHMKYH9i0o0zGiMn7AACEIMKYH9iYU6aIcLu6t2e+GAAAoYYw5gcyckrVs0OSIsI5HAAAhBo+/S1WUV2vnYWVzBcDACBEEcYstml7mSTmiwEAEKoIYxbL2F6qqIgwdW2bYHUpAADAAoQxi2XuKFOvjkkKD+NQAAAQikgAFqqpa1BuYZW6t0+0uhQAAGARwpiFtudXyEiEMQAAQhhhzEJZeQ5JUtd2hDEAAEIVYcxC2bkOtUmKVmJspNWlAAAAixDGLJSd52CIEgCAEEcYs0h5ZZ2KHXXqxhAlAAAhjTBmkey8CkkijAEAEOIIYxbJynPIbrOpSzqLvQIAEMoIYxbJznOoQ2qcoiLDrC4FAABYiDBmAWOMsnMdDFECAADCmBUKSmtUXdfAmZQAAIAwZoW9i712p2cMAICQRxizQHauQ1ERYWrfJs7qUgAAgMUIYxbIznOoS9sE2e02q0sBAAAWI4z5WIPLrZz8SoYoAQCAJMKYz+0srFSDy61uTN4HAAAijPlcdm7j5P1u7VjsFQAAEMZ8LivPocTYCKUkRltdCgAA8AOEMR/LzqtQt3aJstmYvA8AAAhjPlVT16C8oirmiwEAAA/CmA9t210hIxZ7BQAAvyCM+VD2npX3uxLGAADAHoQxH8rOdSitVYziYyKsLgUAAPgJwpgPZeU5GKIEAABNEMZ8pLSiTqUVdepGGAMAAPsgjPnItj3zxTiTEgAA7Isw5iNZeQ6F2W3qnBZvdSkAAMCPEMZ8JDvPoY6p8YqMCLO6FAAA4EcIYz7gNqZx5X2GKAEAwH4IYz6QX1KtmroGLg4OAAAOQBjzgazcxsn7LGsBAAD2Rxjzgew8h6Iiw9QuJc7qUgAAgJ8hjPlAdp5D3domyG63WV0KAADwM4QxL3M2uLU9v5LFXgEAwEERxrxsR0GlXG5DGAMAAAdFGPOy7D0r73dnWQsAAHAQhDEvy8p1KCkuUq0SoqwuBQAA+CHCmJdl5znUrV2ibDYm7wMAgAMRxryoutap3SXVrLwPAAAOiTDmRdm7KySx2CsAADg0wpgXZe9Zeb8rl0ECAACHQBjzouw8h9JbxyouOsLqUgAAgJ8ijHmJMUZZuQ51p1cMAAAcBmHMS0or6lReVc9irwAA4LAIY16yd7FXzqQEAACHQxjzkqw8h8LsNnVOY5gSAAAcGmHMS7JzHeqcHq+IcL7FAADg0EgKXuB2G23bXcF8MQAA0CzCmBfklVSrtt5FGAMAAM0ijHnB3sVeuzN5HwAANIMw5gXZeQ7FRIUpvXWs1aUAAAA/Rxjzgqw8h7q2TZTdZrO6FAAA4OcIYy3M2eDSzoJKhigBAMARIYy1sO35lXK5DZP3AQDAESGMtbCsvSvvE8YAAMARIIy1sOw8h1olRKlVQpTVpQAAgABAGGth2bkOesUAAMARI4y1oMoap/JLa9StHdejBAAAR4Yw1oK27d6z2Cs9YwAA4AgRxlpQdq5DNkld2hLGAADAkSGMtaCsXIfapsQqNjrc6lIAAECAIIy1EGOMsvMcDFECAICjQhhrIcWOWjmqnerGyvsAAOAoEMZaSHZehSQWewUAAEeHMNZCsnMdCg+zqVNavNWlAACAAEIYayFZeQ51Tk9QeBjfUgAAcORIDi3A5XZr225W3gcAAEfPq2Fs0aJFOuecczRu3DjNnz//gP1ZWVm68sorNWXKFF177bUqLy/3Zjlek1dUrXqnmzMpAQDAUfNaGMvPz9fTTz+tN998Ux9++KHeeecdbdmyxbPfGKObbrpJ119/vT766CP169dP8+bN81Y5XpWV17jyPmdSAgCAo+W1MLZixQqNGDFCycnJio2N1YQJE7RkyRLP/g0bNig2Nlann366JOm3v/2tLr/8cm+V41XZeQ7FRoUrrVWM1aUAAIAA47Wl4gsKCpSamuq5n5aWpnXr1nnub9++XW3atNE999yjn3/+Wb1799aDDz54VO+RkuKbMxdTUw9/4e8dBVXq3aWV0tOCr2esubYHs1BuuxTa7Q/ltkuh3X7aHrqsbL/Xwpgx5oBtNpvNc7uhoUHfffed3njjDQ0cOFDPPPOMnnjiCT3xxBNH/B7FxZVyuw98n5aUmpqgwsKKQ+6vc7q0Lc+hc0Z2PuzjAlFzbQ9modx2KbTbH8ptl0K7/bQ9NNsueb/9drvtsB1IXhumTE9PV1FRked+QUGB0tLSPPdTU1PVpUsXDRw4UJI0adKkJj1ngWJ7foXcxnAmJQAAOCZeC2OjRo3SypUrVVJSopqaGi1dutQzP0yShg4dqpKSEm3cuFGS9Pnnn2vAgAHeKsdrsnP3TN4njAEAgGPgtWHK9PR0zZgxQ9OnT5fT6dRFF12kQYMG6frrr9dtt92mgQMH6u9//7seeOAB1dTUqG3btpozZ463yvGarDyHWidGKTk+yupSAABAAPJaGJOkyZMna/LkyU22vfTSS57bgwcP1oIFC7xZgtdl57HYKwAAOHaswH8cKqrrVVhWy2KvAADgmBHGjkN2XuOZF/SMAQCAY0UYOw7ZeQ7ZbFLXdqG9NgsAADh2hLHjkJ3nUPs2cYqO9OrUOwAAEMQIY8fIGKOsXCbvAwCA40MYO0ZF5bWqrHEyeR8AABwXwtgxys5jsVcAAHD8CGPHKCvXoYhwuzqkxlldCgAACGCEsWOUnedQl/QEhYfxLQQAAMeOJHEMXG63cnZXMEQJAACOG2HsGOwqrFJ9g1vd2rO+GAAAOD6EsWOwd/I+Z1ICAIDjRRg7Btl5DsVFhys1OcbqUgAAQIAjjB2DrNwKdWufKJvNZnUpAAAgwBHGjlJdvUu7iioZogQAAC2CMHaUtu12yBgWewUAAC2DMHaUsvMqJBHGAABAyyCMHaWsPIfaJEUrMS7S6lIAAEAQIIwdpexcB71iAACgxRDGjkJ5Vb2KHbWEMQAA0GIIY0fBs9hre8IYAABoGYSxo5Cd65DNJnVJ5zJIAACgZRDGjkJ2nkMd2sQrKjLM6lIAAECQIIwdIWOMsvMc6s7FwQEAQAsijB2hgrIaVdU2MHkfAAC0KMLYEcrObZy8TxgDAAAtiTB2hLLyHIoMt6tDapzVpQAAgCBCGDtC2XkOdWmboDA73zIAANBySBZHoMHlVs7uSoYoAQBAiyOMHYFdhVVqcLlZ7BUAALQ4wtgRyMpj8j4AAPAOwtgRyM51KD4mQm2Soq0uBQAABBnC2BFoXOw1UTabzepSAABAkCGMNaO61qncoiqGKAEAgFcQxpqxdWe5jJgvBgAAvIMw1ozM7aWSpG7tuCYlAABoeYSxZmTuKFVacowSYiOtLgUAAAQhwlgzMreXqRvriwEAAC8hjB1GWWWdispqmC8GAAC8hjB2GNl7FnvtThgDAABeQhg7jOw8h+x2mzqnx1tdCgAACFKEscPIznWoa7tERUaEWV0KAAAIUoSxQ3Abo+y8CvXu3MrqUgAAQBAjjB1CbZ1LNXUNGtCttdWlAACAIEYYO4TY6HDNvu4UjT6xo9WlAACAIEYYO4wObeK4ODgAAPAqwhgAAICFCGMAAAAWIowBAABYiDAGAABgIcIYAACAhQhjAAAAFiKMAQAAWIgwBgAAYCHCGAAAgIUIYwAAABYijAEAAFiIMAYAAGAhwhgAAICFCGMAAAAWIowBAABYiDAGAABgIcIYAACAhQhjAAAAFiKMAQAAWIgwBgAAYCHCGAAAgIXCrS7geNjttqB6H39E20NXKLc/lNsuhXb7aXvo8mb7m3ttmzHGeO3dAQAAcFgMUwIAAFiIMAYAAGAhwhgAAICFCGMAAAAWIowBAABYiDAGAABgIcIYAACAhQhjAAAAFiKMAQAAWIgwJmnRokU655xzNG7cOM2fP/+A/RkZGbrwwgs1YcIE3X///WpoaLCgSu+ZO3euzj33XJ177rmaM2fOQfePGTNGU6dO1dSpUw/6PQpU06dP17nnnutp29q1a5vsX7FihSZPnqzx48fr6aeftqhK73jvvfc87Z46dapOOukkzZ49u8ljgvHYV1ZWatKkSdq5c6ekIzvGubm5uvzyy3X22WfrpptuUlVVlS9LbjH7t/2dd97RpEmTNHnyZN17772qr68/4DkffPCBTjvtNM/PQCD/Huzf/nvvvVfjx4/3tO3TTz894DnB8vd/37Z/+eWXTX73R4wYoRtvvPGA5wTDsT/Y55tf/s6bELd7924zZswYU1paaqqqqszkyZPN5s2bmzzm3HPPNWvWrDHGGHPvvfea+fPnW1Cpdyxfvtxceumlpq6uztTX15vp06ebpUuXNnnMjTfeaFavXm1Rhd7jdrvNqaeeapxO50H319TUmNGjR5vt27cbp9NprrnmGvPFF1/4uErfyMzMNOPGjTPFxcVNtgfbsf/xxx/NpEmTzIABA8yOHTuO+BjfcMMNZvHixcYYY+bOnWvmzJnj69KP2/5tz8rKMuPGjTMVFRXG7Xabu+++27z66qsHPG/27Nlm0aJFvi+4he3ffmOMmTRpksnPzz/s84Lh7//B2r5XQUGBGTt2rMnOzj7geYF+7A/2+bZo0SK//J0P+Z6xFStWaMSIEUpOTlZsbKwmTJigJUuWePbv2rVLtbW1GjJkiCTpggsuaLI/0KWmpmrmzJmKjIxURESEevToodzc3CaP+emnn/TSSy9p8uTJmj17turq6iyqtmVlZWXJZrPp+uuv15QpU/TGG2802b9u3Tp16dJFnTp1Unh4uCZPnhxUx35fs2bN0owZM9S6desm24Pt2L/77rt66KGHlJaWJunIjrHT6dSqVas0YcIESYH7N2D/tkdGRmrWrFmKj4+XzWZT7969D/jdl6T169frgw8+0JQpU3TXXXepvLzc16W3iP3bX11drdzcXD344IOaPHmy/va3v8ntdjd5TrD8/d+/7fuaM2eOpk2bpq5dux6wL9CP/cE+37Zt2+aXv/MhH8YKCgqUmprquZ+Wlqb8/PxD7k9NTW2yP9D16tXL84dm27Zt+ve//63Ro0d79ldVValfv36655579P7778vhcOj555+3qNqW5XA4NHLkSP3973/Xa6+9prffflvLly/37G/uZyNYrFixQrW1tZo4cWKT7cF47B977DENGzbMc/9IjnFpaani4+MVHh4uKXD/Buzf9g4dOmjUqFGSpJKSEs2fP19jx4494Hmpqam69dZb9eGHH6pdu3YHDGUHiv3bX1xcrBEjRujxxx/Xu+++q++//14LFixo8pxg+fu/f9v32rZtm7777jtNnz79oM8L9GN/sM83m83ml7/zIR/GjDEHbLPZbEe8P1hs3rxZ11xzje65554m/0OKi4vTSy+9pC5duig8PFzXXHONvvzyS+sKbUFDhw7VnDlzFBsbq9atW+uiiy5q0rZQOfZvv/22fvOb3xywPZiP/V5HcoyD/ecgPz9fV111lS688EKdcsopB+z/+9//rsGDB8tms+m6667T//73PwuqbHmdOnXS3//+d6WkpCgmJkZXXnnlAT/fwX7s33nnHf36179WZGTkQfcHy7Hf9/Otc+fOB+z3h9/5kA9j6enpKioq8twvKCho0pW7//7CwsKDdvUGsh9++EFXX3217rzzTp1//vlN9uXm5jb536IxxvO/hUD3/fffa+XKlZ77+7etuZ+NYFBfX69Vq1bpzDPPPGBfMB/7vY7kGLdu3VqVlZVyuVySgutvwNatW3XZZZfp/PPP180333zA/oqKCr322mue+8H0M7Bp0yZ98sknnvsHa1uw//1ftmyZzjnnnIPuC5Zjv//nm7/+zod8GBs1apRWrlypkpIS1dTUaOnSpTr99NM9+zt06KCoqCj98MMPkhrPLtl3f6DLy8vTzTffrKeeekrnnnvuAfujo6P15z//WTt27JAxRvPnz9e4ceMsqLTlVVRUaM6cOaqrq1NlZaXef//9Jm0bPHiwsrOzlZOTI5fLpcWLFwfVsZcaP5C6du2q2NjYA/YF87Hf60iOcUREhIYNG6Z///vfkoLnb0BlZaWuvfZa/f73v9c111xz0MfExsbq5Zdf9pxl/MYbbwTNz4AxRo8//rjKy8vldDr1zjvvHNC2YP77X1JSotraWnXq1Omg+4Ph2B/s881vf+e9enpAgPjoo4/Mueeea8aPH2/mzZtnjDHmuuuuM+vWrTPGGJORkWEuvPBCc/bZZ5s77rjD1NXVWVlui3rkkUfMkCFDzJQpUzxfb775ZpP2L1myxPP9mTlzZlC1/+mnnzZnn322GT9+vHnttdeMMcZMmTLF7N692xhjzIoVK8zkyZPN+PHjzWOPPWbcbreV5ba4jz/+2Nx+++1NtoXCsR8zZoznrLJDHeP77rvPfPbZZ8YYY3bu3GmuuOIKM3HiRHPNNdeYsrIyy2o/Xnvb/uqrr5oBAwY0+d1/5plnjDFN275q1Spz3nnnmbPPPtv89re/NQ6Hw8ryj9u+x/6NN94wEydONOPGjTN//vOfPY8J1r//+7Z97dq15uKLLz7gMcF07A/1+eaPv/M2Yw4yOAoAAACfCPlhSgAAACsRxgAAACxEGAMAALAQYQwAAMBChDEAAAALEcYA4Ai89957mj9/viTprbfe0rx58yyuCECwCLzldAHAAj/88IN69eolSbrsssssrgZAMCGMAfBL8+bN04IFCxQXF6dhw4Zp2bJlWrJkiZ566imtWrVKLpdL/fv31wMPPKD4+HideeaZOv/887Vy5Url5eVp4sSJuvvuuyVJn3/+uV544QU5nU5FR0frnnvu0dChQ/Xcc8/pxx9/VEFBgfr06aOZM2fqj3/8o4qLi1VYWKgOHTromWee0erVq/X5559r+fLlio6OVklJiUpLS/XHP/5Rmzdv1uzZs1VWViabzaZrrrlG5513nr799ls9/fTT6tSpkzZv3qz6+nr98Y9/1IgRI/T999/riSeekNvtliTdeOONmjBhgpXfbgAWYpgSgN/56quvtHDhQi1YsEALFy5UVVWVpMaAFhYWpoULF+qjjz5SWlqannrqKc/zqqur9eabb+rtt9/WG2+8oR07dmjbtm16+umnNW/ePH3wwQd65JFHdOutt6q6ulqStGvXLr3//vt66qmn9PHHH2vIkCF65513tGzZMkVHR+vDDz/UuHHjdOaZZ+rqq6/W5Zdf7nm/hoYG3XTTTbryyiu1aNEivfTSS/rrX/+qNWvWSJLWrVuna665Rh988IEuuugizZ07V5L03HPP6Te/+Y0WLlyoxx9/XN98842vvrUA/BA9YwD8zpdffqmzzz5biYmJkqTLL79c33zzjb744gtVVFRoxYoVkiSn06mUlBTP88aOHSup8QLPKSkpKi8v19q1a1VQUKCrr77a8zibzabt27dLkoYMGeK5APJVV12l77//Xq+++qq2bdumzZs3a/DgwYesc9u2baqrq9P48eM97zt+/Hh99dVXOuWUU9S+fXv169dPktS/f3+9//77kqSJEydq9uzZ+vzzzzVq1CjdcccdLfFtAxCgCGMA/E54eLj2vVJbWFiYJMntduu+++7T6NGjJUlVVVWqq6vzPC4qKspz22azyRgjt9utkSNH6plnnvHsy8vLU1pamj799NMmF0n/85//rHXr1unCCy/UKaecooaGBh3uinF7hxn3ZYxRQ0ODpMaLre9fjyRNmzZNY8aM0fLly/XVV19p7ty5+uijj5SQkHBE3x8AwYVhSgB+Z/To0Vq6dKkqKiokSQsWLJAknXbaaZo/f77q6+vldrv14IMP6q9//ethX2vEiBFavny5tm7dKqmx123KlClNQtxeX3/9ta666iqdd955SklJ0YoVK+RyuSQ1BsK9IWuvbt26KSIiQkuXLpUk5efn65NPPtGoUaMOW9O0adOUkZGhCy64QI888ogcDofKy8uP4DsDIBjRMwbA74wcOVKXXHKJLr30UkVHR6tXr16KiYnR7373Oz355JM6//zz5XK51K9fP82cOfOwr9WrVy/Nnj1bd9xxh4wxCg8P1wsvvNCkR2yvm2++WXPmzNHzzz+vsLAwnXjiiZ7hzNNPP12PPPJIk8dHRETo+eef16OPPqrnnntOLpdLN998s0aMGKFvv/32kDXdddddevzxx/XMM8/IbrfrlltuUceOHY/hOwUgGNjM4frgAcAC69ev15o1azR9+nRJ0quvvqq1a9c2GWoEgGBBGAPgdyorK3XfffcpKytLNptN7dq10yOPPKL09HSrSwOAFkcYAwAAsBAT+AEAACxEGAMAALAQYQwAAMBChDEAAAALEcYAAAAsRBgDAACw0P8HKEb86Xv8I/IAAAAASUVORK5CYII=\n" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot = plot_fitness_evolution(evolved_estimator, metric=\"fitness\")\n", + "plt.show()\n" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/docs/release_notes.rst b/docs/release_notes.rst index fccae5c..1ed4c8a 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -3,6 +3,30 @@ Release Notes Some notes on new features in various releases + +What's new in 0.7.0dev0 +----------------------- + +This is the current in-development version, these features are not yet +available via PyPI + +^^^^^^^^^ +Features: +^^^^^^^^^ + +* :class:`~sklearn_genetic.GAFeatureSelectionCV` for feature selection along + with any scikit-learn classifier or regressor. It optimizes the cv-score + while minimizing the number of features to select. + This class is compatible with the mlflow and tensorboard integration, + the Callbacks and the ``plot_fitness_evolution`` function. + +^^^^^^^^^^^^ +API Changes: +^^^^^^^^^^^^ + +* The module :mod:`~sklearn_genetic.mlflow` was renamed to :class:`~sklearn_genetic.mlflow_log` + to avoid unexpected errors on name resolutions + What's new in 0.6.1 ------------------- diff --git a/docs/tutorials/basic_usage.rst b/docs/tutorials/basic_usage.rst index ad51370..a6adfbe 100644 --- a/docs/tutorials/basic_usage.rst +++ b/docs/tutorials/basic_usage.rst @@ -6,7 +6,8 @@ How to Use Sklearn-genetic-opt Introduction ------------ -Sklearn-genetic-opt uses evolutionary algorithms to fine-tune scikit-learn machine learning algorithms. +Sklearn-genetic-opt uses evolutionary algorithms to fine-tune scikit-learn machine learning algorithms +and perform feature selection. It is designed to accept a `scikit-learn `__ regression or classification model (or a pipeline containing on of those). @@ -23,8 +24,8 @@ Then by using evolutionary operators as the mating, mutation, selection and eval it generates new candidates looking to improve the cross-validation score in each generation. It'll continue with this process until a number of generations is reached or until a callback criterion is met. -Example -------- +Fine-tuning Example +------------------- First let's import some dataset and other scikit-learn standard modules, we'll use the `digits dataset `__. @@ -165,10 +166,109 @@ sklearn-genetic-opt comes with a plot function to analyze this log: .. image:: ../images/basic_usage_plot_space_4.png -What this plot shows us, is the distributione of the sampled values for each hyperparameter. +What this plot shows us, is the distribution of the sampled values for each hyperparameter. We can see for example in the *'min_weight_fraction_leaf'* that the algorithm mostly sampled values below 0.15. You can also check every single combination of variables and the contour plot that represents the sampled values. + +Feature Selection Example +------------------------- + +For this example, we are going to use the well-known Iris dataset, it's a classification problem with four features. +We are also going to simulate some random noise to represent non-important features: + +.. code:: python3 + + import matplotlib.pyplot as plt + from sklearn_genetic import GAFeatureSelectionCV + from sklearn_genetic.plots import plot_fitness_evolution + from sklearn.model_selection import train_test_split, StratifiedKFold + from sklearn.svm import SVC + from sklearn.datasets import load_iris + from sklearn.metrics import accuracy_score + import numpy as np + + data = load_iris() + X, y = data["data"], data["target"] + + noise = np.random.uniform(0, 10, size=(X.shape[0], 10)) + + X = np.hstack((X, noise)) + + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=0) + +This should give us 10 extra noisy features with our train and test set. + +Now we can create the GAFeatureSelectionCV object, it's very similar to the GASearchCV and they share +most of the parameters, the main difference is GAFeatureSelectionCV doesn't run hyperparameters optimization +thus the param_grid parameter it's not available, and the estimator should be defined with its hyperparameters. + +The way the feature selection is performed is by creating models with a subsample of features +and evaluate its cv-score, the way the subsets are created is by using the available evolutionary algorithms. +It also tries to minimize the number of selected features, so it's a multi-objective optimization. + +Let's create the feature selection object, the estimator we're going to use is a SVM: + +.. code:: python3 + + clf = SVC(gamma='auto') + + evolved_estimator = GAFeatureSelectionCV( + estimator=clf, + cv=3, + scoring="accuracy", + population_size=30, + generations=20, + n_jobs=-1, + verbose=True, + keep_top_k=2, + elitism=True, + ) + +We are ready to run the optimization routine: + +.. code:: python3 + + # Train and select the features + evolved_estimator.fit(X_train, y_train) + +During the training, the same log format is displayed as before: + +.. image:: ../images/basic_usage_train_log_5.png + +After fitting the model, we have some extra methods to use the model right away. It will use by default the best set of +features it found, remember as the algorithm used only a subset, you have to select them from the +``X_test array``, this is done like this: + +.. code:: python3 + + features = evolved_estimator.best_features_ + + # Predict only with the subset of selected features + y_predict_ga = evolved_estimator.predict(X_test[:, features]) + accuracy = accuracy_score(y_test, y_predict_ga) + +.. image:: ../images/basic_usage_accuracy_6.png + +In this case, we got an accuracy score in the test set of 0.98. + +Notice that the ``best_features_`` is a vector of bool values, each +position represents the index of the feature (column) and the value indicates +if that features was selected (True) or not (False) by the algorithm. +In this example, the algorithm, discarded all the noisy random variables we created +and selected the original variables. + +We can also plot the fitness evolution: + +.. code:: python3 + + from sklearn_genetic.plots import plot_fitness_evolution + plot_fitness_evolution(evolved_estimator) + plt.show() + +.. image:: ../images/basic_usage_fitness_plot_7.png + This concludes our introduction to the basic sklearn-genetic-opt usage. -Further tutorials will cover the GASearchCV parameters, callbacks, -different optimization algorithms and more advanced use cases. \ No newline at end of file +Further tutorials will cover the GASearchCV and GAFeatureSelectionCV parameters, callbacks, +different optimization algorithms and more advanced use cases. + diff --git a/sklearn_genetic/_version.py b/sklearn_genetic/_version.py index 43c4ab0..8531016 100644 --- a/sklearn_genetic/_version.py +++ b/sklearn_genetic/_version.py @@ -1 +1 @@ -__version__ = "0.6.1" +__version__ = "0.7.0dev0" diff --git a/sklearn_genetic/plots.py b/sklearn_genetic/plots.py index 1b47113..ba4b4fa 100644 --- a/sklearn_genetic/plots.py +++ b/sklearn_genetic/plots.py @@ -15,6 +15,7 @@ from .utils import logbook_to_pandas from .parameters import Metrics from .space import Categorical +from .genetic_search import GAFeatureSelectionCV """ This module contains some useful function to explore the results of the optimization routines @@ -75,6 +76,10 @@ def plot_search_space(estimator, height=2, s=25, features: list = None): Pair plot of the used hyperparameters during the search """ + + if isinstance(estimator, GAFeatureSelectionCV): + raise TypeError("Estimator must be a GASearchCV instance, not a GAFeatureSelectionCV instance") + sns.set_style("white") df = logbook_to_pandas(estimator.logbook) @@ -131,6 +136,9 @@ def plot_parallel_coordinates(estimator, features: list = None): """ + if isinstance(estimator, GAFeatureSelectionCV): + raise TypeError("Estimator must be a GASearchCV instance, not a GAFeatureSelectionCV instance") + df = logbook_to_pandas(estimator.logbook) param_grid = estimator.space.param_grid score = df["score"] diff --git a/sklearn_genetic/tests/test_plots.py b/sklearn_genetic/tests/test_plots.py index 1399c1c..7fe8af6 100644 --- a/sklearn_genetic/tests/test_plots.py +++ b/sklearn_genetic/tests/test_plots.py @@ -3,11 +3,10 @@ from sklearn.model_selection import train_test_split from sklearn.tree import DecisionTreeRegressor -from .. import GASearchCV +from .. import GASearchCV, GAFeatureSelectionCV from ..plots import plot_fitness_evolution, plot_search_space, plot_parallel_coordinates from ..space import Integer, Categorical, Continuous - data = load_boston() y = data["target"] @@ -49,9 +48,9 @@ def test_plot_evolution(): plot = plot_fitness_evolution(evolved_estimator, metric="accuracy") assert ( - str(excinfo.value) - == "metric must be one of ['fitness', 'fitness_std', 'fitness_max', 'fitness_min'], " - "but got accuracy instead" + str(excinfo.value) + == "metric must be one of ['fitness', 'fitness_std', 'fitness_max', 'fitness_min'], " + "but got accuracy instead" ) @@ -68,3 +67,35 @@ def test_plot_parallel(): plot = plot_parallel_coordinates( evolved_estimator, features=["ccp_alpha", "criterion"] ) + + +def test_wrong_estimator_space(): + estimator = GAFeatureSelectionCV( + clf, + cv=3, + scoring="accuracy", + population_size=6 + ) + with pytest.raises(Exception) as excinfo: + plot = plot_search_space(estimator) + + assert ( + str(excinfo.value) + == "Estimator must be a GASearchCV instance, not a GAFeatureSelectionCV instance" + ) + + +def test_wrong_estimator_parallel(): + estimator = GAFeatureSelectionCV( + clf, + cv=3, + scoring="accuracy", + population_size=6 + ) + with pytest.raises(Exception) as excinfo: + plot = plot_parallel_coordinates(estimator) + + assert ( + str(excinfo.value) + == "Estimator must be a GASearchCV instance, not a GAFeatureSelectionCV instance" + )