625 lines
49 KiB
Plaintext
625 lines
49 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"outputs": [],
|
|
"source": [
|
|
"import seaborn as sns\n",
|
|
"from pathlib import Path\n",
|
|
"from natsort import natsorted\n",
|
|
"\n",
|
|
"import yaml\n",
|
|
"import shutil\n",
|
|
"\n",
|
|
"import numpy as np\n",
|
|
"\n",
|
|
"import pandas as pd\n"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%% Imports go here\n"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"outputs": [],
|
|
"source": [
|
|
"# Settings and Variables\n",
|
|
"\n",
|
|
"# This Experiment (= Model and Parameter Configuration\n",
|
|
"from yaml.constructor import ConstructorError\n",
|
|
"_ROOT = Path('..')\n",
|
|
"out_path = Path('..') / Path('output')\n",
|
|
"_model_name = 'VisualTransformer'\n",
|
|
"_dataset_name = 'PrimatesLibrosaDatamodule'\n",
|
|
"_param_name = 'heads'"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%%\n"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"outputs": [],
|
|
"source": [
|
|
"def print_stats(data_option, mean_duration, std_duration, min_duration, max_duration, dataset=None):\n",
|
|
" if dataset is not None:\n",
|
|
" print(f'For {data_option}; dataset is: {dataset}')\n",
|
|
" print(f'For {data_option}; statistics are:')\n",
|
|
" print(f'Scores - mean: {mean_duration:.3f}s\\tstd: {std_duration:.3f}s '\n",
|
|
" f'min: {min_duration:.3f}s\\t max: {max_duration:.3f}s')\n",
|
|
"\n",
|
|
"def is_clean(exp_path: Path):\n",
|
|
" if exp_path.exists():\n",
|
|
" runs = list(exp_path.iterdir())\n",
|
|
"\n",
|
|
" if len(runs) <= 3:\n",
|
|
" shutil.rmtree(exp_path)\n",
|
|
" return False\n",
|
|
"\n",
|
|
" else:\n",
|
|
" return True\n",
|
|
" else:\n",
|
|
" return False\n",
|
|
"\n",
|
|
"def print_metrics(exp_path, valid_dataset_name=None):\n",
|
|
" best_scores = []\n",
|
|
" had_errors = []\n",
|
|
" dataset_name = None\n",
|
|
" header_printed = False\n",
|
|
" for idx, run_folder in enumerate([x for x in exp_path.iterdir() if x.is_dir()]):\n",
|
|
" # model_class = locate_and_import_class(model_name, 'models')\n",
|
|
" # sorted_checkpoints = natsorted(run_folder.glob('*.ckpt'))\n",
|
|
" # model = ModelIO.load_from_checkpoint(str(sorted_checkpoints[0]), strict=True)\n",
|
|
" try:\n",
|
|
" yaml_file = run_folder / 'hparams.yaml'\n",
|
|
" if dataset_name is None and yaml_file.exists():\n",
|
|
" with yaml_file.open('rb') as f:\n",
|
|
" configuration = yaml.load(f, yaml.BaseLoader)\n",
|
|
" dataset_name = configuration['data_name']\n",
|
|
"\n",
|
|
" except IOError:\n",
|
|
" pass\n",
|
|
"\n",
|
|
" try:\n",
|
|
" if dataset_name != valid_dataset_name and valid_dataset_name is not None:\n",
|
|
" raise NameError\n",
|
|
" metrics: pd.DataFrame = pd.read_csv(run_folder / 'metrics.csv')\n",
|
|
" # Possible keys are:\n",
|
|
" # -- CE - Losses:\n",
|
|
" # val_max_vote_loss, val_mean_vote_loss, mean_val_loss\n",
|
|
" # -- Fallback:\n",
|
|
" # mean_loss,epoch,step,macro_f1_score, macro_roc_auc_ovr, uar_score, micro_f1_score\n",
|
|
" # Pytorch Metrics:\n",
|
|
" # PL_f1_score,PL_accuracy_score_score, PL_fbeta_score,PL_recall_score,PL_precision_score,\n",
|
|
" score = metrics.PL_recall_score.max()\n",
|
|
" epoch = metrics.PL_recall_score.argmax()\n",
|
|
" if not header_printed:\n",
|
|
" print(f'--------------{exp_path.name}------------------')\n",
|
|
" header_printed=True\n",
|
|
" print(f'{exp_path.name} - {run_folder.name}: {score}, epoch: {epoch / 5}')\n",
|
|
" best_scores.append(score)\n",
|
|
" had_errors.append(False)\n",
|
|
" except (AttributeError, FileNotFoundError, NameError):\n",
|
|
" had_errors.append(True)\n",
|
|
" pass\n",
|
|
" if any(had_errors) or (dataset_name is None and valid_dataset_name is not None):\n",
|
|
" print('-------------- Had Errors ------------------')\n",
|
|
" return\n",
|
|
" else:\n",
|
|
" print('\\n')\n",
|
|
" stats = np.nanmean(best_scores), np.nanstd(best_scores), np.nanmin(best_scores), np.nanmax(best_scores)\n",
|
|
" print_stats(exp_path.name, *stats, dataset=dataset_name)\n",
|
|
" print('--------------------------------------------')\n"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%% Util Functions\n"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"--------------VT_01123c93daaffa92d2ed341bda32426d------------------\n",
|
|
"VT_01123c93daaffa92d2ed341bda32426d - version_0: 0.8587360978126526, epoch: 86.0\n",
|
|
"VT_01123c93daaffa92d2ed341bda32426d - version_1: 0.8587360978126526, epoch: 86.0\n",
|
|
"VT_01123c93daaffa92d2ed341bda32426d - version_2: 0.8587360978126526, epoch: 86.0\n",
|
|
"VT_01123c93daaffa92d2ed341bda32426d - version_3: 0.8587360978126526, epoch: 86.0\n",
|
|
"VT_01123c93daaffa92d2ed341bda32426d - version_4: 0.8504778146743774, epoch: 58.0\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_01123c93daaffa92d2ed341bda32426d; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_01123c93daaffa92d2ed341bda32426d; statistics are:\n",
|
|
"Scores - mean: 0.857s\tstd: 0.003s min: 0.850s\t max: 0.859s\n",
|
|
"--------------------------------------------\n",
|
|
"--------------VT_012aff7c1c667073aedafcbebfa35ec7------------------\n",
|
|
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_0: 0.8631429672241211, epoch: 79.8\n",
|
|
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_1: 0.864475429058075, epoch: 58.0\n",
|
|
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_2: 0.8683117032051086, epoch: 76.0\n",
|
|
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_3: 0.854859471321106, epoch: 79.8\n",
|
|
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_4: 0.8658838272094727, epoch: 76.0\n",
|
|
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_5: 0.8564963340759277, epoch: 79.8\n",
|
|
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_6: 0.8637051582336426, epoch: 50.0\n",
|
|
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_7: 0.8519455194473267, epoch: 38.0\n",
|
|
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_8: 0.8484407663345337, epoch: 54.0\n",
|
|
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_9: 0.8730489611625671, epoch: 74.0\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_012aff7c1c667073aedafcbebfa35ec7; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_012aff7c1c667073aedafcbebfa35ec7; statistics are:\n",
|
|
"Scores - mean: 0.861s\tstd: 0.007s min: 0.848s\t max: 0.873s\n",
|
|
"--------------------------------------------\n",
|
|
"-------------- Had Errors ------------------\n",
|
|
"--------------VT_15cbb349b2b50dbb97beec16af2bedab------------------\n",
|
|
"VT_15cbb349b2b50dbb97beec16af2bedab - version_0: 0.8336991667747498, epoch: 39.8\n",
|
|
"VT_15cbb349b2b50dbb97beec16af2bedab - version_1: 0.836580216884613, epoch: 36.8\n",
|
|
"VT_15cbb349b2b50dbb97beec16af2bedab - version_2: 0.8349334001541138, epoch: 37.8\n",
|
|
"VT_15cbb349b2b50dbb97beec16af2bedab - version_3: 0.8312996029853821, epoch: 30.8\n",
|
|
"VT_15cbb349b2b50dbb97beec16af2bedab - version_4: 0.8381868600845337, epoch: 31.8\n",
|
|
"VT_15cbb349b2b50dbb97beec16af2bedab - version_5: 0.8243923187255859, epoch: 32.8\n",
|
|
"VT_15cbb349b2b50dbb97beec16af2bedab - version_6: 0.8407894372940063, epoch: 36.8\n",
|
|
"VT_15cbb349b2b50dbb97beec16af2bedab - version_7: 0.8342592120170593, epoch: 33.8\n",
|
|
"VT_15cbb349b2b50dbb97beec16af2bedab - version_8: 0.8231534957885742, epoch: 34.8\n",
|
|
"VT_15cbb349b2b50dbb97beec16af2bedab - version_9: 0.8382810950279236, epoch: 39.8\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_15cbb349b2b50dbb97beec16af2bedab; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_15cbb349b2b50dbb97beec16af2bedab; statistics are:\n",
|
|
"Scores - mean: 0.834s\tstd: 0.006s min: 0.823s\t max: 0.841s\n",
|
|
"--------------------------------------------\n",
|
|
"--------------VT_259ee495ee2d2dc0e56bb23d12476f17------------------\n",
|
|
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_0: 0.8342075347900391, epoch: 38.0\n",
|
|
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_1: 0.8403531908988953, epoch: 42.0\n",
|
|
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_2: 0.8468937277793884, epoch: 48.0\n",
|
|
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_3: 0.8312729001045227, epoch: 96.0\n",
|
|
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_4: 0.8404075503349304, epoch: 50.0\n",
|
|
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_5: 0.8485946655273438, epoch: 50.0\n",
|
|
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_6: 0.8351554870605469, epoch: 50.0\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_259ee495ee2d2dc0e56bb23d12476f17; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_259ee495ee2d2dc0e56bb23d12476f17; statistics are:\n",
|
|
"Scores - mean: 0.840s\tstd: 0.006s min: 0.831s\t max: 0.849s\n",
|
|
"--------------------------------------------\n",
|
|
"--------------VT_2c71e1d2106eedaca7bac96abc4848cd------------------\n",
|
|
"VT_2c71e1d2106eedaca7bac96abc4848cd - version_1: 0.8323831558227539, epoch: 76.0\n",
|
|
"VT_2c71e1d2106eedaca7bac96abc4848cd - version_10: 0.8282755017280579, epoch: 66.0\n",
|
|
"VT_2c71e1d2106eedaca7bac96abc4848cd - version_2: 0.8355768322944641, epoch: 58.0\n",
|
|
"VT_2c71e1d2106eedaca7bac96abc4848cd - version_3: 0.8362802863121033, epoch: 78.0\n",
|
|
"VT_2c71e1d2106eedaca7bac96abc4848cd - version_4: 0.8537742495536804, epoch: 79.8\n",
|
|
"VT_2c71e1d2106eedaca7bac96abc4848cd - version_5: 0.8362966775894165, epoch: 78.0\n",
|
|
"VT_2c71e1d2106eedaca7bac96abc4848cd - version_6: 0.8405885696411133, epoch: 60.0\n",
|
|
"VT_2c71e1d2106eedaca7bac96abc4848cd - version_7: 0.8508822321891785, epoch: 76.0\n",
|
|
"VT_2c71e1d2106eedaca7bac96abc4848cd - version_8: 0.8435678482055664, epoch: 74.0\n",
|
|
"VT_2c71e1d2106eedaca7bac96abc4848cd - version_9: 0.8420049548149109, epoch: 76.0\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_2c71e1d2106eedaca7bac96abc4848cd; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_2c71e1d2106eedaca7bac96abc4848cd; statistics are:\n",
|
|
"Scores - mean: 0.840s\tstd: 0.008s min: 0.828s\t max: 0.854s\n",
|
|
"--------------------------------------------\n",
|
|
"--------------VT_2c7afd50e127f5a2339db0ddfd6bfd7c------------------\n",
|
|
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_0: 0.8636038899421692, epoch: 78.0\n",
|
|
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_1: 0.8686699271202087, epoch: 76.0\n",
|
|
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_2: 0.8499867916107178, epoch: 42.0\n",
|
|
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_3: 0.8729345798492432, epoch: 78.0\n",
|
|
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_4: 0.8555077314376831, epoch: 79.8\n",
|
|
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_5: 0.8710847496986389, epoch: 78.0\n",
|
|
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_6: 0.8630585670471191, epoch: 79.8\n",
|
|
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_7: 0.8619015216827393, epoch: 79.8\n",
|
|
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_8: 0.8558077812194824, epoch: 72.0\n",
|
|
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_9: 0.8507344722747803, epoch: 76.0\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_2c7afd50e127f5a2339db0ddfd6bfd7c; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_2c7afd50e127f5a2339db0ddfd6bfd7c; statistics are:\n",
|
|
"Scores - mean: 0.861s\tstd: 0.008s min: 0.850s\t max: 0.873s\n",
|
|
"--------------------------------------------\n",
|
|
"--------------VT_30c0815ba934bff4458141e33dacb15a------------------\n",
|
|
"VT_30c0815ba934bff4458141e33dacb15a - version_0: 0.841953456401825, epoch: 74.0\n",
|
|
"VT_30c0815ba934bff4458141e33dacb15a - version_1: 0.8552379608154297, epoch: 79.8\n",
|
|
"VT_30c0815ba934bff4458141e33dacb15a - version_2: 0.8526695966720581, epoch: 74.0\n",
|
|
"VT_30c0815ba934bff4458141e33dacb15a - version_3: 0.8482565879821777, epoch: 54.0\n",
|
|
"VT_30c0815ba934bff4458141e33dacb15a - version_4: 0.8506109118461609, epoch: 74.0\n",
|
|
"VT_30c0815ba934bff4458141e33dacb15a - version_5: 0.850794792175293, epoch: 76.0\n",
|
|
"VT_30c0815ba934bff4458141e33dacb15a - version_6: 0.8524023294448853, epoch: 62.0\n",
|
|
"VT_30c0815ba934bff4458141e33dacb15a - version_7: 0.8411595225334167, epoch: 79.8\n",
|
|
"VT_30c0815ba934bff4458141e33dacb15a - version_8: 0.8499799370765686, epoch: 79.8\n",
|
|
"VT_30c0815ba934bff4458141e33dacb15a - version_9: 0.8531520366668701, epoch: 76.0\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_30c0815ba934bff4458141e33dacb15a; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_30c0815ba934bff4458141e33dacb15a; statistics are:\n",
|
|
"Scores - mean: 0.850s\tstd: 0.004s min: 0.841s\t max: 0.855s\n",
|
|
"--------------------------------------------\n",
|
|
"--------------VT_378971720b930050ad7662bb96699e20------------------\n",
|
|
"VT_378971720b930050ad7662bb96699e20 - version_0: 0.8287097811698914, epoch: 34.8\n",
|
|
"VT_378971720b930050ad7662bb96699e20 - version_1: 0.8333806395530701, epoch: 30.8\n",
|
|
"VT_378971720b930050ad7662bb96699e20 - version_2: 0.8407015204429626, epoch: 38.8\n",
|
|
"VT_378971720b930050ad7662bb96699e20 - version_3: 0.847841203212738, epoch: 39.8\n",
|
|
"VT_378971720b930050ad7662bb96699e20 - version_4: 0.8400266766548157, epoch: 38.8\n",
|
|
"VT_378971720b930050ad7662bb96699e20 - version_5: 0.8392724990844727, epoch: 35.8\n",
|
|
"VT_378971720b930050ad7662bb96699e20 - version_6: 0.8388294577598572, epoch: 35.8\n",
|
|
"VT_378971720b930050ad7662bb96699e20 - version_7: 0.8410612344741821, epoch: 36.8\n",
|
|
"VT_378971720b930050ad7662bb96699e20 - version_8: 0.8436978459358215, epoch: 37.8\n",
|
|
"VT_378971720b930050ad7662bb96699e20 - version_9: 0.8334627151489258, epoch: 32.8\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_378971720b930050ad7662bb96699e20; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_378971720b930050ad7662bb96699e20; statistics are:\n",
|
|
"Scores - mean: 0.839s\tstd: 0.005s min: 0.829s\t max: 0.848s\n",
|
|
"--------------------------------------------\n",
|
|
"--------------VT_63b9fee765cdda91756af1f35cd320a3------------------\n",
|
|
"VT_63b9fee765cdda91756af1f35cd320a3 - version_0: 0.8603388071060181, epoch: 37.8\n",
|
|
"VT_63b9fee765cdda91756af1f35cd320a3 - version_1: 0.8519773483276367, epoch: 35.8\n",
|
|
"VT_63b9fee765cdda91756af1f35cd320a3 - version_2: 0.8558205962181091, epoch: 37.8\n",
|
|
"VT_63b9fee765cdda91756af1f35cd320a3 - version_3: 0.8519774675369263, epoch: 36.8\n",
|
|
"VT_63b9fee765cdda91756af1f35cd320a3 - version_4: 0.8546129465103149, epoch: 35.8\n",
|
|
"VT_63b9fee765cdda91756af1f35cd320a3 - version_5: 0.8558711409568787, epoch: 36.8\n",
|
|
"VT_63b9fee765cdda91756af1f35cd320a3 - version_6: 0.8663593530654907, epoch: 33.8\n",
|
|
"VT_63b9fee765cdda91756af1f35cd320a3 - version_7: 0.8537712097167969, epoch: 29.8\n",
|
|
"VT_63b9fee765cdda91756af1f35cd320a3 - version_8: 0.8614517450332642, epoch: 32.8\n",
|
|
"VT_63b9fee765cdda91756af1f35cd320a3 - version_9: 0.8647329211235046, epoch: 37.8\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_63b9fee765cdda91756af1f35cd320a3; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_63b9fee765cdda91756af1f35cd320a3; statistics are:\n",
|
|
"Scores - mean: 0.858s\tstd: 0.005s min: 0.852s\t max: 0.866s\n",
|
|
"--------------------------------------------\n",
|
|
"--------------VT_7899c07a4809a45c57cba58047cefb5e------------------\n",
|
|
"VT_7899c07a4809a45c57cba58047cefb5e - version_0: 0.8663597106933594, epoch: 78.0\n",
|
|
"VT_7899c07a4809a45c57cba58047cefb5e - version_1: 0.8652830123901367, epoch: 78.0\n",
|
|
"VT_7899c07a4809a45c57cba58047cefb5e - version_2: 0.8739997744560242, epoch: 79.8\n",
|
|
"VT_7899c07a4809a45c57cba58047cefb5e - version_3: 0.854115903377533, epoch: 60.0\n",
|
|
"VT_7899c07a4809a45c57cba58047cefb5e - version_4: 0.8697185516357422, epoch: 74.0\n",
|
|
"VT_7899c07a4809a45c57cba58047cefb5e - version_5: 0.8741324543952942, epoch: 58.0\n",
|
|
"VT_7899c07a4809a45c57cba58047cefb5e - version_6: 0.8711682558059692, epoch: 70.0\n",
|
|
"VT_7899c07a4809a45c57cba58047cefb5e - version_7: 0.8780345916748047, epoch: 79.8\n",
|
|
"VT_7899c07a4809a45c57cba58047cefb5e - version_8: 0.8690432906150818, epoch: 79.8\n",
|
|
"VT_7899c07a4809a45c57cba58047cefb5e - version_9: 0.8685160875320435, epoch: 78.0\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_7899c07a4809a45c57cba58047cefb5e; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_7899c07a4809a45c57cba58047cefb5e; statistics are:\n",
|
|
"Scores - mean: 0.869s\tstd: 0.006s min: 0.854s\t max: 0.878s\n",
|
|
"--------------------------------------------\n",
|
|
"--------------VT_7e3807b3a99a435453d9b1e5e730bd6f------------------\n",
|
|
"VT_7e3807b3a99a435453d9b1e5e730bd6f - version_1: 0.8437240719795227, epoch: 78.0\n",
|
|
"VT_7e3807b3a99a435453d9b1e5e730bd6f - version_10: 0.8432115912437439, epoch: 60.0\n",
|
|
"VT_7e3807b3a99a435453d9b1e5e730bd6f - version_2: 0.8426862955093384, epoch: 78.0\n",
|
|
"VT_7e3807b3a99a435453d9b1e5e730bd6f - version_3: 0.8437070846557617, epoch: 76.0\n",
|
|
"VT_7e3807b3a99a435453d9b1e5e730bd6f - version_4: 0.8473502993583679, epoch: 68.0\n",
|
|
"VT_7e3807b3a99a435453d9b1e5e730bd6f - version_5: 0.8560628890991211, epoch: 79.8\n",
|
|
"VT_7e3807b3a99a435453d9b1e5e730bd6f - version_6: 0.8335348963737488, epoch: 74.0\n",
|
|
"VT_7e3807b3a99a435453d9b1e5e730bd6f - version_7: 0.8560174107551575, epoch: 68.0\n",
|
|
"VT_7e3807b3a99a435453d9b1e5e730bd6f - version_8: 0.8495742082595825, epoch: 64.0\n",
|
|
"VT_7e3807b3a99a435453d9b1e5e730bd6f - version_9: 0.851801872253418, epoch: 72.0\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_7e3807b3a99a435453d9b1e5e730bd6f; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_7e3807b3a99a435453d9b1e5e730bd6f; statistics are:\n",
|
|
"Scores - mean: 0.847s\tstd: 0.007s min: 0.834s\t max: 0.856s\n",
|
|
"--------------------------------------------\n",
|
|
"-------------- Had Errors ------------------\n",
|
|
"--------------VT_9112187923da994fb9f88f757070bb10------------------\n",
|
|
"VT_9112187923da994fb9f88f757070bb10 - version_1: 0.8414382934570312, epoch: 78.0\n",
|
|
"VT_9112187923da994fb9f88f757070bb10 - version_10: 0.8345999717712402, epoch: 79.8\n",
|
|
"VT_9112187923da994fb9f88f757070bb10 - version_2: 0.8381330370903015, epoch: 60.0\n",
|
|
"VT_9112187923da994fb9f88f757070bb10 - version_3: 0.840214729309082, epoch: 72.0\n",
|
|
"VT_9112187923da994fb9f88f757070bb10 - version_4: 0.8510702252388, epoch: 74.0\n",
|
|
"VT_9112187923da994fb9f88f757070bb10 - version_5: 0.8394740223884583, epoch: 78.0\n",
|
|
"VT_9112187923da994fb9f88f757070bb10 - version_6: 0.8388126492500305, epoch: 76.0\n",
|
|
"VT_9112187923da994fb9f88f757070bb10 - version_7: 0.8453190922737122, epoch: 76.0\n",
|
|
"VT_9112187923da994fb9f88f757070bb10 - version_8: 0.8423439264297485, epoch: 74.0\n",
|
|
"VT_9112187923da994fb9f88f757070bb10 - version_9: 0.839865505695343, epoch: 76.0\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_9112187923da994fb9f88f757070bb10; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_9112187923da994fb9f88f757070bb10; statistics are:\n",
|
|
"Scores - mean: 0.841s\tstd: 0.004s min: 0.835s\t max: 0.851s\n",
|
|
"--------------------------------------------\n",
|
|
"--------------VT_9ee8f70a5104ca683c765cfeeb9eba36------------------\n",
|
|
"VT_9ee8f70a5104ca683c765cfeeb9eba36 - version_3: 0.8427470922470093, epoch: 50.0\n",
|
|
"VT_9ee8f70a5104ca683c765cfeeb9eba36 - version_4: 0.8427470922470093, epoch: 50.0\n",
|
|
"VT_9ee8f70a5104ca683c765cfeeb9eba36 - version_5: 0.8427470922470093, epoch: 50.0\n",
|
|
"VT_9ee8f70a5104ca683c765cfeeb9eba36 - version_6: 0.8427470922470093, epoch: 50.0\n",
|
|
"VT_9ee8f70a5104ca683c765cfeeb9eba36 - version_7: 0.8427470922470093, epoch: 50.0\n",
|
|
"VT_9ee8f70a5104ca683c765cfeeb9eba36 - version_8: 0.8427470922470093, epoch: 50.0\n",
|
|
"VT_9ee8f70a5104ca683c765cfeeb9eba36 - version_9: 0.8427470922470093, epoch: 50.0\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_9ee8f70a5104ca683c765cfeeb9eba36; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_9ee8f70a5104ca683c765cfeeb9eba36; statistics are:\n",
|
|
"Scores - mean: 0.843s\tstd: 0.000s min: 0.843s\t max: 0.843s\n",
|
|
"--------------------------------------------\n",
|
|
"--------------VT_aca900a5b9566af61c91aea6525190e6------------------\n",
|
|
"VT_aca900a5b9566af61c91aea6525190e6 - version_0: 0.8547767400741577, epoch: 78.0\n",
|
|
"VT_aca900a5b9566af61c91aea6525190e6 - version_1: 0.8453981280326843, epoch: 78.0\n",
|
|
"VT_aca900a5b9566af61c91aea6525190e6 - version_2: 0.8628634214401245, epoch: 68.0\n",
|
|
"VT_aca900a5b9566af61c91aea6525190e6 - version_3: 0.8621359467506409, epoch: 78.0\n",
|
|
"VT_aca900a5b9566af61c91aea6525190e6 - version_4: 0.8380126357078552, epoch: 78.0\n",
|
|
"VT_aca900a5b9566af61c91aea6525190e6 - version_5: 0.8667657375335693, epoch: 79.8\n",
|
|
"VT_aca900a5b9566af61c91aea6525190e6 - version_6: 0.8575441241264343, epoch: 72.0\n",
|
|
"VT_aca900a5b9566af61c91aea6525190e6 - version_7: 0.8474754095077515, epoch: 58.0\n",
|
|
"VT_aca900a5b9566af61c91aea6525190e6 - version_8: 0.8613359928131104, epoch: 79.8\n",
|
|
"VT_aca900a5b9566af61c91aea6525190e6 - version_9: 0.8585749268531799, epoch: 76.0\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_aca900a5b9566af61c91aea6525190e6; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_aca900a5b9566af61c91aea6525190e6; statistics are:\n",
|
|
"Scores - mean: 0.855s\tstd: 0.009s min: 0.838s\t max: 0.867s\n",
|
|
"--------------------------------------------\n",
|
|
"-------------- Had Errors ------------------\n",
|
|
"-------------- Had Errors ------------------\n",
|
|
"--------------VT_b948cf3132a0750de99a555f30478885------------------\n",
|
|
"VT_b948cf3132a0750de99a555f30478885 - version_1: 0.8622770309448242, epoch: 79.8\n",
|
|
"VT_b948cf3132a0750de99a555f30478885 - version_2: 0.8648049235343933, epoch: 72.0\n",
|
|
"VT_b948cf3132a0750de99a555f30478885 - version_3: 0.8514904379844666, epoch: 64.0\n",
|
|
"VT_b948cf3132a0750de99a555f30478885 - version_4: 0.851193368434906, epoch: 70.0\n",
|
|
"VT_b948cf3132a0750de99a555f30478885 - version_5: 0.8472116589546204, epoch: 46.0\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_b948cf3132a0750de99a555f30478885; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_b948cf3132a0750de99a555f30478885; statistics are:\n",
|
|
"Scores - mean: 0.855s\tstd: 0.007s min: 0.847s\t max: 0.865s\n",
|
|
"--------------------------------------------\n",
|
|
"-------------- Had Errors ------------------\n",
|
|
"--------------VT_cc64c06847a7ca26f5ea4d465f9cc5bc------------------\n",
|
|
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_0: 0.8569087982177734, epoch: 78.0\n",
|
|
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_1: 0.8442623615264893, epoch: 72.0\n",
|
|
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_2: 0.8571171164512634, epoch: 62.0\n",
|
|
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_3: 0.8498414754867554, epoch: 72.0\n",
|
|
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_4: 0.845399022102356, epoch: 79.8\n",
|
|
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_5: 0.8435630798339844, epoch: 78.0\n",
|
|
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_6: 0.8572231531143188, epoch: 74.0\n",
|
|
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_7: 0.845982551574707, epoch: 60.0\n",
|
|
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_8: 0.8455194234848022, epoch: 62.0\n",
|
|
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_9: 0.8448543548583984, epoch: 58.0\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_cc64c06847a7ca26f5ea4d465f9cc5bc; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_cc64c06847a7ca26f5ea4d465f9cc5bc; statistics are:\n",
|
|
"Scores - mean: 0.849s\tstd: 0.005s min: 0.844s\t max: 0.857s\n",
|
|
"--------------------------------------------\n",
|
|
"--------------VT_d55f1492ff29a3cd1026013948ce7fa7------------------\n",
|
|
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_0: 0.8366813063621521, epoch: 37.8\n",
|
|
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_1: 0.8324360251426697, epoch: 35.8\n",
|
|
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_2: 0.8370164632797241, epoch: 38.8\n",
|
|
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_3: 0.8386826515197754, epoch: 39.8\n",
|
|
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_4: 0.8332125544548035, epoch: 39.8\n",
|
|
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_5: 0.8374781608581543, epoch: 39.8\n",
|
|
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_6: 0.8385945558547974, epoch: 32.8\n",
|
|
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_7: 0.8320286273956299, epoch: 39.8\n",
|
|
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_8: 0.8460721969604492, epoch: 38.8\n",
|
|
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_9: 0.8495808839797974, epoch: 34.8\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_d55f1492ff29a3cd1026013948ce7fa7; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_d55f1492ff29a3cd1026013948ce7fa7; statistics are:\n",
|
|
"Scores - mean: 0.838s\tstd: 0.005s min: 0.832s\t max: 0.850s\n",
|
|
"--------------------------------------------\n",
|
|
"--------------VT_e1ab4fa5a5d0649f411b34d2c45731ae------------------\n",
|
|
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_0: 0.8507610559463501, epoch: 78.0\n",
|
|
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_1: 0.8411756753921509, epoch: 79.8\n",
|
|
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_2: 0.8576055765151978, epoch: 76.0\n",
|
|
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_3: 0.8609339594841003, epoch: 48.0\n",
|
|
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_4: 0.8473474383354187, epoch: 50.0\n",
|
|
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_5: 0.8473350405693054, epoch: 76.0\n",
|
|
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_6: 0.8621013760566711, epoch: 79.8\n",
|
|
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_7: 0.8595266342163086, epoch: 78.0\n",
|
|
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_8: 0.8662147521972656, epoch: 78.0\n",
|
|
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_9: 0.8629968762397766, epoch: 76.0\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_e1ab4fa5a5d0649f411b34d2c45731ae; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_e1ab4fa5a5d0649f411b34d2c45731ae; statistics are:\n",
|
|
"Scores - mean: 0.856s\tstd: 0.008s min: 0.841s\t max: 0.866s\n",
|
|
"--------------------------------------------\n",
|
|
"--------------VT_fb6b96a190455106d29f0630f002ac6f------------------\n",
|
|
"VT_fb6b96a190455106d29f0630f002ac6f - version_0: 0.865719735622406, epoch: 70.0\n",
|
|
"VT_fb6b96a190455106d29f0630f002ac6f - version_1: 0.8261691927909851, epoch: 76.0\n",
|
|
"VT_fb6b96a190455106d29f0630f002ac6f - version_2: 0.8545827865600586, epoch: 78.0\n",
|
|
"VT_fb6b96a190455106d29f0630f002ac6f - version_3: 0.8444902896881104, epoch: 56.0\n",
|
|
"VT_fb6b96a190455106d29f0630f002ac6f - version_4: 0.85297691822052, epoch: 78.0\n",
|
|
"VT_fb6b96a190455106d29f0630f002ac6f - version_5: 0.8555656671524048, epoch: 79.8\n",
|
|
"VT_fb6b96a190455106d29f0630f002ac6f - version_6: 0.8635155558586121, epoch: 78.0\n",
|
|
"VT_fb6b96a190455106d29f0630f002ac6f - version_7: 0.837948739528656, epoch: 70.0\n",
|
|
"VT_fb6b96a190455106d29f0630f002ac6f - version_8: 0.8533784747123718, epoch: 62.0\n",
|
|
"VT_fb6b96a190455106d29f0630f002ac6f - version_9: 0.8541560769081116, epoch: 74.0\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_fb6b96a190455106d29f0630f002ac6f; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_fb6b96a190455106d29f0630f002ac6f; statistics are:\n",
|
|
"Scores - mean: 0.851s\tstd: 0.011s min: 0.826s\t max: 0.866s\n",
|
|
"--------------------------------------------\n",
|
|
"-------------- Had Errors ------------------\n",
|
|
"--------------VT_fdf2a86085b508c1325b181c830a4cf7------------------\n",
|
|
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_0: 0.8728921413421631, epoch: 86.0\n",
|
|
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_1: 0.8609604835510254, epoch: 88.0\n",
|
|
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_2: 0.8636621832847595, epoch: 90.0\n",
|
|
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_3: 0.8558254837989807, epoch: 88.0\n",
|
|
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_4: 0.8657329082489014, epoch: 58.0\n",
|
|
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_5: 0.8612215518951416, epoch: 78.0\n",
|
|
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_6: 0.854997456073761, epoch: 84.0\n",
|
|
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_7: 0.8661960959434509, epoch: 90.0\n",
|
|
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_8: 0.8631933927536011, epoch: 88.0\n",
|
|
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_9: 0.8614727258682251, epoch: 88.0\n",
|
|
"\n",
|
|
"\n",
|
|
"For VT_fdf2a86085b508c1325b181c830a4cf7; dataset is: PrimatesLibrosaDatamodule\n",
|
|
"For VT_fdf2a86085b508c1325b181c830a4cf7; statistics are:\n",
|
|
"Scores - mean: 0.863s\tstd: 0.005s min: 0.855s\t max: 0.873s\n",
|
|
"--------------------------------------------\n",
|
|
"--------------------END------------------------\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for model_configuration in natsorted([x for x in (out_path / _model_name).iterdir() if x.is_dir()]):\n",
|
|
" # Print metrics\n",
|
|
" if is_clean(model_configuration):\n",
|
|
" print_metrics(model_configuration, valid_dataset_name=_dataset_name)\n",
|
|
"print('--------------------END------------------------')\n"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%% Mass - Load Model and read Metrics\n"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"outputs": [],
|
|
"source": [
|
|
"if False:\n",
|
|
" # fingerprint = '012aff7c1c667073aedafcbebfa35ec7'\n",
|
|
" fingerprint = 'fdf2a86085b508c1325b181c830a4cf7'\n",
|
|
" exp_name = f'{\"\".join([x for x in model_name if x.isupper()])}_{fingerprint}'\n",
|
|
"\n",
|
|
" # Print metrics\n",
|
|
" print_metrics(out_path/model_name/exp_name)"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%% Single - Load Model and read Metrics\n"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"outputs": [],
|
|
"source": [
|
|
"if False:\n",
|
|
" predictions_file = out_path/model_name/'VT_fbbe1be4393718c3be98f9ff5f6822fa'/'version_9'/'predictions.csv'\n",
|
|
" df_predictions = pd.read_csv(predictions_file)\n",
|
|
" print(df_predictions.head())\n",
|
|
" # df_predictions = df_predictions[['filenames', 'prediction_named']]\n",
|
|
" # df_predictions.columns = ['filename', 'prediction']\n",
|
|
" df_predictions['filename']: pd.DataFrame = df_predictions['filename'].str.replace('npy', 'wav')\n",
|
|
" predictions_file_new = predictions_file.parent / 'prediction_final.csv'\n",
|
|
" df_predictions.to_csv(index=False, path_or_buf=predictions_file_new)\n"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%% Rename Predictions\n"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"outputs": [],
|
|
"source": [
|
|
"def single_params_score_readout(exp_folder, param_name, monitor='PL_recall_score'):\n",
|
|
" scores = list()\n",
|
|
" param_readout = None\n",
|
|
" for run_folder in [x for x in exp_folder.iterdir() if x.is_dir()]:\n",
|
|
" yaml_file = run_folder / 'hparams.yaml'\n",
|
|
" with yaml_file.open('rb') as f:\n",
|
|
" configuration = yaml.load(f, yaml.BaseLoader)\n",
|
|
" if param_readout is None:\n",
|
|
" param_readout = configuration[param_name]\n",
|
|
" else:\n",
|
|
" if param_readout != configuration[param_name]:\n",
|
|
" raise ValueError\n",
|
|
" metrics: pd.DataFrame = pd.read_csv(run_folder / 'metrics.csv')\n",
|
|
" # Possible keys are:\n",
|
|
" # -- CE - Losses:\n",
|
|
" # val_max_vote_loss, val_mean_vote_loss, mean_val_loss\n",
|
|
" # -- Fallback:\n",
|
|
" # mean_loss,epoch,step,macro_f1_score, macro_roc_auc_ovr, uar_score, micro_f1_score\n",
|
|
" # Pytorch Metrics:\n",
|
|
" # PL_f1_score,PL_accuracy_score_score, PL_fbeta_score,PL_recall_score,PL_precision_score,\n",
|
|
" monitor_score=metrics[monitor].max()\n",
|
|
"\n",
|
|
" scores.append(monitor_score)\n",
|
|
" return dict(mean=np.mean(scores), max=np.max(scores), std=np.std(scores), param_readout=param_readout)\n",
|
|
"\n",
|
|
"\n",
|
|
"if True:\n",
|
|
" scores_for_param = list()\n",
|
|
" for modelname in ['VisualTransformer', 'VerticalVisualTransformer']:\n",
|
|
" for model_configuration in natsorted([x for x in (out_path / modelname).iterdir() if x.is_dir()]):\n",
|
|
" # Print metrics\n",
|
|
" if is_clean(model_configuration):\n",
|
|
" score_for_param=single_params_score_readout(model_configuration, _param_name)\n",
|
|
" score_for_param.update(modelname=modelname)\n",
|
|
" scores_for_param.append(score_for_param)\n",
|
|
"\n",
|
|
" df = pd.DataFrame(scores_for_param)"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%%\n"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": "<Figure size 432x288 with 1 Axes>",
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEHCAYAAAC0pdErAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAqtElEQVR4nO3deXyU5b3//9cnYQmLhCVhkV0BZUkACYsiIuKCilCLesBqRW21p2rPz4qK50eBYm217nUtKuBWOIAUqUpRcUergBiWgIIsEhAIYQlLFpJ8vn/MEIcwQoQMk0zez8cjj8y9zXxIwv2e67qvuW5zd0REREqLi3YBIiJSMSkgREQkLAWEiIiEpYAQEZGwFBAiIhJWtWgXUF6SkpK8TZs20S5DRKRSWbx48XZ3Tw63LWYCok2bNixatCjaZYiIVCpmtuHHtqmLSUREwlJAiIhIWAoIEREJK2auQYRz4MABMjMzycvLi3YpcoIkJCTQokULqlevHu1SRCq9mA6IzMxMTjrpJNq0aYOZRbsciTB3Jzs7m8zMTNq2bRvtckQqvZjuYsrLy6NRo0YKhyrCzGjUqJFajCLlJKYDAlA4VDH6fYuUn5juYhIRiUX7CwrZsjuPLTl5bNmdR0L1eC5JaVbur6OAqIQOfigwKSnpuPYRkYrF3dmde4AtOXl8vzuPrbsD30PDYEtOHrtzDxxyXJfm9RQQIiKVVVGxk703n++DJ/2tOaHfc0tO/nkHig85zgyS6takab0EWjWqTa+2DWmamECzxASa1kugaWLgKxIUECfI+vXrGTRoEH369OHTTz+lZ8+eXH/99YwbN45t27bx6quv0q5dO2644QbWrl1L7dq1mThxIqmpqWRnZzNixAg2bdrEmWeeSehdAF955RX+9re/UVBQQO/evXn66aeJj48/5HUvvvhizj77bD799FOaN2/O66+/Tq1atXjuueeYOHEiBQUFtGvXjpdffpnatWszcuRIatWqxZIlS9i2bRuTJk3ipZde4rPPPqN3795MmTIFgLfffptx48aRn5/PqaeeyuTJk6lbt+6J/tGKRF1+YRHbcgIn/8A7/Vy27M5nS05uSUtg6558iooPvYNn9XijSb3Aib5L80TO79ik5ITfLDGBpom1aHxSTarHR+dyscXKLUfT0tK89FxMK1eupGPHjlGq6FDr16+nXbt2LFmyhM6dO9OzZ0+6du3KCy+8wJw5c5g8eTItW7YkKSmJcePG8d577/H73/+er776it/97nckJSUxduxY3nzzTQYPHkxWVhZZWVncddddzJo1i+rVq/Pb3/6WPn368Mtf/rKki2nv3r20a9eORYsW0a1bN6666iqGDBnCNddcQ3Z2No0aNQJgzJgxNGnShNtuu42RI0eSl5fH1KlTmTNnDtdeey0LFiwoqfuFF16gRYsW/PznP2fu3LnUqVOHBx54gPz8fMaOHRvln3TF+r1L5bc3v7DkhB/6Tj/0e/a+gsOOq10jvuRE36TeDyf8pvV+WNeoTg3i4qI7sMLMFrt7WrhtakGcQG3btiUlJQWAzp07M3DgQMyMlJQU1q9fz4YNG3jttdcAOO+888jOziYnJ4ePPvqIWbNmAXDppZfSoEEDAObPn8/ixYvp2bMnALm5uTRu3Djs63br1g2AHj16sH79egCWL1/OmDFj2LVrF3v37uWiiy4qOeayyy4rqa1JkyaH1L1+/XoyMzPJyMigb9++ABQUFHDmmWeW809MJHLcnR37CsJ09/zwfcvuPPbmFx52bIPa1WmaWItmiQmktqh/SHdPs8QEmiQmcFLNapV+VJ0C4gSqWbNmyeO4uLiS5bi4OAoLC3/yp3/dneuuu46//OUvZX7d+Ph4cnNzARg5ciSzZ8+ma9euTJkyhQ8++OCwY0LrDK01Pj6eCy64gKlTp/6kmkVOhMKiYrbtyf/hnX7w3X7Jhd+cXLbm5FNQeGh/f5xB45MCJ/p2yXU5u11S8J3/DwHQpF4CCdXjf+SVY4sCogLp168fr776Kn/4wx/44IMPSEpKol69epxzzjn84x//YMyYMcydO5edO3cCMHDgQIYOHcrtt99O48aN2bFjB3v27KF169Zler09e/bQrFkzDhw4wKuvvkrz5s3LXGufPn245ZZbWLNmDe3atWPfvn1s2rSJDh06HNO/XaSs8g4UseXg6J6cYF//7txDun2y9uRTqrufGtXiSt7pn9GqQclJP7TrJ6luDapFqb+/IlJAVCDjx4/nhhtuIDU1ldq1a/Piiy8CMG7cOEaMGEHnzp0566yzaNWqFQCdOnXiT3/6ExdeeCHFxcVUr16dp556qswBce+999K7d2+Sk5Pp3bs3e/bsKXOtycnJTJkyhREjRpCfnw/An/70JwWEHDN3JyevMHjyzw3b3bMlJ49d+w8cduxJCdVK+vVPa3rSYX39zRITqF+7eqXv8jnRInqR2swGAY8D8cDz7n5/qe2tgBeB+sF9Rrv7W8FtqcDfgXpAMdDT3X90DoWKfpFaThz93iue4mJn+7788N09ISGQe6DosGOT6tYsdaH30OGdTeslUKem3useq6hcpDazeOAp4AIgE1hoZnPcPSNktzHAdHd/xsw6AW8BbcysGvAKcK27p5tZI+Dwtw0iEnUFhcVszTn8nX7o9605eRSW6vOpFhcc4pmYQMeT6zHg9MaHBUDjkxKoUU1dPtESydjtBaxx97UAZjYNGAqEBoQTaCEAJAKbg48vBJa6ezqAu2dHsE4R+RH78guP+KGuLbvz2L738CGetarH06x+4ETf+5SGpbp7atE0sWIM8ZQji2RANAc2hixnAr1L7TMeeNvMbgPqAOcH13cA3MzmAcnANHf/a+kXMLObgJuAkn55ETk6d2fn/gPBE/0PF3q/L/Xuf09e+CGeB7t7Upon0rRerZKhnQdDoF5C5R/iKdG/SD0CmOLuD5vZmcDLZtYlWNfZQE9gPzA/2E82P/Rgd58ITITANYgTW7pIxVRYVMz2vQVhP9QV2gUUbohn8kk1aZpYi1OT69K3XdIh3T3NqtgQT4lsQGwCWoYstwiuC3UjMAjA3T8zswQgiUBr4yN33w5gZm8BZwDzEanC8g4UhR/dszuP73MCF3237ckLO8Tz4Im+W8v6YS/2JtetqSGecohIBsRCoL2ZtSUQDMOBq0vt8x0wEJhiZh2BBCALmAfcZWa1gQKgP/BoBGsViSp3Z09+4Q+jfErG+Qfn9ckJdAHtDDfEs2a1kpN8h8ZJh3X3NEusRQMN8ZRjELGAcPdCM7uVwMk+Hpjk7ivMbAKwyN3nAHcAz5nZ7QQuWI/0wLjbnWb2CIGQceAtd38zUrVG0oABAxg9evQh01g89thjPP7449x8882MHj263F7r3HPP5aGHHuKWW24hPz+fHTt2kJubW/IBuNmzZ9OmTZtyez2ArKwsBg8eTEFBAX/729/o169fuT5/LCgudrL3FYS848/9YZhnSCtgf0G4IZ41aJqYQPP6CfRoXT/4jr9Wycm/aWICdTXEUyIkon9Zwc80vFVq3diQxxlA3x859hUCQ10rtREjRjBt2rRDAmLatGm8+OKLnHPOORF5zc8//xyAKVOmsGjRIp588slDthcWFlKtWvn86ufPn09KSgrPP/98mY8pKio6ZMbZ41Ge/5ZI+Hh1FvfMWkbmztxD1h8c4tmkXk06Nq3HgNMaHza2v3G9mtSspv5+iSJ3j4mvHj16eGkZGRmHrTvRsrOzPTk52fPz893dfd26dd6yZUufNGmS33LLLe7uPn36dO/cubOnpqZ6v3793N198uTJJdvd3S+99FJ///333d39N7/5jffo0cM7derkY8eOLdmnf//+vnDhwpLl0OcYN26cX3PNNX7WWWf58OHDfd26dX722Wd79+7dvXv37r5gwQJ3d3///fe9f//+PmzYMD/ttNP86quv9uLiYnd3v/vuu71jx46ekpLid9xxhy9ZssRbtmzpSUlJ3rVrV9+/f7//4x//8C5dunjnzp39rrvuKqmlTp06/vvf/95TU1P9448/9jp16vioUaO8U6dOPnDgQP/888+9f//+3rZtW3/99dfd3b2wsNBHjRrlaWlpnpKS4s8++2xJjWeffbZfdtll3r59+8N+5hXh974n74DfM2upt777DT/vofd9yoJ1Pm/5956+cadvzcn1oqLiaJco4u7uBHp0wp5XK+5br3L2x3+tIGNzTrk+Z6eT6zHuss5H3Kdhw4b06tWLuXPnMnToUKZNm8ZVV111SH/whAkTmDdvHs2bN2fXrl1Hfd377ruPhg0bUlRUxMCBA1m6dCmpqalHPS4jI4NPPvmEWrVqsX//ft555x0SEhJYvXo1I0aM4OAn0ZcsWcKKFSs4+eST6du3LwsWLKBjx47885//ZNWqVZgZu3bton79+kyYMKGklbJ582buvvtuFi9eTIMGDbjwwguZPXs2P/vZz9i3bx+9e/fm4YcfBmDfvn2cd955PPjgg1x++eWMGTOGd955h4yMDK677jqGDBnCCy+8QGJiIgsXLiQ/P5++ffty4YUXAvDll1+yfPly2rZte9R/94m2YM127pq5lM27c7n5nFO4/YIOGvkjlZKGLJwAB7uZINC9NGLEiEO29+3bl5EjR/Lcc89RVHR4P3Rp06dP54wzzqB79+6sWLGCjIyMox4DMGTIEGrVqgXAgQMH+PWvf01KSgpXXnnlIc/Rq1cvWrRoQVxcHN26dWP9+vUkJiaSkJDAjTfeyKxZs6hdu/Zhz79w4ULOPfdckpOTqVatGr/4xS/46KOPgMAsssOGDSvZt0aNGgwaNAiAlJQU+vfvT/Xq1UumPofADYleeuklunXrRu/evcnOzmb16tUlNVa0cNiXX8gfZi/nF89/Ts1qccz8zZncc0lHhYNUWlWmBXG0d/qRdHDG1S+//JL9+/fTo0cPli1bVrL92Wef5fPPP+fNN9+kR48eLF68mGrVqlFc/MM49by8wDRU69at46GHHmLhwoU0aNCg5OY+ZVGnTp2Sx48++ihNmjQhPT2d4uJiEhJ+uGVh6enBD/bzf/HFF8yfP5+ZM2fy5JNP8t5775X5Z5CQkHDIdYfq1X8YVRNu6nMIdH8+8cQTh1y/Afjggw8O+bdUBP9Zm82dM9PJ3JnLr85uy6iLTlMwSKWnFsQJULduXQYMGMANN9xwWOsB4Ntvv6V3795MmDCB5ORkNm7cSJs2bfjqq68oLi5m48aNfPHFFwDk5ORQp04dEhMT2bp1K3Pnzj2mmnbv3k2zZs2Ii4vj5ZdfPmrLZe/evezevZtLLrmERx99lPT09MP26dWrFx9++CHbt2+nqKiIqVOn0r9//2OqD+Ciiy7imWee4cCBwNDOb775hn379h3z80XC/oJCxs9ZwfCJ/yHejOk3n8mYwZ0UDhITqkwLItpGjBjB5ZdfXtLVFOrOO+9k9erVuDsDBw6ka9euQOBOcJ06daJjx46cccYZAHTt2pXu3btz+umn07Jly5I7uv1Uv/3tbxk2bBgvvfQSgwYNOuo78j179jB06FDy8vJwdx555JHD9mnWrBn3338/AwYMwN259NJLGTp06DHVB/CrX/2K9evXc8YZZ+DuJCcnM3v27GN+vvL2xbod3DkznQ3Z+7m+bxvuuuh0atVQMEjs0D2pJeZE+veeW1DEg/O+ZvKn62jZoDZ/vSKVPqc0itjriUSS7kktUk4Wb9jBqBlLWbd9H9ed2Zq7Lz6d2jX030hik/6yRcog70ARD7/9Nc9/so7m9Wvxj1/35qxTk6JdlkhExXxAuLvmoKlCItFl+uV3Oxk1I521Wfv4Re9W3HNJR01vIVVCTP+VJyQkkJ2dTaNGjRQSVYC7k52dfciQ3eORd6CIR9/9huc+WkuzxFq8cmNvzm6vVoNUHTEdEC1atCAzM5OsrKxolyInSEJCAi1atDju50nfuIs7ZqSzZtteRvRqxf9ecjonJVQvhwpFKo+YDojq1atXuE/bSsWWX1jE4++u5tkPv6VJvQReuqEX53RIjnZZIlER0wEh8lMszdzFqBnpfLN1L1eltWDM4E7UU6tBqjAFhFR5BYXFPPHeap7+4FuS6tZg8vU9GXBa42iXJRJ1Cgip0pZv2s2oGems2rKHK3q04A+DO5FYS60GEVBASBVVUFjMU++v4an319CwTg1euC6NgR2bRLsskQpFASFVTsbmHEbNSCfj+xwu796ccZd1on7tGtEuS6TCUUBIlXGgqJhnPviWv81fTf3aNZh4bQ8u7Nw02mWJVFgKCKkSvt6yhztmfMXyTTkM7XYy4y/rTIM6ajWIHIkCQmJaYVExf/9oLY+9+w31Eqrz7DVnMKhLs2iXJVIpKCAkZq3euoc7ZqSzNHM3l6Y2Y8KQzjSqW/PoB4oIoICQGFRYVMxzH6/j0Xe+oW5CNZ66+gwuTVWrQeSnUkBITFmzbQ93zFhK+sZdXNylKff+rAtJajWIHBMFhMSEomLnhU/W8tDb31C7RjxPjOjO4NRmmsVX5DgoIKTSW5u1l1Ez0vnyu11c2KkJf7q8C41PKp8pv0WqMgWEVFpFxc7kBet4cN7XJFSP5/Hh3RjS9WS1GkTKiQJCKqV12/dx54x0Fm3YyfkdG/Pny1NoXE+tBpHypICQSqW42Hnxs/U88O9V1IiP45GrunJ59+ZqNYhEgAJCKo3vsvczamY6X6zbwYDTkrl/WCpN1GoQiZi4SD65mQ0ys6/NbI2ZjQ6zvZWZvW9mS8xsqZldEmb7XjMbFck6pWIrLnZe+mw9Fz32ESs35/DgFalMGtlT4SASYRFrQZhZPPAUcAGQCSw0sznunhGy2xhgurs/Y2adgLeANiHbHwHmRqpGqfg27tjPnTPT+c/aHfTvkMz9w1Jollgr2mWJVAmR7GLqBaxx97UAZjYNGAqEBoQD9YKPE4HNBzeY2c+AdcC+CNYoFZS78+rn3/Hnt1YSZ8YDw1K4Kq2lrjWInECRDIjmwMaQ5Uygd6l9xgNvm9ltQB3gfAAzqwvcTaD18aPdS2Z2E3ATQKtWrcqrbomyzJ37Gf3aMj5Zs51+7ZO4f1gqzeur1SByokX0GkQZjACmuHsL4BLgZTOLIxAcj7r73iMd7O4T3T3N3dOSk5OPq5CtOXnHdbwcP3dn6hffMeixj1ny3U7+fHkKL93QS+EgEiWRbEFsAlqGLLcIrgt1IzAIwN0/M7MEIIlAS+MKM/srUB8oNrM8d38yUsVuy8nXRc8o2rwrl7tfW8rHq7dz1qmNeGBYKi0b1o52WSJVWiQDYiHQ3szaEgiG4cDVpfb5DhgITDGzjkACkOXu/Q7uYGbjgb2RDAeJHndn+qKN/OmNlRS5c+/PuvCLXq2Ii9O1BpFoi1hAuHuhmd0KzAPigUnuvsLMJgCL3H0OcAfwnJndTuCC9Uh390jVJBXL97tzGf3aMj78Jos+pzTkwSu6qtUgUoFE9INy7v4WgaGroevGhjzOAPoe5TnGR6Q4iRp357UvN/HHf62gsMj545DOXNuntVoNIhWMPkktJ9TWnDzumbWM91Zto1ebhjx4ZSqtG9WJdlkiEoYCQk4Id+efSzYxfs4KCoqKGTu4EyPPaqNWg0gFpoCQiNu2J4//nbWcd1dupUfrBjx0ZVfaJqnVIFLRKSAkYtydOembGTdnBbkFRYy5tCPX921LvFoNIpWCAkIiImtPPmNmL2Peiq10b1Wfh67syqnJdaNdloj8BAoIKXdvLN3MH2YvZ19BEfdcfDq/6neKWg0ilZACQspN9t58xr6+gjeXfU/XFok8fFVX2jU+KdplicgxUkBIuZi77HvGzF7OnrxC7hp0Gjf1O4Vq8dGe6ktEjocCQo7Ljn0FjH19OW8s/Z6U5ok8dGVXTmuqVoNILFBAyDH79/ItjJm9jN25Bxh1YQdu7n8q1dVqEIkZCgj5yXbuK2D8v1bw+leb6XxyPV6+sTcdm9U7+oEiUqkoIOQneSdjK//7z2Xs3FfA7ed34LcD1GoQiVUKCCmT3fsP8Md/rWDWkk2c3vQkplzfk84nJ0a7LBGJIAWEHNV7q7Zyz6xlbN9bwO8GtufWAe2oUU2tBpFYp4CQH7U79wD3vpHBzMWZnNbkJF64riddmqvVIFJVKCAkrA++3sbo15aRtTefWwe047aB7ahZLT7aZYnICaSAkEPk5B3gvjdW8n+LNtK+cV3+fm0PurasH+2yRCQKFBBS4uPVWdw9cylbcvL473NP5X8GtiehuloNIlWVAkLYm1/IfW+uZOoX33Fqch1e+++z6N6qQbTLEpEoU0BUcQvWbOeumUvZvDuXm885hdsv6KBWg4gACogqa19+IffPXcXL/9nAKUl1mPmbM+nRumG0yxKRCkQBUQV99m02d72WTubOXH51dltGXXSaWg0ichgFRBWyv6CQB+au4sXPNtCmUW2m33wmPduo1SAi4SkgqojP12Zz58ylfLdjP9f3bcNdF51OrRpqNYjIj1NAxLjcgiL+Om8VUz5dT8sGtZl2Ux/6nNIo2mWJSCWggIhhi9bv4M6ZS1m3fR/Xndmauy8+ndo19CsXkbLR2SIG5R0o4qF5X/PCgnU0r1+Lf/y6N2edmhTtskSkklFAxJjFG3Zy54x01m7fxzV9WnHPxR2pU1O/ZhH56XTmiBF5B4p49J1veO7jtTRLrMUrN/bm7PZqNYjIsYvopP5mNsjMvjazNWY2Osz2Vmb2vpktMbOlZnZJcP0FZrbYzJYFv58XyToru6827mLwE5/w94/W8l89W/Hv/6+fwkFEjlvEWhBmFg88BVwAZAILzWyOu2eE7DYGmO7uz5hZJ+AtoA2wHbjM3TebWRdgHtA8UrVWVvmFRTz27mr+/uG3NK2XwEs39OKcDsnRLktEYkQku5h6AWvcfS2AmU0DhgKhAeHAwbvdJwKbAdx9Scg+K4BaZlbT3fMjWG+lsjRzF6NmpPPN1r38V1pL/v/BHamXUD3aZYlIDIlkQDQHNoYsZwK9S+0zHnjbzG4D6gDnh3meYcCXCoeA/MIinpi/hmc+/JakujWYfH1PBpzWONpliUgMivZF6hHAFHd/2MzOBF42sy7uXgxgZp2BB4ALwx1sZjcBNwG0atXqBJUcPcs37WbUjHRWbdnDFT1a8IfBnUispVaDiERGJANiE9AyZLlFcF2oG4FBAO7+mZklAEnANjNrAfwT+KW7fxvuBdx9IjARIC0tzcu3/IqjoLCYJ99fw9Pvr6FhnRpMGpnGeac3iXZZIhLjyhQQZpbg7nml1iW5+/YjHLYQaG9mbQkEw3Dg6lL7fAcMBKaYWUcgAcgys/rAm8Bod19Qpn9JjMrYnMMdM9JZ+X0OP+/enHGXdSaxtloNIhJ5ZR3mutDM+hxcMLNhwKdHOsDdC4FbCYxAWklgtNIKM5tgZkOCu90B/NrM0oGpwEh39+Bx7YCxZvZV8KtKdbQfKCrm8XdXM+TJT8jak8/Ea3vwyH91UziIyAljgfPxUXYySwEmAR8AJwONgF+5e2ZEq/sJ0tLSfNGiRcd8/LLM3aS0SCzHio7dqi053DE9nRWbcxja7WTGX9aZBnVqRLssEYlBZrbY3dPCbStTF5O7LzOz+4CXgT3AORUpHGJFYVExz374LY/PX01ireo8e00PBnVpGu2yRKSKKus1iBeAU4FUoAPwhpk94e5PRbK4quSbrXsYNSOdpZm7GZzajAlDu9BQrQYRiaKyjmJaRqBLyYF1ZtYbeCRyZVUdhUXFTPx4LY+9s5q6CdV46uozuDS1WbTLEhEpcxfTY6WWdxMYoirHYc22PdwxYynpG3dxcZem3PuzLiTVrRntskREgLJ3MbUH/gJ0IjAUFQB3PyVCdcW0omLn+Y/X8vA731CnRjxPjOjO4NRmmFm0SxMRKVHWLqbJwDjgUWAAcD0Rngk2Vn2btZc7Z6Tz5Xe7uLBTE+67PIXkk9RqEJGKp6wBUcvd55uZufsGYLyZLQbGRrC2mFJU7ExesI4H531NQvV4Hh/ejSFdT1arQUQqrLIGRL6ZxQGrzexWAp+Mrhu5smLLuu37uHNGOos27OT8jo358+UpNK6XcPQDRUSiqKwB8T9AbeB3wL0Eupl+GamiYkVxsTPl0/X8dd4qasTH8chVXbm8e3O1GkSkUihrQDiBD8m1Bg7O9fAcgc9FSBgbsvdx58ylfLFuBwNOS+b+Yak0UatBRCqRsgbEq8CdBD4PURy5ciq/4mLn5f9s4P65q6gWZzx4RSpX9GihVoOIVDplDYgsd58T0UpiwMYd+7lzZjr/WbuD/h2SuX9YCs0Sa0W7LBGRY1LWgBhnZs8D84GSO7u5+6yIVFXJFBc7r37xHX95ayVxZjwwLIWr0lqq1SAilVpZA+J64HQC1x8OdjE5UOUDInPnfu5+bSkL1mTTr30S9w9LpXl9tRpEpPIra0D0dPfTIlpJJePuTP1iI/e9mQHAny9PYUQvtRpEJHaUNSA+NbNO7p4R0WoqiU27chn92lI+Xr2ds05txAPDUmnZsHa0yxIRKVdlDYg+wFdmto7ANQgD3N2r1DBXd2f6oo3c+8ZKit2592dd+EWvVsTFqdUgIrGnrAExKKJVVALf785l9GvL+PCbLPqc0pAHr+iqVoOIxLSyTve9IdKFVFTuzszFmUx4I4PCIuePQzpzbZ/WajWISMwrawuiStqak8c9s5bx3qpt9GrTkAevTKV1ozrRLktE5IRQQITh7vxzySbGz1lBQVExYwd3YuRZbdRqEJEqRQFRyrY9efzvrOW8u3Iraa0b8OCVXWmbpFaDiFQ9Coggd+f1rzYxbs4KcguKGHNpR67v25Z4tRpEpIpSQABZe/L5y9xVfLY2m+6t6vPQlV05NVm3uxCRqq3KB8S67fv4+dML2JtfyD0Xn86v+p2iVoOICAoIWjeszdBuzenVpiGXpDaLdjkiIhVGXLQLiLa4OGP8kM760JuISClVPiBERCQ8BYSIiISlgBARkbAiGhBmNsjMvjazNWY2Osz2Vmb2vpktMbOlZnZJyLZ7gsd9bWYXRbJOERE5XMRGMZlZPPAUcAGQCSw0szml7ikxBpju7s+YWSfgLaBN8PFwoDNwMvCumXVw96JI1SsiIoeKZAuiF7DG3de6ewEwDRhaah8H6gUfJwKbg4+HAtPcPd/d1wFrgs8nIiInSCQDojmwMWQ5M7gu1HjgGjPLJNB6uO0nHIuZ3WRmi8xsUVZWVnnVLSIiRP8i9Qhgiru3AC4BXjazMtfk7hPdPc3d05KTkyNWpIhIVRTJT1JvAlqGLLcIrgt1I8G71bn7Z2aWACSV8VgREYmgSLYgFgLtzaytmdUgcNF5Tql9vgMGAphZRyAByAruN9zMappZW6A98EUEaxURkVIi1oJw90IzuxWYB8QDk9x9hZlNABa5+xzgDuA5M7udwAXrke7uwAozmw5kAIXALRrBJCJyYlngfFz5paWl+aJFi475+GWZu0lpkViOFYmIVHxmttjd08Jti/ZFahERqaAUECIiEpYCQkREwlJAiIhIWAoIEREJSwEhIiJhKSBERCQsBYSIiISlgBARkbAUECIiEpYCQkREwlJAiIhIWAoIEREJSwEhIiJhKSBERCQsBYSIiISlgBARkbAUECIiEpYCQkREwlJAiIhIWAoIEREJSwEhIiJhKSBERCQsBYSIiISlgBARkbAUECIiEpYCQkREwlJAiIhIWAoIEREJSwEhIiJhRTQgzGyQmX1tZmvMbHSY7Y+a2VfBr2/MbFfItr+a2QozW2lmfzMzi2StIiJyqGqRemIziweeAi4AMoGFZjbH3TMO7uPut4fsfxvQPfj4LKAvkBrc/AnQH/ggUvWKiMihItmC6AWscfe17l4ATAOGHmH/EcDU4GMHEoAaQE2gOrA1grWKiEgpkQyI5sDGkOXM4LrDmFlroC3wHoC7fwa8D3wf/Jrn7ivDHHeTmS0ys0VZWVnlXL6ISNVWUS5SDwdmunsRgJm1AzoCLQiEynlm1q/0Qe4+0d3T3D0tOTn5hBYsIhLrIhkQm4CWIcstguvCGc4P3UsAlwP/cfe97r4XmAucGZEqRUQkrEgGxEKgvZm1NbMaBEJgTumdzOx0oAHwWcjq74D+ZlbNzKoTuEB9WBeTiIhETsQCwt0LgVuBeQRO7tPdfYWZTTCzISG7DgemubuHrJsJfAssA9KBdHf/V6RqFRGRw0VsmCuAu78FvFVq3dhSy+PDHFcE3BzJ2kRE5MgqykVqERGpYBQQIiISlgJCRETCUkCIiEhYCggREQlLASEiImEpIEREJCwFhIiIhKWAEBGRsBQQIiISlgJCRETCUkCIiEhYCggREQlLASEiImEpIEREJCwFhIiIhKWAEBGRsBQQIiISlgJCRETCUkCIiEhYCggREQlLASEiImEpIEREJCwFhIiIhKWAEBGRsBQQIiISlgIiqHG9mtEuQUSkQlFABDWplxDtEkREKhQFhIiIhBXRgDCzQWb2tZmtMbPRYbY/amZfBb++MbNdIdtamdnbZrbSzDLMrE0kaxURkUNVi9QTm1k88BRwAZAJLDSzOe6ecXAfd789ZP/bgO4hT/EScJ+7v2NmdYHiSNUqIiKHi2QLohewxt3XunsBMA0YeoT9RwBTAcysE1DN3d8BcPe97r4/grWKiEgpkQyI5sDGkOXM4LrDmFlroC3wXnBVB2CXmc0ysyVm9mCwRVL6uJvMbJGZLcrKyirn8kVEqraKcpF6ODDT3YuCy9WAfsAooCdwCjCy9EHuPtHd09w9LTk5+UTVKiJSJUQyIDYBLUOWWwTXhTOcYPdSUCbwVbB7qhCYDZwRiSJFRCS8SAbEQqC9mbU1sxoEQmBO6Z3M7HSgAfBZqWPrm9nBZsF5QEbpY0VEJHIiNorJ3QvN7FZgHhAPTHL3FWY2AVjk7gfDYjgwzd095NgiMxsFzDczAxYDzx3p9RYvXrzdzDYcR8lJwPbjOF7kSPT3JZF0PH9frX9sg4Wcl6s0M1vk7mnRrkNik/6+JJIi9fdVUS5Si4hIBaOAEBGRsBQQP5gY7QIkpunvSyIpIn9fugYhIiJhqQUhIiJhKSBERCSsKh8QZlbfzGaa2arg1OJnRrsmiR1mdruZrTCz5WY21cx0Zyo5LmY2ycy2mdnyUutvC57HVpjZX8vjtap8QACPA/9299OBrsDKKNcjMcLMmgO/A9LcvQuBD4wOj25VEgOmAINCV5jZAAKzZXd1987AQ+XxQlU6IMwsETgHeAHA3QvcfVdUi5JYUw2oZWbVgNrA5ijXI5Wcu38E7Ci1+r+B+909P7jPtvJ4rSodEASmGM8CJgenFX/ezOpEuyiJDe6+icA7ue+A74Hd7v52dKuSGNUB6Gdmn5vZh2bWszyetKoHRDUCs8Q+4+7dgX3AYbdGFTkWZtaAQLO/LXAyUMfMroluVRKjqgENgT7AncD04Dx2x6WqB0QmkOnunweXZ6JpxaX8nA+sc/csdz8AzALOinJNEpsygVke8AWBWzQnHe+TVumAcPctwEYzOy24aiCaVlzKz3dAHzOrHXw3NxANgpDImA0MADCzDkANymH24IhN912J3Aa8GrxnxVrg+ijXIzHC3T83s5nAl0AhsARNuSHHycymAucCSWaWCYwDJgGTgkNfC4DrvBymydBUGyIiElaV7mISEZEfp4AQEZGwFBAiIhKWAkJERMJSQIiISFgKCBERCUsBIVLJmFmb0lM9/8Tj/7c865HYpYAQCRGcdTXSrxEf6dc4CgWElIkCQmJO8B32KjN7NXgTqJnB6S7GmtnC4M17Jh6czMzMPjCzx8xsEfA/ZnZZcFbMJWb2rpk1Ce433sxeNLOPzWyDmf3czP5qZsvM7N9mVv0INa03swfM7EvgSjO70Mw+M7MvzWyGmdUN7vdjNfYws3QzSwduCXneBDObHKxhSfC+AJjZSDN7MmS/N8zsXDO7n8D041+Z2avl/9OXWKKAkFh1GvC0u3cEcoDfAk+6e8/gzXtqAYND9q/h7mnu/jDwCdAnOMPvNOCukP1OBc4DhgCvAO+7ewqQC1x6lJqy3f0M4F1gDHB+cHkR8PvgPj9W42TgNnfvWuo5bwE8WMMI4MUj3bXO3UcDue7ezd1/cZR6pYrTXEwSqza6+4Lg41cI3NltnZndReDGPQ2BFcC/gvv8X8ixLYD/M7NmBCY9Wxeyba67HzCzZQTuEPfv4PplQJuj1HTwNfoAnYAFwQZCDeCz4LYBpWs0s4+B+sEbxQC8DFwcfHw28ASAu68ysw0E7g0gctwUEBKrSk8y5sDTBG7/udHMxgOh77T3hTx+AnjE3eeY2bnA+JBtB+/YVWxmB0ImRCvm6P+fDr6GAe+4+4jQjcF3/keq8aco5NAeAt0LW34ydTFJrGplZmcGH19NoNsIYHuwv/+KIxybCGwKPr4uArX9B+hrZu0AzKxOcIrmgyfxQ2oM3gZ3l5mdHdwe2jX08cHl4HO0Ar4G1gPdzCzOzFoCvUKOOXCk6yUiB6kFIbHqa+AWM5tE4B4fzwANgOXAFmDhEY4dD8wws53AewTuCFdu3D3LzEYCU82sZnD1GHf/xsye+5EarycwnbMDobctfRp4JtjlVQiMdPd8M1tAoGssg8A9KL4MOWYisNTMvtR1CDkSTfctMcfM2gBvBC/0isgxUheTiIiEpRaESDkys39yeJfU3e4+Lxr1iBwPBYSIiISlLiYREQlLASEiImEpIEREJCwFhIiIhPX/AItYt83B4L3aAAAAAElFTkSuQmCC\n"
|
|
},
|
|
"metadata": {
|
|
"needs_background": "light"
|
|
},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"lineplot = sns.lineplot(data=df, x='param_readout', y='max', hue='modelname', legend=True)"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%%\n"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 2
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython2",
|
|
"version": "2.7.6"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 0
|
|
} |