steffen/Mel_Vision_Transformer_ComParE-21
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bug in metric calculation

Browse Source
This commit is contained in:
Steffen Illium 2021-03-27 16:39:07 +01:00
parent 82835295a1
commit 37e36df0a8
12 changed files with 967 additions and 239 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
_parameters.ini
View File

@ -4,20 +4,19 @@ debug = False
eval = True
seed = 69
owner = si11ium
model_name = VisualTransformer
data_name = PrimatesLibrosaDatamodule
model_name = CNNBaseline
data_name = CCSLibrosaDatamodule
[data]
num_worker = 10
data_root = data
variable_length = True
sample_segment_len=0
sample_hop_len=0
target_mel_length_in_seconds = 0.7
n_mels = 128
sr = 16000
hop_length = 128
n_fft = 256
n_fft = 512
random_apply_chance = 0.7
loudness_ratio = 0.0
@ -47,8 +46,7 @@ use_bias = True
use_norm = True
dropout = 0.2
lat_dim = 32
features = 64
filters = [16, 32, 64]
filters = [16, 32, 64, 128]
[VisualTransformer]
weight_init = xavier_normal_
@ -111,7 +109,7 @@ embedding_size = 30
[train]
outpath = output
version = None
sampler = EqualSampler
sampler = WeightedRandomSampler
loss = ce_loss
sto_weight_avg = False
weight_decay = 0

92
datasets/compare_base.py
View File

@ -32,7 +32,7 @@ class CompareBase(_BaseDataModule):
@property
def mel_folder(self):
return self.root / 'mel_folder'
return Path(f'{self.root}_mel_folder')
@property
def wav_folder(self):
@ -58,7 +58,10 @@ class CompareBase(_BaseDataModule):
self.sample_segment_length = target_frames // self.mel_kwargs['hop_length'] + 1
# Utility
self.utility_transforms = Compose([NormalizeLocal(), ToTensor()])
self.utility_transforms = Compose([
NormalizeLocal(),
ToTensor()
])
# Data Augmentations
self.random_apply_chance = random_apply_chance
@ -85,8 +88,11 @@ class CompareBase(_BaseDataModule):
batch_size=self.batch_size, pin_memory=False,
num_workers=self.num_worker)
def _build_subdataset(self, row, build=False):
def _build_subdataset(self, row, build=False, data_option=None):
slice_file_name, class_name = row.strip().split(',')
if data_option is not None:
if data_option not in slice_file_name:
return None, -1, 'no_file'
class_id = self.class_names.get(class_name, -1)
audio_file_path = self.wav_folder / slice_file_name
@ -96,53 +102,54 @@ class CompareBase(_BaseDataModule):
kwargs.update(mel_augmentations=self.utility_transforms)
# DATA OPTION DIFFERENTIATION !!!!!!!!!!! - End
kwargs.update(sample_segment_len=self.sample_segment_length, sample_hop_len=self.sample_segment_length//2)
mel_dataset = LibrosaAudioToMelDataset(audio_file_path, class_id, **kwargs)
if build:
assert mel_dataset.build_mel()
return mel_dataset, class_id, slice_file_name
def manual_setup(self, stag=None):
def manual_setup(self):
datasets = dict()
for data_option in data_options:
with open(Path(self.root) / 'lab' / f'{data_option}.csv', mode='r') as f:
# Exclude the header
_ = next(f)
all_rows = list(f)
chunksize = len(all_rows) // max(self.num_worker, 1)
dataset = list()
with mp.Pool(processes=self.num_worker) as pool:
from itertools import repeat
results = pool.starmap_async(self._build_subdataset, zip(all_rows, repeat(True, len(all_rows))),
chunksize=chunksize)
for sub_dataset in results.get():
dataset.append(sub_dataset[0])
with (Path(self.root) / 'lab' / 'labels.csv') as label_csv_file:
if label_csv_file.exists():
lab_file = label_csv_file.name
else:
lab_file = None
for data_option in data_options:
if lab_file is not None:
if any([x in lab_file for x in data_options]):
lab_file = f'{data_option}.csv'
dataset = self._load_from_file(lab_file, data_option, rebuild=True)
datasets[data_option] = ConcatDataset(dataset)
print(f'{data_option}-dataset prepared.')
self.datasets = datasets
return datasets
def prepare_data(self, *args, rebuild=False, **kwargs):
def prepare_data(self, *args, rebuild=False, subsets=None, **kwargs):
datasets = dict()
samplers = dict()
weights = dict()
for data_option in data_options:
with open(Path(self.root) / 'lab' / f'{data_option}.csv', mode='r') as f:
# Exclude the header
_ = next(f)
all_rows = list(f)
chunksize = len(all_rows) // max(self.num_worker, 1)
dataset = list()
with mp.Pool(processes=self.num_worker) as pool:
with (Path(self.root) / 'lab' / 'labels.csv') as label_csv_file:
if label_csv_file.exists():
lab_file = label_csv_file.name
else:
lab_file = None
for data_option in data_options:
if subsets is not None:
if data_option not in subsets:
print(f'{data_option} skipped...')
continue
if lab_file is not None:
if any([x in lab_file for x in data_options]):
lab_file = f'{data_option}.csv'
dataset = self._load_from_file(lab_file, data_option, rebuild=rebuild)
from itertools import repeat
results = pool.starmap_async(self._build_subdataset, zip(all_rows, repeat(rebuild, len(all_rows))),
chunksize=chunksize)
for sub_dataset in results.get():
dataset.append(sub_dataset[0])
datasets[data_option] = ConcatDataset(dataset)
print(f'{data_option}-dataset set up!')
@ -172,6 +179,27 @@ class CompareBase(_BaseDataModule):
print(f'Dataset {self.__class__.__name__} setup done.')
return datasets
def _load_from_file(self, lab_file, data_option, rebuild=False):
with open(Path(self.root) / 'lab' / lab_file, mode='r') as f:
# Exclude the header
_ = next(f)
all_rows = list(f)
chunksize = len(all_rows) // max(self.num_worker, 1)
dataset = list()
with mp.Pool(processes=self.num_worker) as pool:
from itertools import repeat
results = pool.starmap_async(self._build_subdataset,
zip(all_rows,
repeat(rebuild, len(all_rows)),
repeat(data_option, len(all_rows))
),
chunksize=chunksize)
for sub_dataset in results.get():
if sub_dataset[0] is not None:
dataset.append(sub_dataset[0])
return dataset
def purge(self):
import shutil

19
datasets/mask_librosa_datamodule.py Normal file
View File

@ -0,0 +1,19 @@
from datasets.compare_base import CompareBase
from ml_lib.utils.tools import add_argparse_args
class MaskLibrosaDatamodule(CompareBase):
class_names = ['mask', 'clear']
sub_dataset_name = 'ComParE2020_Mask'
def __init__(self, *args, **kwargs):
super(MaskLibrosaDatamodule, self).__init__(*args, **kwargs)
@classmethod
def add_argparse_args(cls, parent_parser):
return add_argparse_args(CompareBase, parent_parser)
@classmethod
def from_argparse_args(cls, args, **kwargs):
return CompareBase.from_argparse_args(args, class_names=cls.class_names, sub_dataset_name=cls.sub_dataset_name)

3
models/cnn_baseline.py
View File

@ -16,8 +16,7 @@ class CNNBaseline(CombinedModelMixins,
):
def __init__(self, in_shape, n_classes, weight_init, activation,
use_bias, use_norm, dropout, lat_dim, features,
filters,
use_bias, use_norm, dropout, lat_dim, filters,
lr, weight_decay, sto_weight_avg, lr_warm_restart_epochs, opt_reset_interval,
loss, scheduler, lr_scheduler_parameter
):

4
models/transformer_model.py
View File

@ -6,6 +6,7 @@ from torch import nn
from einops import rearrange, repeat
from ml_lib.metrics.binary_class_classifictaion import BinaryScores
from ml_lib.metrics.multi_class_classification import MultiClassScores
from ml_lib.modules.blocks import (TransformerModule, F_x)
from ml_lib.modules.util import (LightningBaseModule, AutoPadToShape)
@ -128,4 +129,7 @@ class VisualTransformer(CombinedModelMixins,
return Namespace(main_out=tensor, attn_weights=attn_weights)
def additional_scores(self, outputs):
if self.params.n_classes <= 2:
return BinaryScores(self)(outputs)
else:
return MultiClassScores(self)(outputs)

10
multi_run.py
View File

@ -10,13 +10,13 @@ import itertools
if __name__ == '__main__':
# Set new values
hparams_dict = dict(seed=range(10),
hparams_dict = dict(seed=range(1, 11),
model_name=['CNNBaseline'],
data_name=['CCSLibrosaDatamodule'], # 'CCSLibrosaDatamodule'],
batch_size=[50],
max_epochs=[200],
variable_length=[False],
target_mel_length_in_seconds=[0.5],
variable_length=[False], # THIS IS NEXT
target_mel_length_in_seconds=[0.7],
random_apply_chance=[0.5], # trial.suggest_float('random_apply_chance', 0.1, 0.5, step=0.1),
loudness_ratio=[0], # trial.suggest_float('loudness_ratio', 0.0, 0.5, step=0.1),
shift_ratio=[0.3], # trial.suggest_float('shift_ratio', 0.0, 0.5, step=0.1),
@ -31,11 +31,11 @@ if __name__ == '__main__':
attn_depth=[12], # trial.suggest_int('attn_depth', 2, 14, step=4),
heads=[6], # trial.suggest_int('heads', 2, 16, step=2),
scheduler=['LambdaLR'], # trial.suggest_categorical('scheduler', [None, 'LambdaLR']),
lr_scheduler_parameter=[0.95], # [0.98],
lr_scheduler_parameter=[0.94, 0.93, 0.95], # [0.98],
embedding_size=[30], # trial.suggest_int('embedding_size', 12, 64, step=12),
loss=['ce_loss'],
sampler=['WeightedRandomSampler'],
# rial.suggest_categorical('sampler', [None, 'WeightedRandomSampler']),
# trial.suggest_categorical('sampler', [None, 'WeightedRandomSampler']),
weight_decay=[0], # trial.suggest_loguniform('weight_decay', 1e-20, 1e-1),
)

731
notebooks/Train Eval.ipynb
View File

@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 25,
"outputs": [],
"source": [
"from collections import defaultdict\n",
@ -33,14 +33,14 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 26,
"outputs": [],
"source": [
"# Settings and Variables\n",
"\n",
"# This Experiment (= Model and Parameter Configuration\n",
"_ROOT = Path('..')\n",
"out_path = Path('..') / Path('output')\n",
"out_path = Path('..') / Path('output') / 'output'\n",
"model_name = 'VisualTransformer'\n"
],
"metadata": {
@ -52,7 +52,7 @@
},
{
"cell_type": "code",
"execution_count": 42,
"execution_count": 27,
"outputs": [],
"source": [
"def print_stats(data_option, mean_duration, std_duration, min_duration, max_duration):\n",
@ -78,7 +78,7 @@
" # mean_loss,epoch,step,macro_f1_score, macro_roc_auc_ovr, uar_score, micro_f1_score\n",
" # Pytorch Metrics:\n",
" # PL_f1_score,PL_accuracy_score_score, PL_fbeta_score,PL_recall_score,PL_precision_score,\n",
" score = metrics.PL_recall_score[-1]\n",
" score = metrics.PL_recall_score.iat[-1]\n",
" print(f'{exp_path.name} - {run_folder.name}: {score}')\n",
" best_scores.append(score)\n",
" had_errors.append(False)\n",
@ -102,183 +102,648 @@
},
{
"cell_type": "code",
"execution_count": 32,
"execution_count": 28,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"--------------VT_259ee495ee2d2dc0e56bb23d12476f17------------------\n",
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_1: 0.8403531908988953\n",
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_3: 0.8312729001045227\n",
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_0: 0.8342075347900391\n",
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_5: 0.8459098935127258\n",
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_2: 0.8468937277793884\n",
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_4: 0.8404075503349304\n",
"--------------VT_01123c93daaffa92d2ed341bda32426d------------------\n",
"VT_01123c93daaffa92d2ed341bda32426d - version_0: 0.8587360978126526\n",
"VT_01123c93daaffa92d2ed341bda32426d - version_1: 0.8587360978126526\n",
"VT_01123c93daaffa92d2ed341bda32426d - version_2: 0.8587360978126526\n",
"VT_01123c93daaffa92d2ed341bda32426d - version_3: 0.8587360978126526\n",
"VT_01123c93daaffa92d2ed341bda32426d - version_4: nan\n",
"\n",
"\n",
"For VT_259ee495ee2d2dc0e56bb23d12476f17; statistics are:\n",
"Scores - mean: 0.840s\tstd: 0.006smin: 0.831s\t max: 0.847s\n",
"For VT_01123c93daaffa92d2ed341bda32426d; statistics are:\n",
"Scores - mean: nans\tstd: nansmin: nans\t max: nans\n",
"--------------------------------------------\n",
"--------------VT_012aff7c1c667073aedafcbebfa35ec7------------------\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_6: 0.8637051582336426\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_1: 0.864475429058075\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_3: 0.854859471321106\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_0: 0.8631429672241211\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_8: 0.8484407663345337\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_5: 0.8564963340759277\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_7: 0.8519455194473267\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_1: 0.864475429058075\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_2: 0.8683117032051086\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_9: 0.8730489611625671\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_3: 0.854859471321106\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_4: 0.8658838272094727\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_5: 0.8564963340759277\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_6: 0.8637051582336426\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_7: 0.8519455194473267\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_8: 0.8484407663345337\n",
"VT_012aff7c1c667073aedafcbebfa35ec7 - version_9: 0.8730489611625671\n",
"\n",
"\n",
"For VT_012aff7c1c667073aedafcbebfa35ec7; statistics are:\n",
"Scores - mean: 0.861s\tstd: 0.007smin: 0.848s\t max: 0.873s\n",
"--------------------------------------------\n",
"--------------VT_fdf2a86085b508c1325b181c830a4cf7------------------\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_6: 0.854997456073761\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_1: 0.8609604835510254\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_3: 0.8558254837989807\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_0: 0.8728921413421631\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_8: 0.8631933927536011\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_5: 0.8612215518951416\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_7: 0.8661960959434509\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_2: 0.8636621832847595\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_9: 0.8614727258682251\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_4: 0.8657329082489014\n",
"--------------VT_028418f4008a3ef47d12924589eae87f------------------\n",
"--------------VT_0791cc2ee5aa32971fb0414616f198dd------------------\n",
"VT_0791cc2ee5aa32971fb0414616f198dd - version_69: 0.5\n",
"\n",
"\n",
"For VT_fdf2a86085b508c1325b181c830a4cf7; statistics are:\n",
"Scores - mean: 0.863s\tstd: 0.005smin: 0.855s\t max: 0.873s\n",
"For VT_0791cc2ee5aa32971fb0414616f198dd; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_cc64c06847a7ca26f5ea4d465f9cc5bc------------------\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_6: 0.8572231531143188\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_1: 0.8442623615264893\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_3: 0.8498414754867554\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_0: 0.8569087982177734\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_8: 0.8455194234848022\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_5: 0.8435630798339844\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_7: 0.845982551574707\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_2: 0.8571171164512634\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_9: 0.8448543548583984\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_4: 0.845399022102356\n",
"--------------VT_0f47a18659846bffd5557edaa03f43b0------------------\n",
"VT_0f47a18659846bffd5557edaa03f43b0 - version_69: 0.5\n",
"\n",
"\n",
"For VT_cc64c06847a7ca26f5ea4d465f9cc5bc; statistics are:\n",
"Scores - mean: 0.849s\tstd: 0.005smin: 0.844s\t max: 0.857s\n",
"For VT_0f47a18659846bffd5557edaa03f43b0; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_0f705777d0da20ebadfd2a1bca77544e------------------\n",
"VT_0f705777d0da20ebadfd2a1bca77544e - version_69: 0.5\n",
"\n",
"\n",
"For VT_0f705777d0da20ebadfd2a1bca77544e; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_10c1c7d5e95892f7b1f2449277d5401d------------------\n",
"VT_10c1c7d5e95892f7b1f2449277d5401d - version_69: 0.5\n",
"\n",
"\n",
"For VT_10c1c7d5e95892f7b1f2449277d5401d; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_15cbb349b2b50dbb97beec16af2bedab------------------\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_0: 0.8336991667747498\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_1: 0.836580216884613\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_2: 0.8349334001541138\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_3: 0.8312996029853821\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_4: 0.8381868600845337\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_5: 0.8243923187255859\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_6: 0.8407894372940063\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_7: 0.8342592120170593\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_8: 0.8231534957885742\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_9: 0.8382810950279236\n",
"\n",
"\n",
"For VT_15cbb349b2b50dbb97beec16af2bedab; statistics are:\n",
"Scores - mean: 0.834s\tstd: 0.006smin: 0.823s\t max: 0.841s\n",
"--------------------------------------------\n",
"--------------VT_15f1df9a738c8f14f1535b5267bf9c35------------------\n",
"VT_15f1df9a738c8f14f1535b5267bf9c35 - version_7: nan\n",
"\n",
"\n",
"For VT_15f1df9a738c8f14f1535b5267bf9c35; statistics are:\n",
"Scores - mean: nans\tstd: nansmin: nans\t max: nans\n",
"--------------------------------------------\n",
"--------------VT_190a66a3e55a8bc5e14f24429702e6b3------------------\n",
"VT_190a66a3e55a8bc5e14f24429702e6b3 - version_69: 0.5\n",
"\n",
"\n",
"For VT_190a66a3e55a8bc5e14f24429702e6b3; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_1988173e9e2bd82b83868260996df192------------------\n",
"VT_1988173e9e2bd82b83868260996df192 - version_69: 0.5\n",
"\n",
"\n",
"For VT_1988173e9e2bd82b83868260996df192; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_1f3896e6f435f3f0b3b043b90356fcaa------------------\n",
"--------------VT_1f4ef79ca183a18cc66a01d6179cbac5------------------\n",
"VT_1f4ef79ca183a18cc66a01d6179cbac5 - version_0: nan\n",
"\n",
"\n",
"For VT_1f4ef79ca183a18cc66a01d6179cbac5; statistics are:\n",
"Scores - mean: nans\tstd: nansmin: nans\t max: nans\n",
"--------------------------------------------\n",
"--------------VT_235111597a471c2c3cd33786f2a96f98------------------\n",
"--------------VT_23fce6bbb050ed1ca3a84e28bec6540a------------------\n",
"--------------VT_259ee495ee2d2dc0e56bb23d12476f17------------------\n",
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_0: 0.8342075347900391\n",
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_1: 0.8403531908988953\n",
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_2: 0.8468937277793884\n",
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_3: 0.8312729001045227\n",
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_4: 0.8404075503349304\n",
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_5: 0.8485946655273438\n",
"VT_259ee495ee2d2dc0e56bb23d12476f17 - version_6: 0.8351554870605469\n",
"\n",
"\n",
"For VT_259ee495ee2d2dc0e56bb23d12476f17; statistics are:\n",
"Scores - mean: 0.840s\tstd: 0.006smin: 0.831s\t max: 0.849s\n",
"--------------------------------------------\n",
"--------------VT_2c7afd50e127f5a2339db0ddfd6bfd7c------------------\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_6: 0.8630585670471191\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_1: 0.8686699271202087\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_3: 0.8729345798492432\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_0: 0.8636038899421692\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_8: 0.8558077812194824\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_5: 0.8710847496986389\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_7: 0.8619015216827393\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_1: 0.8686699271202087\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_2: 0.8499867916107178\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_9: 0.8507344722747803\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_3: 0.8729345798492432\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_4: 0.8555077314376831\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_5: 0.8710847496986389\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_6: 0.8630585670471191\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_7: 0.8619015216827393\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_8: 0.8558077812194824\n",
"VT_2c7afd50e127f5a2339db0ddfd6bfd7c - version_9: 0.8507344722747803\n",
"\n",
"\n",
"For VT_2c7afd50e127f5a2339db0ddfd6bfd7c; statistics are:\n",
"Scores - mean: 0.861s\tstd: 0.008smin: 0.850s\t max: 0.873s\n",
"--------------------------------------------\n",
"--------------VT_63b9fee765cdda91756af1f35cd320a3------------------\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_6: 0.8663593530654907\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_1: 0.8519773483276367\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_3: 0.8519774675369263\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_0: 0.8603388071060181\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_8: 0.8614517450332642\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_5: 0.8558711409568787\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_7: 0.8537712097167969\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_2: 0.8558205962181091\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_9: 0.8647329211235046\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_4: 0.8546129465103149\n",
"--------------VT_2fb996c7394577d90651f381ad06d9b0------------------\n",
"--------------VT_30c0815ba934bff4458141e33dacb15a------------------\n",
"VT_30c0815ba934bff4458141e33dacb15a - version_0: 0.841953456401825\n",
"VT_30c0815ba934bff4458141e33dacb15a - version_1: 0.8552379608154297\n",
"VT_30c0815ba934bff4458141e33dacb15a - version_2: 0.8526695966720581\n",
"VT_30c0815ba934bff4458141e33dacb15a - version_3: 0.8482565879821777\n",
"VT_30c0815ba934bff4458141e33dacb15a - version_4: 0.8506109118461609\n",
"VT_30c0815ba934bff4458141e33dacb15a - version_5: 0.850794792175293\n",
"VT_30c0815ba934bff4458141e33dacb15a - version_6: 0.8524023294448853\n",
"VT_30c0815ba934bff4458141e33dacb15a - version_7: 0.8411595225334167\n",
"VT_30c0815ba934bff4458141e33dacb15a - version_8: 0.8499799370765686\n",
"VT_30c0815ba934bff4458141e33dacb15a - version_9: 0.8531520366668701\n",
"\n",
"\n",
"For VT_63b9fee765cdda91756af1f35cd320a3; statistics are:\n",
"Scores - mean: 0.858s\tstd: 0.005smin: 0.852s\t max: 0.866s\n",
"For VT_30c0815ba934bff4458141e33dacb15a; statistics are:\n",
"Scores - mean: 0.850s\tstd: 0.004smin: 0.841s\t max: 0.855s\n",
"--------------------------------------------\n",
"--------------VT_aca900a5b9566af61c91aea6525190e6------------------\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_6: 0.8575441241264343\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_1: 0.8453981280326843\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_3: 0.8621359467506409\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_0: 0.8547767400741577\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_8: 0.8613359928131104\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_5: 0.8667657375335693\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_7: 0.8474754095077515\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_2: 0.8628634214401245\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_9: 0.8585749268531799\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_4: 0.8380126357078552\n",
"--------------VT_3481f06b90f6d2dd76c70f3fb304e289------------------\n",
"VT_3481f06b90f6d2dd76c70f3fb304e289 - version_69: 0.7722082138061523\n",
"\n",
"\n",
"For VT_aca900a5b9566af61c91aea6525190e6; statistics are:\n",
"Scores - mean: 0.855s\tstd: 0.009smin: 0.838s\t max: 0.867s\n",
"For VT_3481f06b90f6d2dd76c70f3fb304e289; statistics are:\n",
"Scores - mean: 0.772s\tstd: 0.000smin: 0.772s\t max: 0.772s\n",
"--------------------------------------------\n",
"--------------VT_fb6b96a190455106d29f0630f002ac6f------------------\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_6: 0.8635155558586121\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_1: 0.8261691927909851\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_3: 0.8444902896881104\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_0: 0.865719735622406\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_8: 0.8533784747123718\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_5: 0.8555656671524048\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_7: 0.837948739528656\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_2: 0.8545827865600586\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_9: 0.8541560769081116\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_4: 0.85297691822052\n",
"--------------VT_352b6ac0a2cbda72c0ab4e461f3d531b------------------\n",
"VT_352b6ac0a2cbda72c0ab4e461f3d531b - version_69: 0.5\n",
"\n",
"\n",
"For VT_fb6b96a190455106d29f0630f002ac6f; statistics are:\n",
"Scores - mean: 0.851s\tstd: 0.011smin: 0.826s\t max: 0.866s\n",
"For VT_352b6ac0a2cbda72c0ab4e461f3d531b; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_378971720b930050ad7662bb96699e20------------------\n",
"VT_378971720b930050ad7662bb96699e20 - version_6: 0.8388294577598572\n",
"VT_378971720b930050ad7662bb96699e20 - version_1: 0.8333806395530701\n",
"VT_378971720b930050ad7662bb96699e20 - version_3: 0.847841203212738\n",
"VT_378971720b930050ad7662bb96699e20 - version_0: 0.8287097811698914\n",
"VT_378971720b930050ad7662bb96699e20 - version_8: 0.8436978459358215\n",
"VT_378971720b930050ad7662bb96699e20 - version_5: 0.8392724990844727\n",
"VT_378971720b930050ad7662bb96699e20 - version_7: 0.8410612344741821\n",
"VT_378971720b930050ad7662bb96699e20 - version_1: 0.8333806395530701\n",
"VT_378971720b930050ad7662bb96699e20 - version_2: 0.8407015204429626\n",
"VT_378971720b930050ad7662bb96699e20 - version_9: 0.8334627151489258\n",
"VT_378971720b930050ad7662bb96699e20 - version_3: 0.847841203212738\n",
"VT_378971720b930050ad7662bb96699e20 - version_4: 0.8400266766548157\n",
"VT_378971720b930050ad7662bb96699e20 - version_5: 0.8392724990844727\n",
"VT_378971720b930050ad7662bb96699e20 - version_6: 0.8388294577598572\n",
"VT_378971720b930050ad7662bb96699e20 - version_7: 0.8410612344741821\n",
"VT_378971720b930050ad7662bb96699e20 - version_8: 0.8436978459358215\n",
"VT_378971720b930050ad7662bb96699e20 - version_9: 0.8334627151489258\n",
"\n",
"\n",
"For VT_378971720b930050ad7662bb96699e20; statistics are:\n",
"Scores - mean: 0.839s\tstd: 0.005smin: 0.829s\t max: 0.848s\n",
"--------------------------------------------\n",
"--------------VT_38732f1b943508e5c6767bfd2685787f------------------\n",
"VT_38732f1b943508e5c6767bfd2685787f - version_69: 0.5\n",
"\n",
"\n",
"For VT_38732f1b943508e5c6767bfd2685787f; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_43d4a465ae0448602958abb6b56ca98a------------------\n",
"--------------VT_47b626672d6b4618e6251e11807e4552------------------\n",
"VT_47b626672d6b4618e6251e11807e4552 - version_69: nan\n",
"\n",
"\n",
"For VT_47b626672d6b4618e6251e11807e4552; statistics are:\n",
"Scores - mean: nans\tstd: nansmin: nans\t max: nans\n",
"--------------------------------------------\n",
"--------------VT_4943dd404e08bb9aefd11a1fc739a548------------------\n",
"--------------VT_49df1762a14fec8e3bcc1552368ce3f7------------------\n",
"--------------VT_4a5fb0a95c68bd6f0bc665c7430c296d------------------\n",
"VT_4a5fb0a95c68bd6f0bc665c7430c296d - version_69: 0.8368446826934814\n",
"\n",
"\n",
"For VT_4a5fb0a95c68bd6f0bc665c7430c296d; statistics are:\n",
"Scores - mean: 0.837s\tstd: 0.000smin: 0.837s\t max: 0.837s\n",
"--------------------------------------------\n",
"--------------VT_4a9c7a537898f0ca054f7fbc47d42bc6------------------\n",
"VT_4a9c7a537898f0ca054f7fbc47d42bc6 - version_69: 0.5\n",
"\n",
"\n",
"For VT_4a9c7a537898f0ca054f7fbc47d42bc6; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_4c3ef78ea641b992b877403335ae2a3a------------------\n",
"--------------VT_4cb7ad6614a3c2b9e0baeef4d614a98c------------------\n",
"VT_4cb7ad6614a3c2b9e0baeef4d614a98c - version_69: 0.5\n",
"\n",
"\n",
"For VT_4cb7ad6614a3c2b9e0baeef4d614a98c; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_5275e294f2fa694448691a4c251daa82------------------\n",
"VT_5275e294f2fa694448691a4c251daa82 - version_69: 0.5\n",
"\n",
"\n",
"For VT_5275e294f2fa694448691a4c251daa82; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_535681d3bbafc4d5dc371d82d054c146------------------\n",
"--------------VT_53bcfd70e60d332995ec83194de4763d------------------\n",
"VT_53bcfd70e60d332995ec83194de4763d - version_1: 0.8556768298149109\n",
"VT_53bcfd70e60d332995ec83194de4763d - version_2: 0.8515230417251587\n",
"VT_53bcfd70e60d332995ec83194de4763d - version_3: 0.8562380075454712\n",
"\n",
"\n",
"For VT_53bcfd70e60d332995ec83194de4763d; statistics are:\n",
"Scores - mean: 0.854s\tstd: 0.002smin: 0.852s\t max: 0.856s\n",
"--------------------------------------------\n",
"--------------VT_58c410c607ebfbd9ee22030be05da540------------------\n",
"VT_58c410c607ebfbd9ee22030be05da540 - version_69: 0.5\n",
"\n",
"\n",
"For VT_58c410c607ebfbd9ee22030be05da540; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_58dce7138334203ef2703c7a1749f893------------------\n",
"VT_58dce7138334203ef2703c7a1749f893 - version_1: 0.5\n",
"\n",
"\n",
"For VT_58dce7138334203ef2703c7a1749f893; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_5b1cb6144b2f79e9e53804b4e0ee4f22------------------\n",
"--------------VT_63b9fee765cdda91756af1f35cd320a3------------------\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_0: 0.8603388071060181\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_1: 0.8519773483276367\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_2: 0.8558205962181091\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_3: 0.8519774675369263\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_4: 0.8546129465103149\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_5: 0.8558711409568787\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_6: 0.8663593530654907\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_7: 0.8537712097167969\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_8: 0.8614517450332642\n",
"VT_63b9fee765cdda91756af1f35cd320a3 - version_9: 0.8647329211235046\n",
"\n",
"\n",
"For VT_63b9fee765cdda91756af1f35cd320a3; statistics are:\n",
"Scores - mean: 0.858s\tstd: 0.005smin: 0.852s\t max: 0.866s\n",
"--------------------------------------------\n",
"--------------VT_67c06cbaf03ac8624a9aa24afe928e98------------------\n",
"VT_67c06cbaf03ac8624a9aa24afe928e98 - version_69: 0.7984834909439087\n",
"\n",
"\n",
"For VT_67c06cbaf03ac8624a9aa24afe928e98; statistics are:\n",
"Scores - mean: 0.798s\tstd: 0.000smin: 0.798s\t max: 0.798s\n",
"--------------------------------------------\n",
"--------------VT_67e6558f4d240f8100b6993970941b0a------------------\n",
"--------------VT_6b4e9061e68a8f697e2a4755471a7ffd------------------\n",
"--------------VT_6eaec2911947abd6ba0a671f3de4f5bf------------------\n",
"VT_6eaec2911947abd6ba0a671f3de4f5bf - version_69: 0.5\n",
"\n",
"\n",
"For VT_6eaec2911947abd6ba0a671f3de4f5bf; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_72807e86b387f4cd327afcb93f945f5e------------------\n",
"VT_72807e86b387f4cd327afcb93f945f5e - version_69: 0.8392706513404846\n",
"\n",
"\n",
"For VT_72807e86b387f4cd327afcb93f945f5e; statistics are:\n",
"Scores - mean: 0.839s\tstd: 0.000smin: 0.839s\t max: 0.839s\n",
"--------------------------------------------\n",
"--------------VT_7899c07a4809a45c57cba58047cefb5e------------------\n",
"VT_7899c07a4809a45c57cba58047cefb5e - version_0: 0.8663597106933594\n",
"VT_7899c07a4809a45c57cba58047cefb5e - version_1: 0.8652830123901367\n",
"VT_7899c07a4809a45c57cba58047cefb5e - version_2: 0.8739997744560242\n",
"VT_7899c07a4809a45c57cba58047cefb5e - version_3: 0.854115903377533\n",
"VT_7899c07a4809a45c57cba58047cefb5e - version_4: 0.8697185516357422\n",
"VT_7899c07a4809a45c57cba58047cefb5e - version_5: 0.8741324543952942\n",
"VT_7899c07a4809a45c57cba58047cefb5e - version_6: 0.8711682558059692\n",
"VT_7899c07a4809a45c57cba58047cefb5e - version_7: 0.8780345916748047\n",
"VT_7899c07a4809a45c57cba58047cefb5e - version_8: 0.8690432906150818\n",
"VT_7899c07a4809a45c57cba58047cefb5e - version_9: 0.8685160875320435\n",
"\n",
"\n",
"For VT_7899c07a4809a45c57cba58047cefb5e; statistics are:\n",
"Scores - mean: 0.869s\tstd: 0.006smin: 0.854s\t max: 0.878s\n",
"--------------------------------------------\n",
"--------------VT_81fb4e76a4cf8574c6f20badb0fbe3b2------------------\n",
"VT_81fb4e76a4cf8574c6f20badb0fbe3b2 - version_69: nan\n",
"\n",
"\n",
"For VT_81fb4e76a4cf8574c6f20badb0fbe3b2; statistics are:\n",
"Scores - mean: nans\tstd: nansmin: nans\t max: nans\n",
"--------------------------------------------\n",
"--------------VT_8434217cbc9c36788536ce5888fe7037------------------\n",
"VT_8434217cbc9c36788536ce5888fe7037 - version_3: 0.8607174754142761\n",
"\n",
"\n",
"For VT_8434217cbc9c36788536ce5888fe7037; statistics are:\n",
"Scores - mean: 0.861s\tstd: 0.000smin: 0.861s\t max: 0.861s\n",
"--------------------------------------------\n",
"--------------VT_854b9e312d2c3a8aab7de8953b84e9ea------------------\n",
"VT_854b9e312d2c3a8aab7de8953b84e9ea - version_69: nan\n",
"\n",
"\n",
"For VT_854b9e312d2c3a8aab7de8953b84e9ea; statistics are:\n",
"Scores - mean: nans\tstd: nansmin: nans\t max: nans\n",
"--------------------------------------------\n",
"--------------VT_8a19b1652c8c561da9ffca44db7043f3------------------\n",
"--------------VT_8d4128b953ebf41b1d7072fd214a5b53------------------\n",
"VT_8d4128b953ebf41b1d7072fd214a5b53 - version_0: 0.8607174754142761\n",
"VT_8d4128b953ebf41b1d7072fd214a5b53 - version_1: 0.8607174754142761\n",
"VT_8d4128b953ebf41b1d7072fd214a5b53 - version_2: 0.8607174754142761\n",
"\n",
"\n",
"For VT_8d4128b953ebf41b1d7072fd214a5b53; statistics are:\n",
"Scores - mean: 0.861s\tstd: 0.000smin: 0.861s\t max: 0.861s\n",
"--------------------------------------------\n",
"--------------VT_8d522affb43dec302858d19b9ca807d2------------------\n",
"VT_8d522affb43dec302858d19b9ca807d2 - version_1: 0.5\n",
"\n",
"\n",
"For VT_8d522affb43dec302858d19b9ca807d2; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_8d6fe086678573b737243bcbc9e812fd------------------\n",
"--------------VT_965daab4ee04e9d174e694504572e5f8------------------\n",
"VT_965daab4ee04e9d174e694504572e5f8 - version_69: nan\n",
"\n",
"\n",
"For VT_965daab4ee04e9d174e694504572e5f8; statistics are:\n",
"Scores - mean: nans\tstd: nansmin: nans\t max: nans\n",
"--------------------------------------------\n",
"--------------VT_99d788004e558bc20a46a5915ec33d9a------------------\n",
"VT_99d788004e558bc20a46a5915ec33d9a - version_1: 0.5\n",
"\n",
"\n",
"For VT_99d788004e558bc20a46a5915ec33d9a; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_9e1a8791100cc7e14b96c397e8bc7113------------------\n",
"--------------VT_9ee8f70a5104ca683c765cfeeb9eba36------------------\n",
"VT_9ee8f70a5104ca683c765cfeeb9eba36 - version_3: 0.8427470922470093\n",
"VT_9ee8f70a5104ca683c765cfeeb9eba36 - version_4: 0.8427470922470093\n",
"VT_9ee8f70a5104ca683c765cfeeb9eba36 - version_5: 0.8427470922470093\n",
"VT_9ee8f70a5104ca683c765cfeeb9eba36 - version_6: 0.8427470922470093\n",
"VT_9ee8f70a5104ca683c765cfeeb9eba36 - version_7: 0.8427470922470093\n",
"VT_9ee8f70a5104ca683c765cfeeb9eba36 - version_8: 0.8427470922470093\n",
"VT_9ee8f70a5104ca683c765cfeeb9eba36 - version_9: 0.8427470922470093\n",
"\n",
"\n",
"For VT_9ee8f70a5104ca683c765cfeeb9eba36; statistics are:\n",
"Scores - mean: 0.843s\tstd: 0.000smin: 0.843s\t max: 0.843s\n",
"--------------------------------------------\n",
"--------------VT_a26f40b26cb96b4c6cf78b913435dfa9------------------\n",
"VT_a26f40b26cb96b4c6cf78b913435dfa9 - version_69: nan\n",
"\n",
"\n",
"For VT_a26f40b26cb96b4c6cf78b913435dfa9; statistics are:\n",
"Scores - mean: nans\tstd: nansmin: nans\t max: nans\n",
"--------------------------------------------\n",
"--------------VT_a37714ee4c37cf225ff7c56f1250c27c------------------\n",
"VT_a37714ee4c37cf225ff7c56f1250c27c - version_1: 0.5\n",
"\n",
"\n",
"For VT_a37714ee4c37cf225ff7c56f1250c27c; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_a3e77e5cd59d847a711f3703f9225a64------------------\n",
"VT_a3e77e5cd59d847a711f3703f9225a64 - version_69: nan\n",
"\n",
"\n",
"For VT_a3e77e5cd59d847a711f3703f9225a64; statistics are:\n",
"Scores - mean: nans\tstd: nansmin: nans\t max: nans\n",
"--------------------------------------------\n",
"--------------VT_a5ef9966e457973286d73568f161f293------------------\n",
"VT_a5ef9966e457973286d73568f161f293 - version_0: 0.8427470922470093\n",
"VT_a5ef9966e457973286d73568f161f293 - version_1: 0.8427470922470093\n",
"VT_a5ef9966e457973286d73568f161f293 - version_2: 0.8427470922470093\n",
"\n",
"\n",
"For VT_a5ef9966e457973286d73568f161f293; statistics are:\n",
"Scores - mean: 0.843s\tstd: 0.000smin: 0.843s\t max: 0.843s\n",
"--------------------------------------------\n",
"--------------VT_a69b0267162831d5989d11b10f3f0795------------------\n",
"--------------VT_aca900a5b9566af61c91aea6525190e6------------------\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_0: 0.8547767400741577\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_1: 0.8453981280326843\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_2: 0.8628634214401245\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_3: 0.8621359467506409\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_4: 0.8380126357078552\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_5: 0.8667657375335693\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_6: 0.8575441241264343\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_7: 0.8474754095077515\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_8: 0.8613359928131104\n",
"VT_aca900a5b9566af61c91aea6525190e6 - version_9: 0.8585749268531799\n",
"\n",
"\n",
"For VT_aca900a5b9566af61c91aea6525190e6; statistics are:\n",
"Scores - mean: 0.855s\tstd: 0.009smin: 0.838s\t max: 0.867s\n",
"--------------------------------------------\n",
"--------------VT_acd670939b4ffff77dc826dc03dbc9d0------------------\n",
"--------------VT_ada2eb434097122a4177871d47c1f818------------------\n",
"VT_ada2eb434097122a4177871d47c1f818 - version_69: 0.5\n",
"\n",
"\n",
"For VT_ada2eb434097122a4177871d47c1f818; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_b06a84d2430e20f39469a1c1d342f0be------------------\n",
"VT_b06a84d2430e20f39469a1c1d342f0be - version_69: 0.5\n",
"\n",
"\n",
"For VT_b06a84d2430e20f39469a1c1d342f0be; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_b22165b70e7fc3b24a5e7734e9fb5531------------------\n",
"VT_b22165b70e7fc3b24a5e7734e9fb5531 - version_69: 0.5\n",
"\n",
"\n",
"For VT_b22165b70e7fc3b24a5e7734e9fb5531; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_b545432f93932d26236992e6d159a3a0------------------\n",
"--------------VT_b63db9082a8274248a76465ecd2f8bf0------------------\n",
"VT_b63db9082a8274248a76465ecd2f8bf0 - version_69: 0.8024263381958008\n",
"\n",
"\n",
"For VT_b63db9082a8274248a76465ecd2f8bf0; statistics are:\n",
"Scores - mean: 0.802s\tstd: 0.000smin: 0.802s\t max: 0.802s\n",
"--------------------------------------------\n",
"--------------VT_b948cf3132a0750de99a555f30478885------------------\n",
"VT_b948cf3132a0750de99a555f30478885 - version_1: 0.8622770309448242\n",
"VT_b948cf3132a0750de99a555f30478885 - version_2: 0.8648049235343933\n",
"VT_b948cf3132a0750de99a555f30478885 - version_3: 0.8514904379844666\n",
"VT_b948cf3132a0750de99a555f30478885 - version_4: nan\n",
"\n",
"\n",
"For VT_b948cf3132a0750de99a555f30478885; statistics are:\n",
"Scores - mean: nans\tstd: nansmin: nans\t max: nans\n",
"--------------------------------------------\n",
"--------------VT_bb3e61fe791c2013a5a8668cb3d79763------------------\n",
"--------------VT_bb817858c2b2a53f0b85a944d73e6b2f------------------\n",
"VT_bb817858c2b2a53f0b85a944d73e6b2f - version_69: nan\n",
"\n",
"\n",
"For VT_bb817858c2b2a53f0b85a944d73e6b2f; statistics are:\n",
"Scores - mean: nans\tstd: nansmin: nans\t max: nans\n",
"--------------------------------------------\n",
"--------------VT_c66d2e6b7b7bf8ff8c1fcc0dc22994ae------------------\n",
"--------------VT_cc64c06847a7ca26f5ea4d465f9cc5bc------------------\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_0: 0.8569087982177734\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_1: 0.8442623615264893\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_2: 0.8571171164512634\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_3: 0.8498414754867554\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_4: 0.845399022102356\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_5: 0.8435630798339844\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_6: 0.8572231531143188\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_7: 0.845982551574707\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_8: 0.8455194234848022\n",
"VT_cc64c06847a7ca26f5ea4d465f9cc5bc - version_9: 0.8448543548583984\n",
"\n",
"\n",
"For VT_cc64c06847a7ca26f5ea4d465f9cc5bc; statistics are:\n",
"Scores - mean: 0.849s\tstd: 0.005smin: 0.844s\t max: 0.857s\n",
"--------------------------------------------\n",
"--------------VT_d3512eec26876ed3950d6883044ccbd8------------------\n",
"--------------VT_d4431d4990af1c076fc67b2e7fa15475------------------\n",
"VT_d4431d4990af1c076fc67b2e7fa15475 - version_69: 0.5\n",
"\n",
"\n",
"For VT_d4431d4990af1c076fc67b2e7fa15475; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_d4b93b2170ca4d4b41a304ee582b41df------------------\n",
"--------------VT_d50f8d5f2272084ee61273d028fbd2f9------------------\n",
"VT_d50f8d5f2272084ee61273d028fbd2f9 - version_0: nan\n",
"\n",
"\n",
"For VT_d50f8d5f2272084ee61273d028fbd2f9; statistics are:\n",
"Scores - mean: nans\tstd: nansmin: nans\t max: nans\n",
"--------------------------------------------\n",
"--------------VT_d55f1492ff29a3cd1026013948ce7fa7------------------\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_6: 0.8385945558547974\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_1: 0.8324360251426697\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_3: 0.8386826515197754\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_0: 0.8366813063621521\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_8: 0.8460721969604492\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_5: 0.8374781608581543\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_7: 0.8320286273956299\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_1: 0.8324360251426697\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_2: 0.8370164632797241\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_9: 0.8495808839797974\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_3: 0.8386826515197754\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_4: 0.8332125544548035\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_5: 0.8374781608581543\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_6: 0.8385945558547974\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_7: 0.8320286273956299\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_8: 0.8460721969604492\n",
"VT_d55f1492ff29a3cd1026013948ce7fa7 - version_9: 0.8495808839797974\n",
"\n",
"\n",
"For VT_d55f1492ff29a3cd1026013948ce7fa7; statistics are:\n",
"Scores - mean: 0.838s\tstd: 0.005smin: 0.832s\t max: 0.850s\n",
"--------------------------------------------\n",
"--------------VT_15cbb349b2b50dbb97beec16af2bedab------------------\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_6: 0.8407894372940063\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_1: 0.836580216884613\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_3: 0.8312996029853821\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_0: 0.8336991667747498\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_8: 0.8231534957885742\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_5: 0.8243923187255859\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_7: 0.8342592120170593\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_2: 0.8349334001541138\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_9: 0.8382810950279236\n",
"VT_15cbb349b2b50dbb97beec16af2bedab - version_4: 0.8381868600845337\n",
"--------------VT_d6c088aa12591278d31c02f0c05763e2------------------\n",
"VT_d6c088aa12591278d31c02f0c05763e2 - version_1: 0.5\n",
"\n",
"\n",
"For VT_15cbb349b2b50dbb97beec16af2bedab; statistics are:\n",
"Scores - mean: 0.834s\tstd: 0.006smin: 0.823s\t max: 0.841s\n",
"For VT_d6c088aa12591278d31c02f0c05763e2; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_d8637b6c86a31b305c2ea3131834299e------------------\n",
"VT_d8637b6c86a31b305c2ea3131834299e - version_69: 0.5\n",
"\n",
"\n",
"For VT_d8637b6c86a31b305c2ea3131834299e; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_d9e748d535a69eb7592f9fcb697d29f4------------------\n",
"VT_d9e748d535a69eb7592f9fcb697d29f4 - version_69: 0.8140299916267395\n",
"\n",
"\n",
"For VT_d9e748d535a69eb7592f9fcb697d29f4; statistics are:\n",
"Scores - mean: 0.814s\tstd: 0.000smin: 0.814s\t max: 0.814s\n",
"--------------------------------------------\n",
"--------------VT_dc0b4819c1e7642ee819651bfb0a2e63------------------\n",
"--------------VT_ddcd3cf319138ec13f4b0b77b9ab1d92------------------\n",
"VT_ddcd3cf319138ec13f4b0b77b9ab1d92 - version_69: 0.5\n",
"\n",
"\n",
"For VT_ddcd3cf319138ec13f4b0b77b9ab1d92; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_e1ab4fa5a5d0649f411b34d2c45731ae------------------\n",
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_0: 0.8507610559463501\n",
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_1: 0.8411756753921509\n",
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_2: 0.8576055765151978\n",
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_3: 0.8609339594841003\n",
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_4: 0.8473474383354187\n",
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_5: 0.8473350405693054\n",
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_6: 0.8621013760566711\n",
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_7: 0.8595266342163086\n",
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_8: 0.8662147521972656\n",
"VT_e1ab4fa5a5d0649f411b34d2c45731ae - version_9: 0.8629968762397766\n",
"\n",
"\n",
"For VT_e1ab4fa5a5d0649f411b34d2c45731ae; statistics are:\n",
"Scores - mean: 0.856s\tstd: 0.008smin: 0.841s\t max: 0.866s\n",
"--------------------------------------------\n",
"--------------VT_edd2a2ba78735db9d31839071cd1b05e------------------\n",
"VT_edd2a2ba78735db9d31839071cd1b05e - version_69: 0.5\n",
"\n",
"\n",
"For VT_edd2a2ba78735db9d31839071cd1b05e; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_f32f257e1bfe3885c781a17da6ff29bd------------------\n",
"--------------VT_f53f335a7ec043211a1290df42fb723d------------------\n",
"VT_f53f335a7ec043211a1290df42fb723d - version_69: nan\n",
"\n",
"\n",
"For VT_f53f335a7ec043211a1290df42fb723d; statistics are:\n",
"Scores - mean: nans\tstd: nansmin: nans\t max: nans\n",
"--------------------------------------------\n",
"--------------VT_f56c6a9953c5a42b8d11761918762168------------------\n",
"VT_f56c6a9953c5a42b8d11761918762168 - version_1: 0.5\n",
"\n",
"\n",
"For VT_f56c6a9953c5a42b8d11761918762168; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_f9001ab9419a3aa658713ac38cf59ad8------------------\n",
"--------------VT_fae88c1b18c4c4b1708fa9f3bf243230------------------\n",
"VT_fae88c1b18c4c4b1708fa9f3bf243230 - version_69: 0.5\n",
"\n",
"\n",
"For VT_fae88c1b18c4c4b1708fa9f3bf243230; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_fb6b96a190455106d29f0630f002ac6f------------------\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_0: 0.865719735622406\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_1: 0.8261691927909851\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_2: 0.8545827865600586\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_3: 0.8444902896881104\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_4: 0.85297691822052\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_5: 0.8555656671524048\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_6: 0.8635155558586121\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_7: 0.837948739528656\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_8: 0.8533784747123718\n",
"VT_fb6b96a190455106d29f0630f002ac6f - version_9: 0.8541560769081116\n",
"\n",
"\n",
"For VT_fb6b96a190455106d29f0630f002ac6f; statistics are:\n",
"Scores - mean: 0.851s\tstd: 0.011smin: 0.826s\t max: 0.866s\n",
"--------------------------------------------\n",
"--------------VT_fdf2a86085b508c1325b181c830a4cf7------------------\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_0: 0.8728921413421631\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_1: 0.8609604835510254\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_2: 0.8636621832847595\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_3: 0.8558254837989807\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_4: 0.8657329082489014\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_5: 0.8612215518951416\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_6: 0.854997456073761\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_7: 0.8661960959434509\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_8: 0.8631933927536011\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_9: 0.8614727258682251\n",
"\n",
"\n",
"For VT_fdf2a86085b508c1325b181c830a4cf7; statistics are:\n",
"Scores - mean: 0.863s\tstd: 0.005smin: 0.855s\t max: 0.873s\n",
"--------------------------------------------\n",
"--------------VT_ff3a0c864586958f3087f6e207e4fa7f------------------\n",
"VT_ff3a0c864586958f3087f6e207e4fa7f - version_69: 0.5\n",
"\n",
"\n",
"For VT_ff3a0c864586958f3087f6e207e4fa7f; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n",
"--------------VT_ffa54e3ac1508cfd9ac50368c852630d------------------\n",
"VT_ffa54e3ac1508cfd9ac50368c852630d - version_69: 0.5\n",
"\n",
"\n",
"For VT_ffa54e3ac1508cfd9ac50368c852630d; statistics are:\n",
"Scores - mean: 0.500s\tstd: 0.000smin: 0.500s\t max: 0.500s\n",
"--------------------------------------------\n"
]
}
@ -297,25 +762,27 @@
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": 29,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"--------------VT_fdf2a86085b508c1325b181c830a4cf7------------------\n",
"--------------VT_fdf2a86085b508c1325b181c830a4cf7------------------\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_6: 0.854997456073761\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_1: 0.8609604835510254\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_3: 0.8558254837989807\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_0: 0.8728921413421631\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_8: 0.8631933927536011\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_5: 0.8612215518951416\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_7: 0.8661960959434509\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_1: 0.8609604835510254\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_2: 0.8636621832847595\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_9: 0.8614727258682251\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_3: 0.8558254837989807\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_4: 0.8657329082489014\n",
"--------------------------------------------\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_5: 0.8612215518951416\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_6: 0.854997456073761\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_7: 0.8661960959434509\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_8: 0.8631933927536011\n",
"VT_fdf2a86085b508c1325b181c830a4cf7 - version_9: 0.8614727258682251\n",
"\n",
"\n",
"For VT_fdf2a86085b508c1325b181c830a4cf7; statistics are:\n",
"Scores - mean: 0.863s\tstd: 0.005smin: 0.855s\t max: 0.873s\n",
"--------------------------------------------\n"
]
}
@ -338,7 +805,7 @@
},
{
"cell_type": "code",
"execution_count": 39,
"execution_count": 30,
"outputs": [
{
"name": "stdout",

192
notebooks/plot_mels.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

0
reload model.ipynb → notebooks/reload model.ipynb
View File

71
optuna_tune.py
View File

@ -7,50 +7,63 @@ from optuna.integration import PyTorchLightningPruningCallback
from main import run_lightning_loop
from ml_lib.utils.config import parse_comandline_args_add_defaults
import neptunecontrib.monitoring.optuna as opt_utils
def optimize(trial: optuna.Trial):
# Optuna configuration
folder = Path('study')
folder.mkdir(parents=False, exist_ok=True)
scheduler = trial.suggest_categorical('scheduler', [None, 'LambdaLR'])
if scheduler is not None:
lr_scheduler_parameter = trial.suggest_float('lr_scheduler_parameter', 0.8, 1, step=0.01)
else:
lr_scheduler_parameter = None
optuna_suggestions = dict(
model_name='VisualTransformer',
batch_size=trial.suggest_int('batch_size', 30, 100, step=32),
lr_scheduler_parameter=trial.suggest_float('lr_scheduler_parameter', 0.8, 1, step=0.01),
max_epochs=100,
random_apply_chance=0.1, # trial.suggest_float('random_apply_chance', 0.1, 0.5, step=0.1),
loudness_ratio=0.1, # trial.suggest_float('loudness_ratio', 0.0, 0.5, step=0.1),
shift_ratio=0.1, # trial.suggest_float('shift_ratio', 0.0, 0.5, step=0.1),
noise_ratio=0, # trial.suggest_float('noise_ratio', 0.0, 0.5, step=0.1),
mask_ratio=0.2, # trial.suggest_float('mask_ratio', 0.0, 0.5, step=0.1),
lr=trial.suggest_uniform('lr', 1e-3, 3e-3),
dropout=0.05, # trial.suggest_float('dropout', 0.0, 0.3, step=0.05),
lat_dim=32, # 2 ** trial.suggest_int('lat_dim', 1, 5, step=1),
mlp_dim=16, # 2 ** trial.suggest_int('mlp_dim', 1, 5, step=1),
head_dim=8, # 2 ** trial.suggest_int('head_dim', 1, 5, step=1),
patch_size=12, # trial.suggest_int('patch_size', 6, 12, step=3),
attn_depth=10, # trial.suggest_int('attn_depth', 2, 14, step=4),
heads=16, # trial.suggest_int('heads', 2, 16, step=2),
scheduler='LambdaLR', # trial.suggest_categorical('scheduler', [None, 'LambdaLR']),
embedding_size=48, # trial.suggest_int('embedding_size', 12, 64, step=12),
model_name='CNNBaseline',
data_name='MaskLibrosaDatamodule',
batch_size=trial.suggest_int('batch_size', 5, 50, step=5),
max_epochs=75,
target_mel_length_in_seconds=trial.suggest_float('target_mel_length_in_seconds', 0.2, 1.5, step=0.1),
random_apply_chance=trial.suggest_float('random_apply_chance', 0.1, 0.5, step=0.1),
loudness_ratio=trial.suggest_float('loudness_ratio', 0.0, 0.5, step=0.1),
shift_ratio=trial.suggest_float('shift_ratio', 0.0, 0.5, step=0.1),
noise_ratio=trial.suggest_float('noise_ratio', 0.0, 0.5, step=0.1),
mask_ratio=trial.suggest_float('mask_ratio', 0.0, 0.5, step=0.1),
lr=trial.suggest_loguniform('lr', 1e-5, 1e-3),
dropout=trial.suggest_float('dropout', 0.0, 0.3, step=0.05),
lat_dim=2 ** trial.suggest_int('lat_dim', 1, 5, step=1),
scheduler=scheduler,
lr_scheduler_parameter=lr_scheduler_parameter,
loss='ce_loss',
sampler='WeightedRandomSampler', # rial.suggest_categorical('sampler', [None, 'WeightedRandomSampler']),
weight_decay=trial.suggest_loguniform('weight_decay', 1e-20, 1e-1),
sampler=trial.suggest_categorical('sampler', [None, 'WeightedRandomSampler']),
study_name=trial.study.study_name
)
if optuna_suggestions['model_name'] == 'CNNBaseline':
model_depth = trial.suggest_int('model_depth', 1, 6, step=1)
filters = list()
for layer_idx in range(model_depth):
filters.append(2 ** trial.suggest_int(f'filters_{layer_idx}', 2, 6, step=1))
optuna_suggestions.update(filters=filters)
elif optuna_suggestions['model_name'] == 'VisualTransformer':
transformer_dict = dict(
mlp_dim=2 ** trial.suggest_int('mlp_dim', 1, 5, step=1),
head_dim=2 ** trial.suggest_int('head_dim', 1, 5, step=1),
patch_size=trial.suggest_int('patch_size', 6, 12, step=3),
attn_depth=trial.suggest_int('attn_depth', 2, 14, step=4),
heads=trial.suggest_int('heads', 2, 16, step=2),
embedding_size=trial.suggest_int('embedding_size', 12, 64, step=12)
)
optuna_suggestions.update(**transformer_dict)
pruning_callback = PyTorchLightningPruningCallback(trial, monitor="PL_recall_score")
# Parse comandline args, read config and get model
cmd_args, found_data_class, found_model_class = parse_comandline_args_add_defaults('_parameters.ini')
h_params = dict(**cmd_args)
h_params.update(optuna_suggestions)
h_params, found_data_class, found_model_class, seed = parse_comandline_args_add_defaults(
'_parameters.ini', overrides=optuna_suggestions)
h_params = Namespace(**h_params)
try:
best_score = run_lightning_loop(h_params, data_class=found_data_class, model_class=found_model_class,
additional_callbacks=pruning_callback)
additional_callbacks=pruning_callback, seed=seed)
except Exception as e:
print(e)
best_score = 0
@ -60,7 +73,7 @@ def optimize(trial: optuna.Trial):
if __name__ == '__main__':
study = optuna.create_study(direction='maximize', sampler=optuna.samplers.TPESampler(seed=1337))
# study.optimize(optimize, n_trials=50, callbacks=[opt_utils.NeptuneCallback(log_study=True, log_charts=True)])
study.optimize(optimize, n_trials=50)
study.optimize(optimize, n_trials=100)
print("Number of finished trials: {}".format(len(study.trials)))

11
rebuild_dataset.py
View File

@ -2,6 +2,9 @@ from argparse import Namespace
import warnings
from datasets.ccs_librosa_datamodule import CCSLibrosaDatamodule
from datasets.primates_librosa_datamodule import PrimatesLibrosaDatamodule
from datasets.mask_librosa_datamodule import MaskLibrosaDatamodule
from ml_lib.utils.config import parse_comandline_args_add_defaults
warnings.filterwarnings('ignore', category=FutureWarning)
@ -15,20 +18,20 @@ def rebuild_dataset(h_params, data_class):
# Let Datamodule pull what it wants
datamodule = data_class.from_argparse_args(h_params)
assert datamodule.purge()
datasets = datamodule.prepare_data()
datasets = datamodule.setup()
datasets = datamodule.manual_setup()
print(f'Dataset length is: {len(datasets)}')
if __name__ == '__main__':
for dataset in [MaskLibrosaDatamodule]: # [PrimatesLibrosaDatamodule, CCSLibrosaDatamodule]:
# Parse comandline args, read config and get model
cmd_args, found_data_class, _, _ = parse_comandline_args_add_defaults('_parameters.ini')
cmd_args, _, _, _ = parse_comandline_args_add_defaults('_parameters.ini')
# To NameSpace
hparams = Namespace(**cmd_args)
# Start
# -----------------
rebuild_dataset(hparams, found_data_class)
rebuild_dataset(hparams, dataset)
print('done')
pass

27
util/module_mixins.py
View File

@ -56,17 +56,16 @@ class ValMixin:
for file_name in sorted_y:
sorted_y.update({file_name: torch.stack(sorted_y[file_name])})
target_y = torch.stack(tuple(sorted_batch_y.values())).long()
if self.params.n_classes <= 2:
mean_sorted_y = torch.stack([x.mean(dim=0) if x.shape[0] > 1 else x for x in sorted_y.values()])
self.metrics.update(mean_sorted_y, target_y)
else:
y_max = torch.stack(
[torch.argmax(x.mean(dim=0)) if x.shape[0] > 1 else torch.argmax(x) for x in sorted_y.values()]
).squeeze()
y_one_hot = torch.nn.functional.one_hot(y_max, num_classes=self.params.n_classes).float()
target_y = torch.stack(tuple(sorted_batch_y.values())).long()
if self.params.n_classes <= 2:
if y_one_hot.ndim == 1:
y_one_hot = y_one_hot.unsqueeze(0)
if target_y.ndim == 1:
target_y = target_y.unsqueeze(-1)
self.metrics.update(y_one_hot, target_y)
if self.params.n_classes <= 2:
val_loss = self.bce_loss(y.squeeze().float(), batch_y.float())
@ -109,14 +108,15 @@ class ValMixin:
#mean_vote_loss = self.ce_loss(y_mean, sorted_batch_y)
#summary_dict.update(val_mean_vote_loss=mean_vote_loss)
if self.params.n_classes <= 2:
mean_sorted_y = torch.stack([x.mean(dim=0) if x.shape[0] > 1 else x for x in sorted_y.values()])
max_vote_loss = self.bce_loss(mean_sorted_y.float(), sorted_batch_y.float())
else:
y_max = torch.stack(
[torch.argmax(x.mean(dim=0)) if x.shape[0] > 1 else torch.argmax(x) for x in sorted_y.values()]
).squeeze()
y_one_hot = torch.nn.functional.one_hot(y_max, num_classes=self.params.n_classes).float()
if self.params.n_classes >= 2:
max_vote_loss = self.ce_loss(y_one_hot, sorted_batch_y)
else:
max_vote_loss = self.bce_loss(y_one_hot, sorted_batch_y)
summary_dict.update(val_max_vote_loss=max_vote_loss)
summary_dict.update({f'mean_{key}': torch.mean(torch.stack([output[key]
@ -124,6 +124,9 @@ class ValMixin:
for key in keys if 'loss' in key}
)
# Sklearn Scores
if self.params.n_classes <= 2:
additional_scores = self.additional_scores(dict(y=y_max, batch_y=sorted_batch_y))
else:
additional_scores = self.additional_scores(dict(y=y_one_hot, batch_y=sorted_batch_y))
summary_dict.update(**additional_scores)
@ -132,7 +135,9 @@ class ValMixin:
summary_dict.update(**pl_metrics)
summary_dict.update(epoch=self.current_epoch)
self.log_dict(summary_dict, on_epoch=True)
self.log_dict(summary_dict)
# For Debugging:
# print(f'Summary Metrics are: {summary_dict}')
for name, image in pl_images.items():
self.logger.log_image(name, image, step=self.global_step)