93 lines
4.0 KiB
Python
93 lines
4.0 KiB
Python
import pickle
|
|
from argparse import Namespace
|
|
from pathlib import Path
|
|
|
|
import optuna as optuna
|
|
from optuna.integration import PyTorchLightningPruningCallback
|
|
|
|
from main import run_lightning_loop
|
|
from ml_lib.utils.config import parse_comandline_args_add_defaults
|
|
|
|
def optimize(trial: optuna.Trial):
|
|
# Optuna configuration
|
|
folder = Path('study')
|
|
folder.mkdir(parents=False, exist_ok=True)
|
|
scheduler = trial.suggest_categorical('scheduler', [None, 'LambdaLR'])
|
|
if scheduler is not None:
|
|
lr_scheduler_parameter = trial.suggest_float('lr_scheduler_parameter', 0.8, 1, step=0.01)
|
|
else:
|
|
lr_scheduler_parameter = None
|
|
|
|
optuna_suggestions = dict(
|
|
model_name='CNNBaseline',
|
|
data_name='MaskLibrosaDatamodule',
|
|
batch_size=trial.suggest_int('batch_size', 5, 50, step=5),
|
|
max_epochs=75,
|
|
target_mel_length_in_seconds=trial.suggest_float('target_mel_length_in_seconds', 0.2, 1.5, step=0.1),
|
|
random_apply_chance=trial.suggest_float('random_apply_chance', 0.1, 0.5, step=0.1),
|
|
loudness_ratio=trial.suggest_float('loudness_ratio', 0.0, 0.5, step=0.1),
|
|
shift_ratio=trial.suggest_float('shift_ratio', 0.0, 0.5, step=0.1),
|
|
noise_ratio=trial.suggest_float('noise_ratio', 0.0, 0.5, step=0.1),
|
|
mask_ratio=trial.suggest_float('mask_ratio', 0.0, 0.5, step=0.1),
|
|
lr=trial.suggest_loguniform('lr', 1e-5, 1e-3),
|
|
dropout=trial.suggest_float('dropout', 0.0, 0.3, step=0.05),
|
|
lat_dim=2 ** trial.suggest_int('lat_dim', 1, 5, step=1),
|
|
scheduler=scheduler,
|
|
lr_scheduler_parameter=lr_scheduler_parameter,
|
|
loss='ce_loss',
|
|
sampler=trial.suggest_categorical('sampler', [None, 'WeightedRandomSampler']),
|
|
study_name=trial.study.study_name
|
|
)
|
|
if optuna_suggestions['model_name'] == 'CNNBaseline':
|
|
model_depth = trial.suggest_int('model_depth', 1, 6, step=1)
|
|
filters = list()
|
|
for layer_idx in range(model_depth):
|
|
filters.append(2 ** trial.suggest_int(f'filters_{layer_idx}', 2, 6, step=1))
|
|
optuna_suggestions.update(filters=filters)
|
|
elif optuna_suggestions['model_name'] == 'VisualTransformer':
|
|
transformer_dict = dict(
|
|
mlp_dim=2 ** trial.suggest_int('mlp_dim', 1, 5, step=1),
|
|
head_dim=2 ** trial.suggest_int('head_dim', 1, 5, step=1),
|
|
patch_size=trial.suggest_int('patch_size', 6, 12, step=3),
|
|
attn_depth=trial.suggest_int('attn_depth', 2, 14, step=4),
|
|
heads=trial.suggest_int('heads', 2, 16, step=2),
|
|
embedding_size=trial.suggest_int('embedding_size', 12, 64, step=12)
|
|
)
|
|
optuna_suggestions.update(**transformer_dict)
|
|
|
|
pruning_callback = PyTorchLightningPruningCallback(trial, monitor="PL_recall_score")
|
|
|
|
# Parse comandline args, read config and get model
|
|
h_params, found_data_class, found_model_class, seed = parse_comandline_args_add_defaults(
|
|
'_parameters.ini', overrides=optuna_suggestions)
|
|
h_params = Namespace(**h_params)
|
|
try:
|
|
best_score = run_lightning_loop(h_params, data_class=found_data_class, model_class=found_model_class,
|
|
additional_callbacks=pruning_callback, seed=seed)
|
|
except Exception as e:
|
|
print(e)
|
|
best_score = 0
|
|
return best_score
|
|
|
|
|
|
if __name__ == '__main__':
|
|
study = optuna.create_study(direction='maximize', sampler=optuna.samplers.TPESampler(seed=1337))
|
|
# study.optimize(optimize, n_trials=50, callbacks=[opt_utils.NeptuneCallback(log_study=True, log_charts=True)])
|
|
study.optimize(optimize, n_trials=100)
|
|
|
|
print("Number of finished trials: {}".format(len(study.trials)))
|
|
|
|
print("Best trial:")
|
|
trial = study.best_trial
|
|
|
|
print(" Value: {}".format(trial.value))
|
|
|
|
print(" Params: ")
|
|
for key, value in trial.params.items():
|
|
print(" {}: {}".format(key, value))
|
|
|
|
optuna_study_object = Path('study') / 'study.pkl'
|
|
optuna_study_object.parent.mkdir(exist_ok=True)
|
|
with optuna_study_object.open(mode='wb') as f:
|
|
pickle.dump(study, f)
|