import pickle from argparse import Namespace from pathlib import Path import optuna as optuna from optuna.integration import PyTorchLightningPruningCallback from main import run_lightning_loop from ml_lib.utils.config import parse_comandline_args_add_defaults def optimize(trial: optuna.Trial): # Optuna configuration folder = Path('study') folder.mkdir(parents=False, exist_ok=True) scheduler = trial.suggest_categorical('scheduler', [None, 'LambdaLR']) if scheduler is not None: lr_scheduler_parameter = trial.suggest_float('lr_scheduler_parameter', 0.8, 1, step=0.01) else: lr_scheduler_parameter = None optuna_suggestions = dict( model_name='CNNBaseline', data_name='MaskLibrosaDatamodule', batch_size=trial.suggest_int('batch_size', 5, 50, step=5), max_epochs=75, target_mel_length_in_seconds=trial.suggest_float('target_mel_length_in_seconds', 0.2, 1.5, step=0.1), random_apply_chance=trial.suggest_float('random_apply_chance', 0.1, 0.5, step=0.1), loudness_ratio=trial.suggest_float('loudness_ratio', 0.0, 0.5, step=0.1), shift_ratio=trial.suggest_float('shift_ratio', 0.0, 0.5, step=0.1), noise_ratio=trial.suggest_float('noise_ratio', 0.0, 0.5, step=0.1), mask_ratio=trial.suggest_float('mask_ratio', 0.0, 0.5, step=0.1), lr=trial.suggest_loguniform('lr', 1e-5, 1e-3), dropout=trial.suggest_float('dropout', 0.0, 0.3, step=0.05), lat_dim=2 ** trial.suggest_int('lat_dim', 1, 5, step=1), scheduler=scheduler, lr_scheduler_parameter=lr_scheduler_parameter, loss='ce_loss', sampler=trial.suggest_categorical('sampler', [None, 'WeightedRandomSampler']), study_name=trial.study.study_name ) if optuna_suggestions['model_name'] == 'CNNBaseline': model_depth = trial.suggest_int('model_depth', 1, 6, step=1) filters = list() for layer_idx in range(model_depth): filters.append(2 ** trial.suggest_int(f'filters_{layer_idx}', 2, 6, step=1)) optuna_suggestions.update(filters=filters) elif optuna_suggestions['model_name'] == 'VisualTransformer': transformer_dict = dict( mlp_dim=2 ** trial.suggest_int('mlp_dim', 1, 5, step=1), head_dim=2 ** trial.suggest_int('head_dim', 1, 5, step=1), patch_size=trial.suggest_int('patch_size', 6, 12, step=3), attn_depth=trial.suggest_int('attn_depth', 2, 14, step=4), heads=trial.suggest_int('heads', 2, 16, step=2), embedding_size=trial.suggest_int('embedding_size', 12, 64, step=12) ) optuna_suggestions.update(**transformer_dict) pruning_callback = PyTorchLightningPruningCallback(trial, monitor="PL_recall_score") # Parse comandline args, read config and get model h_params, found_data_class, found_model_class, seed = parse_comandline_args_add_defaults( '_parameters.ini', overrides=optuna_suggestions) h_params = Namespace(**h_params) try: best_score = run_lightning_loop(h_params, data_class=found_data_class, model_class=found_model_class, additional_callbacks=pruning_callback, seed=seed) except Exception as e: print(e) best_score = 0 return best_score if __name__ == '__main__': study = optuna.create_study(direction='maximize', sampler=optuna.samplers.TPESampler(seed=1337)) # study.optimize(optimize, n_trials=50, callbacks=[opt_utils.NeptuneCallback(log_study=True, log_charts=True)]) study.optimize(optimize, n_trials=100) print("Number of finished trials: {}".format(len(study.trials))) print("Best trial:") trial = study.best_trial print(" Value: {}".format(trial.value)) print(" Params: ") for key, value in trial.params.items(): print(" {}: {}".format(key, value)) optuna_study_object = Path('study') / 'study.pkl' optuna_study_object.parent.mkdir(exist_ok=True) with optuna_study_object.open(mode='wb') as f: pickle.dump(study, f)