From 7edd3834a1ec9b9031095ce4d12c9f104cdf9ef8 Mon Sep 17 00:00:00 2001 From: Steffen Illium Date: Tue, 16 Feb 2021 10:18:04 +0100 Subject: [PATCH] Dataset rdy --- .gitignore | 1 + _parameters.ini | 63 ++++++++++++++++ datasets/primates_librosa_datamodule.py | 23 ++++-- main.py | 74 ++++++++++++++++--- models/__init__.py | 0 models/cnn_baseline.py | 53 ++++++++++++++ util/__init__.py | 0 util/loss_mixin.py | 9 +++ util/module_mixins.py | 95 +++++++++++++++++++++++++ util/optimizer_mixin.py | 45 ++++++++++++ variables.py | 2 + 11 files changed, 350 insertions(+), 15 deletions(-) create mode 100644 _parameters.ini create mode 100644 models/__init__.py create mode 100644 models/cnn_baseline.py create mode 100644 util/__init__.py create mode 100644 util/loss_mixin.py create mode 100644 util/module_mixins.py create mode 100644 util/optimizer_mixin.py diff --git a/.gitignore b/.gitignore index d462286..f8a9794 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ # My Paths ml_lib /data +/.idea # C extensions diff --git a/_parameters.ini b/_parameters.ini new file mode 100644 index 0000000..99d5a0b --- /dev/null +++ b/_parameters.ini @@ -0,0 +1,63 @@ +[project] +neptune_key = eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vdWkubmVwdHVuZS5haSIsImFwaV91cmwiOiJodHRwczovL3VpLm5lcHR1bmUuYWkiLCJhcGlfa2V5IjoiZmI0OGMzNzUtOTg1NS00Yzg2LThjMzYtMWFiYjUwMDUyMjVlIn0= +debug = 1 +eval = True +seed = 69 +owner = si11ium +model_name = CNNBaseline +data_name = PrimatesLibrosaDatamodule + +[data] +num_worker = 0 +data_root = data +reset = False +n_mels = 64 +sr = 16000 +hop_length = 256 +n_fft = 512 + +loudness_ratio = 0.0 +shift_ratio = 0.0 +noise_ratio = 0 +mask_ratio = 0.3 +speed_amount = 0 +speed_min = 0 +speed_max = 0 + +[model_cnn] +weight_init = xavier_normal_ +activation = gelu +use_bias = True +use_norm = True +dropout = 0.2 +lat_dim = 128 +features = 64 +filters = [32, 64, 128, 64] + +[model_attn] +name = VerticalVisualTransformer +weight_init = xavier_normal_ +activation = gelu +use_bias = True +use_norm = True +dropout = 0.2 +lat_dim = 128 +features = 64 +patch_size = 3 +attn_depth = 3 +heads = 8 +embedding_size = 64 + +[train] +outpath = output +version = None +gpus=0 +sto_weight_avg = False +weight_decay = 0 +opt_reset_interval = 0 +epochs = 100 +batch_size = 30 +lr = 0.01 +lr_warmup_steps = 0 +num_sanity_val_steps = 0 + diff --git a/datasets/primates_librosa_datamodule.py b/datasets/primates_librosa_datamodule.py index c97f085..c817ba4 100644 --- a/datasets/primates_librosa_datamodule.py +++ b/datasets/primates_librosa_datamodule.py @@ -15,11 +15,17 @@ import multiprocessing as mp data_options = [DATA_OPTION_test, DATA_OPTION_train, DATA_OPTION_devel] -class_names = {key: val for val, key in enumerate(['background', 'chimpanze', 'geunon', 'mandrille', 'redcap'])} class PrimatesLibrosaDatamodule(_BaseDataModule): + class_names = {key: val for val, key in enumerate(['background', 'chimpanze', 'geunon', 'mandrille', 'redcap'])} + + @property + def shape(self): + + return self.datasets[DATA_OPTION_train].datasets[0][0][1].shape + @property def mel_folder(self): return self.root / 'mel_folder' @@ -28,14 +34,15 @@ class PrimatesLibrosaDatamodule(_BaseDataModule): def wav_folder(self): return self.root / 'wav' - def __init__(self, root, batch_size, num_worker, sr, n_mels, n_fft, hop_length, + def __init__(self, data_root, batch_size, num_worker, sr, n_mels, n_fft, hop_length, sample_segment_len=40, sample_hop_len=15): super(PrimatesLibrosaDatamodule, self).__init__() self.sample_hop_len = sample_hop_len self.sample_segment_len = sample_segment_len - self.num_worker = num_worker + self.num_worker = num_worker or 1 self.batch_size = batch_size - self.root = Path(root) / 'primates' + self.root = Path(data_root) / 'primates' + self.mel_length_in_seconds = 0.7 # Mel Transforms - will be pushed with all other paramters by self.__dict__ to subdataset-class self.mel_kwargs = dict(sr=sr, n_mels=n_mels, n_fft=n_fft, hop_length=hop_length) @@ -70,13 +77,17 @@ class PrimatesLibrosaDatamodule(_BaseDataModule): def _build_subdataset(self, row, build=False): slice_file_name, class_name = row.strip().split(',') - class_id = class_names.get(class_name, -1) + class_id = self.class_names.get(class_name, -1) audio_file_path = self.wav_folder / slice_file_name # DATA OPTION DIFFERENTIATION !!!!!!!!!!! - Begin kwargs = self.__dict__ if any([x in slice_file_name for x in [DATA_OPTION_devel, DATA_OPTION_test]]): kwargs.update(mel_augmentations=self.utility_transforms) # DATA OPTION DIFFERENTIATION !!!!!!!!!!! - End + + target_frames = self.mel_length_in_seconds * self.mel_kwargs['sr'] + sample_segment_length = target_frames // self.mel_kwargs['hop_length'] + 1 + kwargs.update(sample_segment_len=sample_segment_length, sample_hop_len=sample_segment_length//2) mel_dataset = LibrosaAudioToMelDataset(audio_file_path, class_id, **kwargs) if build: assert mel_dataset.build_mel() @@ -101,7 +112,7 @@ class PrimatesLibrosaDatamodule(_BaseDataModule): chunksize=chunksize) for sub_dataset in results.get(): dataset.append(sub_dataset) - tqdm.update() # FIXME: will i ever get this to work? + update() # FIXME: will i ever get this to work? datasets[data_option] = ConcatDataset(dataset) self.datasets = datasets return datasets diff --git a/main.py b/main.py index 7c18c67..1fc63b2 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,68 @@ -from pathlib import Path +import configparser +from argparse import ArgumentParser, Namespace + +from pytorch_lightning import Trainer +from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint + +from ml_lib.utils.logging import Logger +from ml_lib.utils.tools import locate_and_import_class, auto_cast + import variables as v -from datasets.primates_librosa_datamodule import PrimatesLibrosaDatamodule - -data_root = Path() / 'data' - if __name__ == '__main__': - dataset = PrimatesLibrosaDatamodule(data_root, batch_size=25, num_worker=6, - sr=v.sr, n_mels=64, n_fft=512, hop_length=256) - dataset.prepare_data() - print('done') + # Argument Parser and default Values + # ============================================================================= + # Load Defaults from _parameters.ini file + config = configparser.ConfigParser() + config.read('_parameters.ini') + project = config['project'] + + data_class = locate_and_import_class(project['data_name'], 'datasets') + model_class = locate_and_import_class(project['model_name'], 'models') + + tmp_params = dict() + for key in ['project', 'train', 'data', 'model_cnn']: + defaults = config[key] + tmp_params.update({key: auto_cast(val) for key, val in defaults.items()}) + + # Parse Command Line + parser = ArgumentParser() + for module in [Logger, Trainer, data_class, model_class]: + parser = module.add_argparse_args(parser) + cmd_args, _ = parser.parse_known_args() + tmp_params.update({key: val for key, val in vars(cmd_args).items() if val is not None}) + hparams = Namespace(**tmp_params) + + with Logger.from_argparse_args(hparams) as logger: + # Callbacks + # ============================================================================= + # Checkpoint Saving + ckpt_callback = ModelCheckpoint( + monitor='mean_loss', + filepath=str(logger.log_dir / 'ckpt_weights'), + verbose=False, + save_top_k=3, + ) + + # Learning Rate Logger + lr_logger = LearningRateMonitor(logging_interval='epoch') + + # + # START + # ============================================================================= + # Let Datamodule pull what it wants + datamodule = data_class.from_argparse_args(hparams) + datamodule.setup() + model_in_shape = datamodule.shape + + # Let Trainer pull what it wants and add callbacks + trainer = Trainer.from_argparse_args(hparams, callbacks=[ckpt_callback, lr_logger]) + + # Let Model pull what it wants + model = model_class.from_argparse_args(hparams, in_shape=datamodule.shape, n_classes=v.N_CLASS_multi) + model.init_weights() + + logger.log_hyperparams(dict(model.params)) + trainer.fit(model, datamodule) + + trainer.save_checkpoint(trainer.logger.save_dir) diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/models/cnn_baseline.py b/models/cnn_baseline.py new file mode 100644 index 0000000..87087fd --- /dev/null +++ b/models/cnn_baseline.py @@ -0,0 +1,53 @@ +import inspect +from argparse import Namespace + +import variables as v + +from torch import nn + +from ml_lib.metrics.multi_class_classification import MultiClassScores +from ml_lib.modules.blocks import LinearModule +from ml_lib.modules.model_parts import CNNEncoder +from ml_lib.modules.util import (LightningBaseModule) +from util.module_mixins import CombinedModelMixins + + +class CNNBaseline(CombinedModelMixins, + LightningBaseModule + ): + + def __init__(self, in_shape, n_classes, weight_init, activation, use_bias, use_norm, dropout, lat_dim, features, + filters): + + # TODO: Move this to parent class, or make it much easieer to access.... + a = dict(locals()) + params = {arg: a[arg] for arg in inspect.signature(self.__init__).parameters.keys() if arg != 'self'} + super(CNNBaseline, self).__init__(params) + + # Model + # ============================================================================= + # Additional parameters + self.in_shape = in_shape + assert len(self.in_shape) == 3, 'There need to be three Dimensions' + channels, height, width = self.in_shape + + # Modules with Parameters + self.encoder = CNNEncoder(in_shape=self.in_shape, **self.params.module_kwargs) + + module_kwargs = self.params.module_kwargs + module_kwargs.update(activation=nn.Softmax) + self.classifier = LinearModule(self.encoder.shape, n_classes, **module_kwargs) + + def forward(self, x, mask=None, return_attn_weights=False): + """ + :param x: the sequence to the encoder (required). + :param mask: the mask for the src sequence (optional). + :return: + """ + tensor = self.encoder(x) + + tensor = self.classifier(tensor) + return Namespace(main_out=tensor) + + def additional_scores(self, outputs): + return MultiClassScores(self) diff --git a/util/__init__.py b/util/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/util/loss_mixin.py b/util/loss_mixin.py new file mode 100644 index 0000000..b1b8c1f --- /dev/null +++ b/util/loss_mixin.py @@ -0,0 +1,9 @@ +from torch import nn + + +class LossMixin: + + absolute_loss = nn.L1Loss() + nll_loss = nn.NLLLoss() + bce_loss = nn.BCELoss() + ce_loss = nn.CrossEntropyLoss() \ No newline at end of file diff --git a/util/module_mixins.py b/util/module_mixins.py new file mode 100644 index 0000000..42a5aa3 --- /dev/null +++ b/util/module_mixins.py @@ -0,0 +1,95 @@ +from abc import ABC + +import torch + +from ml_lib.modules.util import LightningBaseModule +from util.loss_mixin import LossMixin +from util.optimizer_mixin import OptimizerMixin + + +class TrainMixin: + + def training_step(self, batch_xy, batch_nb, *args, **kwargs): + assert isinstance(self, LightningBaseModule) + batch_x, batch_y = batch_xy + y = self(batch_x).main_out + loss = self.ce_loss(y.squeeze(), batch_y.long()) + return dict(loss=loss) + + def training_epoch_end(self, outputs): + assert isinstance(self, LightningBaseModule) + keys = list(outputs[0].keys()) + + summary_dict = {f'mean_{key}': torch.mean(torch.stack([output[key] + for output in outputs])) + for key in keys if 'loss' in key} + for key in summary_dict.keys(): + self.log(key, summary_dict[key]) + + +class ValMixin: + + def validation_step(self, batch_xy, batch_idx, *args, **kwargs): + assert isinstance(self, LightningBaseModule) + batch_x, batch_y = batch_xy + model_out = self(batch_x) + y = model_out.main_out + + val_loss = self.ce_loss(y.squeeze(), batch_y.long()) + + return dict(val_loss=val_loss, + batch_idx=batch_idx, y=y, batch_y=batch_y) + + def validation_epoch_end(self, outputs, *_, **__): + assert isinstance(self, LightningBaseModule) + summary_dict = dict() + + keys = list(outputs[0].keys()) + summary_dict.update({f'mean_{key}': torch.mean(torch.stack([output[key] + for output in outputs])) + for key in keys if 'loss' in key} + ) + + additional_scores = self.additional_scores(outputs) + summary_dict.update(**additional_scores) + + for key in summary_dict.keys(): + self.log(key, summary_dict[key]) + + +class TestMixin: + + def test_step(self, batch_xy, batch_idx, *_, **__): + assert isinstance(self, LightningBaseModule) + batch_x, batch_y = batch_xy + model_out = self(batch_x) + y = model_out.main_out + test_loss = self.ce_loss(y.squeeze(), batch_y.long()) + return dict(test_loss=test_loss, + batch_idx=batch_idx, y=y, batch_y=batch_y) + + def test_epoch_end(self, outputs, *_, **__): + assert isinstance(self, LightningBaseModule) + summary_dict = dict() + + keys = list(outputs[0].keys()) + summary_dict.update({f'mean_{key}': torch.mean(torch.stack([output[key] + for output in outputs])) + for key in keys if 'loss' in key} + ) + + additional_scores = self.additional_scores(outputs) + summary_dict.update(**additional_scores) + + for key in summary_dict.keys(): + self.log(key, summary_dict[key]) + + +class CombinedModelMixins(LossMixin, + TrainMixin, + ValMixin, + TestMixin, + OptimizerMixin, + LightningBaseModule, + ABC): + pass diff --git a/util/optimizer_mixin.py b/util/optimizer_mixin.py new file mode 100644 index 0000000..9c361b0 --- /dev/null +++ b/util/optimizer_mixin.py @@ -0,0 +1,45 @@ +from collections import defaultdict + +from torch.optim import Adam +from torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, LambdaLR +from torchcontrib.optim import SWA + +from ml_lib.modules.util import LightningBaseModule + + +class OptimizerMixin: + + def configure_optimizers(self): + assert isinstance(self, LightningBaseModule) + optimizer_dict = dict( + # 'optimizer':optimizer, # The Optimizer + # 'lr_scheduler': scheduler, # The LR scheduler + frequency=1, # The frequency of the scheduler + interval='epoch', # The unit of the scheduler's step size + # 'reduce_on_plateau': False, # For ReduceLROnPlateau scheduler + # 'monitor': 'mean_val_loss' # Metric to monitor + ) + + optimizer = Adam(params=self.parameters(), lr=self.params.lr, weight_decay=self.params.weight_decay) + if self.params.sto_weight_avg: + optimizer = SWA(optimizer, swa_start=10, swa_freq=5, swa_lr=0.05) + optimizer_dict.update(optimizer=optimizer) + if self.params.lr_warmup_steps: + scheduler = CosineAnnealingWarmRestarts(optimizer, self.params.lr_warmup_steps) + else: + scheduler = LambdaLR(optimizer, lr_lambda=lambda epoch: 0.95 ** epoch) + optimizer_dict.update(lr_scheduler=scheduler) + return optimizer_dict + + def on_train_end(self): + assert isinstance(self, LightningBaseModule) + for opt in self.trainer.optimizers: + if isinstance(opt, SWA): + opt.swap_swa_sgd() + + def on_epoch_end(self): + assert isinstance(self, LightningBaseModule) + if self.params.opt_reset_interval: + if self.current_epoch % self.params.opt_reset_interval == 0: + for opt in self.trainer.optimizers: + opt.state = defaultdict(dict) \ No newline at end of file diff --git a/variables.py b/variables.py index 6ce4730..5bb22d7 100644 --- a/variables.py +++ b/variables.py @@ -6,3 +6,5 @@ sr = 16000 PRIMATES_Root = Path(__file__).parent / 'data' / 'primates' + +N_CLASS_multi = 4