Dataset rdy

.gitignore

@@ -8,6 +8,7 @@ __pycache__/
 # My Paths
 ml_lib
 /data
+/.idea
 
 
 # C extensions

_parameters.ini

@@ -0,0 +1,63 @@
+[project]
+neptune_key = eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vdWkubmVwdHVuZS5haSIsImFwaV91cmwiOiJodHRwczovL3VpLm5lcHR1bmUuYWkiLCJhcGlfa2V5IjoiZmI0OGMzNzUtOTg1NS00Yzg2LThjMzYtMWFiYjUwMDUyMjVlIn0=
+debug = 1
+eval = True
+seed = 69
+owner = si11ium
+model_name = CNNBaseline
+data_name = PrimatesLibrosaDatamodule
+
+[data]
+num_worker = 0
+data_root = data
+reset = False
+n_mels = 64
+sr = 16000
+hop_length = 256
+n_fft = 512
+
+loudness_ratio = 0.0
+shift_ratio = 0.0
+noise_ratio = 0
+mask_ratio = 0.3
+speed_amount = 0
+speed_min = 0
+speed_max = 0
+
+[model_cnn]
+weight_init = xavier_normal_
+activation = gelu
+use_bias = True
+use_norm = True
+dropout = 0.2
+lat_dim = 128
+features = 64
+filters = [32, 64, 128, 64]
+
+[model_attn]
+name = VerticalVisualTransformer
+weight_init = xavier_normal_
+activation = gelu
+use_bias = True
+use_norm = True
+dropout = 0.2
+lat_dim = 128
+features = 64
+patch_size = 3
+attn_depth = 3
+heads = 8
+embedding_size = 64
+
+[train]
+outpath = output
+version = None
+gpus=0
+sto_weight_avg = False
+weight_decay = 0
+opt_reset_interval = 0
+epochs = 100
+batch_size = 30
+lr = 0.01
+lr_warmup_steps = 0
+num_sanity_val_steps = 0
+

datasets/primates_librosa_datamodule.py

@@ -15,11 +15,17 @@ import multiprocessing as mp
 
 
 data_options = [DATA_OPTION_test, DATA_OPTION_train, DATA_OPTION_devel]
-class_names = {key: val for val, key in enumerate(['background', 'chimpanze', 'geunon', 'mandrille', 'redcap'])}
 
 
 class PrimatesLibrosaDatamodule(_BaseDataModule):
 
+    class_names = {key: val for val, key in enumerate(['background', 'chimpanze', 'geunon', 'mandrille', 'redcap'])}
+
+    @property
+    def shape(self):
+
+        return self.datasets[DATA_OPTION_train].datasets[0][0][1].shape
+
     @property
     def mel_folder(self):
         return self.root / 'mel_folder'
@@ -28,14 +34,15 @@ class PrimatesLibrosaDatamodule(_BaseDataModule):
     def wav_folder(self):
         return self.root / 'wav'
 
-    def __init__(self, root, batch_size, num_worker, sr, n_mels, n_fft, hop_length,
+    def __init__(self, data_root, batch_size, num_worker, sr, n_mels, n_fft, hop_length,
                  sample_segment_len=40, sample_hop_len=15):
         super(PrimatesLibrosaDatamodule, self).__init__()
         self.sample_hop_len = sample_hop_len
         self.sample_segment_len = sample_segment_len
-        self.num_worker = num_worker
+        self.num_worker = num_worker or 1
         self.batch_size = batch_size
-        self.root = Path(root) / 'primates'
+        self.root = Path(data_root) / 'primates'
+        self.mel_length_in_seconds = 0.7
 
         # Mel Transforms - will be pushed with all other paramters by self.__dict__ to subdataset-class
         self.mel_kwargs = dict(sr=sr, n_mels=n_mels, n_fft=n_fft, hop_length=hop_length)
@@ -70,13 +77,17 @@ class PrimatesLibrosaDatamodule(_BaseDataModule):
 
     def _build_subdataset(self, row, build=False):
         slice_file_name, class_name = row.strip().split(',')
-        class_id = class_names.get(class_name, -1)
+        class_id = self.class_names.get(class_name, -1)
         audio_file_path = self.wav_folder / slice_file_name
         # DATA OPTION DIFFERENTIATION !!!!!!!!!!! - Begin
         kwargs = self.__dict__
         if any([x in slice_file_name for x in [DATA_OPTION_devel, DATA_OPTION_test]]):
             kwargs.update(mel_augmentations=self.utility_transforms)
         # DATA OPTION DIFFERENTIATION !!!!!!!!!!! - End
+
+        target_frames = self.mel_length_in_seconds * self.mel_kwargs['sr']
+        sample_segment_length = target_frames // self.mel_kwargs['hop_length'] + 1
+        kwargs.update(sample_segment_len=sample_segment_length, sample_hop_len=sample_segment_length//2)
         mel_dataset = LibrosaAudioToMelDataset(audio_file_path, class_id, **kwargs)
         if build:
             assert mel_dataset.build_mel()
@@ -101,7 +112,7 @@ class PrimatesLibrosaDatamodule(_BaseDataModule):
                                              chunksize=chunksize)
                 for sub_dataset in results.get():
                     dataset.append(sub_dataset)
-                    tqdm.update()  # FIXME: will i ever get this to work?
+                    update()  # FIXME: will i ever get this to work?
             datasets[data_option] = ConcatDataset(dataset)
         self.datasets = datasets
         return datasets

main.py

@@ -1,12 +1,68 @@
-from pathlib import Path
+import configparser
+from argparse import ArgumentParser, Namespace
+
+from pytorch_lightning import Trainer
+from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint
+
+from ml_lib.utils.logging import Logger
+from ml_lib.utils.tools import locate_and_import_class, auto_cast
+
 import variables as v
-from datasets.primates_librosa_datamodule import PrimatesLibrosaDatamodule
-
-data_root = Path() / 'data'
-
 
 if __name__ == '__main__':
-    dataset = PrimatesLibrosaDatamodule(data_root, batch_size=25, num_worker=6,
+    # Argument Parser and default Values
-                                        sr=v.sr, n_mels=64, n_fft=512, hop_length=256)
+    # =============================================================================
-    dataset.prepare_data()
+    # Load Defaults from _parameters.ini file
-    print('done')
+    config = configparser.ConfigParser()
+    config.read('_parameters.ini')
+    project = config['project']
+
+    data_class = locate_and_import_class(project['data_name'], 'datasets')
+    model_class = locate_and_import_class(project['model_name'], 'models')
+
+    tmp_params = dict()
+    for key in ['project', 'train', 'data', 'model_cnn']:
+        defaults = config[key]
+        tmp_params.update({key: auto_cast(val) for key, val in defaults.items()})
+
+    # Parse Command Line
+    parser = ArgumentParser()
+    for module in [Logger, Trainer, data_class, model_class]:
+        parser = module.add_argparse_args(parser)
+    cmd_args, _ = parser.parse_known_args()
+    tmp_params.update({key: val for key, val in vars(cmd_args).items() if val is not None})
+    hparams = Namespace(**tmp_params)
+
+    with Logger.from_argparse_args(hparams) as logger:
+        # Callbacks
+        # =============================================================================
+        # Checkpoint Saving
+        ckpt_callback = ModelCheckpoint(
+            monitor='mean_loss',
+            filepath=str(logger.log_dir / 'ckpt_weights'),
+            verbose=False,
+            save_top_k=3,
+        )
+
+        # Learning Rate Logger
+        lr_logger = LearningRateMonitor(logging_interval='epoch')
+
+        #
+        # START
+        # =============================================================================
+        # Let Datamodule pull what it wants
+        datamodule = data_class.from_argparse_args(hparams)
+        datamodule.setup()
+        model_in_shape = datamodule.shape
+
+        # Let Trainer pull what it wants and add callbacks
+        trainer = Trainer.from_argparse_args(hparams, callbacks=[ckpt_callback, lr_logger])
+
+        # Let Model pull what it wants
+        model = model_class.from_argparse_args(hparams, in_shape=datamodule.shape, n_classes=v.N_CLASS_multi)
+        model.init_weights()
+
+        logger.log_hyperparams(dict(model.params))
+        trainer.fit(model, datamodule)
+
+        trainer.save_checkpoint(trainer.logger.save_dir)

models/cnn_baseline.py

@@ -0,0 +1,53 @@
+import inspect
+from argparse import Namespace
+
+import variables as v
+
+from torch import nn
+
+from ml_lib.metrics.multi_class_classification import MultiClassScores
+from ml_lib.modules.blocks import LinearModule
+from ml_lib.modules.model_parts import CNNEncoder
+from ml_lib.modules.util import (LightningBaseModule)
+from util.module_mixins import CombinedModelMixins
+
+
+class CNNBaseline(CombinedModelMixins,
+                  LightningBaseModule
+                  ):
+
+    def __init__(self, in_shape, n_classes, weight_init, activation, use_bias, use_norm, dropout, lat_dim, features,
+                 filters):
+
+        # TODO: Move this to parent class, or make it much easieer to access....
+        a = dict(locals())
+        params = {arg: a[arg] for arg in inspect.signature(self.__init__).parameters.keys() if arg != 'self'}
+        super(CNNBaseline, self).__init__(params)
+
+        # Model
+        # =============================================================================
+        # Additional parameters
+        self.in_shape = in_shape
+        assert len(self.in_shape) == 3, 'There need to be three Dimensions'
+        channels, height, width = self.in_shape
+
+        # Modules with Parameters
+        self.encoder = CNNEncoder(in_shape=self.in_shape, **self.params.module_kwargs)
+
+        module_kwargs = self.params.module_kwargs
+        module_kwargs.update(activation=nn.Softmax)
+        self.classifier = LinearModule(self.encoder.shape, n_classes, **module_kwargs)
+
+    def forward(self, x, mask=None, return_attn_weights=False):
+        """
+        :param x: the sequence to the encoder (required).
+        :param mask: the mask for the src sequence (optional).
+        :return:
+        """
+        tensor = self.encoder(x)
+
+        tensor = self.classifier(tensor)
+        return Namespace(main_out=tensor)
+
+    def additional_scores(self, outputs):
+        return MultiClassScores(self)

util/loss_mixin.py

@@ -0,0 +1,9 @@
+from torch import nn
+
+
+class LossMixin:
+
+    absolute_loss = nn.L1Loss()
+    nll_loss = nn.NLLLoss()
+    bce_loss = nn.BCELoss()
+    ce_loss = nn.CrossEntropyLoss()

util/module_mixins.py

@@ -0,0 +1,95 @@
+from abc import ABC
+
+import torch
+
+from ml_lib.modules.util import LightningBaseModule
+from util.loss_mixin import LossMixin
+from util.optimizer_mixin import OptimizerMixin
+
+
+class TrainMixin:
+
+    def training_step(self, batch_xy, batch_nb, *args, **kwargs):
+        assert isinstance(self, LightningBaseModule)
+        batch_x, batch_y = batch_xy
+        y = self(batch_x).main_out
+        loss = self.ce_loss(y.squeeze(), batch_y.long())
+        return dict(loss=loss)
+
+    def training_epoch_end(self, outputs):
+        assert isinstance(self, LightningBaseModule)
+        keys = list(outputs[0].keys())
+
+        summary_dict = {f'mean_{key}': torch.mean(torch.stack([output[key]
+                                                               for output in outputs]))
+                        for key in keys if 'loss' in key}
+        for key in summary_dict.keys():
+            self.log(key, summary_dict[key])
+
+
+class ValMixin:
+
+    def validation_step(self, batch_xy, batch_idx, *args, **kwargs):
+        assert isinstance(self, LightningBaseModule)
+        batch_x, batch_y = batch_xy
+        model_out = self(batch_x)
+        y = model_out.main_out
+
+        val_loss = self.ce_loss(y.squeeze(), batch_y.long())
+
+        return dict(val_loss=val_loss,
+                    batch_idx=batch_idx, y=y, batch_y=batch_y)
+
+    def validation_epoch_end(self, outputs, *_, **__):
+        assert isinstance(self, LightningBaseModule)
+        summary_dict = dict()
+
+        keys = list(outputs[0].keys())
+        summary_dict.update({f'mean_{key}': torch.mean(torch.stack([output[key]
+                                                                    for output in outputs]))
+                             for key in keys if 'loss' in key}
+                            )
+
+        additional_scores = self.additional_scores(outputs)
+        summary_dict.update(**additional_scores)
+
+        for key in summary_dict.keys():
+            self.log(key, summary_dict[key])
+
+
+class TestMixin:
+
+    def test_step(self, batch_xy, batch_idx, *_, **__):
+        assert isinstance(self, LightningBaseModule)
+        batch_x, batch_y = batch_xy
+        model_out = self(batch_x)
+        y = model_out.main_out
+        test_loss = self.ce_loss(y.squeeze(), batch_y.long())
+        return dict(test_loss=test_loss,
+                    batch_idx=batch_idx, y=y, batch_y=batch_y)
+
+    def test_epoch_end(self, outputs, *_, **__):
+        assert isinstance(self, LightningBaseModule)
+        summary_dict = dict()
+
+        keys = list(outputs[0].keys())
+        summary_dict.update({f'mean_{key}': torch.mean(torch.stack([output[key]
+                                                                    for output in outputs]))
+                             for key in keys if 'loss' in key}
+                            )
+
+        additional_scores = self.additional_scores(outputs)
+        summary_dict.update(**additional_scores)
+
+        for key in summary_dict.keys():
+            self.log(key, summary_dict[key])
+
+
+class CombinedModelMixins(LossMixin,
+                          TrainMixin,
+                          ValMixin,
+                          TestMixin,
+                          OptimizerMixin,
+                          LightningBaseModule,
+                          ABC):
+    pass

util/optimizer_mixin.py

@@ -0,0 +1,45 @@
+from collections import defaultdict
+
+from torch.optim import Adam
+from torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, LambdaLR
+from torchcontrib.optim import SWA
+
+from ml_lib.modules.util import LightningBaseModule
+
+
+class OptimizerMixin:
+
+    def configure_optimizers(self):
+        assert isinstance(self, LightningBaseModule)
+        optimizer_dict = dict(
+            # 'optimizer':optimizer,        # The Optimizer
+            # 'lr_scheduler': scheduler,       # The LR scheduler
+            frequency=1,                    # The frequency of the scheduler
+            interval='epoch',               # The unit of the scheduler's step size
+            # 'reduce_on_plateau': False,   # For ReduceLROnPlateau scheduler
+            # 'monitor': 'mean_val_loss'    # Metric to monitor
+        )
+
+        optimizer = Adam(params=self.parameters(), lr=self.params.lr, weight_decay=self.params.weight_decay)
+        if self.params.sto_weight_avg:
+            optimizer = SWA(optimizer, swa_start=10, swa_freq=5, swa_lr=0.05)
+        optimizer_dict.update(optimizer=optimizer)
+        if self.params.lr_warmup_steps:
+            scheduler = CosineAnnealingWarmRestarts(optimizer, self.params.lr_warmup_steps)
+        else:
+            scheduler = LambdaLR(optimizer, lr_lambda=lambda epoch: 0.95 ** epoch)
+        optimizer_dict.update(lr_scheduler=scheduler)
+        return optimizer_dict
+
+    def on_train_end(self):
+        assert isinstance(self, LightningBaseModule)
+        for opt in self.trainer.optimizers:
+            if isinstance(opt, SWA):
+                opt.swap_swa_sgd()
+
+    def on_epoch_end(self):
+        assert isinstance(self, LightningBaseModule)
+        if self.params.opt_reset_interval:
+            if self.current_epoch % self.params.opt_reset_interval == 0:
+                for opt in self.trainer.optimizers:
+                    opt.state = defaultdict(dict)

variables.py

@@ -6,3 +6,5 @@ sr = 16000
 
 
 PRIMATES_Root = Path(__file__).parent / 'data' / 'primates'
+
+N_CLASS_multi = 4