transition

audio_toolset/audio_io.py

@@ -1,5 +1,3 @@
-from typing import Union
-
 import numpy as np
 
 try:

audio_toolset/audio_to_mel_dataset.py

@@ -20,7 +20,7 @@ class _AudioToMelDataset(Dataset, ABC):
     def sampling_rate(self):
         raise NotImplementedError
 
-    def __init__(self, audio_file_path, label, sample_segment_len=1, sample_hop_len=1, reset=False,
+    def __init__(self, audio_file_path, label, sample_segment_len=0, sample_hop_len=0, reset=False,
                  audio_augmentations=None, mel_augmentations=None, mel_kwargs=None, **kwargs):
         self.ignored_kwargs = kwargs
         self.mel_kwargs = mel_kwargs
@@ -46,7 +46,7 @@ class _AudioToMelDataset(Dataset, ABC):
             return self.dataset[item]
         except FileNotFoundError:
             assert self._build_mel()
-        return self.dataset[item]
+            return self.dataset[item]
 
     def __len__(self):
         return len(self.dataset)
@@ -79,7 +79,6 @@ class LibrosaAudioToMelDataset(_AudioToMelDataset):
                                        MelToImage()
                                        ])
 
-
     def _build_mel(self):
         if self.reset:
             self.mel_file_path.unlink(missing_ok=True)

audio_toolset/mel_dataset.py

@@ -13,13 +13,16 @@ class TorchMelDataset(Dataset):
         super(TorchMelDataset, self).__init__()
         self.sampling_rate = sampling_rate
         self.audio_file_len = audio_file_len
-        self.padding = AutoPadToShape((n_mels , sub_segment_len)) if auto_pad_to_shape else None
+        self.padding = AutoPadToShape((n_mels, sub_segment_len)) if auto_pad_to_shape and sub_segment_len else None
         self.path = Path(mel_path)
         self.sub_segment_len = sub_segment_len
         self.mel_hop_len = mel_hop_len
         self.sub_segment_hop_len = sub_segment_hop_len
         self.n = int((self.sampling_rate / self.mel_hop_len) * self.audio_file_len + 1)
-        self.offsets = list(range(0, self.n - self.sub_segment_len, self.sub_segment_hop_len))
+        if self.sub_segment_len and self.sub_segment_hop_len:
+            self.offsets = list(range(0, self.n - self.sub_segment_len, self.sub_segment_hop_len))
+        else:
+            self.offsets = [0]
         self.label = label
         self.transform = transform
 
@@ -29,7 +32,8 @@ class TorchMelDataset(Dataset):
         with self.path.open('rb') as mel_file:
             mel_spec = pickle.load(mel_file, fix_imports=True)
         start = self.offsets[item]
-        snippet = mel_spec[: , start: start + self.sub_segment_len]
+        duration = self.sub_segment_len if self.sub_segment_len and self.sub_segment_hop_len else mel_spec.shape[1]
+        snippet = mel_spec[:, start: start + duration]
         if self.transform:
             snippet = self.transform(snippet)
         if self.padding:

metrics/generative_task_evaluation.py

@@ -0,0 +1,68 @@
+from itertools import cycle
+
+import numpy as np
+import torch
+from sklearn.metrics import roc_curve, auc, roc_auc_score, ConfusionMatrixDisplay, confusion_matrix
+from scipy.spatial.distance import cdist
+
+from ml_lib.metrics._base_score import _BaseScores
+
+from matplotlib import pyplot as plt
+
+
+class GenerativeTaskEval(_BaseScores):
+
+    def __init__(self, *args):
+        super(GenerativeTaskEval, self).__init__(*args)
+        pass
+
+    def __call__(self, outputs):
+        summary_dict = dict()
+        #######################################################################################
+        # Additional Score  -  Histogram Distances - Image Plotting
+        #######################################################################################
+        #
+        # INIT
+        y_true = torch.cat([output['batch_y'] for output in outputs]).cpu().numpy()
+
+        y_pred = torch.cat([output['y'] for output in outputs]).squeeze().cpu().numpy()
+
+        attn_weights = torch.cat([output['attn_weights'] for output in outputs]).squeeze().cpu().numpy()
+
+        ######################################################################################
+        #
+        # Histogram comparission
+
+        y_true_hist = np.histogram(y_true, bins=128)[0]  # Todo: Find a better value
+        y_pred_hist = np.histogram(y_pred, bins=128)[0]  # Todo: Find a better value
+
+        # L2 norm == euclidean distance
+        hist_euc_dist = cdist(np.expand_dims(y_true_hist, axis=0), np.expand_dims(y_pred_hist, axis=0),
+                              metric='euclidean')
+
+        # Manhattan Distance
+        hist_manhattan_dist = cdist(np.expand_dims(y_true_hist, axis=0), np.expand_dims(y_pred_hist, axis=0),
+                                    metric='cityblock')
+
+        summary_dict.update(hist_manhattan_dist=hist_manhattan_dist, hist_euc_dist=hist_euc_dist)
+
+        #######################################################################################
+        #
+        idx = np.random.choice(np.arange(y_true.shape[0]), 1).item()
+
+        ax = plt.imshow(y_true[idx].squeeze())
+        # Plot using a small number of colors, with unevenly spaced boundaries.
+        ax2 = plt.imshow(attn_weights[idx].sq, interpolation='nearest', aspect='auto', extent=ax.get_extent())
+        self.model.logger.log_image('ROC', image=plt.gcf(), step=self.model.current_epoch)
+        plt.clf()
+
+
+        #######################################################################################
+        #
+
+
+        #######################################################################################
+        #
+
+        plt.close('all')
+        return summary_dict

modules/blocks.py

@@ -12,7 +12,7 @@ from einops import rearrange
 import sys
 sys.path.append(str(Path(__file__).parent))
 
-from .util import AutoPad, Interpolate, ShapeMixin, F_x, Flatten, ResidualBlock, PreNorm
+from .util import AutoPad, Interpolate, ShapeMixin, F_x, Flatten
 
 DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 
@@ -85,7 +85,6 @@ class ConvModule(ShapeMixin, nn.Module):
         else:
             pass
 
-
     def forward(self, x):
         tensor = self.norm(x)
         tensor = self.conv(tensor)
@@ -100,12 +99,13 @@ class PreInitializedConvModule(ShapeMixin, nn.Module):
     def __init__(self, in_shape, weight_matrix):
         super(PreInitializedConvModule, self).__init__()
         self.in_shape = in_shape
+        self.weight_matrix = weight_matrix
         raise NotImplementedError
         # ToDo Get the weight_matrix shape and init a conv_module of similar size,
         #      override the weights then.
 
     def forward(self, x):
-
+        x = torch.matmul(x, self.weight_matrix)  # ToDo: This is an Placeholder
         return x
 
 
@@ -214,8 +214,9 @@ class RecurrentModule(ShapeMixin, nn.Module):
         tensor = self.rnn(x)
         return tensor
 
+
 class FeedForward(nn.Module):
-    def __init__(self, dim, hidden_dim, dropout = 0.):
+    def __init__(self, dim, hidden_dim, dropout=0.):
         super().__init__()
         self.net = nn.Sequential(
             nn.Linear(dim, hidden_dim),
@@ -224,31 +225,35 @@ class FeedForward(nn.Module):
             nn.Linear(hidden_dim, dim),
             nn.Dropout(dropout)
         )
+
     def forward(self, x):
         return self.net(x)
 
+
 class Attention(nn.Module):
-    def __init__(self, dim, heads = 8, dropout = 0.):
+    def __init__(self, dim, heads=8, dropout=0.):
         super().__init__()
         self.heads = heads
-        self.scale = dim ** -0.5
+        self.scale = dim / heads ** -0.5
 
-        self.to_qkv = nn.Linear(dim, dim * 3, bias = False)
+        self.to_qkv = nn.Linear(dim, dim * 3, bias=False)
         self.to_out = nn.Sequential(
             nn.Linear(dim, dim),
             nn.Dropout(dropout)
         )
 
-    def forward(self, x, mask = None):
+    def forward(self, x, mask=None, return_attn_weights=False):
+        # noinspection PyTupleAssignmentBalance
         b, n, _, h = *x.shape, self.heads
-        qkv = self.to_qkv(x).chunk(3, dim = -1)
+
-        q, k, v = [rearrange(t, 'b n (h d) -> b h n d', h = h) for t in qkv]
+        qkv = self.to_qkv(x).chunk(3, dim=-1)
+        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h=h), qkv)
 
         dots = torch.einsum('bhid,bhjd->bhij', q, k) * self.scale
         mask_value = -torch.finfo(dots.dtype).max
 
         if mask is not None:
-            mask = F.pad(mask.flatten(1), [1, 0], value = True)
+            mask = F.pad(mask.flatten(1), (1, 0), value=True)
             assert mask.shape[-1] == dots.shape[-1], 'mask has incorrect dimensions'
             mask = mask[:, None, :] * mask[:, :, None]
             dots.masked_fill_(~mask, mask_value)
@@ -258,39 +263,47 @@ class Attention(nn.Module):
 
         out = torch.einsum('bhij,bhjd->bhid', attn, v)
         out = rearrange(out, 'b h n d -> b n (h d)')
-        out =  self.to_out(out)
+        out = self.to_out(out)
-        return out
+        if return_attn_weights:
-
+            return out, attn
-class Transformer(nn.Module):
+        else:
-    def __init__(self, dim, depth, heads, mlp_dim, dropout):
+            return out
-        super().__init__()
-        self.layers = nn.ModuleList([])
-        for _ in range(depth):
-            self.layers.append(nn.ModuleList([
-                ResidualBlock(PreNorm(dim, Attention(dim, heads = heads, dropout = dropout))),
-                ResidualBlock(PreNorm(dim, FeedForward(dim, mlp_dim, dropout = dropout)))
-            ]))
-
-    def forward(self, x, mask = None, *_, **__):
-        for attn, ff in self.layers:
-            x = attn(x, mask = mask)
-            x = ff(x)
-        return x
 
 
 class TransformerModule(ShapeMixin, nn.Module):
 
-    def __init__(self, in_shape, hidden_size, n_heads, num_layers=1, dropout=None, use_norm=False, activation='gelu'):
+    def __init__(self, in_shape, depth, heads, mlp_dim, dropout=None, use_norm=False, activation='gelu'):
         super(TransformerModule, self).__init__()
 
         self.in_shape = in_shape
 
         self.flat = Flatten(self.in_shape) if isinstance(self.in_shape, (tuple, list)) else F_x(in_shape)
 
-        self.transformer = Transformer(dim=self.flat.flat_shape, depth=num_layers, heads=n_heads,
+        self.layers = nn.ModuleList([])
-                                       mlp_dim=hidden_size, dropout=dropout)
+        self.embedding_dim = self.flat.flat_shape
+        self.norm = nn.LayerNorm(self.embedding_dim)
+        self.attns = nn.ModuleList([Attention(self.embedding_dim, heads=heads, dropout=dropout) for _ in range(depth)])
+        self.mlps = nn.ModuleList([FeedForward(self.embedding_dim, mlp_dim, dropout=dropout) for _ in range(depth)])
 
-    def forward(self, x, mask=None, key_padding_mask=None):
+    def forward(self, x, mask=None, return_attn_weights=False, **_):
         tensor = self.flat(x)
-        tensor = self.transformer(tensor, mask, key_padding_mask)
+        attn_weights = list()
-        return tensor
+
+        for attn, mlp in zip(self.attns, self.mlps):
+            # Attention
+            skip_connection = tensor.clone()
+            tensor = self.norm(tensor)
+            if return_attn_weights:
+                tensor, attn_weight = attn(tensor, mask=mask, return_attn_weights=return_attn_weights)
+                attn_weights.append(attn_weight)
+            else:
+                tensor = attn(tensor, mask=mask)
+            tensor = tensor + skip_connection
+
+            # MLP
+            skip_connection = tensor.clone()
+            tensor = self.norm(tensor)
+            tensor = mlp(tensor)
+            tensor = tensor + skip_connection
+
+        return (tensor, attn_weights) if return_attn_weights else tensor

modules/model_parts.py

@@ -96,6 +96,7 @@ class Generator(ShapeMixin, nn.Module):
         super(Generator, self).__init__()
         assert filters, '"Filters" has to be a list of int.'
         assert filters, '"Filters" has to be a list of int.'
+        kernels = kernels if kernels else [3] * len(filters)
         assert len(filters) == len(kernels), '"Filters" and "Kernels" has to be of same length.'
 
         interpolations = interpolations or [2, 2, 2]

modules/util.py

@@ -150,23 +150,6 @@ class F_x(ShapeMixin, nn.Module):
         return x
 
 
-class ResidualBlock(nn.Module):
-    def __init__(self, fn):
-        super().__init__()
-        self.fn = fn
-    def forward(self, x, **kwargs):
-        return self.fn(x, **kwargs) + x
-
-
-class PreNorm(nn.Module):
-    def __init__(self, dim, fn):
-        super().__init__()
-        self.norm = nn.LayerNorm(dim)
-        self.fn = fn
-    def forward(self, x, **kwargs):
-        return self.fn(self.norm(x), **kwargs)
-
-
 class SlidingWindow(ShapeMixin, nn.Module):
     def __init__(self, in_shape, kernel, stride=1, padding=0, keepdim=False):
         super(SlidingWindow, self).__init__()

utils/logging.py

@@ -3,6 +3,7 @@ from pathlib import Path
 
 from pytorch_lightning.loggers.base import LightningLoggerBase
 from pytorch_lightning.loggers.neptune import NeptuneLogger
+from neptune.api_exceptions import ProjectNotFound
 # noinspection PyUnresolvedReferences
 from pytorch_lightning.loggers.csv_logs import CSVLogger
 
@@ -71,7 +72,12 @@ class Logger(LightningLoggerBase, ABC):
                                     experiment_name=self.name,
                                     project_name=self.project_name,
                                     params=self.config.model_paramters)
-        self.neptunelogger = NeptuneLogger(**self._neptune_kwargs)
+        try:
+            self.neptunelogger = NeptuneLogger(**self._neptune_kwargs)
+        except ProjectNotFound as e:
+            print(f'The project "{self.project_name}"')
+            print(e)
+
         self.csvlogger = CSVLogger(**self._csvlogger_kwargs)
         self.log_config_as_ini()