Done: AE, VAE, AAE

ToDo: Double AAE, Visualization
All Modularized

.idea/ae_toolbox_torch.iml

@@ -1,8 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
+    <content url="file://$MODULE_DIR$">
-    <orderEntry type="inheritedJdk" />
+      <excludeFolder url="file://$MODULE_DIR$/data" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.7 (torch)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyDocumentationSettings">

.idea/workspace.xml

@@ -1,16 +1,16 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="ChangeListManager">
-    <list default="true" id="5955480a-c876-43d5-afd7-8717f51f413e" name="Default Changelist" comment="">
+    <list default="true" id="5955480a-c876-43d5-afd7-8717f51f413e" name="Default Changelist" comment="Lightning integration basic ae, dataloaders and dataset">
-      <change afterPath="$PROJECT_DIR$/.idea/dictionaries/illium.xml" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/networks/adverserial_auto_encoder.py" afterDir="false" />
-      <change afterPath="$PROJECT_DIR$/.idea/other.xml" afterDir="false" />
-      <change afterPath="$PROJECT_DIR$/.idea/vcs.xml" afterDir="false" />
-      <change afterPath="$PROJECT_DIR$/basic_ae_lightning_torch.py" afterDir="false" />
-      <change afterPath="$PROJECT_DIR$/data/dataset.py" afterDir="false" />
-      <change afterPath="$PROJECT_DIR$/networks/basic_ae.py" afterDir="false" />
-      <change afterPath="$PROJECT_DIR$/networks/modules.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.idea/ae_toolbox_torch.iml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/ae_toolbox_torch.iml" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/dataset.py" beforeDir="false" afterPath="$PROJECT_DIR$/dataset.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/networks/basic_ae.py" beforeDir="false" afterPath="$PROJECT_DIR$/networks/auto_encoder.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/networks/basic_vae.py" beforeDir="false" afterPath="$PROJECT_DIR$/networks/variational_auto_encoder.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/networks/modules.py" beforeDir="false" afterPath="$PROJECT_DIR$/networks/modules.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/run_basic_ae.py" beforeDir="false" afterPath="$PROJECT_DIR$/run_models.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/run_basic_vae.py" beforeDir="false" />
     </list>
     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
     <option name="SHOW_DIALOG" value="false" />
@@ -34,6 +34,7 @@
     <property name="ASKED_ADD_EXTERNAL_FILES" value="true" />
     <property name="SHARE_PROJECT_CONFIGURATION_FILES" value="true" />
     <property name="WebServerToolWindowFactoryState" value="false" />
+    <property name="last_opened_file_path" value="$PROJECT_DIR$/networks" />
     <property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
     <property name="settings.editor.selected.configurable" value="pyconsole" />
   </component>
@@ -46,6 +47,16 @@
     </option>
     <option name="myShowDebugConsoleByDefault" value="true" />
   </component>
+  <component name="RecentsManager">
+    <key name="CopyFile.RECENT_KEYS">
+      <recent name="C:\Users\illium\Google Drive\LMU\Research\ae_toolbox_torch\networks" />
+      <recent name="C:\Users\illium\Google Drive\LMU\Research\ae_toolbox_torch\viz" />
+    </key>
+    <key name="MoveFile.RECENT_KEYS">
+      <recent name="C:\Users\illium\Google Drive\LMU\Research\ae_toolbox_torch\data\processed" />
+      <recent name="C:\Users\illium\Google Drive\LMU\Research\ae_toolbox_torch" />
+    </key>
+  </component>
   <component name="RunDashboard">
     <option name="ruleStates">
       <list>
@@ -58,7 +69,7 @@
       </list>
     </option>
   </component>
-  <component name="RunManager" selected="Python.basic_ae_lightning_torch">
+  <component name="RunManager" selected="Python.run_models">
     <configuration default="true" type="PythonConfigurationType" factoryName="Python">
       <module name="ae_toolbox_torch" />
       <option name="INTERPRETER_OPTIONS" value="" />
@@ -81,20 +92,17 @@
       <option name="INPUT_FILE" value="" />
       <method v="2" />
     </configuration>
-    <configuration name="basic_ae_lightning_torch" type="PythonConfigurationType" factoryName="Python" temporary="true">
+    <configuration name="run_basic_ae" type="PythonConfigurationType" factoryName="Python" temporary="true">
       <module name="ae_toolbox_torch" />
       <option name="INTERPRETER_OPTIONS" value="" />
       <option name="PARENT_ENVS" value="true" />
-      <envs>
-        <env name="PYTHONUNBUFFERED" value="1" />
-      </envs>
       <option name="SDK_HOME" value="" />
       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
       <option name="IS_MODULE_SDK" value="true" />
       <option name="ADD_CONTENT_ROOTS" value="true" />
       <option name="ADD_SOURCE_ROOTS" value="true" />
       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
-      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/basic_ae_lightning_torch.py" />
+      <option name="SCRIPT_NAME" value="C:\Users\illium\Google Drive\LMU\Research\ae_toolbox_torch\run_models.py" />
       <option name="PARAMETERS" value="" />
       <option name="SHOW_COMMAND_LINE" value="true" />
       <option name="EMULATE_TERMINAL" value="false" />
@@ -103,20 +111,17 @@
       <option name="INPUT_FILE" value="" />
       <method v="2" />
     </configuration>
-    <configuration name="dataset" type="PythonConfigurationType" factoryName="Python" temporary="true">
+    <configuration name="run_models" type="PythonConfigurationType" factoryName="Python" temporary="true">
       <module name="ae_toolbox_torch" />
       <option name="INTERPRETER_OPTIONS" value="" />
       <option name="PARENT_ENVS" value="true" />
-      <envs>
-        <env name="PYTHONUNBUFFERED" value="1" />
-      </envs>
       <option name="SDK_HOME" value="" />
-      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/data" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
       <option name="IS_MODULE_SDK" value="true" />
       <option name="ADD_CONTENT_ROOTS" value="true" />
       <option name="ADD_SOURCE_ROOTS" value="true" />
       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
-      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/data/dataset.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/run_models.py" />
       <option name="PARAMETERS" value="" />
       <option name="SHOW_COMMAND_LINE" value="true" />
       <option name="EMULATE_TERMINAL" value="false" />
@@ -125,10 +130,14 @@
       <option name="INPUT_FILE" value="" />
       <method v="2" />
     </configuration>
+    <list>
+      <item itemvalue="Python.run_basic_ae" />
+      <item itemvalue="Python.run_models" />
+    </list>
     <recent_temporary>
       <list>
-        <item itemvalue="Python.basic_ae_lightning_torch" />
+        <item itemvalue="Python.run_models" />
-        <item itemvalue="Python.dataset" />
+        <item itemvalue="Python.run_basic_ae" />
       </list>
     </recent_temporary>
   </component>
@@ -145,7 +154,23 @@
       <workItem from="1564587420277" duration="6891000" />
       <workItem from="1565364574595" duration="1092000" />
       <workItem from="1565592214301" duration="53660000" />
+      <workItem from="1565793671730" duration="30373000" />
     </task>
+    <task id="LOCAL-00001" summary="Lightning integration basic ae, dataloaders and dataset">
+      <created>1565793753423</created>
+      <option name="number" value="00001" />
+      <option name="presentableId" value="LOCAL-00001" />
+      <option name="project" value="LOCAL" />
+      <updated>1565793753423</updated>
+    </task>
+    <task id="LOCAL-00002" summary="Lightning integration basic ae, dataloaders and dataset">
+      <created>1565958589041</created>
+      <option name="number" value="00002" />
+      <option name="presentableId" value="LOCAL-00002" />
+      <option name="project" value="LOCAL" />
+      <updated>1565958589041</updated>
+    </task>
+    <option name="localTasksCounter" value="3" />
     <servers />
   </component>
   <component name="TypeScriptGeneratedFilesManager">
@@ -164,8 +189,34 @@
       </map>
     </option>
   </component>
+  <component name="VcsManagerConfiguration">
+    <MESSAGE value="Lightning integration basic ae, dataloaders and dataset" />
+    <option name="LAST_COMMIT_MESSAGE" value="Lightning integration basic ae, dataloaders and dataset" />
+  </component>
+  <component name="XDebuggerManager">
+    <breakpoint-manager>
+      <breakpoints>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/run_models.py</url>
+          <line>20</line>
+          <option name="timeStamp" value="27" />
+        </line-breakpoint>
+      </breakpoints>
+      <default-breakpoints>
+        <breakpoint type="python-exception">
+          <properties notifyOnTerminate="true" exception="BaseException">
+            <option name="notifyOnTerminate" value="true" />
+          </properties>
+        </breakpoint>
+      </default-breakpoints>
+    </breakpoint-manager>
+  </component>
   <component name="com.intellij.coverage.CoverageDataManagerImpl">
-    <SUITE FILE_PATH="coverage/ae_toolbox_torch$basic_ae_lightning_torch.coverage" NAME="basic_ae_lightning_torch Coverage Results" MODIFIED="1565790288699" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/ae_toolbox_torch$basic_ae_lightning_torch.coverage" NAME="basic_ae_lightning_torch Coverage Results" MODIFIED="1565937164457" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/ae_toolbox_torch$basic_ae_lightning.coverage" NAME="basic_ae_lightning Coverage Results" MODIFIED="1565956491159" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/ae_toolbox_torch$basic_vae_lightning.coverage" NAME="basic_vae_lightning Coverage Results" MODIFIED="1565955311009" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/ae_toolbox_torch$run_basic_ae.coverage" NAME="run_basic_ae Coverage Results" MODIFIED="1565966122607" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/ae_toolbox_torch$run_models.coverage" NAME="run_models Coverage Results" MODIFIED="1565987843914" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
     <SUITE FILE_PATH="coverage/ae_toolbox_torch$dataset.coverage" NAME="dataset Coverage Results" MODIFIED="1565772669750" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/data" />
   </component>
 </project>

dataset.py

@@ -125,7 +125,7 @@ class DataContainer(AbstractDataset):
 
     def process(self, filepath):
         dataDict = defaultdict(list)
-        total_lines = len(open(filepath,'r').readlines())
+        total_lines = len(open(filepath, 'r').readlines())
         with open(filepath, 'r') as f:
             delimiter = ','
             # Separate the header
@@ -176,7 +176,6 @@ class Trajectories(Dataset):
             data[:, 2:] = transformation(data[:, 2:])
         return data
 
-
     def __iter__(self):
         for i in range(len(self)):
             yield self[i]

networks/adverserial_auto_encoder.py

@@ -0,0 +1,66 @@
+from networks.auto_encoder import AutoEncoder
+from torch.nn.functional import mse_loss
+from torch.nn import Sequential, Linear, ReLU, Dropout, Sigmoid
+from torch.distributions import Normal
+from networks.modules import *
+import torch
+
+
+class AdversarialAutoEncoder(AutoEncoder):
+
+    def __init__(self, *args, **kwargs):
+        super(AdversarialAutoEncoder, self).__init__(*args, **kwargs)
+        self.discriminator = Discriminator(self.latent_dim, self.dataParams)
+
+    def forward(self, batch):
+        # Encoder
+        #  outputs, hidden (Batch, Timesteps aka. Size, Features / Latent Dim Size)
+        z = self.encoder(batch)
+        # Decoder
+        # First repeat the data accordingly to the batch size
+        z_repeatet = Repeater((batch.shape[0], self.dataParams['size'], -1))(z)
+        x_hat = self.decoder(z_repeatet)
+        return z, x_hat
+
+
+class AdversarialAELightningOverrides:
+
+    def forward(self, x):
+        return self.network.forward(x)
+
+    def training_step(self, batch, _, optimizer_i):
+        if optimizer_i == 0:
+            # ---------------------
+            #  Train Discriminator
+            # ---------------------
+            # latent_fake, reconstruction
+            latent_fake, _ = self.network.forward(batch)
+            latent_real = self.normal.sample(latent_fake.shape)
+
+            # Evaluate the input
+            d_real_prediction = self.network.discriminator.forward(latent_real)
+            d_fake_prediction = self.network.discriminator.forward(latent_fake)
+
+            # Train the discriminator
+            d_loss_real = mse_loss(d_real_prediction, torch.zeros(d_real_prediction.shape))
+            d_loss_fake = mse_loss(d_fake_prediction, torch.ones(d_fake_prediction.shape))
+
+            # Calculate the mean over both the real and the fake acc
+            d_loss = 0.5 * torch.add(d_loss_real, d_loss_fake)
+            return {'loss': d_loss}
+
+        elif optimizer_i == 1:
+            # ---------------------
+            #  Train AutoEncoder
+            # ---------------------
+            # z, x_hat
+            _, batch_hat = self.forward(batch)
+            loss = mse_loss(batch, batch_hat)
+            return {'loss': loss}
+
+        else:
+            raise RuntimeError('This should not have happened, catch me if u can.')
+
+
+if __name__ == '__main__':
+    raise PermissionError('Get out of here - never run this module')

networks/auto_encoder.py

@@ -0,0 +1,45 @@
+from .modules import *
+from torch.nn.functional import mse_loss
+from torch import Tensor
+
+
+#######################
+# Basic AE-Implementation
+class AutoEncoder(Module, ABC):
+
+    @property
+    def name(self):
+        return self.__class__.__name__
+
+    def __init__(self, dataParams, **kwargs):
+        super(AutoEncoder, self).__init__()
+        self.dataParams = dataParams
+        self.latent_dim = kwargs.get('latent_dim', 2)
+        self.encoder = Encoder(self.latent_dim)
+        self.decoder = Decoder(self.latent_dim, self.dataParams['features'])
+
+    def forward(self, batch: Tensor):
+        # Encoder
+        #  outputs, hidden (Batch, Timesteps aka. Size, Features / Latent Dim Size)
+        z = self.encoder(batch)
+        # Decoder
+        # First repeat the data accordingly to the batch size
+        z_repeatet = Repeater((batch.shape[0], self.dataParams['size'], -1))(z)
+        x_hat = self.decoder(z_repeatet)
+        return z, x_hat
+
+
+class AutoEncoderLightningOverrides:
+
+    def forward(self, x):
+        return self.network.forward(x)
+
+    def training_step(self, x, batch_nb):
+        # z, x_hat
+        _, x_hat = self.forward(x)
+        loss = mse_loss(x, x_hat)
+        return {'loss': loss}
+
+
+if __name__ == '__main__':
+    raise PermissionError('Get out of here - never run this module')

networks/basic_ae.py

@@ -1,73 +0,0 @@
-from torch.nn import Sequential, Linear, GRU, ReLU, Tanh
-from .modules import *
-from torch.nn.functional import mse_loss
-
-
-
-#######################
-# Basic AE-Implementation
-class BasicAE(Module, ABC):
-
-    @property
-    def name(self):
-        return self.__class__.__name__
-
-    def __init__(self, dataParams, **kwargs):
-        super(BasicAE, self).__init__()
-        self.dataParams = dataParams
-        self.latent_dim = kwargs.get('latent_dim', 2)
-        self.encoder = self._build_encoder()
-        self.decoder = self._build_decoder(out_shape=self.dataParams['features'])
-
-    def _build_encoder(self):
-        encoder = Sequential(
-            Linear(6, 100, bias=True),
-            ReLU(),
-            Linear(100, 10, bias=True),
-            ReLU()
-        )
-        gru = Sequential(
-            TimeDistributed(encoder),
-            GRU(10, 10, batch_first=True),
-            RNNOutputFilter(only_last=True),
-            Linear(10, self.latent_dim)
-        )
-        return gru
-
-    def _build_decoder(self, out_shape):
-        decoder = Sequential(
-            Linear(10, 100, bias=True),
-            ReLU(),
-            Linear(100, out_shape, bias=True),
-            Tanh()
-        )
-
-        gru = Sequential(
-            GRU(self.latent_dim, 10,batch_first=True),
-            RNNOutputFilter(),
-            TimeDistributed(decoder)
-        )
-        return gru
-
-    def forward(self, batch: torch.Tensor):
-        # Encoder
-        #  outputs, hidden (Batch, Timesteps aka. Size, Features / Latent Dim Size)
-        z = self.encoder(batch)
-        # Decoder
-        # First repeat the data accordingly to the batch size
-        z = Repeater((batch.shape[0], self.dataParams['size'], -1))(z)
-        x_hat = self.decoder(z)
-        return z, x_hat
-
-
-class AELightningOverrides:
-
-    def training_step(self, x, batch_nb):
-        # z, x_hat
-        _, x_hat = self.forward(x)
-        loss = mse_loss(x, x_hat)
-        return {'loss': loss}
-
-
-if __name__ == '__main__':
-    raise PermissionError('Get out of here - never run this module')

networks/basic_vae.py

@@ -1,81 +0,0 @@
-from torch.nn import Sequential, Linear, GRU, ReLU
-from .modules import *
-from torch.nn.functional import mse_loss
-
-
-#######################
-# Basic AE-Implementation
-class BasicVAE(Module, ABC):
-
-    @property
-    def name(self):
-        return self.__class__.__name__
-
-    def __init__(self, dataParams, **kwargs):
-        super(BasicVAE, self).__init__()
-        self.dataParams = dataParams
-        self.latent_dim = kwargs.get('latent_dim', 2)
-        self.encoder = self._build_encoder()
-        self.decoder = self._build_decoder(out_shape=self.dataParams['features'])
-        self.mu, self.logvar = Linear(10, self.latent_dim), Linear(10, self.latent_dim)
-
-    def _build_encoder(self):
-        linear_stack = Sequential(
-            Linear(6, 100, bias=True),
-            ReLU(),
-            Linear(100, 10, bias=True),
-            ReLU()
-        )
-        encoder = Sequential(
-            TimeDistributed(linear_stack),
-            GRU(10, 10, batch_first=True),
-            RNNOutputFilter(only_last=True),
-        )
-        return encoder
-
-    def reparameterize(self, mu, logvar):
-        # Lambda Layer, add gaussian noise
-        std = torch.exp(0.5*logvar)
-        eps = torch.randn_like(std)
-        return mu + eps*std
-
-    def _build_decoder(self, out_shape):
-        decoder = Sequential(
-            Linear(10, 100, bias=True),
-            ReLU(),
-            Linear(100, out_shape, bias=True),
-            ReLU()
-        )
-
-        sequential_decoder = Sequential(
-            GRU(self.latent_dim, 10, batch_first=True),
-            RNNOutputFilter(),
-            TimeDistributed(decoder)
-        )
-        return sequential_decoder
-
-    def forward(self, batch):
-        encoding = self.encoder(batch)
-        mu_logvar = self.mu(encoding), self.logvar(encoding)
-        z = self.reparameterize(*mu_logvar)
-        repeat = Repeater((batch.shape[0], self.dataParams['size'], -1))
-        x_hat = self.decoder(repeat(z))
-        return (x_hat, *mu_logvar)
-
-
-class VAELightningOverrides:
-
-    def training_step(self, x, batch_nb):
-        x_hat, logvar, mu = self.forward(x)
-        BCE = mse_loss(x_hat, x, reduction='mean')
-
-        # see Appendix B from VAE paper:
-        # Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014
-        # https://arxiv.org/abs/1312.6114
-        # 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)
-        KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
-        return {'loss': BCE + KLD}
-
-
-if __name__ == '__main__':
-    raise PermissionError('Get out of here - never run this module')

networks/modules.py

@@ -1,6 +1,6 @@
 import torch
 import pytorch_lightning as pl
-from torch.nn import Module
+from torch.nn import Module, Linear, ReLU, Tanh, Sigmoid, Dropout, GRU
 
 from abc import ABC, abstractmethod
 
@@ -85,9 +85,10 @@ class Repeater(Module):
         self.shape = shape
 
     def forward(self, x: torch.Tensor):
-        x.unsqueeze_(-2)
+        x = x.unsqueeze(-2)
         return x.expand(self.shape)
 
+
 class RNNOutputFilter(Module):
 
     def __init__(self, return_output=True, only_last=False):
@@ -101,5 +102,108 @@ class RNNOutputFilter(Module):
         return out if not self.only_last else out[:, -1, :]
 
 
+#######################
+# Network Modules
+# Generators, Decoders, Encoders, Discriminators
+class Discriminator(Module):
+
+    def __init__(self, latent_dim, dataParams, dropout=.0, activation=ReLU):
+        super(Discriminator, self).__init__()
+        self.dataParams = dataParams
+        self.latent_dim = latent_dim
+        self.l1 = Linear(self.latent_dim, self.dataParams['features'] * 10)
+        self.l2 = Linear(self.dataParams['features']*10, self.dataParams['features'] * 20)
+        self.lout = Linear(self.dataParams['features']*20, 1)
+        self.dropout = Dropout(dropout)
+        self.activation = activation()
+        self.sigmoid = Sigmoid()
+
+    def forward(self, x, **kwargs):
+        tensor = self.l1(x)
+        tensor = self.dropout(self.activation(tensor))
+        tensor = self.l2(tensor)
+        tensor = self.dropout(self.activation(tensor))
+        tensor = self.lout(tensor)
+        tensor = self.sigmoid(tensor)
+        return tensor
+
+
+class DecoderLinearStack(Module):
+
+    def __init__(self, out_shape):
+        super(DecoderLinearStack, self).__init__()
+        self.l1 = Linear(10, 100, bias=True)
+        self.l2 = Linear(100, out_shape, bias=True)
+        self.activation = ReLU()
+        self.activation_out = Tanh()
+
+    def forward(self, x):
+        tensor = self.l1(x)
+        tensor = self.activation(tensor)
+        tensor = self.l2(tensor)
+        tensor = self.activation_out(tensor)
+        return tensor
+
+
+class EncoderLinearStack(Module):
+
+    def __init__(self):
+        super(EncoderLinearStack, self).__init__()
+        self.l1 = Linear(6, 100, bias=True)
+        self.l2 = Linear(100, 10, bias=True)
+        self.activation = ReLU()
+
+    def forward(self, x):
+        tensor = self.l1(x)
+        tensor = self.activation(tensor)
+        tensor = self.l2(tensor)
+        tensor = self.activation(tensor)
+        return tensor
+
+
+class Encoder(Module):
+
+    def __init__(self, lat_dim, variational=False):
+        self.lat_dim = lat_dim
+        self.variational = variational
+
+        super(Encoder, self).__init__()
+        self.l_stack = TimeDistributed(EncoderLinearStack())
+        self.gru = GRU(10, 10, batch_first=True)
+        self.filter = RNNOutputFilter(only_last=True)
+        if variational:
+            self.mu = Linear(10, self.lat_dim)
+            self.logvar = Linear(10, self.lat_dim)
+        else:
+            self.lat_dim_layer = Linear(10, self.lat_dim)
+
+    def forward(self, x):
+        tensor = self.l_stack(x)
+        tensor = self.gru(tensor)
+        tensor = self.filter(tensor)
+        if self.variational:
+            tensor = self.mu(tensor), self.logvar(tensor)
+        else:
+            tensor = self.lat_dim_layer(tensor)
+        return tensor
+
+
+class Decoder(Module):
+
+    def __init__(self, latent_dim, *args, variational=False):
+        self.variational = variational
+        super(Decoder, self).__init__()
+        self.g = GRU(latent_dim, 10, batch_first=True)
+        self.filter = RNNOutputFilter()
+        self.l_stack = TimeDistributed(DecoderLinearStack(*args))
+        pass
+
+    def forward(self, x):
+        tensor = self.g(x)
+        tensor = self.filter(tensor)
+        tensor = self.l_stack(tensor)
+        return tensor
+
+
 if __name__ == '__main__':
     raise PermissionError('Get out of here - never run this module')

networks/variational_auto_encoder.py

@@ -0,0 +1,53 @@
+from .modules import *
+from torch.nn.functional import mse_loss
+
+
+#######################
+# Basic AE-Implementation
+class VariationalAutoEncoder(Module, ABC):
+
+    @property
+    def name(self):
+        return self.__class__.__name__
+
+    def __init__(self, dataParams, **kwargs):
+        super(VariationalAutoEncoder, self).__init__()
+        self.dataParams = dataParams
+        self.latent_dim = kwargs.get('latent_dim', 2)
+        self.encoder = Encoder(self.latent_dim, variational=True)
+        self.decoder = Decoder(self.latent_dim, self.dataParams['features'], variational=True)
+
+    @staticmethod
+    def reparameterize(mu, logvar):
+        # Lambda Layer, add gaussian noise
+        std = torch.exp(0.5*logvar)
+        eps = torch.randn_like(std)
+        return mu + eps*std
+
+    def forward(self, batch):
+        mu, logvar = self.encoder(batch)
+        z = self.reparameterize(mu, logvar)
+        repeat = Repeater((batch.shape[0], self.dataParams['size'], -1))
+        x_hat = self.decoder(repeat(z))
+        return x_hat, mu, logvar
+
+
+class VariationalAutoEncoderLightningOverrides:
+
+    def forward(self, x):
+        return self.network.forward(x)
+
+    def training_step(self, x, _):
+        x_hat, logvar, mu = self.forward(x)
+        BCE = mse_loss(x_hat, x, reduction='mean')
+
+        # see Appendix B from VAE paper:
+        # Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014
+        # https://arxiv.org/abs/1312.6114
+        # 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)
+        KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
+        return {'loss': BCE + KLD}
+
+
+if __name__ == '__main__':
+    raise PermissionError('Get out of here - never run this module')

run_basic_ae.py

@@ -1,41 +0,0 @@
-from networks.basic_ae import BasicAE, AELightningOverrides
-from networks.modules import LightningModule
-from torch.optim import Adam
-from torch.utils.data import DataLoader
-from pytorch_lightning import data_loader
-from dataset import DataContainer
-
-from torch.nn import BatchNorm1d
-from pytorch_lightning import Trainer
-
-
-class AEModel(AELightningOverrides, LightningModule):
-
-    def __init__(self, dataParams: dict):
-        super(AEModel, self).__init__()
-        self.dataParams = dataParams
-        # noinspection PyUnresolvedReferences
-        self.network = BasicAE(self.dataParams)
-
-
-    def configure_optimizers(self):
-        return [Adam(self.parameters(), lr=0.02)]
-
-
-    @data_loader
-    def tng_dataloader(self):
-        return DataLoader(DataContainer('data', **self.dataParams), shuffle=True, batch_size=100)
-
-
-    def forward(self, x):
-        return self.network.forward(x)
-
-
-if __name__ == '__main__':
-    features = 6
-    ae = AEModel(
-        dataParams=dict(refresh=False, size=5, step=5, features=features, transforms=[BatchNorm1d(features)])
-    )
-
-    trainer = Trainer()
-    trainer.fit(ae)

run_basic_vae.py

@@ -1,42 +0,0 @@
-from networks.basic_vae import BasicVAE, VAELightningOverrides
-from networks.modules import LightningModule
-import pytorch_lightning as pl
-from torch.nn.functional import mse_loss
-from torch.optim import Adam
-import torch
-from torch.nn import BatchNorm1d
-
-from torch.utils.data import DataLoader
-from dataset import DataContainer
-
-from pytorch_lightning import Trainer
-
-
-class AEModel(VAELightningOverrides, LightningModule):
-
-    def __init__(self, dataParams: dict):
-        super(AEModel, self).__init__()
-        self.dataParams = dataParams
-        # noinspection PyUnresolvedReferences
-        self.network = BasicVAE(self.dataParams)
-
-    def forward(self, x):
-        return self.network.forward(x)
-
-    def configure_optimizers(self):
-        # ToDo: Where do i get the Paramers from?
-        return [Adam(self.parameters(), lr=0.02)]
-
-    @pl.data_loader
-    def tng_dataloader(self):
-        return DataLoader(DataContainer('data', **self.dataParams), shuffle=True, batch_size=100)
-
-
-if __name__ == '__main__':
-    features = 6
-    ae = AEModel(
-        dataParams=dict(refresh=False, size=5, step=5, features=features, transforms=[BatchNorm1d(features)])
-    )
-
-    trainer = Trainer()
-    trainer.fit(ae)

run_models.py

@@ -0,0 +1,60 @@
+from networks.auto_encoder import *
+from networks.variational_auto_encoder import *
+from networks.adverserial_auto_encoder import *
+from networks.modules import LightningModule
+from torch.optim import Adam
+from torch.utils.data import DataLoader
+from pytorch_lightning import data_loader
+from dataset import DataContainer
+
+from torch.nn import BatchNorm1d
+from pytorch_lightning import Trainer
+
+
+# ToDo: How to implement this better?
+# other_classes = [AutoEncoder, AutoEncoderLightningOverrides]
+class Model(VariationalAutoEncoderLightningOverrides, LightningModule):
+
+    def __init__(self, dataParams: dict):
+        super(Model, self).__init__()
+        self.dataParams = dataParams
+        self.network = VariationalAutoEncoder(self.dataParams)
+
+    def configure_optimizers(self):
+        return [Adam(self.parameters(), lr=0.02)]
+
+    @data_loader
+    def tng_dataloader(self):
+        return DataLoader(DataContainer('data', **self.dataParams), shuffle=True, batch_size=100)
+
+
+class AdversarialModel(AdversarialAELightningOverrides, LightningModule):
+
+    def __init__(self, dataParams: dict):
+        super(AdversarialModel, self).__init__()
+        self.dataParams = dataParams
+        self.normal = Normal(0, 1)
+        self.network = AdversarialAutoEncoder(self.dataParams)
+        pass
+
+    # This is Fucked up, why do i need to put an additional empty list here?
+    def configure_optimizers(self):
+        return [Adam(self.network.discriminator.parameters(), lr=0.02),
+                Adam([*self.network.encoder.parameters(), *self.network.decoder.parameters()], lr=0.02)],\
+               []
+
+    @data_loader
+    def tng_dataloader(self):
+        return DataLoader(DataContainer('data', **self.dataParams), shuffle=True, batch_size=100)
+
+
+if __name__ == '__main__':
+    features = 6
+    ae = AdversarialModel(
+        dataParams=dict(refresh=False, size=5, step=5,
+                        features=features, transforms=[BatchNorm1d(features)]
+                        )
+    )
+
+    trainer = Trainer()
+    trainer.fit(ae)