Done: AE, VAE, AAE
ToDo: Double AAE, Visualization All Modularized
This commit is contained in:
Si11ium
parent
265c900f33
commit
f2cb9b7c42
6
.idea/ae_toolbox_torch.iml
generated
6
.idea/ae_toolbox_torch.iml
generated
@ -1,8 +1,10 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="PYTHON_MODULE" version="4">
|
||||
<component name="NewModuleRootManager">
|
||||
<content url="file://$MODULE_DIR$" />
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<content url="file://$MODULE_DIR$">
|
||||
<excludeFolder url="file://$MODULE_DIR$/data" />
|
||||
</content>
|
||||
<orderEntry type="jdk" jdkName="Python 3.7 (torch)" jdkType="Python SDK" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
</component>
|
||||
<component name="PyDocumentationSettings">
|
||||
|
||||
97
.idea/workspace.xml
generated
97
.idea/workspace.xml
generated
@ -1,16 +1,16 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ChangeListManager">
|
||||
<list default="true" id="5955480a-c876-43d5-afd7-8717f51f413e" name="Default Changelist" comment="">
|
||||
<change afterPath="$PROJECT_DIR$/.idea/dictionaries/illium.xml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/.idea/other.xml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/.idea/vcs.xml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/basic_ae_lightning_torch.py" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/data/dataset.py" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/networks/basic_ae.py" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/networks/modules.py" afterDir="false" />
|
||||
<list default="true" id="5955480a-c876-43d5-afd7-8717f51f413e" name="Default Changelist" comment="Lightning integration basic ae, dataloaders and dataset">
|
||||
<change afterPath="$PROJECT_DIR$/networks/adverserial_auto_encoder.py" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/.idea/ae_toolbox_torch.iml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/ae_toolbox_torch.iml" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/dataset.py" beforeDir="false" afterPath="$PROJECT_DIR$/dataset.py" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/networks/basic_ae.py" beforeDir="false" afterPath="$PROJECT_DIR$/networks/auto_encoder.py" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/networks/basic_vae.py" beforeDir="false" afterPath="$PROJECT_DIR$/networks/variational_auto_encoder.py" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/networks/modules.py" beforeDir="false" afterPath="$PROJECT_DIR$/networks/modules.py" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/run_basic_ae.py" beforeDir="false" afterPath="$PROJECT_DIR$/run_models.py" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/run_basic_vae.py" beforeDir="false" />
|
||||
</list>
|
||||
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
|
||||
<option name="SHOW_DIALOG" value="false" />
|
||||
@ -34,6 +34,7 @@
|
||||
<property name="ASKED_ADD_EXTERNAL_FILES" value="true" />
|
||||
<property name="SHARE_PROJECT_CONFIGURATION_FILES" value="true" />
|
||||
<property name="WebServerToolWindowFactoryState" value="false" />
|
||||
<property name="last_opened_file_path" value="$PROJECT_DIR$/networks" />
|
||||
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
|
||||
<property name="settings.editor.selected.configurable" value="pyconsole" />
|
||||
</component>
|
||||
@ -46,6 +47,16 @@
|
||||
</option>
|
||||
<option name="myShowDebugConsoleByDefault" value="true" />
|
||||
</component>
|
||||
<component name="RecentsManager">
|
||||
<key name="CopyFile.RECENT_KEYS">
|
||||
<recent name="C:\Users\illium\Google Drive\LMU\Research\ae_toolbox_torch\networks" />
|
||||
<recent name="C:\Users\illium\Google Drive\LMU\Research\ae_toolbox_torch\viz" />
|
||||
</key>
|
||||
<key name="MoveFile.RECENT_KEYS">
|
||||
<recent name="C:\Users\illium\Google Drive\LMU\Research\ae_toolbox_torch\data\processed" />
|
||||
<recent name="C:\Users\illium\Google Drive\LMU\Research\ae_toolbox_torch" />
|
||||
</key>
|
||||
</component>
|
||||
<component name="RunDashboard">
|
||||
<option name="ruleStates">
|
||||
<list>
|
||||
@ -58,7 +69,7 @@
|
||||
</list>
|
||||
</option>
|
||||
</component>
|
||||
<component name="RunManager" selected="Python.basic_ae_lightning_torch">
|
||||
<component name="RunManager" selected="Python.run_models">
|
||||
<configuration default="true" type="PythonConfigurationType" factoryName="Python">
|
||||
<module name="ae_toolbox_torch" />
|
||||
<option name="INTERPRETER_OPTIONS" value="" />
|
||||
@ -81,20 +92,17 @@
|
||||
<option name="INPUT_FILE" value="" />
|
||||
<method v="2" />
|
||||
</configuration>
|
||||
<configuration name="basic_ae_lightning_torch" type="PythonConfigurationType" factoryName="Python" temporary="true">
|
||||
<configuration name="run_basic_ae" type="PythonConfigurationType" factoryName="Python" temporary="true">
|
||||
<module name="ae_toolbox_torch" />
|
||||
<option name="INTERPRETER_OPTIONS" value="" />
|
||||
<option name="PARENT_ENVS" value="true" />
|
||||
<envs>
|
||||
<env name="PYTHONUNBUFFERED" value="1" />
|
||||
</envs>
|
||||
<option name="SDK_HOME" value="" />
|
||||
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
|
||||
<option name="IS_MODULE_SDK" value="true" />
|
||||
<option name="ADD_CONTENT_ROOTS" value="true" />
|
||||
<option name="ADD_SOURCE_ROOTS" value="true" />
|
||||
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
|
||||
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/basic_ae_lightning_torch.py" />
|
||||
<option name="SCRIPT_NAME" value="C:\Users\illium\Google Drive\LMU\Research\ae_toolbox_torch\run_models.py" />
|
||||
<option name="PARAMETERS" value="" />
|
||||
<option name="SHOW_COMMAND_LINE" value="true" />
|
||||
<option name="EMULATE_TERMINAL" value="false" />
|
||||
@ -103,20 +111,17 @@
|
||||
<option name="INPUT_FILE" value="" />
|
||||
<method v="2" />
|
||||
</configuration>
|
||||
<configuration name="dataset" type="PythonConfigurationType" factoryName="Python" temporary="true">
|
||||
<configuration name="run_models" type="PythonConfigurationType" factoryName="Python" temporary="true">
|
||||
<module name="ae_toolbox_torch" />
|
||||
<option name="INTERPRETER_OPTIONS" value="" />
|
||||
<option name="PARENT_ENVS" value="true" />
|
||||
<envs>
|
||||
<env name="PYTHONUNBUFFERED" value="1" />
|
||||
</envs>
|
||||
<option name="SDK_HOME" value="" />
|
||||
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/data" />
|
||||
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
|
||||
<option name="IS_MODULE_SDK" value="true" />
|
||||
<option name="ADD_CONTENT_ROOTS" value="true" />
|
||||
<option name="ADD_SOURCE_ROOTS" value="true" />
|
||||
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
|
||||
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/data/dataset.py" />
|
||||
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/run_models.py" />
|
||||
<option name="PARAMETERS" value="" />
|
||||
<option name="SHOW_COMMAND_LINE" value="true" />
|
||||
<option name="EMULATE_TERMINAL" value="false" />
|
||||
@ -125,10 +130,14 @@
|
||||
<option name="INPUT_FILE" value="" />
|
||||
<method v="2" />
|
||||
</configuration>
|
||||
<list>
|
||||
<item itemvalue="Python.run_basic_ae" />
|
||||
<item itemvalue="Python.run_models" />
|
||||
</list>
|
||||
<recent_temporary>
|
||||
<list>
|
||||
<item itemvalue="Python.basic_ae_lightning_torch" />
|
||||
<item itemvalue="Python.dataset" />
|
||||
<item itemvalue="Python.run_models" />
|
||||
<item itemvalue="Python.run_basic_ae" />
|
||||
</list>
|
||||
</recent_temporary>
|
||||
</component>
|
||||
@ -145,7 +154,23 @@
|
||||
<workItem from="1564587420277" duration="6891000" />
|
||||
<workItem from="1565364574595" duration="1092000" />
|
||||
<workItem from="1565592214301" duration="53660000" />
|
||||
<workItem from="1565793671730" duration="30373000" />
|
||||
</task>
|
||||
<task id="LOCAL-00001" summary="Lightning integration basic ae, dataloaders and dataset">
|
||||
<created>1565793753423</created>
|
||||
<option name="number" value="00001" />
|
||||
<option name="presentableId" value="LOCAL-00001" />
|
||||
<option name="project" value="LOCAL" />
|
||||
<updated>1565793753423</updated>
|
||||
</task>
|
||||
<task id="LOCAL-00002" summary="Lightning integration basic ae, dataloaders and dataset">
|
||||
<created>1565958589041</created>
|
||||
<option name="number" value="00002" />
|
||||
<option name="presentableId" value="LOCAL-00002" />
|
||||
<option name="project" value="LOCAL" />
|
||||
<updated>1565958589041</updated>
|
||||
</task>
|
||||
<option name="localTasksCounter" value="3" />
|
||||
<servers />
|
||||
</component>
|
||||
<component name="TypeScriptGeneratedFilesManager">
|
||||
@ -164,8 +189,34 @@
|
||||
</map>
|
||||
</option>
|
||||
</component>
|
||||
<component name="VcsManagerConfiguration">
|
||||
<MESSAGE value="Lightning integration basic ae, dataloaders and dataset" />
|
||||
<option name="LAST_COMMIT_MESSAGE" value="Lightning integration basic ae, dataloaders and dataset" />
|
||||
</component>
|
||||
<component name="XDebuggerManager">
|
||||
<breakpoint-manager>
|
||||
<breakpoints>
|
||||
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
|
||||
<url>file://$PROJECT_DIR$/run_models.py</url>
|
||||
<line>20</line>
|
||||
<option name="timeStamp" value="27" />
|
||||
</line-breakpoint>
|
||||
</breakpoints>
|
||||
<default-breakpoints>
|
||||
<breakpoint type="python-exception">
|
||||
<properties notifyOnTerminate="true" exception="BaseException">
|
||||
<option name="notifyOnTerminate" value="true" />
|
||||
</properties>
|
||||
</breakpoint>
|
||||
</default-breakpoints>
|
||||
</breakpoint-manager>
|
||||
</component>
|
||||
<component name="com.intellij.coverage.CoverageDataManagerImpl">
|
||||
<SUITE FILE_PATH="coverage/ae_toolbox_torch$basic_ae_lightning_torch.coverage" NAME="basic_ae_lightning_torch Coverage Results" MODIFIED="1565790288699" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
|
||||
<SUITE FILE_PATH="coverage/ae_toolbox_torch$basic_ae_lightning_torch.coverage" NAME="basic_ae_lightning_torch Coverage Results" MODIFIED="1565937164457" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
|
||||
<SUITE FILE_PATH="coverage/ae_toolbox_torch$basic_ae_lightning.coverage" NAME="basic_ae_lightning Coverage Results" MODIFIED="1565956491159" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
|
||||
<SUITE FILE_PATH="coverage/ae_toolbox_torch$basic_vae_lightning.coverage" NAME="basic_vae_lightning Coverage Results" MODIFIED="1565955311009" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
|
||||
<SUITE FILE_PATH="coverage/ae_toolbox_torch$run_basic_ae.coverage" NAME="run_basic_ae Coverage Results" MODIFIED="1565966122607" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
|
||||
<SUITE FILE_PATH="coverage/ae_toolbox_torch$run_models.coverage" NAME="run_models Coverage Results" MODIFIED="1565987843914" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
|
||||
<SUITE FILE_PATH="coverage/ae_toolbox_torch$dataset.coverage" NAME="dataset Coverage Results" MODIFIED="1565772669750" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/data" />
|
||||
</component>
|
||||
</project>
|
||||
@ -176,7 +176,6 @@ class Trajectories(Dataset):
|
||||
data[:, 2:] = transformation(data[:, 2:])
|
||||
return data
|
||||
|
||||
|
||||
def __iter__(self):
|
||||
for i in range(len(self)):
|
||||
yield self[i]
|
||||
|
||||
66
networks/adverserial_auto_encoder.py
Normal file
66
networks/adverserial_auto_encoder.py
Normal file
@ -0,0 +1,66 @@
|
||||
from networks.auto_encoder import AutoEncoder
|
||||
from torch.nn.functional import mse_loss
|
||||
from torch.nn import Sequential, Linear, ReLU, Dropout, Sigmoid
|
||||
from torch.distributions import Normal
|
||||
from networks.modules import *
|
||||
import torch
|
||||
|
||||
|
||||
class AdversarialAutoEncoder(AutoEncoder):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(AdversarialAutoEncoder, self).__init__(*args, **kwargs)
|
||||
self.discriminator = Discriminator(self.latent_dim, self.dataParams)
|
||||
|
||||
def forward(self, batch):
|
||||
# Encoder
|
||||
# outputs, hidden (Batch, Timesteps aka. Size, Features / Latent Dim Size)
|
||||
z = self.encoder(batch)
|
||||
# Decoder
|
||||
# First repeat the data accordingly to the batch size
|
||||
z_repeatet = Repeater((batch.shape[0], self.dataParams['size'], -1))(z)
|
||||
x_hat = self.decoder(z_repeatet)
|
||||
return z, x_hat
|
||||
|
||||
|
||||
class AdversarialAELightningOverrides:
|
||||
|
||||
def forward(self, x):
|
||||
return self.network.forward(x)
|
||||
|
||||
def training_step(self, batch, _, optimizer_i):
|
||||
if optimizer_i == 0:
|
||||
# ---------------------
|
||||
# Train Discriminator
|
||||
# ---------------------
|
||||
# latent_fake, reconstruction
|
||||
latent_fake, _ = self.network.forward(batch)
|
||||
latent_real = self.normal.sample(latent_fake.shape)
|
||||
|
||||
# Evaluate the input
|
||||
d_real_prediction = self.network.discriminator.forward(latent_real)
|
||||
d_fake_prediction = self.network.discriminator.forward(latent_fake)
|
||||
|
||||
# Train the discriminator
|
||||
d_loss_real = mse_loss(d_real_prediction, torch.zeros(d_real_prediction.shape))
|
||||
d_loss_fake = mse_loss(d_fake_prediction, torch.ones(d_fake_prediction.shape))
|
||||
|
||||
# Calculate the mean over both the real and the fake acc
|
||||
d_loss = 0.5 * torch.add(d_loss_real, d_loss_fake)
|
||||
return {'loss': d_loss}
|
||||
|
||||
elif optimizer_i == 1:
|
||||
# ---------------------
|
||||
# Train AutoEncoder
|
||||
# ---------------------
|
||||
# z, x_hat
|
||||
_, batch_hat = self.forward(batch)
|
||||
loss = mse_loss(batch, batch_hat)
|
||||
return {'loss': loss}
|
||||
|
||||
else:
|
||||
raise RuntimeError('This should not have happened, catch me if u can.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise PermissionError('Get out of here - never run this module')
|
||||
45
networks/auto_encoder.py
Normal file
45
networks/auto_encoder.py
Normal file
@ -0,0 +1,45 @@
|
||||
from .modules import *
|
||||
from torch.nn.functional import mse_loss
|
||||
from torch import Tensor
|
||||
|
||||
|
||||
#######################
|
||||
# Basic AE-Implementation
|
||||
class AutoEncoder(Module, ABC):
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.__class__.__name__
|
||||
|
||||
def __init__(self, dataParams, **kwargs):
|
||||
super(AutoEncoder, self).__init__()
|
||||
self.dataParams = dataParams
|
||||
self.latent_dim = kwargs.get('latent_dim', 2)
|
||||
self.encoder = Encoder(self.latent_dim)
|
||||
self.decoder = Decoder(self.latent_dim, self.dataParams['features'])
|
||||
|
||||
def forward(self, batch: Tensor):
|
||||
# Encoder
|
||||
# outputs, hidden (Batch, Timesteps aka. Size, Features / Latent Dim Size)
|
||||
z = self.encoder(batch)
|
||||
# Decoder
|
||||
# First repeat the data accordingly to the batch size
|
||||
z_repeatet = Repeater((batch.shape[0], self.dataParams['size'], -1))(z)
|
||||
x_hat = self.decoder(z_repeatet)
|
||||
return z, x_hat
|
||||
|
||||
|
||||
class AutoEncoderLightningOverrides:
|
||||
|
||||
def forward(self, x):
|
||||
return self.network.forward(x)
|
||||
|
||||
def training_step(self, x, batch_nb):
|
||||
# z, x_hat
|
||||
_, x_hat = self.forward(x)
|
||||
loss = mse_loss(x, x_hat)
|
||||
return {'loss': loss}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise PermissionError('Get out of here - never run this module')
|
||||
@ -1,73 +0,0 @@
|
||||
from torch.nn import Sequential, Linear, GRU, ReLU, Tanh
|
||||
from .modules import *
|
||||
from torch.nn.functional import mse_loss
|
||||
|
||||
|
||||
|
||||
#######################
|
||||
# Basic AE-Implementation
|
||||
class BasicAE(Module, ABC):
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.__class__.__name__
|
||||
|
||||
def __init__(self, dataParams, **kwargs):
|
||||
super(BasicAE, self).__init__()
|
||||
self.dataParams = dataParams
|
||||
self.latent_dim = kwargs.get('latent_dim', 2)
|
||||
self.encoder = self._build_encoder()
|
||||
self.decoder = self._build_decoder(out_shape=self.dataParams['features'])
|
||||
|
||||
def _build_encoder(self):
|
||||
encoder = Sequential(
|
||||
Linear(6, 100, bias=True),
|
||||
ReLU(),
|
||||
Linear(100, 10, bias=True),
|
||||
ReLU()
|
||||
)
|
||||
gru = Sequential(
|
||||
TimeDistributed(encoder),
|
||||
GRU(10, 10, batch_first=True),
|
||||
RNNOutputFilter(only_last=True),
|
||||
Linear(10, self.latent_dim)
|
||||
)
|
||||
return gru
|
||||
|
||||
def _build_decoder(self, out_shape):
|
||||
decoder = Sequential(
|
||||
Linear(10, 100, bias=True),
|
||||
ReLU(),
|
||||
Linear(100, out_shape, bias=True),
|
||||
Tanh()
|
||||
)
|
||||
|
||||
gru = Sequential(
|
||||
GRU(self.latent_dim, 10,batch_first=True),
|
||||
RNNOutputFilter(),
|
||||
TimeDistributed(decoder)
|
||||
)
|
||||
return gru
|
||||
|
||||
def forward(self, batch: torch.Tensor):
|
||||
# Encoder
|
||||
# outputs, hidden (Batch, Timesteps aka. Size, Features / Latent Dim Size)
|
||||
z = self.encoder(batch)
|
||||
# Decoder
|
||||
# First repeat the data accordingly to the batch size
|
||||
z = Repeater((batch.shape[0], self.dataParams['size'], -1))(z)
|
||||
x_hat = self.decoder(z)
|
||||
return z, x_hat
|
||||
|
||||
|
||||
class AELightningOverrides:
|
||||
|
||||
def training_step(self, x, batch_nb):
|
||||
# z, x_hat
|
||||
_, x_hat = self.forward(x)
|
||||
loss = mse_loss(x, x_hat)
|
||||
return {'loss': loss}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise PermissionError('Get out of here - never run this module')
|
||||
@ -1,81 +0,0 @@
|
||||
from torch.nn import Sequential, Linear, GRU, ReLU
|
||||
from .modules import *
|
||||
from torch.nn.functional import mse_loss
|
||||
|
||||
|
||||
#######################
|
||||
# Basic AE-Implementation
|
||||
class BasicVAE(Module, ABC):
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.__class__.__name__
|
||||
|
||||
def __init__(self, dataParams, **kwargs):
|
||||
super(BasicVAE, self).__init__()
|
||||
self.dataParams = dataParams
|
||||
self.latent_dim = kwargs.get('latent_dim', 2)
|
||||
self.encoder = self._build_encoder()
|
||||
self.decoder = self._build_decoder(out_shape=self.dataParams['features'])
|
||||
self.mu, self.logvar = Linear(10, self.latent_dim), Linear(10, self.latent_dim)
|
||||
|
||||
def _build_encoder(self):
|
||||
linear_stack = Sequential(
|
||||
Linear(6, 100, bias=True),
|
||||
ReLU(),
|
||||
Linear(100, 10, bias=True),
|
||||
ReLU()
|
||||
)
|
||||
encoder = Sequential(
|
||||
TimeDistributed(linear_stack),
|
||||
GRU(10, 10, batch_first=True),
|
||||
RNNOutputFilter(only_last=True),
|
||||
)
|
||||
return encoder
|
||||
|
||||
def reparameterize(self, mu, logvar):
|
||||
# Lambda Layer, add gaussian noise
|
||||
std = torch.exp(0.5*logvar)
|
||||
eps = torch.randn_like(std)
|
||||
return mu + eps*std
|
||||
|
||||
def _build_decoder(self, out_shape):
|
||||
decoder = Sequential(
|
||||
Linear(10, 100, bias=True),
|
||||
ReLU(),
|
||||
Linear(100, out_shape, bias=True),
|
||||
ReLU()
|
||||
)
|
||||
|
||||
sequential_decoder = Sequential(
|
||||
GRU(self.latent_dim, 10, batch_first=True),
|
||||
RNNOutputFilter(),
|
||||
TimeDistributed(decoder)
|
||||
)
|
||||
return sequential_decoder
|
||||
|
||||
def forward(self, batch):
|
||||
encoding = self.encoder(batch)
|
||||
mu_logvar = self.mu(encoding), self.logvar(encoding)
|
||||
z = self.reparameterize(*mu_logvar)
|
||||
repeat = Repeater((batch.shape[0], self.dataParams['size'], -1))
|
||||
x_hat = self.decoder(repeat(z))
|
||||
return (x_hat, *mu_logvar)
|
||||
|
||||
|
||||
class VAELightningOverrides:
|
||||
|
||||
def training_step(self, x, batch_nb):
|
||||
x_hat, logvar, mu = self.forward(x)
|
||||
BCE = mse_loss(x_hat, x, reduction='mean')
|
||||
|
||||
# see Appendix B from VAE paper:
|
||||
# Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014
|
||||
# https://arxiv.org/abs/1312.6114
|
||||
# 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)
|
||||
KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
|
||||
return {'loss': BCE + KLD}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise PermissionError('Get out of here - never run this module')
|
||||
@ -1,6 +1,6 @@
|
||||
import torch
|
||||
import pytorch_lightning as pl
|
||||
from torch.nn import Module
|
||||
from torch.nn import Module, Linear, ReLU, Tanh, Sigmoid, Dropout, GRU
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
@ -85,9 +85,10 @@ class Repeater(Module):
|
||||
self.shape = shape
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
x.unsqueeze_(-2)
|
||||
x = x.unsqueeze(-2)
|
||||
return x.expand(self.shape)
|
||||
|
||||
|
||||
class RNNOutputFilter(Module):
|
||||
|
||||
def __init__(self, return_output=True, only_last=False):
|
||||
@ -101,5 +102,108 @@ class RNNOutputFilter(Module):
|
||||
return out if not self.only_last else out[:, -1, :]
|
||||
|
||||
|
||||
#######################
|
||||
# Network Modules
|
||||
# Generators, Decoders, Encoders, Discriminators
|
||||
class Discriminator(Module):
|
||||
|
||||
def __init__(self, latent_dim, dataParams, dropout=.0, activation=ReLU):
|
||||
super(Discriminator, self).__init__()
|
||||
self.dataParams = dataParams
|
||||
self.latent_dim = latent_dim
|
||||
self.l1 = Linear(self.latent_dim, self.dataParams['features'] * 10)
|
||||
self.l2 = Linear(self.dataParams['features']*10, self.dataParams['features'] * 20)
|
||||
self.lout = Linear(self.dataParams['features']*20, 1)
|
||||
self.dropout = Dropout(dropout)
|
||||
self.activation = activation()
|
||||
self.sigmoid = Sigmoid()
|
||||
|
||||
def forward(self, x, **kwargs):
|
||||
tensor = self.l1(x)
|
||||
tensor = self.dropout(self.activation(tensor))
|
||||
tensor = self.l2(tensor)
|
||||
tensor = self.dropout(self.activation(tensor))
|
||||
tensor = self.lout(tensor)
|
||||
tensor = self.sigmoid(tensor)
|
||||
return tensor
|
||||
|
||||
|
||||
class DecoderLinearStack(Module):
|
||||
|
||||
def __init__(self, out_shape):
|
||||
super(DecoderLinearStack, self).__init__()
|
||||
self.l1 = Linear(10, 100, bias=True)
|
||||
self.l2 = Linear(100, out_shape, bias=True)
|
||||
self.activation = ReLU()
|
||||
self.activation_out = Tanh()
|
||||
|
||||
def forward(self, x):
|
||||
tensor = self.l1(x)
|
||||
tensor = self.activation(tensor)
|
||||
tensor = self.l2(tensor)
|
||||
tensor = self.activation_out(tensor)
|
||||
return tensor
|
||||
|
||||
|
||||
class EncoderLinearStack(Module):
|
||||
|
||||
def __init__(self):
|
||||
super(EncoderLinearStack, self).__init__()
|
||||
self.l1 = Linear(6, 100, bias=True)
|
||||
self.l2 = Linear(100, 10, bias=True)
|
||||
self.activation = ReLU()
|
||||
|
||||
def forward(self, x):
|
||||
tensor = self.l1(x)
|
||||
tensor = self.activation(tensor)
|
||||
tensor = self.l2(tensor)
|
||||
tensor = self.activation(tensor)
|
||||
return tensor
|
||||
|
||||
|
||||
class Encoder(Module):
|
||||
|
||||
def __init__(self, lat_dim, variational=False):
|
||||
self.lat_dim = lat_dim
|
||||
self.variational = variational
|
||||
|
||||
super(Encoder, self).__init__()
|
||||
self.l_stack = TimeDistributed(EncoderLinearStack())
|
||||
self.gru = GRU(10, 10, batch_first=True)
|
||||
self.filter = RNNOutputFilter(only_last=True)
|
||||
if variational:
|
||||
self.mu = Linear(10, self.lat_dim)
|
||||
self.logvar = Linear(10, self.lat_dim)
|
||||
else:
|
||||
self.lat_dim_layer = Linear(10, self.lat_dim)
|
||||
|
||||
def forward(self, x):
|
||||
tensor = self.l_stack(x)
|
||||
tensor = self.gru(tensor)
|
||||
tensor = self.filter(tensor)
|
||||
if self.variational:
|
||||
tensor = self.mu(tensor), self.logvar(tensor)
|
||||
else:
|
||||
tensor = self.lat_dim_layer(tensor)
|
||||
return tensor
|
||||
|
||||
|
||||
class Decoder(Module):
|
||||
|
||||
def __init__(self, latent_dim, *args, variational=False):
|
||||
self.variational = variational
|
||||
super(Decoder, self).__init__()
|
||||
self.g = GRU(latent_dim, 10, batch_first=True)
|
||||
self.filter = RNNOutputFilter()
|
||||
self.l_stack = TimeDistributed(DecoderLinearStack(*args))
|
||||
pass
|
||||
|
||||
def forward(self, x):
|
||||
tensor = self.g(x)
|
||||
tensor = self.filter(tensor)
|
||||
tensor = self.l_stack(tensor)
|
||||
return tensor
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise PermissionError('Get out of here - never run this module')
|
||||
|
||||
53
networks/variational_auto_encoder.py
Normal file
53
networks/variational_auto_encoder.py
Normal file
@ -0,0 +1,53 @@
|
||||
from .modules import *
|
||||
from torch.nn.functional import mse_loss
|
||||
|
||||
|
||||
#######################
|
||||
# Basic AE-Implementation
|
||||
class VariationalAutoEncoder(Module, ABC):
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.__class__.__name__
|
||||
|
||||
def __init__(self, dataParams, **kwargs):
|
||||
super(VariationalAutoEncoder, self).__init__()
|
||||
self.dataParams = dataParams
|
||||
self.latent_dim = kwargs.get('latent_dim', 2)
|
||||
self.encoder = Encoder(self.latent_dim, variational=True)
|
||||
self.decoder = Decoder(self.latent_dim, self.dataParams['features'], variational=True)
|
||||
|
||||
@staticmethod
|
||||
def reparameterize(mu, logvar):
|
||||
# Lambda Layer, add gaussian noise
|
||||
std = torch.exp(0.5*logvar)
|
||||
eps = torch.randn_like(std)
|
||||
return mu + eps*std
|
||||
|
||||
def forward(self, batch):
|
||||
mu, logvar = self.encoder(batch)
|
||||
z = self.reparameterize(mu, logvar)
|
||||
repeat = Repeater((batch.shape[0], self.dataParams['size'], -1))
|
||||
x_hat = self.decoder(repeat(z))
|
||||
return x_hat, mu, logvar
|
||||
|
||||
|
||||
class VariationalAutoEncoderLightningOverrides:
|
||||
|
||||
def forward(self, x):
|
||||
return self.network.forward(x)
|
||||
|
||||
def training_step(self, x, _):
|
||||
x_hat, logvar, mu = self.forward(x)
|
||||
BCE = mse_loss(x_hat, x, reduction='mean')
|
||||
|
||||
# see Appendix B from VAE paper:
|
||||
# Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014
|
||||
# https://arxiv.org/abs/1312.6114
|
||||
# 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)
|
||||
KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
|
||||
return {'loss': BCE + KLD}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise PermissionError('Get out of here - never run this module')
|
||||
@ -1,41 +0,0 @@
|
||||
from networks.basic_ae import BasicAE, AELightningOverrides
|
||||
from networks.modules import LightningModule
|
||||
from torch.optim import Adam
|
||||
from torch.utils.data import DataLoader
|
||||
from pytorch_lightning import data_loader
|
||||
from dataset import DataContainer
|
||||
|
||||
from torch.nn import BatchNorm1d
|
||||
from pytorch_lightning import Trainer
|
||||
|
||||
|
||||
class AEModel(AELightningOverrides, LightningModule):
|
||||
|
||||
def __init__(self, dataParams: dict):
|
||||
super(AEModel, self).__init__()
|
||||
self.dataParams = dataParams
|
||||
# noinspection PyUnresolvedReferences
|
||||
self.network = BasicAE(self.dataParams)
|
||||
|
||||
|
||||
def configure_optimizers(self):
|
||||
return [Adam(self.parameters(), lr=0.02)]
|
||||
|
||||
|
||||
@data_loader
|
||||
def tng_dataloader(self):
|
||||
return DataLoader(DataContainer('data', **self.dataParams), shuffle=True, batch_size=100)
|
||||
|
||||
|
||||
def forward(self, x):
|
||||
return self.network.forward(x)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
features = 6
|
||||
ae = AEModel(
|
||||
dataParams=dict(refresh=False, size=5, step=5, features=features, transforms=[BatchNorm1d(features)])
|
||||
)
|
||||
|
||||
trainer = Trainer()
|
||||
trainer.fit(ae)
|
||||
@ -1,42 +0,0 @@
|
||||
from networks.basic_vae import BasicVAE, VAELightningOverrides
|
||||
from networks.modules import LightningModule
|
||||
import pytorch_lightning as pl
|
||||
from torch.nn.functional import mse_loss
|
||||
from torch.optim import Adam
|
||||
import torch
|
||||
from torch.nn import BatchNorm1d
|
||||
|
||||
from torch.utils.data import DataLoader
|
||||
from dataset import DataContainer
|
||||
|
||||
from pytorch_lightning import Trainer
|
||||
|
||||
|
||||
class AEModel(VAELightningOverrides, LightningModule):
|
||||
|
||||
def __init__(self, dataParams: dict):
|
||||
super(AEModel, self).__init__()
|
||||
self.dataParams = dataParams
|
||||
# noinspection PyUnresolvedReferences
|
||||
self.network = BasicVAE(self.dataParams)
|
||||
|
||||
def forward(self, x):
|
||||
return self.network.forward(x)
|
||||
|
||||
def configure_optimizers(self):
|
||||
# ToDo: Where do i get the Paramers from?
|
||||
return [Adam(self.parameters(), lr=0.02)]
|
||||
|
||||
@pl.data_loader
|
||||
def tng_dataloader(self):
|
||||
return DataLoader(DataContainer('data', **self.dataParams), shuffle=True, batch_size=100)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
features = 6
|
||||
ae = AEModel(
|
||||
dataParams=dict(refresh=False, size=5, step=5, features=features, transforms=[BatchNorm1d(features)])
|
||||
)
|
||||
|
||||
trainer = Trainer()
|
||||
trainer.fit(ae)
|
||||
60
run_models.py
Normal file
60
run_models.py
Normal file
@ -0,0 +1,60 @@
|
||||
from networks.auto_encoder import *
|
||||
from networks.variational_auto_encoder import *
|
||||
from networks.adverserial_auto_encoder import *
|
||||
from networks.modules import LightningModule
|
||||
from torch.optim import Adam
|
||||
from torch.utils.data import DataLoader
|
||||
from pytorch_lightning import data_loader
|
||||
from dataset import DataContainer
|
||||
|
||||
from torch.nn import BatchNorm1d
|
||||
from pytorch_lightning import Trainer
|
||||
|
||||
|
||||
# ToDo: How to implement this better?
|
||||
# other_classes = [AutoEncoder, AutoEncoderLightningOverrides]
|
||||
class Model(VariationalAutoEncoderLightningOverrides, LightningModule):
|
||||
|
||||
def __init__(self, dataParams: dict):
|
||||
super(Model, self).__init__()
|
||||
self.dataParams = dataParams
|
||||
self.network = VariationalAutoEncoder(self.dataParams)
|
||||
|
||||
def configure_optimizers(self):
|
||||
return [Adam(self.parameters(), lr=0.02)]
|
||||
|
||||
@data_loader
|
||||
def tng_dataloader(self):
|
||||
return DataLoader(DataContainer('data', **self.dataParams), shuffle=True, batch_size=100)
|
||||
|
||||
|
||||
class AdversarialModel(AdversarialAELightningOverrides, LightningModule):
|
||||
|
||||
def __init__(self, dataParams: dict):
|
||||
super(AdversarialModel, self).__init__()
|
||||
self.dataParams = dataParams
|
||||
self.normal = Normal(0, 1)
|
||||
self.network = AdversarialAutoEncoder(self.dataParams)
|
||||
pass
|
||||
|
||||
# This is Fucked up, why do i need to put an additional empty list here?
|
||||
def configure_optimizers(self):
|
||||
return [Adam(self.network.discriminator.parameters(), lr=0.02),
|
||||
Adam([*self.network.encoder.parameters(), *self.network.decoder.parameters()], lr=0.02)],\
|
||||
[]
|
||||
|
||||
@data_loader
|
||||
def tng_dataloader(self):
|
||||
return DataLoader(DataContainer('data', **self.dataParams), shuffle=True, batch_size=100)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
features = 6
|
||||
ae = AdversarialModel(
|
||||
dataParams=dict(refresh=False, size=5, step=5,
|
||||
features=features, transforms=[BatchNorm1d(features)]
|
||||
)
|
||||
)
|
||||
|
||||
trainer = Trainer()
|
||||
trainer.fit(ae)
|
||||
Loading…
x
Reference in New Issue
Block a user