steffen/ml_lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Templates finalized

Browse Source
This commit is contained in:
Steffen Illium 2020-05-17 22:05:20 +02:00
parent 398504aee6
commit fc93f71608
20 changed files with 459 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -1 +1,6 @@
/.idea/
# my own stuff
/data
/.idea
/ml_lib

0
examples/__init__.py → _templates/__init__.py
View File

0
_templates/new_project/__init__.py Normal file
View File

57
_templates/new_project/_parameters.py Normal file
View File

@ -0,0 +1,57 @@
# Imports
# =============================================================================
import os
from distutils.util import strtobool
from argparse import ArgumentParser, Namespace
# Parameter Configuration
# =============================================================================
# Argument Parser
main_arg_parser = ArgumentParser(description="parser for fast-neural-style")
# Main Parameters
main_arg_parser.add_argument("--main_debug", type=strtobool, default=False, help="")
main_arg_parser.add_argument("--main_eval", type=strtobool, default=True, help="")
main_arg_parser.add_argument("--main_seed", type=int, default=69, help="")
# Project
main_arg_parser.add_argument("--project_name", type=str, default='traj-gen', help="")
main_arg_parser.add_argument("--project_owner", type=str, default='si11ium', help="")
main_arg_parser.add_argument("--project_neptune_key", type=str, default=os.getenv('NEPTUNE_KEY'), help="")
# Data Parameters
main_arg_parser.add_argument("--data_worker", type=int, default=10, help="")
main_arg_parser.add_argument("--data_dataset_length", type=int, default=10000, help="")
main_arg_parser.add_argument("--data_root", type=str, default='data', help="")
main_arg_parser.add_argument("--data_additional_resource_root", type=str, default='res/resource/root', help="")
main_arg_parser.add_argument("--data_use_preprocessed", type=strtobool, default=True, help="")
# Transformations
main_arg_parser.add_argument("--transformations_to_tensor", type=strtobool, default=False, help="")
main_arg_parser.add_argument("--transformations_normalize", type=strtobool, default=False, help="")
# Transformations
main_arg_parser.add_argument("--train_outpath", type=str, default="output", help="")
main_arg_parser.add_argument("--train_version", type=strtobool, required=False, help="")
main_arg_parser.add_argument("--train_epochs", type=int, default=500, help="")
main_arg_parser.add_argument("--train_batch_size", type=int, default=200, help="")
main_arg_parser.add_argument("--train_lr", type=float, default=1e-3, help="")
main_arg_parser.add_argument("--train_num_sanity_val_steps", type=int, default=0, help="")
# Model
main_arg_parser.add_argument("--model_type", type=str, default="CNNRouteGenerator", help="")
main_arg_parser.add_argument("--model_activation", type=str, default="leaky_relu", help="")
main_arg_parser.add_argument("--model_use_bias", type=strtobool, default=True, help="")
main_arg_parser.add_argument("--model_use_norm", type=strtobool, default=False, help="")
main_arg_parser.add_argument("--model_dropout", type=float, default=0.00, help="")
# Model 2: Layer Specific Stuff
main_arg_parser.add_argument("--model_filters", type=str, default="[16, 32, 64]", help="")
main_arg_parser.add_argument("--model_features", type=int, default=16, help="")
# Parse it
args: Namespace = main_arg_parser.parse_args()
if __name__ == '__main__':
pass

6
_templates/new_project/datasets/template_dataset.py Normal file
View File

@ -0,0 +1,6 @@
from torch.utils.data import Dataset
class TemplateDataset(Dataset):
def __init__(self, *args, **kwargs):
super(TemplateDataset, self).__init__()

86
_templates/new_project/main.py Normal file
View File

@ -0,0 +1,86 @@
# Imports
# =============================================================================
import warnings
import torch
from pytorch_lightning import Trainer
from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping
from modules.utils import LightningBaseModule
from utils.config import Config
from utils.logging import Logger
from utils.model_io import SavedLightningModels
warnings.filterwarnings('ignore', category=FutureWarning)
warnings.filterwarnings('ignore', category=UserWarning)
def run_lightning_loop(config_obj):
# Logging
# ================================================================================
# Logger
with Logger(config_obj) as logger:
# Callbacks
# =============================================================================
# Checkpoint Saving
checkpoint_callback = ModelCheckpoint(
filepath=str(logger.log_dir / 'ckpt_weights'),
verbose=True, save_top_k=0,
)
# =============================================================================
# Early Stopping
# TODO: For This to work, one must set a validation step and End Eval and Score
early_stopping_callback = EarlyStopping(
monitor='val_loss',
min_delta=0.0,
patience=0,
)
# Model
# =============================================================================
# Init
model: LightningBaseModule = config_obj.model_class(config_obj.model_paramters)
model.init_weights(torch.nn.init.xavier_normal_)
if model.name == 'CNNRouteGeneratorDiscriminated':
# ToDo: Make this dependent on the used seed
path = logger.outpath / 'classifier_cnn' / 'version_0'
disc_model = SavedLightningModels.load_checkpoint(path).restore()
model.set_discriminator(disc_model)
# Trainer
# =============================================================================
trainer = Trainer(max_epochs=config_obj.train.epochs,
show_progress_bar=True,
weights_save_path=logger.log_dir,
gpus=[0] if torch.cuda.is_available() else None,
check_val_every_n_epoch=10,
# num_sanity_val_steps=config_obj.train.num_sanity_val_steps,
# row_log_interval=(model.n_train_batches * 0.1), # TODO: Better Value / Setting
# log_save_interval=(model.n_train_batches * 0.2), # TODO: Better Value / Setting
checkpoint_callback=checkpoint_callback,
logger=logger,
fast_dev_run=config_obj.main.debug,
early_stop_callback=None
)
# Train It
trainer.fit(model)
# Save the last state & all parameters
trainer.save_checkpoint(logger.log_dir / 'weights.ckpt')
model.save_to_disk(logger.log_dir)
# Evaluate It
if config_obj.main.eval:
trainer.test()
return model
if __name__ == "__main__":
from _templates.new_project._parameters import args
config = Config.read_namespace(args)
trained_model = run_lightning_loop(config)

4
examples/multi_run.py → _templates/new_project/multi_run.py
View File

@ -1,6 +1,6 @@
import warnings
from ml_lib.utils.config import Config
from utils.config import Config
warnings.filterwarnings('ignore', category=FutureWarning)
warnings.filterwarnings('ignore', category=UserWarning)
@ -8,7 +8,7 @@ warnings.filterwarnings('ignore', category=UserWarning)
# Imports
# =============================================================================
from main import run_lightning_loop, args
from _templates.new_project.main import run_lightning_loop, args
if __name__ == '__main__':

0
_templates/new_project/utils/__init__.py Normal file
View File

172
_templates/new_project/utils/module_mixins.py Normal file
View File

@ -0,0 +1,172 @@
from collections import defaultdict
from abc import ABC
from argparse import Namespace
import torch
from torch import nn
from torch.optim import Adam
from torch.utils.data import DataLoader
from torchcontrib.optim import SWA
from torchvision.transforms import Compose
from _templates.new_project.datasets.template_dataset import TemplateDataset
from audio_toolset.audio_io import NormalizeLocal
from modules.utils import LightningBaseModule
from utils.transforms import ToTensor
from _templates.new_project.utils.project_config import GlobalVar as GlobalVars
class BaseOptimizerMixin:
def configure_optimizers(self):
assert isinstance(self, LightningBaseModule)
opt = Adam(params=self.parameters(), lr=self.params.lr, weight_decay=self.params.weight_decay)
if self.params.sto_weight_avg:
# TODO: Make this glabaly available.
opt = SWA(opt, swa_start=10, swa_freq=5, swa_lr=0.05)
return opt
def on_train_end(self):
assert isinstance(self, LightningBaseModule)
for opt in self.trainer.optimizers:
if isinstance(opt, SWA):
opt.swap_swa_sgd()
def on_epoch_end(self):
assert isinstance(self, LightningBaseModule)
if self.params.opt_reset_interval:
if self.current_epoch % self.params.opt_reset_interval == 0:
for opt in self.trainer.optimizers:
opt.state = defaultdict(dict)
class BaseTrainMixin:
absolute_loss = nn.L1Loss()
nll_loss = nn.NLLLoss()
bce_loss = nn.BCELoss()
def training_step(self, batch_xy, batch_nb, *_, **__):
assert isinstance(self, LightningBaseModule)
batch_x, batch_y = batch_xy
y = self(batch_x).main_out
bce_loss = self.bce_loss(y, batch_y)
return dict(loss=bce_loss, log=dict(batch_nb=batch_nb))
def training_epoch_end(self, outputs):
assert isinstance(self, LightningBaseModule)
keys = list(outputs[0].keys())
summary_dict = dict(log={f'mean_{key}': torch.mean(torch.stack([output[key]
for output in outputs]))
for key in keys if 'loss' in key})
return summary_dict
class BaseValMixin:
absolute_loss = nn.L1Loss()
nll_loss = nn.NLLLoss()
bce_loss = nn.BCELoss()
def validation_step(self, batch_xy, batch_idx, _, *__, **___):
assert isinstance(self, LightningBaseModule)
batch_x, batch_y = batch_xy
y = self(batch_x).main_out
val_bce_loss = self.bce_loss(y, batch_y)
return dict(val_bce_loss=val_bce_loss,
batch_idx=batch_idx, y=y, batch_y=batch_y)
def validation_epoch_end(self, outputs, *_, **__):
assert isinstance(self, LightningBaseModule)
summary_dict = dict(log=dict())
# In case of Multiple given dataloader this will outputs will be: list[list[dict[]]]
# for output_idx, output in enumerate(outputs):
# else:list[dict[]]
keys = list(outputs.keys())
# Add Every Value das has a "loss" in it, by calc. mean over all occurences.
summary_dict['log'].update({f'mean_{key}': torch.mean(torch.stack([output[key]
for output in outputs]))
for key in keys if 'loss' in key}
)
"""
# Additional Score like the unweighted Average Recall:
# UnweightedAverageRecall
y_true = torch.cat([output['batch_y'] for output in outputs]) .cpu().numpy()
y_pred = torch.cat([output['y'] for output in outputs]).squeeze().cpu().numpy()
y_pred = (y_pred >= 0.5).astype(np.float32)
uar_score = sklearn.metrics.recall_score(y_true, y_pred, labels=[0, 1], average='macro',
sample_weight=None, zero_division='warn')
summary_dict['log'].update({f'uar_score': uar_score})
"""
return summary_dict
class BinaryMaskDatasetMixin:
def build_dataset(self):
assert isinstance(self, LightningBaseModule)
# Dataset
# =============================================================================
# Data Augmentations or Utility Transformations
transforms = Compose([NormalizeLocal(), ToTensor()])
# Dataset
dataset = Namespace(
**dict(
# TRAIN DATASET
train_dataset=TemplateDataset(self.params.root, setting=GlobalVars.DATA_OPTIONS.train,
transforms=transforms
),
# VALIDATION DATASET
val_dataset=TemplateDataset(self.params.root, setting=GlobalVars.vali,
),
# TEST DATASET
test_dataset=TemplateDataset(self.params.root, setting=GlobalVars.test,
),
)
)
return dataset
class BaseDataloadersMixin(ABC):
# Dataloaders
# ================================================================================
# Train Dataloader
def train_dataloader(self):
assert isinstance(self, LightningBaseModule)
# In case you want to implement bootstraping
# sampler = RandomSampler(self.dataset.train_dataset, True, len(self.dataset.train_dataset))
sampler = None
return DataLoader(dataset=self.dataset.train_dataset, shuffle=True if not sampler else None, sampler=sampler,
batch_size=self.params.batch_size,
num_workers=self.params.worker)
# Test Dataloader
def test_dataloader(self):
assert isinstance(self, LightningBaseModule)
return DataLoader(dataset=self.dataset.test_dataset, shuffle=False,
batch_size=self.params.batch_size,
num_workers=self.params.worker)
# Validation Dataloader
def val_dataloader(self):
assert isinstance(self, LightningBaseModule)
val_dataloader = DataLoader(dataset=self.dataset.val_dataset, shuffle=False,
batch_size=self.params.batch_size, num_workers=self.params.worker)
# Alternative return [val_dataloader, alternative dataloader], there will be a dataloader_idx in validation_step
return val_dataloader

30
_templates/new_project/utils/project_config.py Normal file
View File

@ -0,0 +1,30 @@
from argparse import Namespace
from utils.config import Config
class GlobalVar(Namespace):
# Labels for classes
LEFT = 1
RIGHT = 0
WRONG = -1
# Colors for img files
WHITE = 255
BLACK = 0
# Variables for plotting
PADDING = 0.25
DPI = 50
# DATAOPTIONS
train='train',
vali='vali',
test='test'
class ThisConfig(Config):
@property
def _model_map(self):
return dict()

2
audio_toolset/mel_augmentation.py
View File

@ -1,5 +1,3 @@
from ctypes import Union
import numpy as np

142
examples/main.py
View File

@ -1,142 +0,0 @@
# Imports
# =============================================================================
import os
from distutils.util import strtobool
from pathlib import Path
from argparse import ArgumentParser, Namespace
import warnings
import torch
from pytorch_lightning import Trainer
from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping
from ml_lib.modules.utils import LightningBaseModule
from ml_lib.utils.config import Config
from ml_lib.utils.logging import Logger
from ml_lib.utils.model_io import SavedLightningModels
warnings.filterwarnings('ignore', category=FutureWarning)
warnings.filterwarnings('ignore', category=UserWarning)
_ROOT = Path(__file__).parent
# Parameter Configuration
# =============================================================================
# Argument Parser
main_arg_parser = ArgumentParser(description="parser for fast-neural-style")
# Main Parameters
main_arg_parser.add_argument("--main_debug", type=strtobool, default=False, help="")
main_arg_parser.add_argument("--main_eval", type=strtobool, default=True, help="")
main_arg_parser.add_argument("--main_seed", type=int, default=69, help="")
# Data Parameters
main_arg_parser.add_argument("--data_worker", type=int, default=10, help="")
main_arg_parser.add_argument("--data_dataset_length", type=int, default=10000, help="")
main_arg_parser.add_argument("--data_root", type=str, default='data', help="")
main_arg_parser.add_argument("--data_map_root", type=str, default='res/shapes', help="")
main_arg_parser.add_argument("--data_normalized", type=strtobool, default=True, help="")
main_arg_parser.add_argument("--data_use_preprocessed", type=strtobool, default=True, help="")
main_arg_parser.add_argument("--data_mode", type=str, default='vae_no_label_in_map', help="")
# Transformations
main_arg_parser.add_argument("--transformations_to_tensor", type=strtobool, default=False, help="")
# Transformations
main_arg_parser.add_argument("--train_outpath", type=str, default="output", help="")
main_arg_parser.add_argument("--train_version", type=strtobool, required=False, help="")
main_arg_parser.add_argument("--train_epochs", type=int, default=500, help="")
main_arg_parser.add_argument("--train_batch_size", type=int, default=200, help="")
main_arg_parser.add_argument("--train_lr", type=float, default=1e-3, help="")
main_arg_parser.add_argument("--train_num_sanity_val_steps", type=int, default=0, help="")
# Model
main_arg_parser.add_argument("--model_type", type=str, default="CNNRouteGenerator", help="")
main_arg_parser.add_argument("--model_activation", type=str, default="leaky_relu", help="")
main_arg_parser.add_argument("--model_filters", type=str, default="[16, 32, 64]", help="")
main_arg_parser.add_argument("--model_classes", type=int, default=2, help="")
main_arg_parser.add_argument("--model_lat_dim", type=int, default=16, help="")
main_arg_parser.add_argument("--model_use_bias", type=strtobool, default=True, help="")
main_arg_parser.add_argument("--model_use_norm", type=strtobool, default=False, help="")
main_arg_parser.add_argument("--model_use_res_net", type=strtobool, default=False, help="")
main_arg_parser.add_argument("--model_dropout", type=float, default=0.00, help="")
# Project
main_arg_parser.add_argument("--project_name", type=str, default='traj-gen', help="")
main_arg_parser.add_argument("--project_owner", type=str, default='si11ium', help="")
main_arg_parser.add_argument("--project_neptune_key", type=str, default=os.getenv('NEPTUNE_KEY'), help="")
# Parse it
args: Namespace = main_arg_parser.parse_args()
def run_lightning_loop(config_obj):
# Logging
# ================================================================================
# Logger
with Logger(config_obj) as logger:
# Callbacks
# =============================================================================
# Checkpoint Saving
checkpoint_callback = ModelCheckpoint(
filepath=str(logger.log_dir / 'ckpt_weights'),
verbose=True, save_top_k=0,
)
# =============================================================================
# Early Stopping
# TODO: For This to work, one must set a validation step and End Eval and Score
early_stopping_callback = EarlyStopping(
monitor='val_loss',
min_delta=0.0,
patience=0,
)
# Model
# =============================================================================
# Init
model: LightningBaseModule = config_obj.model_class(config_obj.model_paramters)
model.init_weights(torch.nn.init.xavier_normal_)
if model.name == 'CNNRouteGeneratorDiscriminated':
# ToDo: Make this dependent on the used seed
path = logger.outpath / 'classifier_cnn' / 'version_0'
disc_model = SavedLightningModels.load_checkpoint(path).restore()
model.set_discriminator(disc_model)
# Trainer
# =============================================================================
trainer = Trainer(max_epochs=config_obj.train.epochs,
show_progress_bar=True,
weights_save_path=logger.log_dir,
gpus=[0] if torch.cuda.is_available() else None,
check_val_every_n_epoch=10,
# num_sanity_val_steps=config_obj.train.num_sanity_val_steps,
# row_log_interval=(model.n_train_batches * 0.1), # TODO: Better Value / Setting
# log_save_interval=(model.n_train_batches * 0.2), # TODO: Better Value / Setting
checkpoint_callback=checkpoint_callback,
logger=logger,
fast_dev_run=config_obj.main.debug,
early_stop_callback=None
)
# Train It
trainer.fit(model)
# Save the last state & all parameters
trainer.save_checkpoint(logger.log_dir / 'weights.ckpt')
model.save_to_disk(logger.log_dir)
# Evaluate It
if config_obj.main.eval:
trainer.test()
return model
if __name__ == "__main__":
config = Config.read_namespace(args)
trained_model = run_lightning_loop(config)

12
examples/variables.py
View File

@ -1,12 +0,0 @@
# Labels for classes
HOMOTOPIC = 1
ALTERNATIVE = 0
ANY = -1
# Colors for img files
WHITE = 255
BLACK = 0
# Variables for plotting
PADDING = 0.25
DPI = 50

2
modules/blocks.py
View File

@ -5,7 +5,7 @@ import warnings
from torch import nn
from ml_lib.modules.utils import AutoPad, Interpolate, ShapeMixin, F_x, Flatten
from modules.utils import AutoPad, Interpolate, ShapeMixin, F_x, Flatten
DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

2
modules/model_parts.py
View File

@ -4,7 +4,7 @@
import torch
from torch import nn
from ml_lib.modules.utils import ShapeMixin
from modules.utils import ShapeMixin
class Generator(nn.Module):

2
modules/utils.py
View File

@ -10,7 +10,7 @@ import pytorch_lightning as pl
# Utility - Modules
###################
from ml_lib.utils.model_io import ModelParameters
from utils.model_io import ModelParameters
class ShapeMixin:

92
requirements.txt Normal file
View File

@ -0,0 +1,92 @@
absl-py==0.9.0
appdirs==1.4.3
attrs==19.3.0
audioread==2.1.8
bravado==10.6.0
bravado-core==5.17.0
CacheControl==0.12.6
cachetools==4.1.0
certifi==2019.11.28
cffi==1.14.0
chardet==3.0.4
click==7.1.2
colorama==0.4.3
contextlib2==0.6.0
cycler==0.10.0
decorator==4.4.2
distlib==0.3.0
distro==1.4.0
future==0.18.2
gitdb==4.0.5
GitPython==3.1.2
google-auth==1.14.3
google-auth-oauthlib==0.4.1
grpcio==1.29.0
html5lib==1.0.1
idna==2.8
ipaddr==2.2.0
joblib==0.15.1
jsonpointer==2.0
jsonref==0.2
jsonschema==3.2.0
kiwisolver==1.2.0
librosa==0.7.2
llvmlite==0.32.1
lockfile==0.12.2
Markdown==3.2.2
matplotlib==3.2.1
monotonic==1.5
msgpack==0.6.2
msgpack-python==0.5.6
natsort==7.0.1
neptune-client==0.4.113
numba==0.49.1
numpy==1.18.4
oauthlib==3.1.0
packaging==20.3
pandas==1.0.3
pep517==0.8.2
Pillow==7.1.2
progress==1.5
protobuf==3.12.0
py3nvml==0.2.6
pyasn1==0.4.8
pyasn1-modules==0.2.8
pycparser==2.20
PyJWT==1.7.1
pyparsing==2.4.6
pyrsistent==0.16.0
python-dateutil==2.8.1
pytoml==0.1.21
pytorch-lightning==0.7.6
pytz==2020.1
PyYAML==5.3.1
requests==2.22.0
requests-oauthlib==1.3.0
resampy==0.2.2
retrying==1.3.3
rfc3987==1.3.8
rsa==4.0
scikit-learn==0.23.0
scipy==1.4.1
simplejson==3.17.0
six==1.14.0
smmap==3.0.4
SoundFile==0.10.3.post1
strict-rfc3339==0.7
swagger-spec-validator==2.5.0
tensorboard==2.2.1
tensorboard-plugin-wit==1.6.0.post3
threadpoolctl==2.0.0
torch==1.5.0+cu101
torchvision==0.6.0+cu101
tqdm==4.46.0
typing-extensions==3.7.4.2
urllib3==1.25.8
webcolors==1.11.1
webencodings==0.5.1
websocket-client==0.57.0
Werkzeug==1.0.1
xmltodict==0.12.0
torchcontrib~=0.0.2

3
utils/config.py
View File

@ -70,7 +70,8 @@ class Config(ConfigParser, ABC):
try:
return self._model_map[self.model.type]
except KeyError:
raise KeyError(rf'The model alias you provided ("{self.get("model", "type")}") does not exist! Try one of these: {list(self._model_map.keys())}')
raise KeyError(f'The model alias you provided ("{self.get("model", "type")}")' +
'does not exist! Try one of these: {list(self._model_map.keys())}')
# TODO: Do this programmatically; This did not work:
# Initialize Default Sections as Property

2
utils/logging.py
View File

@ -5,7 +5,7 @@ from pytorch_lightning.loggers.base import LightningLoggerBase
from pytorch_lightning.loggers.neptune import NeptuneLogger
from pytorch_lightning.loggers.test_tube import TestTubeLogger
from ml_lib.utils.config import Config
from utils.config import Config
class Logger(LightningLoggerBase, ABC):

6
utils/transforms.py
View File

@ -1,8 +1,8 @@
from torchvision.transforms import ToTensor as TorchvisionToTensor
from torchvision.transforms import ToTensor as TorchVisionToTensor
class ToTensor(TorchvisionToTensor):
class ToTensor(TorchVisionToTensor):
def __call__(self, pic):
tensor = super(ToTensor, self).__call__(pic).float()
return tensor
return tensor