steffen/Mel_Vision_Transformer_ComParE-21
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Final Train Runs

Browse Source
This commit is contained in:
Steffen Illium 2021-03-18 07:45:07 +01:00
parent ad254dae92
commit fecf4923c2
14 changed files with 672 additions and 362 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

295
.idea/workspace.xml generated
View File

@ -1,295 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="AutoImportSettings">
<option name="autoReloadType" value="SELECTIVE" />
</component>
<component name="ChangeListManager">
<list default="true" id="2be1f675-29fe-4a7d-9fe6-9e96cd7c8055" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/ml_lib/additions/__init__.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/ml_lib/additions/losses.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/ml_lib/utils/equal_sampler.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/models/performer.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/models/transformer_model.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/models/transformer_model_horizontal.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/multi_run.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/rebuild_dataset.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/_parameters.ini" beforeDir="false" afterPath="$PROJECT_DIR$/_parameters.ini" afterDir="false" />
<change beforePath="$PROJECT_DIR$/datasets/primates_librosa_datamodule.py" beforeDir="false" afterPath="$PROJECT_DIR$/datasets/primates_librosa_datamodule.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/ml_lib/_templates/new_project/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/ml_lib/_templates/new_project/main.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/ml_lib/audio_toolset/audio_to_mel_dataset.py" beforeDir="false" afterPath="$PROJECT_DIR$/ml_lib/audio_toolset/audio_to_mel_dataset.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/ml_lib/audio_toolset/mel_dataset.py" beforeDir="false" afterPath="$PROJECT_DIR$/ml_lib/audio_toolset/mel_dataset.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/ml_lib/metrics/multi_class_classification.py" beforeDir="false" afterPath="$PROJECT_DIR$/ml_lib/metrics/multi_class_classification.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/ml_lib/modules/blocks.py" beforeDir="false" afterPath="$PROJECT_DIR$/ml_lib/modules/blocks.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/ml_lib/modules/model_parts.py" beforeDir="false" afterPath="$PROJECT_DIR$/ml_lib/modules/model_parts.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/ml_lib/modules/util.py" beforeDir="false" afterPath="$PROJECT_DIR$/ml_lib/modules/util.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/ml_lib/utils/_basedatamodule.py" beforeDir="false" afterPath="$PROJECT_DIR$/ml_lib/utils/_basedatamodule.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/ml_lib/utils/config.py" beforeDir="false" afterPath="$PROJECT_DIR$/ml_lib/utils/config.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/ml_lib/utils/logging.py" beforeDir="false" afterPath="$PROJECT_DIR$/ml_lib/utils/loggers.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/ml_lib/utils/tools.py" beforeDir="false" afterPath="$PROJECT_DIR$/ml_lib/utils/tools.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/models/cnn_baseline.py" beforeDir="false" afterPath="$PROJECT_DIR$/models/cnn_baseline.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/util/loss_mixin.py" beforeDir="false" afterPath="$PROJECT_DIR$/util/loss_mixin.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/util/module_mixins.py" beforeDir="false" afterPath="$PROJECT_DIR$/util/module_mixins.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/util/optimizer_mixin.py" beforeDir="false" afterPath="$PROJECT_DIR$/util/optimizer_mixin.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/variables.py" beforeDir="false" afterPath="$PROJECT_DIR$/variables.py" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Python Script" />
</list>
</option>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="GitSEFilterConfiguration">
<file-type-list>
<filtered-out-file-type name="LOCAL_BRANCH" />
<filtered-out-file-type name="REMOTE_BRANCH" />
<filtered-out-file-type name="TAG" />
<filtered-out-file-type name="COMMIT_BY_MESSAGE" />
</file-type-list>
</component>
<component name="ProjectId" id="1oTEXjx0b8UPBPmOIGceYxEch8r" />
<component name="ProjectLevelVcsManager" settingsEditedManually="true" />
<component name="ProjectViewState">
<option name="hideEmptyMiddlePackages" value="true" />
<option name="showLibraryContents" value="true" />
</component>
<component name="PropertiesComponent">
<property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
<property name="RunOnceActivity.ShowReadmeOnStart" value="true" />
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="WebServerToolWindowPanel.toolwindow.highlight.mappings" value="true" />
<property name="WebServerToolWindowPanel.toolwindow.highlight.symlinks" value="true" />
<property name="WebServerToolWindowPanel.toolwindow.show.date" value="false" />
<property name="WebServerToolWindowPanel.toolwindow.show.permissions" value="false" />
<property name="WebServerToolWindowPanel.toolwindow.show.size" value="false" />
<property name="credentialsType com.jetbrains.python.remote.PyCreateRemoteInterpreterDialog$PyCreateRemoteSdkForm" value="Web Deployment" />
<property name="last_opened_file_path" value="$PROJECT_DIR$/../inter_challenge_2020" />
<property name="restartRequiresConfirmation" value="false" />
<property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
</component>
<component name="PyDebuggerOptionsProvider">
<option name="mySaveCallSignatures" value="true" />
</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="C:\Users\steff\projects\compare_21\models" />
<recent name="C:\Users\steff\projects\compare_21\util" />
<recent name="C:\Users\steff\projects\compare_21" />
</key>
<key name="MoveFile.RECENT_KEYS">
<recent name="C:\Users\steff\projects\compare_21\util" />
</key>
</component>
<component name="RunManager" selected="Python.rebuild_dataset">
<configuration name="equal_sampler" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
<module name="compare_21" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/ml_lib/utils" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/ml_lib/utils/equal_sampler.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
<configuration name="main" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
<module name="compare_21" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/main.py" />
<option name="PARAMETERS" value="--debug=True --num_worker=0 --sampler=WeightedRandomSampler --model_name=VisualTransformer" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
<configuration name="metadata_readout" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
<module name="compare_21" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/datasets" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/datasets/metadata_readout.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
<configuration name="multi_run" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
<module name="compare_21" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/multi_run.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
<configuration name="rebuild_dataset" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
<module name="compare_21" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/rebuild_dataset.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
<list>
<item itemvalue="Python.main" />
<item itemvalue="Python.metadata_readout" />
<item itemvalue="Python.multi_run" />
<item itemvalue="Python.equal_sampler" />
<item itemvalue="Python.rebuild_dataset" />
</list>
<recent_temporary>
<list>
<item itemvalue="Python.rebuild_dataset" />
<item itemvalue="Python.multi_run" />
<item itemvalue="Python.equal_sampler" />
<item itemvalue="Python.metadata_readout" />
</list>
</recent_temporary>
</component>
<component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="2be1f675-29fe-4a7d-9fe6-9e96cd7c8055" name="Default Changelist" comment="" />
<created>1613302221903</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1613302221903</updated>
<workItem from="1613302223434" duration="2570000" />
<workItem from="1613305247599" duration="11387000" />
<workItem from="1613381449219" duration="21710000" />
<workItem from="1613634319983" duration="160704000" />
<workItem from="1614498086696" duration="17996000" />
<workItem from="1614671803611" duration="595000" />
<workItem from="1614679476632" duration="11417000" />
<workItem from="1614760180356" duration="12583000" />
<workItem from="1614788597119" duration="8215000" />
</task>
<task id="LOCAL-00001" summary="Dataset rdy">
<created>1613467084268</created>
<option name="number" value="00001" />
<option name="presentableId" value="LOCAL-00001" />
<option name="project" value="LOCAL" />
<updated>1613467084268</updated>
</task>
<option name="localTasksCounter" value="2" />
<servers />
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="3" />
</component>
<component name="VcsManagerConfiguration">
<MESSAGE value="Dataset rdy" />
<option name="LAST_COMMIT_MESSAGE" value="Dataset rdy" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
<breakpoints>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/ml_lib/modules/util.py</url>
<line>231</line>
<option name="timeStamp" value="70" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/models/transformer_model_horizontal.py</url>
<line>68</line>
<option name="timeStamp" value="72" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/ml_lib/audio_toolset/mel_dataset.py</url>
<line>29</line>
<option name="timeStamp" value="83" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/main.py</url>
<line>46</line>
<option name="timeStamp" value="84" />
</line-breakpoint>
</breakpoints>
<default-breakpoints>
<breakpoint type="python-exception">
<properties notifyOnTerminate="true" exception="BaseException">
<option name="notifyOnTerminate" value="true" />
</properties>
</breakpoint>
</default-breakpoints>
</breakpoint-manager>
</component>
<component name="com.intellij.coverage.CoverageDataManagerImpl">
<SUITE FILE_PATH="coverage/compare_21$equal_sampler.coverage" NAME="equal_sampler Coverage Results" MODIFIED="1614070702295" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/ml_lib/utils" />
<SUITE FILE_PATH="coverage/compare_21$rebuild_dataset.coverage" NAME="rebuild_dataset Coverage Results" MODIFIED="1614849857426" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
<SUITE FILE_PATH="coverage/compare_21$metadata_readout.coverage" NAME="metadata_readout Coverage Results" MODIFIED="1613306122664" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/datasets" />
<SUITE FILE_PATH="coverage/compare_21$main.coverage" NAME="main Coverage Results" MODIFIED="1614695189720" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
<SUITE FILE_PATH="coverage/compare_21$multi_run.coverage" NAME="multi_run Coverage Results" MODIFIED="1614702462483" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
</component>
</project>

57
_parameters.ini
View File

@ -4,14 +4,13 @@ debug = False
eval = True eval = True
seed = 69 seed = 69
owner = si11ium owner = si11ium
model_name = CNNBaseline model_name = VisualTransformer
data_name = PrimatesLibrosaDatamodule data_name = PrimatesLibrosaDatamodule
[data] [data]
num_worker = 10 num_worker = 10
data_root = data data_root = data
reset = False n_mels = 128
n_mels = 64
sr = 16000 sr = 16000
hop_length = 128 hop_length = 128
n_fft = 256 n_fft = 256
@ -22,6 +21,21 @@ shift_ratio = 0.3
noise_ratio = 0.0 noise_ratio = 0.0
mask_ratio = 0.0 mask_ratio = 0.0
[Tester]
weight_init = xavier_normal_
activation = gelu
use_bias = True
use_norm = True
use_residual = True
dropout = 0.2
lat_dim = 32
patch_size = 8
attn_depth = 12
heads = 4
embedding_size = 128
[CNNBaseline] [CNNBaseline]
weight_init = xavier_normal_ weight_init = xavier_normal_
activation = gelu activation = gelu
@ -40,6 +54,22 @@ use_norm = True
use_residual = True use_residual = True
dropout = 0.2 dropout = 0.2
lat_dim = 32
mlp_dim = 32
head_dim = 32
patch_size = 8
attn_depth = 12
heads = 4
embedding_size = 33
[VerticalVisualTransformer]
weight_init = xavier_normal_
activation = gelu
use_bias = True
use_norm = True
use_residual = True
dropout = 0.2
lat_dim = 32 lat_dim = 32
patch_size = 8 patch_size = 8
attn_depth = 12 attn_depth = 12
@ -60,19 +90,34 @@ attn_depth = 12
heads = 6 heads = 6
embedding_size = 32 embedding_size = 32
[VisualPerformer]
weight_init = xavier_normal_
activation = gelu
use_bias = True
use_norm = True
use_residual = True
dropout = 0.2
lat_dim = 32
patch_size = 8
attn_depth = 12
heads = 4
embedding_size = 30
[train] [train]
outpath = output outpath = output
version = None version = None
sampler = EqualSampler sampler = EqualSampler
loss = focal_loss_rob loss = ce_loss
sto_weight_avg = False sto_weight_avg = False
weight_decay = 0 weight_decay = 0
opt_reset_interval = 0 opt_reset_interval = 0
max_epochs = 200 max_epochs = 150
batch_size = 30 batch_size = 30
lr = 0.001 lr = 0.001
scheduler='LambdaLR'
use_residual = True use_residual = True
lr_warm_restart_epochs = 0 lr_scheduler_parameter = 0.97
num_sanity_val_steps = 2 num_sanity_val_steps = 2
check_val_every_n_epoch = 5 check_val_every_n_epoch = 5
checkpoint_callback = True checkpoint_callback = True

11
datasets/primates_librosa_datamodule.py
View File

@ -4,7 +4,6 @@ from pathlib import Path
from torch.utils.data import DataLoader, ConcatDataset, WeightedRandomSampler from torch.utils.data import DataLoader, ConcatDataset, WeightedRandomSampler
from torchvision.transforms import Compose, RandomApply from torchvision.transforms import Compose, RandomApply
from tqdm import tqdm
from ml_lib.audio_toolset.audio_io import NormalizeLocal from ml_lib.audio_toolset.audio_io import NormalizeLocal
from ml_lib.audio_toolset.audio_to_mel_dataset import LibrosaAudioToMelDataset from ml_lib.audio_toolset.audio_to_mel_dataset import LibrosaAudioToMelDataset
@ -69,13 +68,14 @@ class PrimatesLibrosaDatamodule(_BaseDataModule):
# Validation Dataloader # Validation Dataloader
def val_dataloader(self): def val_dataloader(self):
return DataLoader(dataset=self.datasets[DATA_OPTION_devel], num_workers=self.num_worker, pin_memory=True, return DataLoader(dataset=self.datasets[DATA_OPTION_devel], shuffle=False,
sampler=self.samplers[DATA_OPTION_devel], batch_size=self.batch_size) batch_size=self.batch_size, pin_memory=False,
num_workers=self.num_worker)
# Test Dataloader # Test Dataloader
def test_dataloader(self): def test_dataloader(self):
return DataLoader(dataset=self.datasets[DATA_OPTION_test], shuffle=False, return DataLoader(dataset=self.datasets[DATA_OPTION_test], shuffle=False,
batch_size=self.batch_size, pin_memory=True, batch_size=self.batch_size, pin_memory=False,
num_workers=self.num_worker) num_workers=self.num_worker)
def _build_subdataset(self, row, build=False): def _build_subdataset(self, row, build=False):
@ -134,7 +134,7 @@ class PrimatesLibrosaDatamodule(_BaseDataModule):
datasets[data_option] = ConcatDataset(dataset) datasets[data_option] = ConcatDataset(dataset)
# Build Weighted Sampler for train and val # Build Weighted Sampler for train and val
if data_option in [DATA_OPTION_train, DATA_OPTION_devel]: if data_option in [DATA_OPTION_train]:
if self.sampler == EqualSampler.__name__: if self.sampler == EqualSampler.__name__:
class_idxs = [[idx for idx, (_, __, label) in enumerate(datasets[data_option]) if label == class_idx] class_idxs = [[idx for idx, (_, __, label) in enumerate(datasets[data_option]) if label == class_idx]
for class_idx in range(len(self.class_names)) for class_idx in range(len(self.class_names))
@ -147,6 +147,7 @@ class PrimatesLibrosaDatamodule(_BaseDataModule):
len_largest_class = max(class_counts.values()) len_largest_class = max(class_counts.values())
weights[data_option] = [1 / class_counts[x] for x in range(len(class_counts))] weights[data_option] = [1 / class_counts[x] for x in range(len(class_counts))]
##############################################################################
weights[data_option] = [weights[data_option][datasets[data_option][i][-1]] weights[data_option] = [weights[data_option][datasets[data_option][i][-1]]
for i in range(len(datasets[data_option]))] for i in range(len(datasets[data_option]))]
samplers[data_option] = WeightedRandomSampler(weights[data_option], samplers[data_option] = WeightedRandomSampler(weights[data_option],

44
main.py
View File

@ -2,12 +2,12 @@ from argparse import Namespace
import warnings import warnings
from pytorch_lightning import Trainer from pytorch_lightning import Trainer, Callback
from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint
from ml_lib.utils.callbacks import BestScoresCallback
from ml_lib.utils.config import parse_comandline_args_add_defaults from ml_lib.utils.config import parse_comandline_args_add_defaults
from ml_lib.utils.loggers import Logger from ml_lib.utils.loggers import Logger
from ml_lib.utils.tools import locate_and_import_class, auto_cast
import variables as v import variables as v
@ -15,22 +15,38 @@ warnings.filterwarnings('ignore', category=FutureWarning)
warnings.filterwarnings('ignore', category=UserWarning) warnings.filterwarnings('ignore', category=UserWarning)
def run_lightning_loop(h_params, data_class, model_class): def run_lightning_loop(h_params, data_class, model_class, additional_callbacks=None):
with Logger.from_argparse_args(h_params) as logger: with Logger.from_argparse_args(h_params) as logger:
# Callbacks # Callbacks
# ============================================================================= # =============================================================================
# Checkpoint Saving # Checkpoint Saving
ckpt_callback = ModelCheckpoint( ckpt_callback = ModelCheckpoint(
monitor='mean_loss', monitor='PL_recall_score',
dirpath=str(logger.log_dir), dirpath=str(logger.log_dir),
filename='ckpt_weights', filename='ckpt_weights',
mode='max',
verbose=False, verbose=False,
save_top_k=3, save_top_k=3,
save_last=True
) )
# Learning Rate Logger # Learning Rate Logger
lr_logger = LearningRateMonitor(logging_interval='epoch') lr_logger = LearningRateMonitor(logging_interval='epoch')
# Track best scores
score_callback = BestScoresCallback(['PL_recall_score'])
callbacks = [ckpt_callback, lr_logger, score_callback]
if additional_callbacks and isinstance(additional_callbacks, Callback):
callbacks.append(additional_callbacks)
elif additional_callbacks and isinstance(additional_callbacks, list):
callbacks.extend(additional_callbacks)
else:
pass
# START # START
# ============================================================================= # =============================================================================
# Let Datamodule pull what it wants # Let Datamodule pull what it wants
@ -38,19 +54,27 @@ def run_lightning_loop(h_params, data_class, model_class):
datamodule.setup() datamodule.setup()
# Let Trainer pull what it wants and add callbacks # Let Trainer pull what it wants and add callbacks
trainer = Trainer.from_argparse_args(h_params, logger=logger, callbacks=[ckpt_callback, lr_logger]) trainer = Trainer.from_argparse_args(h_params, logger=logger, callbacks=callbacks)
# Let Model pull what it wants # Let Model pull what it wants
model = model_class.from_argparse_args(h_params, in_shape=datamodule.shape, n_classes=v.N_CLASS_multi) model = model_class.from_argparse_args(h_params, in_shape=datamodule.shape, n_classes=v.N_CLASS_multi)
model.init_weights() model.init_weights()
# trainer.test(model=model, datamodule=datamodule)
trainer.fit(model, datamodule) trainer.fit(model, datamodule)
trainer.save_checkpoint(logger.save_dir / 'last_weights.ckpt')
# Log paramters try:
pytorch_total_params = sum(p.numel() for p in model.parameters()) trainer.test(model=model, datamodule=datamodule)
# logger.log_text('n_parameters', pytorch_total_params) except:
print('Test did not Suceed!')
trainer.save_checkpoint(logger.save_dir / 'weights.ckpt') pass
try:
logger.log_metrics(score_callback.best_scores, step=trainer.global_step+1)
except:
print('debug max_score_logging')
return score_callback.best_scores['PL_recall_score']
if __name__ == '__main__': if __name__ == '__main__':

4
models/cnn_baseline.py
View File

@ -15,7 +15,9 @@ class CNNBaseline(CombinedModelMixins,
): ):
def __init__(self, in_shape, n_classes, weight_init, activation, use_bias, use_norm, dropout, lat_dim, features, def __init__(self, in_shape, n_classes, weight_init, activation, use_bias, use_norm, dropout, lat_dim, features,
filters, lr, weight_decay, sto_weight_avg, lr_warm_restart_epochs, opt_reset_interval, loss): filters,
lr, weight_decay, sto_weight_avg, lr_warm_restart_epochs, opt_reset_interval,
loss, scheduler):
# TODO: Move this to parent class, or make it much easieer to access.... # TODO: Move this to parent class, or make it much easieer to access....
a = dict(locals()) a = dict(locals())

133
models/performer.py
View File

@ -0,0 +1,133 @@
import inspect
from argparse import Namespace
import warnings
import torch
from performer_pytorch import Performer
from torch import nn
from einops import rearrange, repeat
from ml_lib.metrics.multi_class_classification import MultiClassScores
from ml_lib.modules.util import (LightningBaseModule, AutoPadToShape, F_x)
from util.module_mixins import CombinedModelMixins
MIN_NUM_PATCHES = 16
class VisualPerformer(CombinedModelMixins,
LightningBaseModule
):
def __init__(self, in_shape, n_classes, weight_init, activation,
embedding_size, heads, attn_depth, patch_size, use_residual,
use_bias, use_norm, dropout, lat_dim, loss, scheduler,
lr, weight_decay, sto_weight_avg, lr_warm_restart_epochs, opt_reset_interval):
# TODO: Move this to parent class, or make it much easieer to access... But How...
a = dict(locals())
params = {arg: a[arg] for arg in inspect.signature(self.__init__).parameters.keys() if arg != 'self'}
super(VisualPerformer, self).__init__(params)
self.in_shape = in_shape
assert len(self.in_shape) == 3, 'There need to be three Dimensions'
channels, height, width = self.in_shape
# Model Paramters
# =============================================================================
# Additional parameters
self.embed_dim = self.params.embedding_size
# Automatic Image Shaping
self.patch_size = self.params.patch_size
image_size = (max(height, width) // self.patch_size) * self.patch_size
self.image_size = image_size + self.patch_size if image_size < max(height, width) else image_size
# This should be obsolete
assert self.image_size % self.patch_size == 0, 'image dimensions must be divisible by the patch size'
num_patches = (self.image_size // self.patch_size) ** 2
patch_dim = channels * self.patch_size ** 2
assert num_patches >= MIN_NUM_PATCHES, f'your number of patches ({num_patches}) is way too small for ' + \
f'attention. Try decreasing your patch size'
# Correct the Embedding Dim
if not self.embed_dim % self.params.heads == 0:
self.embed_dim = (self.embed_dim // self.params.heads) * self.params.heads
message = ('Embedding Dimension was fixed to be devideable by the number' +
f' of attention heads, is now: {self.embed_dim}')
for func in print, warnings.warn:
func(message)
# Utility Modules
self.autopad = AutoPadToShape((self.image_size, self.image_size))
# Modules with Parameters
self.performer = Performer(
dim=self.embed_dim, # dimension
depth=self.params.attn_depth, # layers
heads=self.params.heads, # heads
causal=True, # auto-regressive or not
nb_features=None, # 256, # number of random features, if not set, will default to
# (d * log(d)), where d is the dimension of each head
feature_redraw_interval=1000, # how frequently to redraw the projection matrix,
# the more frequent, the slower the training
generalized_attention=False, # defaults to softmax approximation,
# but can be set to True for generalized attention
kernel_fn=self.params.activation(), # the kernel function to be used,
# if generalized attention is turned on, defaults to Relu
reversible=True, # reversible layers, from Reformer paper
ff_chunks=10, # chunk feedforward layer, from Reformer paper
use_scalenorm=False, # use scale norm, from 'Transformers without Tears' paper
use_rezero=False, # use rezero, from 'Rezero is all you need' paper
ff_glu=True, # use GLU variant for feedforward
ff_dropout=self.params.dropout, # feedforward dropout
attn_dropout=self.params.dropout, # post-attn dropout
local_attn_heads=self.params.heads // 2, # 4 heads are local attention, 4 others are global performers
local_window_size=(patch_dim // self.params.heads) * 2 # window size of local attention
)
self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, self.embed_dim))
self.patch_to_embedding = nn.Linear(patch_dim, self.embed_dim) if self.params.embedding_size \
else F_x(self.embed_dim)
self.cls_token = nn.Parameter(torch.randn(1, 1, self.embed_dim))
self.dropout = nn.Dropout(self.params.dropout)
self.to_cls_token = nn.Identity()
self.mlp_head = nn.Sequential(
nn.LayerNorm(self.embed_dim),
nn.Linear(self.embed_dim, self.params.lat_dim),
nn.GELU(),
nn.Dropout(self.params.dropout),
nn.Linear(self.params.lat_dim, n_classes),
nn.Softmax()
)
def forward(self, x):
"""
:param x: the sequence to the encoder (required).
:return:
"""
tensor = self.autopad(x)
p = self.params.patch_size
tensor = rearrange(tensor, 'b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1=p, p2=p)
tensor = self.patch_to_embedding(tensor)
b, n, _ = tensor.shape
cls_tokens = repeat(self.cls_token, '() n d -> b n d', b=b)
tensor = torch.cat((cls_tokens, tensor), dim=1)
tensor += self.pos_embedding[:, :(n + 1)]
tensor = self.dropout(tensor)
tensor = self.performer(tensor)
tensor = self.to_cls_token(tensor[:, 0])
tensor = self.mlp_head(tensor)
return Namespace(main_out=tensor)
def additional_scores(self, outputs):
return MultiClassScores(self)(outputs)

120
models/testing.py Normal file
View File

@ -0,0 +1,120 @@
import inspect
from argparse import Namespace
import warnings
import torch
from torch import nn
from einops import rearrange, repeat
from ml_lib.metrics.multi_class_classification import MultiClassScores
from ml_lib.modules.blocks import TransformerModule
from ml_lib.modules.util import (LightningBaseModule, AutoPadToShape, F_x)
from util.module_mixins import CombinedModelMixins
MIN_NUM_PATCHES = 16
class Tester(CombinedModelMixins,
LightningBaseModule
):
def __init__(self, in_shape, n_classes, weight_init, activation,
embedding_size, heads, attn_depth, patch_size, use_residual,
use_bias, use_norm, dropout, lat_dim, loss, scheduler, mlp_dim,
lr, weight_decay, sto_weight_avg, lr_scheduler_parameter, opt_reset_interval):
# TODO: Move this to parent class, or make it much easieer to access... But How...
a = dict(locals())
params = {arg: a[arg] for arg in inspect.signature(self.__init__).parameters.keys() if arg != 'self'}
super(Tester, self).__init__(params)
self.in_shape = in_shape
assert len(self.in_shape) == 3, 'There need to be three Dimensions'
channels, height, width = self.in_shape
# Model Paramters
# =============================================================================
# Additional parameters
self.embed_dim = self.params.embedding_size
# Automatic Image Shaping
self.patch_size = self.params.patch_size
image_size = (max(height, width) // self.patch_size) * self.patch_size
self.image_size = image_size + self.patch_size if image_size < max(height, width) else image_size
# This should be obsolete
assert self.image_size % self.patch_size == 0, 'image dimensions must be divisible by the patch size'
num_patches = (self.image_size // self.patch_size) ** 2
patch_dim = channels * self.patch_size ** 2
assert num_patches >= MIN_NUM_PATCHES, f'your number of patches ({num_patches}) is way too small for ' + \
f'attention. Try decreasing your patch size'
# Correct the Embedding Dim
if not self.embed_dim % self.params.heads == 0:
self.embed_dim = (self.embed_dim // self.params.heads) * self.params.heads
message = ('Embedding Dimension was fixed to be devideable by the number' +
f' of attention heads, is now: {self.embed_dim}')
for func in print, warnings.warn:
func(message)
# Utility Modules
self.autopad = AutoPadToShape((self.image_size, self.image_size))
# Modules with Parameters
self.transformer = TransformerModule(in_shape=self.embed_dim, mlp_dim=self.params.mlp_dim,
heads=self.params.heads, depth=self.params.attn_depth,
dropout=self.params.dropout, use_norm=self.params.use_norm,
activation=self.params.activation, use_residual=self.params.use_residual
)
self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, self.embed_dim))
self.patch_to_embedding = nn.Linear(patch_dim, self.embed_dim) if self.params.embedding_size \
else F_x(self.embed_dim)
self.cls_token = nn.Parameter(torch.randn(1, 1, self.embed_dim))
self.dropout = nn.Dropout(self.params.dropout)
self.to_cls_token = nn.Identity()
self.mlp_head = nn.Sequential(
nn.LayerNorm(self.embed_dim),
nn.Linear(self.embed_dim, self.params.lat_dim),
nn.GELU(),
nn.Dropout(self.params.dropout),
nn.Linear(self.params.lat_dim, n_classes),
nn.Softmax()
)
def forward(self, x, mask=None, return_attn_weights=False):
"""
:param x: the sequence to the encoder (required).
:param mask: the mask for the src sequence (optional).
:return:
"""
tensor = self.autopad(x)
p = self.params.patch_size
tensor = rearrange(tensor, 'b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1=p, p2=p)
tensor = self.patch_to_embedding(tensor)
b, n, _ = tensor.shape
cls_tokens = repeat(self.cls_token, '() n d -> b n d', b=b)
tensor = torch.cat((cls_tokens, tensor), dim=1)
tensor += self.pos_embedding[:, :(n + 1)]
tensor = self.dropout(tensor)
if return_attn_weights:
tensor, attn_weights = self.transformer(tensor, mask, return_attn_weights)
else:
attn_weights = None
tensor = self.transformer(tensor, mask)
tensor = self.to_cls_token(tensor[:, 0])
tensor = self.mlp_head(tensor)
return Namespace(main_out=tensor, attn_weights=attn_weights)
def additional_scores(self, outputs):
return MultiClassScores(self)(outputs)

26
models/transformer_model.py
View File

@ -21,9 +21,9 @@ class VisualTransformer(CombinedModelMixins,
): ):
def __init__(self, in_shape, n_classes, weight_init, activation, def __init__(self, in_shape, n_classes, weight_init, activation,
embedding_size, heads, attn_depth, patch_size,use_residual, embedding_size, heads, attn_depth, patch_size, use_residual,
use_bias, use_norm, dropout, lat_dim, loss, use_bias, use_norm, dropout, lat_dim, loss, scheduler, mlp_dim, head_dim,
lr, weight_decay, sto_weight_avg, lr_warm_restart_epochs, opt_reset_interval): lr, weight_decay, sto_weight_avg, lr_scheduler_parameter, opt_reset_interval):
# TODO: Move this to parent class, or make it much easieer to access... But How... # TODO: Move this to parent class, or make it much easieer to access... But How...
a = dict(locals()) a = dict(locals())
@ -53,26 +53,26 @@ class VisualTransformer(CombinedModelMixins,
f'attention. Try decreasing your patch size' f'attention. Try decreasing your patch size'
# Correct the Embedding Dim # Correct the Embedding Dim
if not self.embed_dim % self.params.heads == 0: #if not self.embed_dim % self.params.heads == 0:
self.embed_dim = (self.embed_dim // self.params.heads) * self.params.heads # self.embed_dim = (self.embed_dim // self.params.heads) * self.params.heads
message = ('Embedding Dimension was fixed to be devideable by the number' + # message = ('Embedding Dimension was fixed to be devideable by the number' +
f' of attention heads, is now: {self.embed_dim}') # f' of attention heads, is now: {self.embed_dim}')
for func in print, warnings.warn: # for func in print, warnings.warn:
func(message) # func(message)
# Utility Modules # Utility Modules
self.autopad = AutoPadToShape((self.image_size, self.image_size)) self.autopad = AutoPadToShape((self.image_size, self.image_size))
# Modules with Parameters # Modules with Parameters
self.transformer = TransformerModule(in_shape=self.embed_dim, mlp_dim=self.params.lat_dim, self.transformer = TransformerModule(in_shape=self.embed_dim, mlp_dim=self.params.mlp_dim,
head_dim=self.params.head_dim,
heads=self.params.heads, depth=self.params.attn_depth, heads=self.params.heads, depth=self.params.attn_depth,
dropout=self.params.dropout, use_norm=self.params.use_norm, dropout=self.params.dropout, use_norm=self.params.use_norm,
activation=self.params.activation, use_residual=self.params.use_residual activation=self.params.activation, use_residual=self.params.use_residual
) )
self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, self.embed_dim)) self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, self.embed_dim))
self.patch_to_embedding = nn.Linear(patch_dim, self.embed_dim) if self.params.embedding_size \ self.patch_to_embedding = nn.Linear(patch_dim, self.embed_dim)
else F_x(self.embed_dim)
self.cls_token = nn.Parameter(torch.randn(1, 1, self.embed_dim)) self.cls_token = nn.Parameter(torch.randn(1, 1, self.embed_dim))
self.dropout = nn.Dropout(self.params.dropout) self.dropout = nn.Dropout(self.params.dropout)
@ -117,4 +117,4 @@ class VisualTransformer(CombinedModelMixins,
return Namespace(main_out=tensor, attn_weights=attn_weights) return Namespace(main_out=tensor, attn_weights=attn_weights)
def additional_scores(self, outputs): def additional_scores(self, outputs):
return MultiClassScores(self)(outputs) return MultiClassScores(self)(outputs)

4
models/transformer_model_horizontal.py
View File

@ -21,8 +21,8 @@ class HorizontalVisualTransformer(CombinedModelMixins,
): ):
def __init__(self, in_shape, n_classes, weight_init, activation, def __init__(self, in_shape, n_classes, weight_init, activation,
embedding_size, heads, attn_depth, patch_size,use_residual, embedding_size, heads, attn_depth, patch_size, use_residual,
use_bias, use_norm, dropout, lat_dim, features, loss, use_bias, use_norm, dropout, lat_dim, features, loss, scheduler,
lr, weight_decay, sto_weight_avg, lr_warm_restart_epochs, opt_reset_interval): lr, weight_decay, sto_weight_avg, lr_warm_restart_epochs, opt_reset_interval):
# TODO: Move this to parent class, or make it much easieer to access... But How... # TODO: Move this to parent class, or make it much easieer to access... But How...

116
models/transformer_model_vertical.py Normal file
View File

@ -0,0 +1,116 @@
import inspect
from argparse import Namespace
import warnings
import torch
from einops import repeat
from torch import nn
from ml_lib.metrics.multi_class_classification import MultiClassScores
from ml_lib.modules.blocks import TransformerModule
from ml_lib.modules.util import (LightningBaseModule, AutoPadToShape, F_x, SlidingWindow)
from util.module_mixins import CombinedModelMixins
MIN_NUM_PATCHES = 16
class VerticalVisualTransformer(CombinedModelMixins, LightningBaseModule):
def __init__(self, in_shape, n_classes, weight_init, activation,
embedding_size, heads, attn_depth, patch_size, use_residual,
use_bias, use_norm, dropout, lat_dim, features, loss, scheduler,
lr, weight_decay, sto_weight_avg, lr_warm_restart_epochs, opt_reset_interval):
# TODO: Move this to parent class, or make it much easieer to access... But How...
a = dict(locals())
params = {arg: a[arg] for arg in inspect.signature(self.__init__).parameters.keys() if arg != 'self'}
super(VerticalVisualTransformer, self).__init__(params)
self.in_shape = in_shape
self.n_classes = n_classes
assert len(self.in_shape) == 3, 'There need to be three Dimensions'
channels, height, width = self.in_shape
# Model Paramters
# =============================================================================
# Additional parameters
self.embed_dim = self.params.embedding_size
self.height = height
self.channels = channels
self.new_width = ((width - self.params.patch_size)//1) + 1
num_patches = self.new_width - (self.params.patch_size // 2)
patch_dim = channels * self.params.patch_size * self.height
assert num_patches >= MIN_NUM_PATCHES, f'your number of patches ({num_patches}) is way too small for ' + \
f'attention. Try decreasing your patch size'
# Correct the Embedding Dim
if not self.embed_dim % self.params.heads == 0:
self.embed_dim = (self.embed_dim // self.params.heads) * self.params.heads
message = ('Embedding Dimension was fixed to be devideable by the number' +
f' of attention heads, is now: {self.embed_dim}')
for func in print, warnings.warn:
func(message)
# Utility Modules
self.autopad = AutoPadToShape((self.height, self.new_width))
self.dropout = nn.Dropout(self.params.dropout)
self.slider = SlidingWindow((channels, *self.autopad.target_shape), (self.height, self.params.patch_size),
keepdim=False)
# Modules with Parameters
self.transformer = TransformerModule(in_shape=self.embed_dim, mlp_dim=self.params.lat_dim,
heads=self.params.heads, depth=self.params.attn_depth,
dropout=self.params.dropout, use_norm=self.params.use_norm,
activation=self.params.activation
)
self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, self.embed_dim))
self.patch_to_embedding = nn.Linear(patch_dim, self.embed_dim) if self.params.embedding_size \
else F_x(self.embed_dim)
self.cls_token = nn.Parameter(torch.randn(1, 1, self.embed_dim))
self.to_cls_token = nn.Identity()
self.mlp_head = nn.Sequential(
nn.LayerNorm(self.embed_dim),
nn.Linear(self.embed_dim, self.params.lat_dim),
nn.GELU(),
nn.Dropout(self.params.dropout),
nn.Linear(self.params.lat_dim, self.n_classes),
nn.Softmax()
)
def forward(self, x, mask=None, return_attn_weights=False):
"""
:param x: the sequence to the encoder (required).
:param mask: the mask for the src sequence (optional).
:param return_attn_weights: wether to return the attn weights (optional)
:return:
"""
tensor = self.autopad(x)
tensor = self.slider(tensor)
tensor = self.patch_to_embedding(tensor)
b, n, _ = tensor.shape
cls_tokens = repeat(self.cls_token, '() n d -> b n d', b=b)
tensor = torch.cat((cls_tokens, tensor), dim=1)
tensor += self.pos_embedding[:, :(n + 1)]
tensor = self.dropout(tensor)
if return_attn_weights:
tensor, attn_weights = self.transformer(tensor, mask, return_attn_weights)
else:
attn_weights = None
tensor = self.transformer(tensor, mask)
tensor = self.to_cls_token(tensor[:, 0])
tensor = self.mlp_head(tensor)
return Namespace(main_out=tensor, attn_weights=attn_weights)
def additional_scores(self, outputs):
return MultiClassScores(self)(outputs)

37
multi_run.py
View File

@ -10,23 +10,30 @@ import itertools
if __name__ == '__main__': if __name__ == '__main__':
# Set new values # Set new values
hparams_dict = dict(model_name=['VisualTransformer'], hparams_dict = dict(seed=range(10),
max_epochs=[150], model_name=['VisualTransformer'],
batch_size=[50], batch_size=[50],
random_apply_chance=[0.5], max_epochs=[250],
loudness_ratio=[0], random_apply_chance=[0.3], # trial.suggest_float('random_apply_chance', 0.1, 0.5, step=0.1),
shift_ratio=[0.3], loudness_ratio=[0], # trial.suggest_float('loudness_ratio', 0.0, 0.5, step=0.1),
noise_ratio=[0.3], shift_ratio=[0.3], # trial.suggest_float('shift_ratio', 0.0, 0.5, step=0.1),
mask_ratio=[0.3], noise_ratio=[0.3], # trial.suggest_float('noise_ratio', 0.0, 0.5, step=0.1),
lr=[0.001], mask_ratio=[0.3], # trial.suggest_float('mask_ratio', 0.0, 0.5, step=0.1),
dropout=[0.2], lr=[5e-3], # trial.suggest_uniform('lr', 1e-3, 3e-3),
lat_dim=[32, 64], dropout=[0.2], # trial.suggest_float('dropout', 0.0, 0.3, step=0.05),
patch_size=[8, 12], lat_dim=[32], # 2 ** trial.suggest_int('lat_dim', 1, 5, step=1),
attn_depth=[12], mlp_dim=[16], # 2 ** trial.suggest_int('mlp_dim', 1, 5, step=1),
heads=[6], head_dim=[6], # 2 ** trial.suggest_int('head_dim', 1, 5, step=1),
embedding_size=[16, 32], patch_size=[12], # trial.suggest_int('patch_size', 6, 12, step=3),
attn_depth=[10], # trial.suggest_int('attn_depth', 2, 14, step=4),
heads=[6], # trial.suggest_int('heads', 2, 16, step=2),
scheduler=['CosineAnnealingWarmRestarts'], # trial.suggest_categorical('scheduler', [None, 'LambdaLR']),
lr_scheduler_parameter=[25], # [0.98],
embedding_size=[30], # trial.suggest_int('embedding_size', 12, 64, step=12),
loss=['ce_loss'], loss=['ce_loss'],
sampler=['WeightedRandomSampler'] sampler=['WeightedRandomSampler'],
# rial.suggest_categorical('sampler', [None, 'WeightedRandomSampler']),
weight_decay=[0], # trial.suggest_loguniform('weight_decay', 1e-20, 1e-1),
) )
keys, values = zip(*hparams_dict.items()) keys, values = zip(*hparams_dict.items())

79
optuna_tune.py Normal file
View File

@ -0,0 +1,79 @@
import pickle
from argparse import Namespace
from pathlib import Path
import optuna as optuna
from optuna.integration import PyTorchLightningPruningCallback
from main import run_lightning_loop
from ml_lib.utils.config import parse_comandline_args_add_defaults
import neptunecontrib.monitoring.optuna as opt_utils
def optimize(trial: optuna.Trial):
# Optuna configuration
folder = Path('study')
folder.mkdir(parents=False, exist_ok=True)
optuna_suggestions = dict(
model_name='VisualTransformer',
batch_size=trial.suggest_int('batch_size', 30, 100, step=32),
lr_scheduler_parameter=trial.suggest_float('lr_scheduler_parameter', 0.8, 1, step=0.01),
max_epochs=100,
random_apply_chance=0.1, # trial.suggest_float('random_apply_chance', 0.1, 0.5, step=0.1),
loudness_ratio=0.1, # trial.suggest_float('loudness_ratio', 0.0, 0.5, step=0.1),
shift_ratio=0.1, # trial.suggest_float('shift_ratio', 0.0, 0.5, step=0.1),
noise_ratio=0, # trial.suggest_float('noise_ratio', 0.0, 0.5, step=0.1),
mask_ratio=0.2, # trial.suggest_float('mask_ratio', 0.0, 0.5, step=0.1),
lr=trial.suggest_uniform('lr', 1e-3, 3e-3),
dropout=0.05, # trial.suggest_float('dropout', 0.0, 0.3, step=0.05),
lat_dim=32, # 2 ** trial.suggest_int('lat_dim', 1, 5, step=1),
mlp_dim=16, # 2 ** trial.suggest_int('mlp_dim', 1, 5, step=1),
head_dim=8, # 2 ** trial.suggest_int('head_dim', 1, 5, step=1),
patch_size=12, # trial.suggest_int('patch_size', 6, 12, step=3),
attn_depth=10, # trial.suggest_int('attn_depth', 2, 14, step=4),
heads=16, # trial.suggest_int('heads', 2, 16, step=2),
scheduler='LambdaLR', # trial.suggest_categorical('scheduler', [None, 'LambdaLR']),
embedding_size=48, # trial.suggest_int('embedding_size', 12, 64, step=12),
loss='ce_loss',
sampler='WeightedRandomSampler', # rial.suggest_categorical('sampler', [None, 'WeightedRandomSampler']),
weight_decay=trial.suggest_loguniform('weight_decay', 1e-20, 1e-1),
study_name=trial.study.study_name
)
pruning_callback = PyTorchLightningPruningCallback(trial, monitor="PL_recall_score")
# Parse comandline args, read config and get model
cmd_args, found_data_class, found_model_class = parse_comandline_args_add_defaults('_parameters.ini')
h_params = dict(**cmd_args)
h_params.update(optuna_suggestions)
h_params = Namespace(**h_params)
try:
best_score = run_lightning_loop(h_params, data_class=found_data_class, model_class=found_model_class,
additional_callbacks=pruning_callback)
except Exception as e:
print(e)
best_score = 0
return best_score
if __name__ == '__main__':
study = optuna.create_study(direction='maximize', sampler=optuna.samplers.TPESampler(seed=1337))
# study.optimize(optimize, n_trials=50, callbacks=[opt_utils.NeptuneCallback(log_study=True, log_charts=True)])
study.optimize(optimize, n_trials=50)
print("Number of finished trials: {}".format(len(study.trials)))
print("Best trial:")
trial = study.best_trial
print(" Value: {}".format(trial.value))
print(" Params: ")
for key, value in trial.params.items():
print(" {}: {}".format(key, value))
optuna_study_object = Path('study') / 'study.pkl'
optuna_study_object.parent.mkdir(exist_ok=True)
with optuna_study_object.open(mode='wb') as f:
pickle.dump(study, f)

95
util/module_mixins.py
View File

@ -1,3 +1,6 @@
from collections import defaultdict
from pathlib import Path
from abc import ABC from abc import ABC
import torch import torch
@ -42,24 +45,67 @@ class ValMixin:
model_out = self(batch_x) model_out = self(batch_x)
y = model_out.main_out y = model_out.main_out
sorted_y = defaultdict(list)
sorted_batch_y = dict()
for idx, file_name in enumerate(batch_files):
sorted_y[file_name].append(y[idx])
sorted_batch_y.update({file_name: batch_y[idx]})
sorted_y = dict(sorted_y)
for file_name in sorted_y:
sorted_y.update({file_name: torch.stack(sorted_y[file_name])})
y_max = torch.stack(
[torch.argmax(x.mean(dim=0)) if x.shape[0] > 1 else torch.argmax(x) for x in sorted_y.values()]
).squeeze()
y_one_hot = torch.nn.functional.one_hot(y_max, num_classes=5).float()
self.metrics.update(y_one_hot, torch.stack(tuple(sorted_batch_y.values())).long())
val_loss = self.ce_loss(y, batch_y.long()) val_loss = self.ce_loss(y, batch_y.long())
self.metrics.update(y, batch_y) # torch.argmax(y, -1), batch_y) return dict(batch_files=batch_files, val_loss=val_loss,
return dict(val_loss=val_loss, batch_files=batch_files,
batch_idx=batch_idx, y=y, batch_y=batch_y) batch_idx=batch_idx, y=y, batch_y=batch_y)
def validation_epoch_end(self, outputs, *_, **__): def validation_epoch_end(self, outputs, *_, **__):
assert isinstance(self, LightningBaseModule) assert isinstance(self, LightningBaseModule)
summary_dict = dict() summary_dict = dict()
keys = list(outputs[0].keys()) keys = list(outputs[0].keys())
summary_dict.update({f'mean_{key}': torch.mean(torch.stack([output[key]
for output in outputs]))
for key in keys if 'loss' in key}
)
sorted_y = defaultdict(list)
sorted_batch_y = dict()
for output in outputs:
for idx, file_name in enumerate(output['batch_files']):
sorted_y[file_name].append(output['y'][idx])
sorted_batch_y.update({file_name: output['batch_y'][idx]})
sorted_y = dict(sorted_y)
sorted_batch_y = torch.stack(tuple(sorted_batch_y.values())).long()
for file_name in sorted_y:
sorted_y.update({file_name: torch.stack(sorted_y[file_name])})
y_mean = torch.stack(
[torch.mean(x, dim=0, keepdim=True) if x.shape[0] > 1 else x for x in sorted_y.values()]
).squeeze()
mean_vote_loss = self.ce_loss(y_mean, sorted_batch_y)
summary_dict.update(val_mean_vote_loss=mean_vote_loss)
y_max = torch.stack(
[torch.argmax(x.mean(dim=0)) if x.shape[0] > 1 else torch.argmax(x) for x in sorted_y.values()]
).squeeze()
y_one_hot = torch.nn.functional.one_hot(y_max, num_classes=5).float()
max_vote_loss = self.ce_loss(y_one_hot, sorted_batch_y)
summary_dict.update(val_max_vote_loss=max_vote_loss)
summary_dict.update({f'mean_{key}': torch.mean(torch.stack([output[key] summary_dict.update({f'mean_{key}': torch.mean(torch.stack([output[key]
for output in outputs])) for output in outputs]))
for key in keys if 'loss' in key} for key in keys if 'loss' in key}
) )
# Sklearn Scores # Sklearn Scores
additional_scores = self.additional_scores(outputs) additional_scores = self.additional_scores(dict(y=y_one_hot, batch_y=sorted_batch_y))
summary_dict.update(**additional_scores) summary_dict.update(**additional_scores)
pl_metrics, pl_images = self.metrics.compute_and_prepare() pl_metrics, pl_images = self.metrics.compute_and_prepare()
@ -85,13 +131,40 @@ class TestMixin:
def test_epoch_end(self, outputs, *_, **__): def test_epoch_end(self, outputs, *_, **__):
assert isinstance(self, LightningBaseModule) assert isinstance(self, LightningBaseModule)
y_arg_max = torch.argmax(outputs[0]['y'])
pd.DataFrame(data=dict(filenames=outputs[0]['batch_files'], predtiction=y_arg_max))
# No logging, just inference. # No logging, just inference.
# self.log_dict(summary_dict, on_epoch=True)
sorted_y = defaultdict(list)
for output in outputs:
for idx, file_name in enumerate(output['batch_files']):
sorted_y[file_name].append(output['y'][idx].cpu())
sorted_y = dict(sorted_y)
for file_name in sorted_y:
sorted_y.update({file_name: torch.stack(sorted_y[file_name])})
y_max = torch.stack(
[torch.argmax(x.mean(dim=0)) if x.shape[0] > 1 else torch.argmax(x) for x in sorted_y.values()]
).squeeze().cpu()
class_names = {val: key for val, key in enumerate(['background', 'chimpanze', 'geunon', 'mandrille', 'redcap'])}
df = pd.DataFrame(data=dict(filenames=[Path(x).stem for x in sorted_y.keys()],
prediction=y_max.cpu().numpy(),
prediction_named=[class_names[x.item()] for x in y_max.cpu().numpy()]))
result_file = Path(self.logger.log_dir / 'predictions.csv')
if result_file.exists():
try:
result_file.unlink()
except:
print('File allready existed')
pass
with result_file.open(mode='wb') as csv_file:
df.to_csv(index=False, path_or_buf=csv_file)
with result_file.open(mode='rb') as csv_file:
try:
self.logger.neptunelogger.log_artifact(csv_file)
except:
print('No possible to send to neptune')
pass
class CombinedModelMixins(LossMixin, class CombinedModelMixins(LossMixin,

13
util/optimizer_mixin.py
View File

@ -24,11 +24,16 @@ class OptimizerMixin:
if self.params.sto_weight_avg: if self.params.sto_weight_avg:
optimizer = SWA(optimizer, swa_start=10, swa_freq=5, swa_lr=0.05) optimizer = SWA(optimizer, swa_start=10, swa_freq=5, swa_lr=0.05)
optimizer_dict.update(optimizer=optimizer) optimizer_dict.update(optimizer=optimizer)
if self.params.lr_warm_restart_epochs:
scheduler = CosineAnnealingWarmRestarts(optimizer, self.params.lr_warm_restart_epochs) if self.params.scheduler == CosineAnnealingWarmRestarts.__name__:
scheduler = CosineAnnealingWarmRestarts(optimizer, self.params.lr_scheduler_parameter)
elif self.params.scheduler == LambdaLR.__name__:
lr_reduce_ratio = self.params.lr_scheduler_parameter
scheduler = LambdaLR(optimizer, lr_lambda=lambda epoch: lr_reduce_ratio ** epoch)
else: else:
scheduler = LambdaLR(optimizer, lr_lambda=lambda epoch: 0.95 ** epoch) scheduler = None
optimizer_dict.update(lr_scheduler=scheduler) if scheduler:
optimizer_dict.update(lr_scheduler=scheduler)
return optimizer_dict return optimizer_dict
def on_train_end(self): def on_train_end(self):