refactoring and running experiments
This commit is contained in:
Steffen Illium
@ -1,12 +1,5 @@
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import torchmetrics
|
||||
|
||||
from torch.utils.data import Dataset
|
||||
from tqdm import tqdm
|
||||
|
||||
from experiments.meta_task_utility import set_checkpoint
|
||||
|
||||
DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
||||
|
||||
@ -24,31 +17,6 @@ class AddTaskDataset(Dataset):
|
||||
return ab, ab.sum(axis=-1, keepdims=True)
|
||||
|
||||
|
||||
def validate(checkpoint_path, valid_d, ratio=1, validmetric=torchmetrics.MeanAbsoluteError()):
|
||||
checkpoint_path = Path(checkpoint_path)
|
||||
|
||||
# initialize metric
|
||||
model = torch.load(checkpoint_path, map_location=DEVICE).eval()
|
||||
n_samples = int(len(valid_d) * ratio)
|
||||
|
||||
with tqdm(total=n_samples, desc='Validation Run: ') as pbar:
|
||||
for idx, (valid_batch_x, valid_batch_y) in enumerate(valid_d):
|
||||
valid_batch_x, valid_batch_y = valid_batch_x.to(DEVICE), valid_batch_y.to(DEVICE)
|
||||
y_valid = model(valid_batch_x)
|
||||
|
||||
# metric on current batch
|
||||
acc = validmetric(y_valid.cpu(), valid_batch_y.cpu())
|
||||
pbar.set_postfix_str(f'Acc: {acc}')
|
||||
pbar.update()
|
||||
if idx == n_samples:
|
||||
break
|
||||
|
||||
# metric on all batches using custom accumulation
|
||||
acc = validmetric.compute()
|
||||
tqdm.write(f"Avg. Accuracy on all data: {acc}")
|
||||
return acc
|
||||
|
||||
|
||||
def train_task(model, optimizer, loss_func, btch_x, btch_y) -> (dict, torch.Tensor):
|
||||
# Zero your gradients for every batch!
|
||||
optimizer.zero_grad()
|
||||
@ -66,12 +34,5 @@ def train_task(model, optimizer, loss_func, btch_x, btch_y) -> (dict, torch.Tens
|
||||
return stp_log, y_prd
|
||||
|
||||
|
||||
def checkpoint_and_validate(model, out_path, epoch_n, valid_d, final_model=False):
|
||||
out_path = Path(out_path)
|
||||
ckpt_path = set_checkpoint(model, out_path, epoch_n, final_model=final_model)
|
||||
result = validate(ckpt_path, valid_d)
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise(NotImplementedError('Get out of here'))
|
||||
|
||||
@ -3,57 +3,30 @@ import re
|
||||
import shutil
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import platform
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import torch
|
||||
import torchmetrics
|
||||
from matplotlib import pyplot as plt
|
||||
import seaborn as sns
|
||||
from torch.nn import Flatten
|
||||
from torch.utils.data import Dataset, DataLoader
|
||||
from torchvision.datasets import MNIST
|
||||
from torchvision.transforms import ToTensor, Compose, Resize
|
||||
from torch.utils.data import Dataset
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
# noinspection DuplicatedCode
|
||||
if platform.node() == 'CarbonX':
|
||||
debug = True
|
||||
print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
|
||||
print("@ Warning, Debugging Config@!!!!!! @")
|
||||
print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
|
||||
else:
|
||||
debug = False
|
||||
try:
|
||||
# noinspection PyUnboundLocalVariable
|
||||
if __package__ is None:
|
||||
DIR = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(DIR.parent))
|
||||
__package__ = DIR.name
|
||||
else:
|
||||
DIR = None
|
||||
except NameError:
|
||||
DIR = None
|
||||
pass
|
||||
|
||||
from network import FixTypes as ft
|
||||
from functionalities_test import test_for_fixpoints
|
||||
|
||||
WORKER = 10 if not debug else 0
|
||||
debug = False
|
||||
BATCHSIZE = 500 if not debug else 50
|
||||
WORKER = 10
|
||||
BATCHSIZE = 500
|
||||
EPOCH = 50
|
||||
VALIDATION_FRQ = 3 if not debug else 1
|
||||
SELF_TRAIN_FRQ = 1 if not debug else 1
|
||||
VALIDATION_FRQ = 3
|
||||
|
||||
DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
||||
|
||||
DATA_PATH = Path('data')
|
||||
DATA_PATH.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
if debug:
|
||||
torch.autograd.set_detect_anomaly(True)
|
||||
|
||||
|
||||
class ToFloat:
|
||||
|
||||
@ -93,39 +66,28 @@ def set_checkpoint(model, out_path, epoch_n, final_model=False):
|
||||
return ckpt_path
|
||||
|
||||
|
||||
def validate(checkpoint_path, ratio=0.1):
|
||||
# noinspection PyProtectedMember
|
||||
def validate(checkpoint_path, valid_loader, metric_class=torchmetrics.Accuracy):
|
||||
checkpoint_path = Path(checkpoint_path)
|
||||
import torchmetrics
|
||||
|
||||
# initialize metric
|
||||
validmetric = torchmetrics.Accuracy()
|
||||
ut = Compose([ToTensor(), ToFloat(), Resize((15, 15)), Flatten(start_dim=0)])
|
||||
|
||||
try:
|
||||
datas = MNIST(str(DATA_PATH), transform=ut, train=False)
|
||||
except RuntimeError:
|
||||
datas = MNIST(str(DATA_PATH), transform=ut, train=False, download=True)
|
||||
valid_d = DataLoader(datas, batch_size=BATCHSIZE, shuffle=True, drop_last=True, num_workers=WORKER)
|
||||
|
||||
validmetric = metric_class()
|
||||
model = torch.load(checkpoint_path, map_location=DEVICE).eval()
|
||||
n_samples = int(len(valid_d) * ratio)
|
||||
|
||||
with tqdm(total=n_samples, desc='Validation Run: ') as pbar:
|
||||
for idx, (valid_batch_x, valid_batch_y) in enumerate(valid_d):
|
||||
with tqdm(total=len(valid_loader), desc='Validation Run: ') as pbar:
|
||||
for idx, (valid_batch_x, valid_batch_y) in enumerate(valid_loader):
|
||||
valid_batch_x, valid_batch_y = valid_batch_x.to(DEVICE), valid_batch_y.to(DEVICE)
|
||||
y_valid = model(valid_batch_x)
|
||||
|
||||
# metric on current batch
|
||||
acc = validmetric(y_valid.cpu(), valid_batch_y.cpu())
|
||||
pbar.set_postfix_str(f'Acc: {acc}')
|
||||
measure = validmetric(y_valid.cpu(), valid_batch_y.cpu())
|
||||
pbar.set_postfix_str(f'Measure: {measure}')
|
||||
pbar.update()
|
||||
if idx == n_samples:
|
||||
break
|
||||
|
||||
# metric on all batches using custom accumulation
|
||||
acc = validmetric.compute()
|
||||
tqdm.write(f"Avg. accuracy on all data: {acc}")
|
||||
return acc
|
||||
measure = validmetric.compute()
|
||||
tqdm.write(f"Avg. {validmetric._get_name()} on all data: {measure}")
|
||||
return measure
|
||||
|
||||
|
||||
def new_storage_df(identifier, weight_count):
|
||||
@ -135,14 +97,16 @@ def new_storage_df(identifier, weight_count):
|
||||
return pd.DataFrame(columns=['Epoch', 'Weight', *(f'weight_{x}' for x in range(weight_count))])
|
||||
|
||||
|
||||
def checkpoint_and_validate(model, out_path, epoch_n, final_model=False):
|
||||
def checkpoint_and_validate(model, valid_loader, out_path, epoch_n, final_model=False,
|
||||
validation_metric=torchmetrics.Accuracy):
|
||||
out_path = Path(out_path)
|
||||
ckpt_path = set_checkpoint(model, out_path, epoch_n, final_model=final_model)
|
||||
result = validate(ckpt_path)
|
||||
result = validate(ckpt_path, valid_loader, metric_class=validation_metric)
|
||||
return result
|
||||
|
||||
|
||||
def plot_training_particle_types(path_to_dataframe):
|
||||
plt.close('all')
|
||||
plt.clf()
|
||||
# load from Drive
|
||||
df = pd.read_csv(path_to_dataframe, index_col=False)
|
||||
@ -158,17 +122,19 @@ def plot_training_particle_types(path_to_dataframe):
|
||||
|
||||
fig.legend(loc="center right", title='Particle Type', bbox_to_anchor=(0.85, 0.5))
|
||||
plt.tight_layout()
|
||||
if debug:
|
||||
plt.show()
|
||||
else:
|
||||
plt.savefig(Path(path_to_dataframe.parent / 'training_particle_type_lp.png'), dpi=300)
|
||||
|
||||
|
||||
def plot_training_result(path_to_dataframe):
|
||||
def plot_training_result(path_to_dataframe, metric='Accuracy', plot_name=None):
|
||||
plt.clf()
|
||||
# load from Drive
|
||||
df = pd.read_csv(path_to_dataframe, index_col=False)
|
||||
|
||||
# Check if this is a single lineplot or if aggregated
|
||||
group = ['Epoch', 'Metric']
|
||||
if 'Seed' in df.columns:
|
||||
group.append('Seed')
|
||||
|
||||
# Set up figure
|
||||
fig, ax1 = plt.subplots() # initializes figure and plots
|
||||
ax2 = ax1.twinx() # applies twinx to ax2, which is the second y-axis.
|
||||
@ -176,26 +142,27 @@ def plot_training_result(path_to_dataframe):
|
||||
# plots the first set of data
|
||||
data = df[(df['Metric'] == 'Task Loss') | (df['Metric'] == 'Self Train Loss')].groupby(['Epoch', 'Metric']).mean()
|
||||
palette = sns.color_palette()[1:data.reset_index()['Metric'].unique().shape[0]+1]
|
||||
sns.lineplot(data=data.groupby(['Epoch', 'Metric']).mean(), x='Epoch', y='Score', hue='Metric',
|
||||
palette=palette, ax=ax1)
|
||||
|
||||
sns.lineplot(data=data.groupby(group).mean(), x='Epoch', y='Score', hue='Metric',
|
||||
palette=palette, ax=ax1, ci='sd')
|
||||
|
||||
# plots the second set of data
|
||||
data = df[(df['Metric'] == 'Test Accuracy') | (df['Metric'] == 'Train Accuracy')]
|
||||
data = df[(df['Metric'] == f'Test {metric}') | (df['Metric'] == f'Train {metric}')]
|
||||
palette = sns.color_palette()[len(palette)+1:data.reset_index()['Metric'].unique().shape[0] + len(palette)+1]
|
||||
sns.lineplot(data=data, x='Epoch', y='Score', marker='o', hue='Metric', palette=palette)
|
||||
sns.lineplot(data=data, x='Epoch', y='Score', marker='o', hue='Metric', palette=palette, ci='sd')
|
||||
|
||||
ax1.set(yscale='log', ylabel='Losses')
|
||||
ax1.set_title('Training Lineplot')
|
||||
ax2.set(ylabel='Accuracy')
|
||||
# ax1.set_title('Training Lineplot')
|
||||
ax2.set(ylabel=metric)
|
||||
if metric != 'MAE':
|
||||
ax2.set(yscale='log')
|
||||
|
||||
fig.legend(loc="center right", title='Metric', bbox_to_anchor=(0.85, 0.5))
|
||||
ax1.get_legend().remove()
|
||||
ax2.get_legend().remove()
|
||||
for ax in [ax1, ax2]:
|
||||
if legend := ax.get_legend():
|
||||
legend.remove()
|
||||
plt.tight_layout()
|
||||
if debug:
|
||||
plt.show()
|
||||
else:
|
||||
plt.savefig(Path(path_to_dataframe.parent / 'training_lineplot.png'), dpi=300)
|
||||
plt.savefig(Path(path_to_dataframe.parent / ('training_lineplot.png' if plot_name is None else plot_name)), dpi=300)
|
||||
|
||||
|
||||
def plot_network_connectivity_by_fixtype(path_to_trained_model):
|
||||
@ -224,31 +191,29 @@ def plot_network_connectivity_by_fixtype(path_to_trained_model):
|
||||
lines = ax.get_lines()
|
||||
for line in lines:
|
||||
line.set_color(sns.color_palette()[n])
|
||||
if debug:
|
||||
plt.show()
|
||||
else:
|
||||
plt.savefig(Path(path_to_trained_model.parent / f'net_connectivity_{fixtype}.png'), dpi=300)
|
||||
tqdm.write(f'Connectivity plottet: {fixtype} - n = {df[df["type"] == fixtype].shape[0] // 2}')
|
||||
else:
|
||||
tqdm.write(f'No Connectivity {fixtype}')
|
||||
|
||||
|
||||
def run_particle_dropout_test(model_path):
|
||||
# noinspection PyProtectedMember
|
||||
def run_particle_dropout_test(model_path, valid_loader, metric_class=torchmetrics.Accuracy):
|
||||
diff_store_path = model_path.parent / 'diff_store.csv'
|
||||
latest_model = torch.load(model_path, map_location=DEVICE).eval()
|
||||
prtcl_dict = defaultdict(lambda: 0)
|
||||
_ = test_for_fixpoints(prtcl_dict, list(latest_model.particles))
|
||||
tqdm.write(str(dict(prtcl_dict)))
|
||||
diff_df = pd.DataFrame(columns=['Particle Type', 'Accuracy', 'Diff'])
|
||||
diff_df = pd.DataFrame(columns=['Particle Type', metric_class()._get_name(), 'Diff'])
|
||||
|
||||
acc_pre = validate(model_path, ratio=1).item()
|
||||
acc_pre = validate(model_path, valid_loader, metric_class=metric_class).item()
|
||||
diff_df.loc[diff_df.shape[0]] = ('All Organism', acc_pre, 0)
|
||||
|
||||
for fixpoint_type in ft.all_types():
|
||||
new_model = torch.load(model_path, map_location=DEVICE).eval().replace_with_zero(fixpoint_type)
|
||||
if [x for x in new_model.particles if x.is_fixpoint == fixpoint_type]:
|
||||
new_ckpt = set_checkpoint(new_model, model_path.parent, fixpoint_type, final_model=True)
|
||||
acc_post = validate(new_ckpt, ratio=1).item()
|
||||
acc_post = validate(new_ckpt, valid_loader, metric_class=metric_class).item()
|
||||
acc_diff = abs(acc_post - acc_pre)
|
||||
tqdm.write(f'Zero_ident diff = {acc_diff}')
|
||||
diff_df.loc[diff_df.shape[0]] = (fixpoint_type, acc_post, acc_diff)
|
||||
@ -257,8 +222,9 @@ def run_particle_dropout_test(model_path):
|
||||
return diff_store_path
|
||||
|
||||
|
||||
def plot_dropout_stacked_barplot(mdl_path, diff_store_path):
|
||||
|
||||
# noinspection PyProtectedMember
|
||||
def plot_dropout_stacked_barplot(mdl_path, diff_store_path, metric_class=torchmetrics.Accuracy):
|
||||
metric_name = metric_class()._get_name()
|
||||
diff_df = pd.read_csv(diff_store_path)
|
||||
particle_dict = defaultdict(lambda: 0)
|
||||
latest_model = torch.load(mdl_path, map_location=DEVICE).eval()
|
||||
@ -267,24 +233,21 @@ def plot_dropout_stacked_barplot(mdl_path, diff_store_path):
|
||||
plt.clf()
|
||||
fig, ax = plt.subplots(ncols=2)
|
||||
colors = sns.color_palette()[1:diff_df.shape[0]+1]
|
||||
_ = sns.barplot(data=diff_df, y='Accuracy', x='Particle Type', ax=ax[0], palette=colors)
|
||||
_ = sns.barplot(data=diff_df, y=metric_name, x='Particle Type', ax=ax[0], palette=colors)
|
||||
|
||||
ax[0].set_title('Accuracy after particle dropout')
|
||||
ax[0].set_title(f'{metric_name} after particle dropout')
|
||||
ax[0].set_xlabel('Particle Type')
|
||||
|
||||
ax[1].pie(particle_dict.values(), labels=particle_dict.keys(), colors=list(reversed(colors)), )
|
||||
ax[1].set_title('Particle Count')
|
||||
|
||||
plt.tight_layout()
|
||||
if debug:
|
||||
plt.show()
|
||||
else:
|
||||
plt.savefig(Path(diff_store_path.parent / 'dropout_stacked_barplot.png'), dpi=300)
|
||||
|
||||
|
||||
def run_particle_dropout_and_plot(model_path):
|
||||
diff_store_path = run_particle_dropout_test(model_path)
|
||||
plot_dropout_stacked_barplot(model_path, diff_store_path)
|
||||
def run_particle_dropout_and_plot(model_path, valid_loader, metric_class=torchmetrics.Accuracy):
|
||||
diff_store_path = run_particle_dropout_test(model_path, valid_loader=valid_loader, metric_class=metric_class)
|
||||
plot_dropout_stacked_barplot(model_path, diff_store_path, metric_class=metric_class)
|
||||
|
||||
|
||||
def flat_for_store(parameters):
|
||||
|
||||
@ -4,7 +4,7 @@ from pathlib import Path
|
||||
|
||||
import platform
|
||||
|
||||
|
||||
import pandas as pd
|
||||
import torchmetrics
|
||||
import numpy as np
|
||||
import torch
|
||||
@ -17,9 +17,9 @@ from torchvision.transforms import ToTensor, Compose, Resize
|
||||
from tqdm import tqdm
|
||||
|
||||
# noinspection DuplicatedCode
|
||||
from experiments.meta_task_utility import ToFloat, new_storage_df, train_task, checkpoint_and_validate, flat_for_store, \
|
||||
plot_training_result, plot_training_particle_types, plot_network_connectivity_by_fixtype, \
|
||||
run_particle_dropout_and_plot
|
||||
from experiments.meta_task_utility import (ToFloat, new_storage_df, train_task, checkpoint_and_validate, flat_for_store,
|
||||
plot_training_result, plot_training_particle_types,
|
||||
plot_network_connectivity_by_fixtype, run_particle_dropout_and_plot)
|
||||
|
||||
if platform.node() == 'CarbonX':
|
||||
debug = True
|
||||
@ -37,7 +37,9 @@ debug = False
|
||||
BATCHSIZE = 2000 if not debug else 50
|
||||
EPOCH = 50
|
||||
VALIDATION_FRQ = 3 if not debug else 1
|
||||
SELF_TRAIN_FRQ = 1 if not debug else 1
|
||||
VALIDATION_METRIC = torchmetrics.Accuracy
|
||||
# noinspection PyProtectedMember
|
||||
VAL_METRIC_NAME = VALIDATION_METRIC()._get_name()
|
||||
DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
||||
|
||||
DATA_PATH = Path('data')
|
||||
@ -50,15 +52,15 @@ if debug:
|
||||
if __name__ == '__main__':
|
||||
|
||||
training = True
|
||||
n_st = 300
|
||||
n_st = 150 # per batch !!
|
||||
activation = None # nn.ReLU()
|
||||
|
||||
for weight_hidden_size in [4, 5]:
|
||||
for weight_hidden_size in [4, 5, 6]:
|
||||
|
||||
weight_hidden_size = weight_hidden_size
|
||||
residual_skip = True
|
||||
n_seeds = 3
|
||||
depth = 3
|
||||
depth = 5
|
||||
width = 3
|
||||
out = 10
|
||||
|
||||
@ -96,6 +98,12 @@ if __name__ == '__main__':
|
||||
train_dataset = MNIST(str(DATA_PATH), transform=utility_transforms, download=True)
|
||||
train_loader = DataLoader(train_dataset, batch_size=BATCHSIZE, shuffle=True,
|
||||
drop_last=True, num_workers=WORKER)
|
||||
try:
|
||||
valid_dataset = MNIST(str(DATA_PATH), transform=utility_transforms, train=False)
|
||||
except RuntimeError:
|
||||
valid_dataset = MNIST(str(DATA_PATH), transform=utility_transforms, train=False, download=True)
|
||||
valid_loader = DataLoader(valid_dataset, batch_size=BATCHSIZE, shuffle=True,
|
||||
drop_last=True, num_workers=WORKER)
|
||||
|
||||
interface = np.prod(train_dataset[0][0].shape)
|
||||
metanet = MetaNet(interface, depth=depth, width=width, out=out,
|
||||
@ -111,11 +119,10 @@ if __name__ == '__main__':
|
||||
|
||||
for epoch in tqdm(range(EPOCH), desc=f'Train - Epochs'):
|
||||
is_validation_epoch = epoch % VALIDATION_FRQ == 0 if not debug else True
|
||||
is_self_train_epoch = epoch % SELF_TRAIN_FRQ == 0 if not debug else True
|
||||
metanet = metanet.train()
|
||||
|
||||
# Init metrics, even we do not need:
|
||||
metric = torchmetrics.Accuracy()
|
||||
metric = VALIDATION_METRIC()
|
||||
n_st_per_batch = n_st // len(train_loader)
|
||||
|
||||
for batch, (batch_x, batch_y) in tqdm(enumerate(train_loader),
|
||||
@ -139,14 +146,14 @@ if __name__ == '__main__':
|
||||
metanet = metanet.eval()
|
||||
try:
|
||||
validation_log = dict(Epoch=int(epoch), Batch=BATCHSIZE,
|
||||
Metric='Train Accuracy', Score=metric.compute().item())
|
||||
Metric=f'Train {VAL_METRIC_NAME}', Score=metric.compute().item())
|
||||
train_store.loc[train_store.shape[0]] = validation_log
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
accuracy = checkpoint_and_validate(metanet, seed_path, epoch).item()
|
||||
accuracy = checkpoint_and_validate(metanet, valid_loader, seed_path, epoch).item()
|
||||
validation_log = dict(Epoch=int(epoch), Batch=BATCHSIZE,
|
||||
Metric='Test Accuracy', Score=accuracy)
|
||||
Metric=f'Test {VAL_METRIC_NAME}', Score=accuracy)
|
||||
train_store.loc[train_store.shape[0]] = validation_log
|
||||
|
||||
if is_validation_epoch:
|
||||
@ -181,9 +188,9 @@ if __name__ == '__main__':
|
||||
for key, value in dict(counter_dict).items():
|
||||
step_log = dict(Epoch=int(EPOCH)+1, Batch=BATCHSIZE, Metric=key, Score=value)
|
||||
train_store.loc[train_store.shape[0]] = step_log
|
||||
accuracy = checkpoint_and_validate(metanet, seed_path, EPOCH, final_model=True)
|
||||
accuracy = checkpoint_and_validate(metanet, valid_loader, seed_path, EPOCH, final_model=True)
|
||||
validation_log = dict(Epoch=EPOCH, Batch=BATCHSIZE,
|
||||
Metric='Test Accuracy', Score=accuracy.item())
|
||||
Metric=f'Test {VAL_METRIC_NAME}', Score=accuracy.item())
|
||||
train_store.loc[train_store.shape[0]] = validation_log
|
||||
for particle in metanet.particles:
|
||||
weight_log = (EPOCH, particle.name, *(flat_for_store(particle.parameters())))
|
||||
@ -206,13 +213,25 @@ if __name__ == '__main__':
|
||||
|
||||
try:
|
||||
# noinspection PyUnboundLocalVariable
|
||||
run_particle_dropout_and_plot(model_path)
|
||||
run_particle_dropout_and_plot(model_path, valid_loader=valid_loader, metric_class=VALIDATION_METRIC)
|
||||
except (ValueError, NameError) as e:
|
||||
print(e)
|
||||
try:
|
||||
plot_network_connectivity_by_fixtype(model_path)
|
||||
except (ValueError, NameError)as e:
|
||||
except (ValueError, NameError) as e:
|
||||
print(e)
|
||||
|
||||
if n_seeds >= 2:
|
||||
pass
|
||||
combined_df_store_path = exp_path.parent / f'comb_train_{exp_path.stem[:-1]}n.csv'
|
||||
# noinspection PyUnboundLocalVariable
|
||||
found_train_stores = exp_path.rglob(df_store_path.name)
|
||||
train_dfs = []
|
||||
for found_train_store in found_train_stores:
|
||||
train_store_df = pd.read_csv(found_train_store, index_col=False)
|
||||
train_store_df['Seed'] = int(found_train_store.parent.name)
|
||||
train_dfs.append(train_store_df)
|
||||
combined_train_df = pd.concat(train_dfs)
|
||||
combined_train_df.to_csv(combined_df_store_path, index=False)
|
||||
plot_training_result(combined_df_store_path, metric=VAL_METRIC_NAME,
|
||||
plot_name=f"{combined_df_store_path.stem}.png"
|
||||
)
|
||||
|
||||
@ -2,37 +2,44 @@ from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
import torchmetrics
|
||||
from torch import nn
|
||||
from torch.utils.data import DataLoader
|
||||
from tqdm import tqdm
|
||||
|
||||
from experiments.meta_task_small_utility import AddTaskDataset, checkpoint_and_validate, train_task
|
||||
from experiments.meta_task_small_utility import AddTaskDataset, train_task
|
||||
from network import MetaNet
|
||||
from functionalities_test import test_for_fixpoints
|
||||
from functionalities_test import test_for_fixpoints, FixTypes as ft
|
||||
from experiments.meta_task_utility import new_storage_df, flat_for_store, plot_training_result, \
|
||||
plot_training_particle_types, run_particle_dropout_and_plot, plot_network_connectivity_by_fixtype
|
||||
plot_training_particle_types, run_particle_dropout_and_plot, plot_network_connectivity_by_fixtype, \
|
||||
checkpoint_and_validate
|
||||
from plot_3d_trajectories import plot_single_3d_trajectories_by_layer, plot_grouped_3d_trajectories_by_layer
|
||||
|
||||
WORKER = 0
|
||||
BATCHSIZE = 50
|
||||
EPOCH = 30
|
||||
VALIDATION_FRQ = 3
|
||||
VALIDATION_METRIC = torchmetrics.MeanAbsoluteError
|
||||
# noinspection PyProtectedMember
|
||||
VAL_METRIC_NAME = VALIDATION_METRIC()._get_name()
|
||||
DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
training = True
|
||||
n_st = 500
|
||||
plotting = True
|
||||
n_st = 700
|
||||
activation = None # nn.ReLU()
|
||||
|
||||
for weight_hidden_size in [3,4,5]:
|
||||
for weight_hidden_size in [3, 4, 5, 6]:
|
||||
|
||||
tsk_threshold = 0.85
|
||||
weight_hidden_size = weight_hidden_size
|
||||
residual_skip = True
|
||||
n_seeds = 3
|
||||
n_seeds = 10
|
||||
depth = 3
|
||||
width = 3
|
||||
out = 1
|
||||
@ -48,9 +55,9 @@ if __name__ == '__main__':
|
||||
config_str = f'{res_str}'
|
||||
exp_path = Path('output') / f'add_st_{EPOCH}_{weight_hidden_size}{config_str}{ac_str}'
|
||||
|
||||
if not training:
|
||||
# noinspection PyRedeclaration
|
||||
exp_path = Path('output') / 'mn_st_n_2_100_4'
|
||||
# if not training:
|
||||
# # noinspection PyRedeclaration
|
||||
# exp_path = Path('output') / f'add_st_{n_st}_{weight_hidden_size}'
|
||||
|
||||
for seed in range(n_seeds):
|
||||
seed_path = exp_path / str(seed)
|
||||
@ -60,17 +67,18 @@ if __name__ == '__main__':
|
||||
weight_store_path = seed_path / 'weight_store.csv'
|
||||
srnn_parameters = dict()
|
||||
|
||||
valid_data = AddTaskDataset()
|
||||
vali_load = DataLoader(valid_data, batch_size=BATCHSIZE, shuffle=True,
|
||||
drop_last=True, num_workers=WORKER)
|
||||
|
||||
if training:
|
||||
# Check if files do exist on project location, warn and break.
|
||||
for path in [model_path, df_store_path, weight_store_path]:
|
||||
assert not path.exists(), f'Path "{path}" already exists. Check your configuration!'
|
||||
|
||||
train_data = AddTaskDataset()
|
||||
valid_data = AddTaskDataset()
|
||||
train_load = DataLoader(train_data, batch_size=BATCHSIZE, shuffle=True,
|
||||
drop_last=True, num_workers=WORKER)
|
||||
vali_load = DataLoader(valid_data, batch_size=BATCHSIZE, shuffle=False,
|
||||
drop_last=True, num_workers=WORKER)
|
||||
|
||||
interface = np.prod(train_data[0][0].shape)
|
||||
metanet = MetaNet(interface, depth=depth, width=width, out=out,
|
||||
@ -89,7 +97,7 @@ if __name__ == '__main__':
|
||||
metanet = metanet.train()
|
||||
|
||||
# Init metrics, even we do not need:
|
||||
metric = torchmetrics.MeanAbsoluteError()
|
||||
metric = VALIDATION_METRIC()
|
||||
n_st_per_batch = n_st // len(train_load)
|
||||
|
||||
for batch, (batch_x, batch_y) in tqdm(enumerate(train_load),
|
||||
@ -113,12 +121,13 @@ if __name__ == '__main__':
|
||||
metanet = metanet.eval()
|
||||
if metric.total.item():
|
||||
validation_log = dict(Epoch=int(epoch), Batch=BATCHSIZE,
|
||||
Metric='Train Accuracy', Score=metric.compute().item())
|
||||
Metric=f'Train {VAL_METRIC_NAME}', Score=metric.compute().item())
|
||||
train_store.loc[train_store.shape[0]] = validation_log
|
||||
|
||||
accuracy = checkpoint_and_validate(metanet, seed_path, epoch, vali_load).item()
|
||||
mae = checkpoint_and_validate(metanet, vali_load, seed_path, epoch,
|
||||
validation_metric=VALIDATION_METRIC).item()
|
||||
validation_log = dict(Epoch=int(epoch), Batch=BATCHSIZE,
|
||||
Metric='Test Accuracy', Score=accuracy)
|
||||
Metric=f'Test {VAL_METRIC_NAME}', Score=mae)
|
||||
train_store.loc[train_store.shape[0]] = validation_log
|
||||
|
||||
if is_validation_epoch:
|
||||
@ -153,9 +162,10 @@ if __name__ == '__main__':
|
||||
for key, value in dict(counter_dict).items():
|
||||
step_log = dict(Epoch=int(EPOCH), Batch=BATCHSIZE, Metric=key, Score=value)
|
||||
train_store.loc[train_store.shape[0]] = step_log
|
||||
accuracy = checkpoint_and_validate(metanet, seed_path, EPOCH, vali_load, final_model=True)
|
||||
accuracy = checkpoint_and_validate(metanet, vali_load, seed_path, EPOCH, final_model=True,
|
||||
validation_metric=VALIDATION_METRIC)
|
||||
validation_log = dict(Epoch=EPOCH, Batch=BATCHSIZE,
|
||||
Metric='Test Accuracy', Score=accuracy.item())
|
||||
Metric=f'Test {VAL_METRIC_NAME}', Score=accuracy.item())
|
||||
for particle in metanet.particles:
|
||||
weight_log = (EPOCH, particle.name, *(flat_for_store(particle.parameters())))
|
||||
weight_store.loc[weight_store.shape[0]] = weight_log
|
||||
@ -163,26 +173,48 @@ if __name__ == '__main__':
|
||||
train_store.loc[train_store.shape[0]] = validation_log
|
||||
train_store.to_csv(df_store_path, mode='a', header=not df_store_path.exists(), index=False)
|
||||
weight_store.to_csv(weight_store_path, mode='a', header=not weight_store_path.exists(), index=False)
|
||||
if plotting:
|
||||
|
||||
plot_training_result(df_store_path)
|
||||
plot_training_result(df_store_path, metric=VAL_METRIC_NAME)
|
||||
plot_training_particle_types(df_store_path)
|
||||
|
||||
try:
|
||||
model_path = next(seed_path.glob(f'*e{EPOCH}.tp'))
|
||||
except StopIteration:
|
||||
print('Model pattern did not trigger.')
|
||||
print(f'Search path was: {seed_path}:')
|
||||
print(f'Found Models are: {list(seed_path.rglob(".tp"))}')
|
||||
print('####################################################')
|
||||
print('ERROR: Model pattern did not trigger.')
|
||||
print(f'INFO: Search path was: {seed_path}:')
|
||||
print(f'INFO: Found Models are: {list(seed_path.rglob(".tp"))}')
|
||||
print('####################################################')
|
||||
exit(1)
|
||||
|
||||
try:
|
||||
run_particle_dropout_and_plot(model_path)
|
||||
run_particle_dropout_and_plot(model_path, valid_loader=vali_load, metric_class=VALIDATION_METRIC)
|
||||
except ValueError as e:
|
||||
print(e)
|
||||
print('ERROR:', e)
|
||||
try:
|
||||
plot_network_connectivity_by_fixtype(model_path)
|
||||
except ValueError as e:
|
||||
print(e)
|
||||
print('ERROR:', e)
|
||||
try:
|
||||
plot_single_3d_trajectories_by_layer(model_path, weight_store_path, status_type=ft.identity_func)
|
||||
plot_single_3d_trajectories_by_layer(model_path, weight_store_path, status_type=ft.other_func)
|
||||
plot_grouped_3d_trajectories_by_layer(model_path, weight_store_path, status_type=ft.identity_func)
|
||||
plot_grouped_3d_trajectories_by_layer(model_path, weight_store_path, status_type=ft.other_func)
|
||||
except ValueError as e:
|
||||
print('ERROR:', e)
|
||||
|
||||
if n_seeds >= 2:
|
||||
pass
|
||||
combined_df_store_path = exp_path.parent / f'comb_train_{exp_path.stem[:-1]}n.csv'
|
||||
# noinspection PyUnboundLocalVariable
|
||||
found_train_stores = exp_path.rglob(df_store_path.name)
|
||||
train_dfs = []
|
||||
for found_train_store in found_train_stores:
|
||||
train_store_df = pd.read_csv(found_train_store, index_col=False)
|
||||
train_store_df['Seed'] = int(found_train_store.parent.name)
|
||||
train_dfs.append(train_store_df)
|
||||
combined_train_df = pd.concat(train_dfs)
|
||||
combined_train_df.to_csv(combined_df_store_path, index=False)
|
||||
plot_training_result(combined_df_store_path, metric=VAL_METRIC_NAME,
|
||||
plot_name=f"{combined_df_store_path.stem}.png"
|
||||
)
|
||||
|
||||
@ -1,13 +1,20 @@
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import numpy as np
|
||||
from network import MetaNet, FixTypes
|
||||
from network import FixTypes
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.decomposition import PCA
|
||||
|
||||
def plot_single_3d_trajectories_by_layer(model:MetaNet, all_weights:pd.DataFrame, save_path:Path, status_type:FixTypes):
|
||||
''' This plots one PCA for every net (over its n epochs) as one trajectory and then combines all of them in one plot '''
|
||||
|
||||
def plot_single_3d_trajectories_by_layer(model_path, all_weights_path, status_type: FixTypes):
|
||||
"""
|
||||
This plots one PCA for every net (over its n epochs) as one trajectory
|
||||
and then combines all of them in one plot
|
||||
"""
|
||||
model = torch.load(model_path, map_location=torch.device('cpu')).eval()
|
||||
all_weights = pd.read_csv(all_weights_path, index_col=False)
|
||||
save_path = model_path.parent / 'trajec_plots'
|
||||
|
||||
all_epochs = all_weights.Epoch.unique()
|
||||
pca = PCA(n_components=2, whiten=True)
|
||||
@ -18,8 +25,9 @@ def plot_single_3d_trajectories_by_layer(model:MetaNet, all_weights:pd.DataFrame
|
||||
fixpoint_statuses = [net.is_fixpoint for net in model_layer.particles]
|
||||
num_status_of_layer = sum([net.is_fixpoint == status_type for net in model_layer.particles])
|
||||
layer = all_weights[all_weights.Weight.str.startswith(f"L{layer_idx}")]
|
||||
weight_batches = [np.array(layer[layer.Weight == name].values.tolist())[:,2:] for name in layer.Weight.unique()]
|
||||
|
||||
weight_batches = [np.array(layer[layer.Weight == name].values.tolist())[:, 2:]
|
||||
for name in layer.Weight.unique()]
|
||||
plt.clf()
|
||||
fig = plt.figure()
|
||||
ax = plt.axes(projection='3d')
|
||||
fig.set_figheight(10)
|
||||
@ -39,15 +47,19 @@ def plot_single_3d_trajectories_by_layer(model:MetaNet, all_weights:pd.DataFrame
|
||||
ax.set_title(f"Layer {layer_idx}: {num_status_of_layer}-{status_type}", fontsize=20)
|
||||
ax.set_xlabel('PCA Transformed x-axis', fontsize=20)
|
||||
ax.set_ylabel('PCA Transformed y-axis', fontsize=20)
|
||||
ax.set_zlabel('Epochs', fontsize=30, rotation = 0)
|
||||
ax.set_zlabel('Epochs', fontsize=30, rotation=0)
|
||||
file_path = save_path / f"layer_{layer_idx}_{num_status_of_layer}_{status_type}.png"
|
||||
plt.savefig(file_path, bbox_inches="tight", dpi=300, format="png")
|
||||
plt.clf()
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def plot_grouped_3d_trajectories_by_layer(model:MetaNet, all_weights:pd.DataFrame, save_path:Path, status_type:FixTypes):
|
||||
''' This computes the PCA over all the net-weights at once and then plots that.'''
|
||||
def plot_grouped_3d_trajectories_by_layer(model_path, all_weights_path, status_type: FixTypes):
|
||||
""" This computes the PCA over all the net-weights at once and then plots that."""
|
||||
|
||||
model = torch.load(model_path, map_location=torch.device('cpu')).eval()
|
||||
save_path = model_path.parent / 'trajec_plots'
|
||||
all_weights = pd.read_csv(all_weights_path, index_col=False)
|
||||
all_epochs = all_weights.Epoch.unique()
|
||||
pca = PCA(n_components=2, whiten=True)
|
||||
save_path.mkdir(exist_ok=True, parents=True)
|
||||
@ -57,8 +69,9 @@ def plot_grouped_3d_trajectories_by_layer(model:MetaNet, all_weights:pd.DataFram
|
||||
fixpoint_statuses = [net.is_fixpoint for net in model_layer.particles]
|
||||
num_status_of_layer = sum([net.is_fixpoint == status_type for net in model_layer.particles])
|
||||
layer = all_weights[all_weights.Weight.str.startswith(f"L{layer_idx}")]
|
||||
weight_batches = np.vstack([np.array(layer[layer.Weight == name].values.tolist())[:,2:] for name in layer.Weight.unique()])
|
||||
|
||||
weight_batches = np.vstack([np.array(layer[layer.Weight == name].values.tolist())[:, 2:]
|
||||
for name in layer.Weight.unique()])
|
||||
plt.clf()
|
||||
fig = plt.figure()
|
||||
fig.set_figheight(10)
|
||||
fig.set_figwidth(12)
|
||||
@ -67,7 +80,8 @@ def plot_grouped_3d_trajectories_by_layer(model:MetaNet, all_weights:pd.DataFram
|
||||
|
||||
pca.fit(weight_batches)
|
||||
w_transformed = pca.transform(weight_batches)
|
||||
for transformed_trajectory,status in zip(np.split(w_transformed, len(layer.Weight.unique())), fixpoint_statuses):
|
||||
for transformed_trajectory, status in zip(
|
||||
np.split(w_transformed, len(layer.Weight.unique())), fixpoint_statuses):
|
||||
if status == status_type:
|
||||
xdata = transformed_trajectory[:, 0]
|
||||
ydata = transformed_trajectory[:, 1]
|
||||
@ -78,13 +92,16 @@ def plot_grouped_3d_trajectories_by_layer(model:MetaNet, all_weights:pd.DataFram
|
||||
ax.set_title(f"Layer {layer_idx}: {num_status_of_layer}-{status_type}", fontsize=20)
|
||||
ax.set_xlabel('PCA Transformed x-axis', fontsize=20)
|
||||
ax.set_ylabel('PCA Transformed y-axis', fontsize=20)
|
||||
ax.set_zlabel('Epochs', fontsize=30, rotation = 0)
|
||||
ax.set_zlabel('Epochs', fontsize=30, rotation=0)
|
||||
file_path = save_path / f"layer_{layer_idx}_{num_status_of_layer}_{status_type}_grouped.png"
|
||||
plt.savefig(file_path, bbox_inches="tight", dpi=300, format="png")
|
||||
plt.clf()
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise (NotImplementedError('Get out of here'))
|
||||
"""
|
||||
weight_path = Path("weight_store.csv")
|
||||
model_path = Path("trained_model_ckpt_e100.tp")
|
||||
save_path = Path("figures/3d_trajectories/")
|
||||
@ -95,5 +112,6 @@ if __name__ == '__main__':
|
||||
|
||||
plot_single_3d_trajectories_by_layer(model, weight_df, save_path, status_type=FixTypes.identity_func)
|
||||
plot_single_3d_trajectories_by_layer(model, weight_df, save_path, status_type=FixTypes.other_func)
|
||||
#plot_grouped_3d_trajectories_by_layer(model, weight_df, save_path, FixTypes.identity_func)
|
||||
plot_grouped_3d_trajectories_by_layer(model, weight_df, save_path, FixTypes.identity_func)
|
||||
#plot_grouped_3d_trajectories_by_layer(model, weight_df, save_path, FixTypes.other_func)
|
||||
"""
|
||||
Reference in New Issue
Block a user