journal linspace basins
This commit is contained in:
steffen-illium
parent
cf6eec639f
commit
14d9a533cb
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
/output/
|
||||
@ -8,15 +8,12 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
from functionalities_test import is_identity_function, test_status
|
||||
from journal_basins import SpawnExperiment, prng, mean_invariate_manhattan_distance
|
||||
from journal_basins import SpawnExperiment, mean_invariate_manhattan_distance
|
||||
from network import Net
|
||||
|
||||
from sklearn.metrics import mean_absolute_error as MAE
|
||||
from sklearn.metrics import mean_squared_error as MSE
|
||||
|
||||
import seaborn as sns
|
||||
from matplotlib import pyplot as plt
|
||||
|
||||
|
||||
class SpawnLinspaceExperiment(SpawnExperiment):
|
||||
|
||||
@ -28,6 +25,12 @@ class SpawnLinspaceExperiment(SpawnExperiment):
|
||||
'status_post'])
|
||||
|
||||
# For every initial net {i} after populating (that is fixpoint after first epoch);
|
||||
# parent = self.parents[0]
|
||||
# parent_clone = clone = Net(parent.input_size, parent.hidden_size, parent.out_size,
|
||||
# name=f"{parent.name}_clone_{0}", start_time=self.ST_steps)
|
||||
# parent_clone.apply_weights(torch.as_tensor(parent.create_target_weights(parent.input_weight_matrix())))
|
||||
# parent_clone = parent_clone.apply_noise(self.noise)
|
||||
# self.parents.append(parent_clone)
|
||||
pairwise_net_list = itertools.combinations(self.parents, 2)
|
||||
for net1, net2 in pairwise_net_list:
|
||||
# We set parent start_time to just before this epoch ended, so plotting is zoomed in. Comment out to
|
||||
@ -42,11 +45,12 @@ class SpawnLinspaceExperiment(SpawnExperiment):
|
||||
net2_target_data = net2.create_target_weights(net2_input_data)
|
||||
|
||||
if is_identity_function(net1) and is_identity_function(net2):
|
||||
# if True:
|
||||
# Clone the fixpoint x times and add (+-)self.noise to weight-sets randomly;
|
||||
# To plot clones starting after first epoch (z=ST_steps), set that as start_time!
|
||||
# To make sure PCA will plot the same trajectory up until this point, we clone the
|
||||
# parent-net's weight history as well.
|
||||
in_between_weights = np.linspace(net1_target_data, net2_target_data, number_clones,endpoint=False)
|
||||
in_between_weights = np.linspace(net1_target_data, net2_target_data, number_clones, endpoint=False)
|
||||
|
||||
for j, in_between_weight in enumerate(in_between_weights):
|
||||
clone = Net(net1.input_size, net1.hidden_size, net1.out_size,
|
||||
@ -89,7 +93,6 @@ class SpawnLinspaceExperiment(SpawnExperiment):
|
||||
for _ in range(self.epochs - 1):
|
||||
for _ in range(self.ST_steps):
|
||||
parent.self_train(1, self.log_step_size, self.net_learning_rate)
|
||||
|
||||
|
||||
self.df = df
|
||||
|
||||
@ -106,7 +109,7 @@ if __name__ == '__main__':
|
||||
ST_log_step_size = 10
|
||||
|
||||
# Define number of networks & their architecture
|
||||
nr_clones = 3
|
||||
nr_clones = 20
|
||||
ST_population_size = 3
|
||||
ST_net_hidden_size = 2
|
||||
ST_net_learning_rate = 0.04
|
||||
@ -123,7 +126,7 @@ if __name__ == '__main__':
|
||||
epochs=ST_epochs,
|
||||
st_steps=ST_steps,
|
||||
nr_clones=nr_clones,
|
||||
noise=None,
|
||||
noise=1e-8,
|
||||
directory=Path('output') / 'spawn_basin' / f'{ST_name_hash}' / f'linage'
|
||||
)
|
||||
df = exp.df
|
||||
@ -133,10 +136,10 @@ if __name__ == '__main__':
|
||||
print(f"\nSaved experiment to {directory}.")
|
||||
|
||||
# Boxplot with counts of nr_fixpoints, nr_other, nr_etc. on y-axis
|
||||
sns.countplot(data=df, x="noise", hue="status_post")
|
||||
plt.savefig(f"output/spawn_basin/{ST_name_hash}/fixpoint_status_countplot.png")
|
||||
# sns.countplot(data=df, x="noise", hue="status_post")
|
||||
# plt.savefig(f"output/spawn_basin/{ST_name_hash}/fixpoint_status_countplot.png")
|
||||
|
||||
# Catplot (either kind="point" or "box") that shows before-after training distances to parent
|
||||
mlt = df[["MIM_pre", "MIM_post", "noise"]].melt("noise", var_name="time", value_name='Average Distance')
|
||||
sns.catplot(data=mlt, x="time", y="Average Distance", col="noise", kind="point", col_wrap=5, sharey=False)
|
||||
plt.savefig(f"output/spawn_basin/{ST_name_hash}/clone_distance_catplot.png")
|
||||
# mlt = df[["MIM_pre", "MIM_post", "noise"]].melt("noise", var_name="time", value_name='Average Distance')
|
||||
# sns.catplot(data=mlt, x="time", y="Average Distance", col="noise", kind="point", col_wrap=5, sharey=False)
|
||||
# plt.savefig(f"output/spawn_basin/{ST_name_hash}/clone_distance_catplot.png")
|
||||
|
||||
@ -84,21 +84,6 @@ def distance_from_parent(nets, distance="MIM", print_it=True):
|
||||
|
||||
class SpawnExperiment:
|
||||
|
||||
@staticmethod
|
||||
def apply_noise(network, noise: int):
|
||||
""" Changing the weights of a network to values + noise """
|
||||
|
||||
for layer_id, layer_name in enumerate(network.state_dict()):
|
||||
for line_id, line_values in enumerate(network.state_dict()[layer_name]):
|
||||
for weight_id, weight_value in enumerate(network.state_dict()[layer_name][line_id]):
|
||||
# network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise
|
||||
if prng() < 0.5:
|
||||
network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise
|
||||
else:
|
||||
network.state_dict()[layer_name][line_id][weight_id] = weight_value - noise
|
||||
|
||||
return network
|
||||
|
||||
def __init__(self, population_size, log_step_size, net_input_size, net_hidden_size, net_out_size, net_learning_rate,
|
||||
epochs, st_steps, nr_clones, noise, directory) -> None:
|
||||
self.population_size = population_size
|
||||
@ -171,7 +156,7 @@ class SpawnExperiment:
|
||||
f"ST_net_{str(i)}_clone_{str(j)}", start_time=self.ST_steps)
|
||||
clone.load_state_dict(copy.deepcopy(net.state_dict()))
|
||||
rand_noise = prng() * self.noise
|
||||
clone = self.apply_noise(clone, rand_noise)
|
||||
clone = clone.apply_noise(rand_noise)
|
||||
clone.s_train_weights_history = copy.deepcopy(net.s_train_weights_history)
|
||||
clone.number_trained = copy.deepcopy(net.number_trained)
|
||||
|
||||
|
||||
@ -91,7 +91,6 @@ class RobustnessComparisonExperiment:
|
||||
self.time_to_vergence, self.time_as_fixpoint = self.test_robustness(
|
||||
seeds=population_size if self.is_synthetic else 1)
|
||||
|
||||
|
||||
def populate_environment(self):
|
||||
nets = []
|
||||
if self.is_synthetic:
|
||||
@ -125,8 +124,8 @@ class RobustnessComparisonExperiment:
|
||||
# This checks wether to use synthetic setting with multiple seeds
|
||||
# or multi network settings with a singlee seed
|
||||
|
||||
df = pd.DataFrame(columns=['setting', 'Noise Level', 'steps', 'absolute_loss',
|
||||
'time_to_vergence', 'time_as_fixpoint'])
|
||||
df = pd.DataFrame(columns=['setting', 'Noise Level', 'Self Train Steps', 'absolute_loss',
|
||||
'Time to vergence', 'Time as fixpoint'])
|
||||
with tqdm(total=max(len(self.id_functions), seeds)) as pbar:
|
||||
for i, fixpoint in enumerate(self.id_functions): # 1 / n
|
||||
row_headers.append(fixpoint.name)
|
||||
@ -138,8 +137,7 @@ class RobustnessComparisonExperiment:
|
||||
clone = Net(fixpoint.input_size, fixpoint.hidden_size, fixpoint.out_size,
|
||||
f"{fixpoint.name}_clone_noise10e-{noise_level}")
|
||||
clone.load_state_dict(copy.deepcopy(fixpoint.state_dict()))
|
||||
rand_noise = prng() * pow(10, -noise_level) # n / 1
|
||||
clone = self.apply_noise(clone, rand_noise)
|
||||
clone = clone.apply_noise(pow(10, -noise_level))
|
||||
|
||||
while not is_zero_fixpoint(clone) and not is_divergent(clone):
|
||||
# -> before
|
||||
@ -154,7 +152,6 @@ class RobustnessComparisonExperiment:
|
||||
|
||||
absolute_loss = F.l1_loss(target_data_pre_application, target_data_post_application).item()
|
||||
|
||||
|
||||
if is_identity_function(clone):
|
||||
time_as_fixpoint[setting][noise_level] += 1
|
||||
# When this raises a Type Error, we found a second order fixpoint!
|
||||
@ -166,26 +163,24 @@ class RobustnessComparisonExperiment:
|
||||
pbar.update(1)
|
||||
|
||||
# Get the measuremts at the highest time_time_to_vergence
|
||||
df_sorted = df.sort_values('Steps', ascending=False).drop_duplicates(['setting', 'Noise Level'])
|
||||
df_melted = df_sorted.reset_index().melt(id_vars=['setting', 'Noise Level', 'Steps'],
|
||||
df_sorted = df.sort_values('Self Train Steps', ascending=False).drop_duplicates(['setting', 'Noise Level'])
|
||||
df_melted = df_sorted.reset_index().melt(id_vars=['setting', 'Noise Level', 'Self Train Steps'],
|
||||
value_vars=['Time to vergence', 'Time as fixpoint'],
|
||||
var_name="Measurement",
|
||||
value_name="Steps")
|
||||
value_name="Steps").sort_values('Noise Level')
|
||||
# Plotting
|
||||
sns.set(style='whitegrid', font_scale=2)
|
||||
bf = sns.boxplot(data=df_melted, y='Steps', x='Noise Level', hue='Measurement', palette=PALETTE)
|
||||
synthetic = 'synthetic' if self.is_synthetic else 'natural'
|
||||
bf.set_title(f'Robustness as self application steps per noise level for {synthetic} fixpoints.')
|
||||
# bf.set_title(f'Robustness as self application steps per noise level for {synthetic} fixpoints.')
|
||||
plt.tight_layout()
|
||||
|
||||
# sns.set(rc={'figure.figsize': (10, 50)})
|
||||
# bx = sns.catplot(data=df[df['absolute_loss'] < 1], y='absolute_loss', x='application_step', kind='box',
|
||||
# col='noise_level', col_wrap=3, showfliers=False)
|
||||
directory = Path('output') / 'robustness'
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
filename = f"absolute_loss_perapplication_boxplot_grid.png"
|
||||
filepath = directory / filename
|
||||
|
||||
filename = f"absolute_loss_perapplication_boxplot_grid_{'synthetic' if self.is_synthetic else 'wild'}.png"
|
||||
filepath = self.directory / filename
|
||||
plt.savefig(str(filepath))
|
||||
|
||||
if print_it:
|
||||
@ -219,11 +214,11 @@ if __name__ == "__main__":
|
||||
ST_steps = 1000
|
||||
ST_epochs = 5
|
||||
ST_log_step_size = 10
|
||||
ST_population_size = 2
|
||||
ST_population_size = 500
|
||||
ST_net_hidden_size = 2
|
||||
ST_net_learning_rate = 0.004
|
||||
ST_name_hash = random.getrandbits(32)
|
||||
ST_synthetic = True
|
||||
ST_synthetic = False
|
||||
|
||||
print(f"Running the robustness comparison experiment:")
|
||||
exp = RobustnessComparisonExperiment(
|
||||
|
||||
@ -1,14 +1,12 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
import pickle
|
||||
from torch import mean
|
||||
|
||||
from tqdm import tqdm
|
||||
import random
|
||||
import copy
|
||||
from functionalities_test import is_identity_function, test_status, test_for_fixpoints, is_zero_fixpoint, is_divergent, is_secondary_fixpoint
|
||||
from functionalities_test import is_identity_function, test_status, is_zero_fixpoint, is_divergent, is_secondary_fixpoint
|
||||
from network import Net
|
||||
from visualization import plot_3d_self_train, plot_loss, plot_3d_soup
|
||||
from visualization import plot_loss, plot_3d_soup
|
||||
import numpy as np
|
||||
from tabulate import tabulate
|
||||
from sklearn.metrics import mean_absolute_error as MAE
|
||||
@ -18,10 +16,6 @@ import seaborn as sns
|
||||
from matplotlib import pyplot as plt
|
||||
|
||||
|
||||
def prng():
|
||||
return random.random()
|
||||
|
||||
|
||||
def l1(tup):
|
||||
a, b = tup
|
||||
return abs(a - b)
|
||||
@ -88,20 +82,6 @@ def distance_from_parent(nets, distance="MIM", print_it=True):
|
||||
|
||||
class SoupSpawnExperiment:
|
||||
|
||||
@staticmethod
|
||||
def apply_noise(network, noise: int):
|
||||
""" Changing the weights of a network to values + noise """
|
||||
|
||||
for layer_id, layer_name in enumerate(network.state_dict()):
|
||||
for line_id, line_values in enumerate(network.state_dict()[layer_name]):
|
||||
for weight_id, weight_value in enumerate(network.state_dict()[layer_name][line_id]):
|
||||
# network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise
|
||||
if prng() < 0.5:
|
||||
network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise
|
||||
else:
|
||||
network.state_dict()[layer_name][line_id][weight_id] = weight_value - noise
|
||||
|
||||
return network
|
||||
|
||||
def __init__(self, population_size, log_step_size, net_input_size, net_hidden_size, net_out_size, net_learning_rate,
|
||||
epochs, st_steps, attack_chance, nr_clones, noise, directory) -> None:
|
||||
@ -220,8 +200,7 @@ class SoupSpawnExperiment:
|
||||
clone = Net(net.input_size, net.hidden_size, net.out_size,
|
||||
f"net_{str(i)}_clone_{str(j)}", start_time=self.ST_steps)
|
||||
clone.load_state_dict(copy.deepcopy(net.state_dict()))
|
||||
rand_noise = prng() * self.noise
|
||||
clone = self.apply_noise(clone, rand_noise)
|
||||
clone = clone.apply_noise(self.noise)
|
||||
clone.s_train_weights_history = copy.deepcopy(net.s_train_weights_history)
|
||||
clone.number_trained = copy.deepcopy(net.number_trained)
|
||||
|
||||
@ -262,9 +241,9 @@ class SoupSpawnExperiment:
|
||||
f"\nMSE({i},{j}): {MSE_post}"
|
||||
f"\nMAE({i},{j}): {MAE_post}"
|
||||
f"\nMIM({i},{j}): {MIM_post}\n")
|
||||
self.parents_clones_id_functions.append(clone):
|
||||
self.parents_clones_id_functions.append(clone)
|
||||
|
||||
df.loc[df.name==clone.name, ["MAE_post", "MSE_post", "MIM_post", "status_post"]] = [MAE_post, MSE_post, MIM_post, clone.is_fixpoint]
|
||||
df.loc[df.name == clone.name, ["MAE_post", "MSE_post", "MIM_post", "status_post"]] = [MAE_post, MSE_post, MIM_post, clone.is_fixpoint]
|
||||
|
||||
# Finally take parent net {i} and finish it's training for comparison to clone development.
|
||||
for _ in range(self.epochs - 1):
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
import copy
|
||||
import random
|
||||
import os.path
|
||||
import pickle
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Union
|
||||
|
||||
@ -13,7 +12,6 @@ from matplotlib import pyplot as plt
|
||||
from torch.nn import functional as F
|
||||
from tabulate import tabulate
|
||||
|
||||
from experiments.helpers import check_folder, summary_fixpoint_percentage, summary_fixpoint_experiment
|
||||
from functionalities_test import test_for_fixpoints, is_zero_fixpoint, is_divergent, is_identity_function
|
||||
from network import Net
|
||||
from visualization import plot_loss, bar_chart_fixpoints, plot_3d_soup, line_chart_fixpoints
|
||||
@ -25,20 +23,6 @@ def prng():
|
||||
|
||||
class SoupRobustnessExperiment:
|
||||
|
||||
@staticmethod
|
||||
def apply_noise(network, noise: int):
|
||||
""" Changing the weights of a network to values + noise """
|
||||
for layer_id, layer_name in enumerate(network.state_dict()):
|
||||
for line_id, line_values in enumerate(network.state_dict()[layer_name]):
|
||||
for weight_id, weight_value in enumerate(network.state_dict()[layer_name][line_id]):
|
||||
# network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise
|
||||
if prng() < 0.5:
|
||||
network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise
|
||||
else:
|
||||
network.state_dict()[layer_name][line_id][weight_id] = weight_value - noise
|
||||
|
||||
return network
|
||||
|
||||
def __init__(self, population_size, net_i_size, net_h_size, net_o_size, learning_rate, attack_chance,
|
||||
train_nets, ST_steps, epochs, log_step_size, directory: Union[str, Path]):
|
||||
super().__init__()
|
||||
@ -146,8 +130,7 @@ class SoupRobustnessExperiment:
|
||||
clone = Net(fixpoint.input_size, fixpoint.hidden_size, fixpoint.out_size,
|
||||
f"{fixpoint.name}_clone_noise10e-{noise_level}")
|
||||
clone.load_state_dict(copy.deepcopy(fixpoint.state_dict()))
|
||||
rand_noise = prng() * pow(10, -noise_level) # n / 1
|
||||
clone = self.apply_noise(clone, rand_noise)
|
||||
clone = clone.apply_noise(pow(10, -noise_level))
|
||||
|
||||
while not is_zero_fixpoint(clone) and not is_divergent(clone):
|
||||
if is_identity_function(clone):
|
||||
|
||||
19
network.py
19
network.py
@ -1,5 +1,6 @@
|
||||
# from __future__ import annotations
|
||||
import copy
|
||||
import random
|
||||
from typing import Union
|
||||
|
||||
import torch
|
||||
@ -9,7 +10,12 @@ import numpy as np
|
||||
from torch import optim, Tensor
|
||||
|
||||
|
||||
def prng():
|
||||
return random.random()
|
||||
|
||||
|
||||
class Net(nn.Module):
|
||||
|
||||
@staticmethod
|
||||
def create_target_weights(input_weight_matrix: Tensor) -> Tensor:
|
||||
""" Outputting a tensor with the target weights. """
|
||||
@ -171,3 +177,16 @@ class Net(nn.Module):
|
||||
SA_steps = 1
|
||||
|
||||
return other_net.apply_weights(my_evaluation)
|
||||
|
||||
def apply_noise(self, noise_size: float):
|
||||
""" Changing the weights of a network to values + noise """
|
||||
for layer_id, layer_name in enumerate(self.state_dict()):
|
||||
for line_id, line_values in enumerate(self.state_dict()[layer_name]):
|
||||
for weight_id, weight_value in enumerate(self.state_dict()[layer_name][line_id]):
|
||||
# network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise
|
||||
if prng() < 0.5:
|
||||
self.state_dict()[layer_name][line_id][weight_id] = weight_value + noise_size * prng()
|
||||
else:
|
||||
self.state_dict()[layer_name][line_id][weight_id] = weight_value - noise_size * prng()
|
||||
|
||||
return self
|
||||
|
||||
@ -9,6 +9,9 @@ from sklearn.decomposition import PCA
|
||||
import random
|
||||
import string
|
||||
|
||||
from matplotlib import rcParams
|
||||
rcParams['axes.labelpad'] = 20
|
||||
|
||||
|
||||
def plot_output(output):
|
||||
""" Plotting the values of the final output """
|
||||
@ -65,6 +68,7 @@ def bar_chart_fixpoints(fixpoint_counter: Dict, population_size: int, directory:
|
||||
plt.xticks(range(len(fixpoint_counter)), list(fixpoint_counter.keys()))
|
||||
|
||||
directory = Path(directory)
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
filename = f"{str(population_size)}_nets_fixpoints_barchart.png"
|
||||
filepath = directory / filename
|
||||
plt.savefig(str(filepath))
|
||||
@ -139,19 +143,19 @@ def plot_3d(matrices_weights_history, directory: Union[str, Path], population_si
|
||||
|
||||
#steps = mpatches.Patch(color="white", label=f"{z_axis_legend}: {len(matrices_weights_history)} steps")
|
||||
population_size = mpatches.Patch(color="white", label=f"Population: {population_size} networks")
|
||||
|
||||
if z_axis_legend == "Self-application":
|
||||
if is_trained == '_trained':
|
||||
trained = mpatches.Patch(color="white", label=f"Trained: true")
|
||||
if False:
|
||||
if z_axis_legend == "Self-application":
|
||||
if is_trained == '_trained':
|
||||
trained = mpatches.Patch(color="white", label=f"Trained: true")
|
||||
else:
|
||||
trained = mpatches.Patch(color="white", label=f"Trained: false")
|
||||
ax.legend(handles=[population_size, trained])
|
||||
else:
|
||||
trained = mpatches.Patch(color="white", label=f"Trained: false")
|
||||
ax.legend(handles=[population_size, trained])
|
||||
else:
|
||||
ax.legend(handles=[population_size])
|
||||
ax.legend(handles=[population_size])
|
||||
|
||||
ax.set_title(f"PCA Transformed Weight Trajectories")
|
||||
ax.set_xlabel("PCA Transformed X-Axis")
|
||||
ax.set_ylabel("PCA Transformed Y-Axis")
|
||||
# ax.set_xlabel("PCA Transformed X-Axis")
|
||||
# ax.set_ylabel("PCA Transformed Y-Axis")
|
||||
ax.set_zlabel(f"Self Training Steps")
|
||||
|
||||
# FIXME: Replace this kind of operation with pathlib.Path() object interactions
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user