From 042188f15a0848ac0919a186b098326fb36a6eb3 Mon Sep 17 00:00:00 2001 From: steffen-illium Date: Sun, 16 May 2021 11:30:34 +0200 Subject: [PATCH] journal_basins.py debugged --- experiments/__init__.py | 5 ++ experiments/robustness_exp.py | 1 - journal_basins.py | 141 ++++++++++++++++++---------------- network.py | 2 +- visualization.py | 15 ++-- 5 files changed, 89 insertions(+), 75 deletions(-) diff --git a/experiments/__init__.py b/experiments/__init__.py index e69de29..ab7187f 100644 --- a/experiments/__init__.py +++ b/experiments/__init__.py @@ -0,0 +1,5 @@ +from mixed_setting_exp import run_mixed_experiment +from robustness_exp import run_robustness_experiment +from self_application_exp import run_SA_experiment +from self_train_exp import run_ST_experiment +from soup_exp import run_soup_experiment \ No newline at end of file diff --git a/experiments/robustness_exp.py b/experiments/robustness_exp.py index 74c81c9..2f6c09b 100644 --- a/experiments/robustness_exp.py +++ b/experiments/robustness_exp.py @@ -20,7 +20,6 @@ def add_noise(input_data, epsilon = pow(10, -5)): return output - class RobustnessExperiment: def __init__(self, population_size, log_step_size, net_input_size, net_hidden_size, net_out_size, net_learning_rate, ST_steps, directory_name) -> None: diff --git a/journal_basins.py b/journal_basins.py index 0c18ee3..0f501ff 100644 --- a/journal_basins.py +++ b/journal_basins.py @@ -1,4 +1,6 @@ import os +from pathlib import Path + from tqdm import tqdm import random import copy @@ -14,37 +16,41 @@ from sklearn.metrics import mean_squared_error as MSE def prng(): return random.random() -def l1 (tup): + +def l1(tup): a, b = tup return abs(a-b) -def mean_invariate_manhattan_distance(X,Y): + +def mean_invariate_manhattan_distance(x, y): # One of these one-liners that might be smart or really dumb. Goal is to find pairwise # distances of ascending values, ie. sum (abs(min1_X-min1_Y), abs(min2_X-min2Y) ...) / mean. # Idea was to find weight sets that have same values but just in different positions, that would # make this distance 0. - return np.mean(list(map(l1, zip(sorted(X),sorted(Y))))) + return np.mean(list(map(l1, zip(sorted(x), sorted(y))))) + def distance_matrix(nets, distance="MIM", print_it=True): matrix = [[0 for _ in range(len(nets))] for _ in range(len(nets))] for net in range(len(nets)): - weights = nets[net].input_weight_matrix()[:,0] - for other_net in range(len(nets)): - other_weights = nets[other_net].input_weight_matrix()[:,0] - if distance in ["MSE"]: - matrix[net][other_net] = MSE(weights, other_weights) - elif distance in ["MAE"]: - matrix[net][other_net] = MAE(weights, other_weights) - elif distance in ["MIM"]: - matrix[net][other_net] = mean_invariate_manhattan_distance(weights, other_weights) + weights = nets[net].input_weight_matrix()[:, 0] + for other_net in range(len(nets)): + other_weights = nets[other_net].input_weight_matrix()[:, 0] + if distance in ["MSE"]: + matrix[net][other_net] = MSE(weights, other_weights) + elif distance in ["MAE"]: + matrix[net][other_net] = MAE(weights, other_weights) + elif distance in ["MIM"]: + matrix[net][other_net] = mean_invariate_manhattan_distance(weights, other_weights) if print_it: - print(f"\nDistance matrix [{distance}]:") - [print(row) for row in matrix] + print(f"\nDistance matrix [{distance}]:") + [print(row) for row in matrix] return matrix + class SpawnExperiment: - + @staticmethod def apply_noise(network, noise: int): """ Changing the weights of a network to values + noise """ @@ -52,16 +58,16 @@ class SpawnExperiment: for layer_id, layer_name in enumerate(network.state_dict()): for line_id, line_values in enumerate(network.state_dict()[layer_name]): for weight_id, weight_value in enumerate(network.state_dict()[layer_name][line_id]): - #network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise + # network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise if prng() < 0.5: - network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise + network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise else: - network.state_dict()[layer_name][line_id][weight_id] = weight_value - noise + network.state_dict()[layer_name][line_id][weight_id] = weight_value - noise return network def __init__(self, population_size, log_step_size, net_input_size, net_hidden_size, net_out_size, net_learning_rate, - epochs, ST_steps, noise, directory_name) -> None: + epochs, st_steps, noise, directory_name) -> None: self.population_size = population_size self.log_step_size = log_step_size self.net_input_size = net_input_size @@ -69,19 +75,19 @@ class SpawnExperiment: self.net_out_size = net_out_size self.net_learning_rate = net_learning_rate self.epochs = epochs - self.ST_steps = ST_steps + self.ST_steps = st_steps self.loss_history = [] self.nets = [] self.noise = noise or 10e-5 print("\nNOISE:", self.noise) - - self.directory_name = directory_name - os.mkdir(self.directory_name) + + self.directory = Path(directory_name) + self.directory.mkdir(parents=True, exist_ok=True) self.populate_environment() self.spawn_and_continue() self.weights_evolution_3d_experiment() - #self.visualize_loss() + # self.visualize_loss() distance_matrix(self.nets) def populate_environment(self): @@ -93,66 +99,67 @@ class SpawnExperiment: net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name) for _ in range(self.ST_steps): - input_data = net.input_weight_matrix() - target_data = net.create_target_weights(input_data) - net.self_train(1, self.log_step_size, self.net_learning_rate, input_data, target_data) + net.self_train(1, self.log_step_size, self.net_learning_rate) - #print(f"\nLast weight matrix (epoch: {self.epochs}):\n{net.input_weight_matrix()}\nLossHistory: {net.loss_history[-10:]}") + # print(f"\nLast weight matrix (epoch: {self.epochs}):\n + # {net.input_weight_matrix()}\nLossHistory: {net.loss_history[-10:]}") self.nets.append(net) - - def spawn_and_continue(self, number_spawns:int = 5): + def spawn_and_continue(self, number_spawns: int = 5): # For every initial net {i} after populating (that is fixpoint after first epoch); for i in range(self.population_size): - net = self.nets[i] - - net_input_data = net.input_weight_matrix() - net_target_data = net.create_target_weights(net_input_data) - if is_identity_function(net, net_input_data, net_target_data): - print(f"\nNet {i} is fixpoint") - #print("\nNet weights before training\n", target_data) + net = self.nets[i] - # Clone the fixpoint x times and add (+-)self.noise to weight-sets randomly; - # To plot clones starting after first epoch (z=ST_steps), set that as start_time! - for j in range(number_spawns): - clone = Net(net.input_size, net.hidden_size, net.out_size, f"ST_net_{str(i)}_clone_{str(j)}", start_time=self.ST_steps) - clone.load_state_dict(copy.deepcopy(net.state_dict())) - rand_noise = prng() * self.noise - clone = self.apply_noise(clone, rand_noise) + net_input_data = net.input_weight_matrix() + net_target_data = net.create_target_weights(net_input_data) + if is_identity_function(net): + print(f"\nNet {i} is fixpoint") + # print("\nNet weights before training\n", target_data) - # Then finish training each clone {j} (for remaining epoch-1 * ST_steps) and add to nets for plotting; - for _ in range(self.epochs - 1): - for _ in range(self.ST_steps): - input_data = clone.input_weight_matrix() - target_data = clone.create_target_weights(input_data) - clone.self_train(1, self.log_step_size, self.net_learning_rate, input_data, target_data) - #print(f"clone {j} last weights: {target_data}, noise {noise}") - if is_identity_function(clone, input_data, target_data): - print(f"Clone {j} (of net_{i}) is fixpoint. \nMSE(j,i): {MSE(net_target_data, target_data)}, \nMAE(j,i): {MAE(net_target_data, target_data)}\n") - self.nets.append(clone) + # Clone the fixpoint x times and add (+-)self.noise to weight-sets randomly; + # To plot clones starting after first epoch (z=ST_steps), set that as start_time! + for j in range(number_spawns): + clone = Net(net.input_size, net.hidden_size, net.out_size, + f"ST_net_{str(i)}_clone_{str(j)}", + start_time=self.ST_steps) + clone.load_state_dict(copy.deepcopy(net.state_dict())) + rand_noise = prng() * self.noise + clone = self.apply_noise(clone, rand_noise) - # Finally take parent net {i} and finish it's training for comparison to clone development. - for _ in range(self.epochs - 1): - for _ in range(self.ST_steps): - input_data = net.input_weight_matrix() - target_data = net.create_target_weights(input_data) - net.self_train(1, self.log_step_size, self.net_learning_rate, input_data, target_data) - #print("\nNet weights after training \n", target_data) + # Then finish training each clone {j} (for remaining epoch-1 * ST_steps) + # and add to nets for plotting; + for _ in range(self.epochs - 1): + for _ in range(self.ST_steps): + clone.self_train(1, self.log_step_size, self.net_learning_rate) + # print(f"clone {j} last weights: {target_data}, noise {noise}") + if is_identity_function(clone): + input_data = clone.input_weight_matrix() + target_data = clone.create_target_weights(input_data) + print(f"Clone {j} (of net_{i}) is fixpoint. \nMSE(j,i): " + f"{MSE(net_target_data, target_data)}, \nMAE(j,i): {MAE(net_target_data, target_data)}\n") + self.nets.append(clone) + + # Finally take parent net {i} and finish it's training for comparison to clone development. + for _ in range(self.epochs - 1): + for _ in range(self.ST_steps): + net.self_train(1, self.log_step_size, self.net_learning_rate) + # print("\nNet weights after training \n", target_data) else: - print("No fixpoints found.") + print("No fixpoints found.") def weights_evolution_3d_experiment(self): exp_name = f"ST_{str(len(self.nets))}_nets_3d_weights_PCA" - return plot_3d_self_train(self.nets, exp_name, self.directory_name, self.log_step_size) + return plot_3d_self_train(self.nets, exp_name, self.directory.name, self.log_step_size) def visualize_loss(self): for i in range(len(self.nets)): net_loss_history = self.nets[i].loss_history self.loss_history.append(net_loss_history) - plot_loss(self.loss_history, self.directory_name) + plot_loss(self.loss_history, self.directory.name) -if __name__=="__main__": + +if __name__ == "__main__": NET_INPUT_SIZE = 4 NET_OUT_SIZE = 1 @@ -180,7 +187,7 @@ if __name__=="__main__": net_out_size=NET_OUT_SIZE, net_learning_rate=ST_net_learning_rate, epochs=ST_epochs, - ST_steps=ST_steps, - noise=pow(10,-noise_factor), + st_steps=ST_steps, + noise=pow(10, -noise_factor), directory_name=f"./experiments/spawn_basin/{ST_name_hash}_10e-{noise_factor}" ) diff --git a/network.py b/network.py index 745bf6a..77a59be 100644 --- a/network.py +++ b/network.py @@ -1,4 +1,4 @@ -#from __future__ import annotations +# from __future__ import annotations import copy from typing import Union diff --git a/visualization.py b/visualization.py index 85e4961..9de9c7e 100644 --- a/visualization.py +++ b/visualization.py @@ -1,7 +1,7 @@ +from pathlib import Path from tokenize import String from typing import List, Dict -import numpy from tqdm import tqdm import matplotlib.pyplot as plt import matplotlib.patches as mpatches @@ -120,14 +120,17 @@ def plot_3d(matrices_weights_history, folder_name, population_size, z_axis_legen ax.set_ylabel("PCA Y") ax.set_zlabel(f"Epochs") - filepath = f"./{folder_name}" - filename = f"{filepath}/{exp_name}{is_trained}.png" - if os.path.isfile(filename): + # FIXME: Replace this kind of operation with pathlib.Path() object interactions + folder = Path(folder_name) + folder.mkdir(parents=True, exist_ok=True) + filename = f"{exp_name}{is_trained}.png" + filepath = folder / filename + if filepath.exists(): letters = string.ascii_lowercase random_letters = ''.join(random.choice(letters) for _ in range(5)) - plt.savefig(f"{filename}_{random_letters}") + plt.savefig(f"{filepath.stem}_{random_letters}.png") else: - plt.savefig(f"{filename}") + plt.savefig(str(filepath)) plt.show() #plt.clf()