From 55bdd706b6b28c1ddf05b9efb4c41f7bddcf5b92 Mon Sep 17 00:00:00 2001 From: steffen-illium Date: Sun, 23 May 2021 13:46:21 +0200 Subject: [PATCH] journal_robustness.py redone, now is sensitive to seeds and plots --- README.md | 9 ++++++ journal_basins.py | 10 +++--- journal_robustness.py | 71 ++++++++++++++++++++++++++++--------------- requirements.txt | 14 ++++++--- 4 files changed, 70 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index fd08a80..7be6661 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,16 @@ - [ ] Adjust Self Training so that it favors second order fixpoints-> Second order test implementation (?) +- [ ] Barplot over clones -> how many become a fixpoint cs how many diverge per noise level +- [ ] Box-Plot of Avg. Distance of clones from parent + +# Future Todos: + +- [ ] Find a statistik over weight space that provides a better init function +- [ ] Test this init function on a mnist classifier - just for the lolz +- [ ] + --- ## Notes: diff --git a/journal_basins.py b/journal_basins.py index ea6fc61..d49ce09 100644 --- a/journal_basins.py +++ b/journal_basins.py @@ -28,7 +28,7 @@ def mean_invariate_manhattan_distance(x, y): # distances of ascending values, ie. sum (abs(min1_X-min1_Y), abs(min2_X-min2Y) ...) / mean. # Idea was to find weight sets that have same values but just in different positions, that would # make this distance 0. - return np.mean(list(map(l1, zip(sorted(x), sorted(y))))) + return np.mean(list(map(l1, zip(sorted(x.numpy()), sorted(y.numpy()))))) def distance_matrix(nets, distance="MIM", print_it=True): @@ -212,19 +212,19 @@ if __name__ == "__main__": # Define number of runs & name: ST_runs = 1 ST_runs_name = "test-27" - ST_steps = 1700 + ST_steps = 2500 ST_epochs = 2 ST_log_step_size = 10 # Define number of networks & their architecture - nr_clones = 5 - ST_population_size = 1 + nr_clones = 10 + ST_population_size = 3 ST_net_hidden_size = 2 ST_net_learning_rate = 0.04 ST_name_hash = random.getrandbits(32) print(f"Running the Spawn experiment:") - for noise_factor in range(2,3): + for noise_factor in [1]: SpawnExperiment( population_size=ST_population_size, log_step_size=ST_log_step_size, diff --git a/journal_robustness.py b/journal_robustness.py index 0fbb296..c4e9ac1 100644 --- a/journal_robustness.py +++ b/journal_robustness.py @@ -1,7 +1,10 @@ import pickle + +import pandas as pd import torch import random import copy +import numpy as np from pathlib import Path from tqdm import tqdm @@ -14,6 +17,8 @@ from functionalities_test import is_identity_function, is_zero_fixpoint, test_fo from network import Net from torch.nn import functional as F from visualization import plot_loss, bar_chart_fixpoints +import seaborn as sns +from matplotlib import pyplot as plt def prng(): @@ -31,7 +36,6 @@ class RobustnessComparisonExperiment: @staticmethod def apply_noise(network, noise: int): """ Changing the weights of a network to values + noise """ - for layer_id, layer_name in enumerate(network.state_dict()): for line_id, line_values in enumerate(network.state_dict()[layer_name]): for weight_id, weight_value in enumerate(network.state_dict()[layer_name][line_id]): @@ -77,41 +81,48 @@ class RobustnessComparisonExperiment: def populate_environment(self): loop_population_size = tqdm(range(self.population_size)) nets = [] + if self.synthetic: + ''' Either use perfect / hand-constructed fixpoint ... ''' + net_name = f"net_{str(0)}_synthetic" + net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name) + net.apply_weights(generate_perfekt_synthetic_fixpoint_weights()) + nets.append(net) - for i in loop_population_size: - loop_population_size.set_description("Populating experiment %s" % i) + else: + for i in loop_population_size: + loop_population_size.set_description("Populating experiment %s" % i) - if self.synthetic: - ''' Either use perfect / hand-constructed fixpoint ... ''' - net_name = f"net_{str(i)}_synthetic" - net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name) - net.apply_weights(generate_perfekt_synthetic_fixpoint_weights()) - - else: ''' .. or use natural approach to train fixpoints from random initialisation. ''' net_name = f"net_{str(i)}" net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name) for _ in range(self.epochs): net.self_train(self.ST_steps, self.log_step_size, self.net_learning_rate) - nets.append(net) + nets.append(net) + return nets - def test_robustness(self, print_it=True): - avg_time_to_vergence = [[0 for _ in range(10)] for _ in range(len(self.id_functions))] - avg_time_as_fixpoint = [[0 for _ in range(10)] for _ in range(len(self.id_functions))] - avg_loss_per_application = [[0 for _ in range(10)] for _ in range(len(self.id_functions))] - noise_range = range(10) + def test_robustness(self, print_it=True, noise_levels=10, seeds=10): + assert (len(self.id_functions) == 1 and seeds > 1) or (len(self.id_functions) > 1 and seeds == 1) + is_synthetic = True if len(self.id_functions) > 1 and seeds == 1 else False + avg_time_to_vergence = [[0 for _ in range(noise_levels)] for _ in + range(seeds if is_synthetic else len(self.id_functions))] + avg_time_as_fixpoint = [[0 for _ in range(noise_levels)] for _ in + range(seeds if is_synthetic else len(self.id_functions))] row_headers = [] + data_pos = 0 + # This checks wether to use synthetic setting with multiple seeds + # or multi network settings with a singlee seed - for i, fixpoint in enumerate(self.id_functions): + df = pd.DataFrame(columns=['seed', 'noise_level', 'application_step', 'absolute_loss']) + for i, fixpoint in enumerate(self.id_functions): #1 / n row_headers.append(fixpoint.name) - loss_per_application = [[0 for _ in range(10)] for _ in range(len(self.id_functions))] - for seed in range(10): - for noise_level in noise_range: + for seed in range(seeds): #n / 1 + for noise_level in range(noise_levels): + self_application_steps = 1 clone = Net(fixpoint.input_size, fixpoint.hidden_size, fixpoint.out_size, f"{fixpoint.name}_clone_noise10e-{noise_level}") clone.load_state_dict(copy.deepcopy(fixpoint.state_dict())) - rand_noise = prng() * pow(10, -noise_level) + rand_noise = prng() * pow(10, -noise_level) #n / 1 clone = self.apply_noise(clone, rand_noise) while not is_zero_fixpoint(clone) and not is_divergent(clone): @@ -128,12 +139,24 @@ class RobustnessComparisonExperiment: clone_weight_post_application = clone.input_weight_matrix() target_data_post_application = clone.create_target_weights(clone_weight_post_application) - loss_per_application[seed][noise_level] = (F.l1_loss(target_data_pre_application, - target_data_post_application)) + absolute_loss = F.l1_loss(target_data_pre_application, target_data_post_application).item() + setting = i if is_synthetic else seed + + df.loc[data_pos] = [setting, noise_level, self_application_steps, absolute_loss] + data_pos += 1 + self_application_steps += 1 + + # calculate the average: + df = df.replace([np.inf, -np.inf], np.nan) + df = df.dropna() + # sns.set(rc={'figure.figsize': (10, 50)}) + bx = sns.catplot(data=df[df['absolute_loss'] < 1], y='absolute_loss', x='application_step', kind='box', + col='noise_level', col_wrap=3, showfliers=False) + plt.show() if print_it: - col_headers = [str(f"10e-{d}") for d in noise_range] + col_headers = [str(f"10e-{d}") for d in range(noise_levels)] print(f"\nAppplications steps until divergence / zero: ") print(tabulate(avg_time_to_vergence, showindex=row_headers, headers=col_headers, tablefmt='orgtbl')) diff --git a/requirements.txt b/requirements.txt index 254276a..feffb28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,11 @@ -torch -tqdm -numpy==1.19.0 -matplotlib +torch~=1.8.1+cpu +tqdm~=4.60.0 +numpy~=1.20.3 +matplotlib~=3.4.2 sklearn scipy -tabulate +tabulate~=0.8.9 + +scikit-learn~=0.24.2 +pandas~=1.2.4 +seaborn~=0.11.1 \ No newline at end of file