From c9efe0a31b9bbfd1b1edee152f1162e0545a5241 Mon Sep 17 00:00:00 2001 From: steffen-illium Date: Tue, 25 May 2021 17:07:24 +0200 Subject: [PATCH 1/4] journal_robustness.py redone, now is sensitive to seeds and plots --- journal_basins.py | 9 +++++---- journal_robustness.py | 38 +++++++++++++++++++++----------------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/journal_basins.py b/journal_basins.py index d49ce09..8809f3b 100644 --- a/journal_basins.py +++ b/journal_basins.py @@ -173,6 +173,7 @@ class SpawnExperiment: # and add to nets for plotting if they are fixpoints themselves; for _ in range(self.epochs - 1): for _ in range(self.ST_steps): + # soup Evolve clone.self_train(1, self.log_step_size, self.net_learning_rate) if is_identity_function(clone): input_data = clone.input_weight_matrix() @@ -212,19 +213,19 @@ if __name__ == "__main__": # Define number of runs & name: ST_runs = 1 ST_runs_name = "test-27" - ST_steps = 2500 + ST_steps = 2000 ST_epochs = 2 ST_log_step_size = 10 # Define number of networks & their architecture - nr_clones = 10 - ST_population_size = 3 + nr_clones = 50 + ST_population_size = 1 ST_net_hidden_size = 2 ST_net_learning_rate = 0.04 ST_name_hash = random.getrandbits(32) print(f"Running the Spawn experiment:") - for noise_factor in [1]: + for noise_factor in [9]: SpawnExperiment( population_size=ST_population_size, log_step_size=ST_log_step_size, diff --git a/journal_robustness.py b/journal_robustness.py index b9f44e2..ad7224c 100644 --- a/journal_robustness.py +++ b/journal_robustness.py @@ -95,7 +95,6 @@ class RobustnessComparisonExperiment: for _ in range(self.epochs): net.self_train(self.ST_steps, self.log_step_size, self.net_learning_rate) nets.append(net) - return nets def test_robustness(self, print_it=True, noise_levels=10, seeds=10): @@ -110,12 +109,12 @@ class RobustnessComparisonExperiment: # This checks wether to use synthetic setting with multiple seeds # or multi network settings with a singlee seed - df = pd.DataFrame(columns=['seed', 'noise_level', 'application_step', 'absolute_loss']) + df = pd.DataFrame(columns=['setting', 'noise_level', 'application_step', 'absolute_loss', 'time_to_vergence']) for i, fixpoint in enumerate(self.id_functions): #1 / n row_headers.append(fixpoint.name) for seed in range(seeds): #n / 1 for noise_level in range(noise_levels): - self_application_steps = 1 + self_application_steps = 0 clone = Net(fixpoint.input_size, fixpoint.hidden_size, fixpoint.out_size, f"{fixpoint.name}_clone_noise10e-{noise_level}") clone.load_state_dict(copy.deepcopy(fixpoint.state_dict())) @@ -123,9 +122,6 @@ class RobustnessComparisonExperiment: clone = self.apply_noise(clone, rand_noise) while not is_zero_fixpoint(clone) and not is_divergent(clone): - if is_identity_function(clone): - avg_time_as_fixpoint[i][noise_level] += 1 - # -> before clone_weight_pre_application = clone.input_weight_matrix() target_data_pre_application = clone.create_target_weights(clone_weight_pre_application) @@ -140,16 +136,27 @@ class RobustnessComparisonExperiment: setting = i if is_synthetic else seed - df.loc[data_pos] = [setting, noise_level, self_application_steps, absolute_loss] - data_pos += 1 - self_application_steps += 1 + if is_identity_function(clone): + avg_time_as_fixpoint[i][noise_level] += 1 + # When this raises a Type Error, we found a second order fixpoint! + self_application_steps += 1 + else: + self_application_steps = pd.NA # Not a Number! + + df.loc[df.shape[0]] = [setting, noise_level, self_application_steps, + absolute_loss, avg_time_to_vergence[i][noise_level]] + # calculate the average: - df = df.replace([np.inf, -np.inf], np.nan) - df = df.dropna() + # df = df.replace([np.inf, -np.inf], np.nan) + # df = df.dropna() + bf = sns.boxplot(data=df, y='self_application_steps', x='noise_level', ) + bf.set_title('Robustness as self application steps per noise level') + plt.tight_layout() + # sns.set(rc={'figure.figsize': (10, 50)}) - bx = sns.catplot(data=df[df['absolute_loss'] < 1], y='absolute_loss', x='application_step', kind='box', - col='noise_level', col_wrap=3, showfliers=False) + # bx = sns.catplot(data=df[df['absolute_loss'] < 1], y='absolute_loss', x='application_step', kind='box', + # col='noise_level', col_wrap=3, showfliers=False) directory = Path('output') / 'robustness' filename = f"absolute_loss_perapplication_boxplot_grid.png" filepath = directory / filename @@ -167,21 +174,18 @@ class RobustnessComparisonExperiment: return avg_time_as_fixpoint, avg_time_to_vergence - def count_fixpoints(self): exp_details = f"ST steps: {self.ST_steps}" self.id_functions = test_for_fixpoints(self.fixpoint_counters, self.nets) bar_chart_fixpoints(self.fixpoint_counters, self.population_size, self.directory, self.net_learning_rate, exp_details) - def visualize_loss(self): for i in range(len(self.nets)): net_loss_history = self.nets[i].loss_history self.loss_history.append(net_loss_history) plot_loss(self.loss_history, self.directory) - def save(self): pickle.dump(self, open(f"{self.directory}/experiment_pickle.p", "wb")) print(f"\nSaved experiment to {self.directory}.") @@ -211,5 +215,5 @@ if __name__ == "__main__": epochs=ST_epochs, st_steps=ST_steps, synthetic=ST_synthetic, - directory=Path('output') / 'robustness' / f'{ST_name_hash}' + directory=Path('output') / 'journal_robustness' / f'{ST_name_hash}' ) From 9abde030af485f0909c3e778cddeb086ca18e221 Mon Sep 17 00:00:00 2001 From: steffen-illium Date: Fri, 4 Jun 2021 14:03:03 +0200 Subject: [PATCH 2/4] robustness cleaned --- journal_robustness.py | 126 +++++++++++++++++++++++++----------------- 1 file changed, 74 insertions(+), 52 deletions(-) diff --git a/journal_robustness.py b/journal_robustness.py index ad7224c..c6c6ff6 100644 --- a/journal_robustness.py +++ b/journal_robustness.py @@ -4,7 +4,6 @@ import pandas as pd import torch import random import copy -import numpy as np from pathlib import Path from tqdm import tqdm @@ -21,6 +20,7 @@ from matplotlib import pyplot as plt def prng(): return random.random() + def generate_perfekt_synthetic_fixpoint_weights(): return torch.tensor([[1.0], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0], [1.0], [0.0], [0.0], [0.0], @@ -28,15 +28,32 @@ def generate_perfekt_synthetic_fixpoint_weights(): ], dtype=torch.float32) +PALETTE = 10 * ( + "#377eb8", + "#4daf4a", + "#984ea3", + "#e41a1c", + "#ff7f00", + "#a65628", + "#f781bf", + "#888888", + "#a6cee3", + "#b2df8a", + "#cab2d6", + "#fb9a99", + "#fdbf6f", +) + + class RobustnessComparisonExperiment: @staticmethod def apply_noise(network, noise: int): - """ Changing the weights of a network to values + noise """ + # Changing the weights of a network to values + noise for layer_id, layer_name in enumerate(network.state_dict()): for line_id, line_values in enumerate(network.state_dict()[layer_name]): for weight_id, weight_value in enumerate(network.state_dict()[layer_name][line_id]): - #network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise + # network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise if prng() < 0.5: network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise else: @@ -55,7 +72,7 @@ class RobustnessComparisonExperiment: self.epochs = epochs self.ST_steps = st_steps self.loss_history = [] - self.synthetic = synthetic + self.is_synthetic = synthetic self.fixpoint_counters = { "identity_func": 0, "divergent": 0, @@ -71,14 +88,14 @@ class RobustnessComparisonExperiment: self.id_functions = [] self.nets = self.populate_environment() self.count_fixpoints() - self.time_to_vergence, self.time_as_fixpoint = self.test_robustness() + self.time_to_vergence, self.time_as_fixpoint = self.test_robustness( + seeds=population_size if self.is_synthetic else 1) self.save() def populate_environment(self): - loop_population_size = tqdm(range(self.population_size)) nets = [] - if self.synthetic: + if self.is_synthetic: ''' Either use perfect / hand-constructed fixpoint ... ''' net_name = f"net_{str(0)}_synthetic" net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name) @@ -86,6 +103,7 @@ class RobustnessComparisonExperiment: nets.append(net) else: + loop_population_size = tqdm(range(self.population_size)) for i in loop_population_size: loop_population_size.set_description("Populating experiment %s" % i) @@ -99,58 +117,61 @@ class RobustnessComparisonExperiment: def test_robustness(self, print_it=True, noise_levels=10, seeds=10): assert (len(self.id_functions) == 1 and seeds > 1) or (len(self.id_functions) > 1 and seeds == 1) - is_synthetic = True if len(self.id_functions) > 1 and seeds == 1 else False - avg_time_to_vergence = [[0 for _ in range(noise_levels)] for _ in - range(seeds if is_synthetic else len(self.id_functions))] - avg_time_as_fixpoint = [[0 for _ in range(noise_levels)] for _ in - range(seeds if is_synthetic else len(self.id_functions))] + time_to_vergence = [[0 for _ in range(noise_levels)] for _ in + range(seeds if self.is_synthetic else len(self.id_functions))] + time_as_fixpoint = [[0 for _ in range(noise_levels)] for _ in + range(seeds if self.is_synthetic else len(self.id_functions))] row_headers = [] - data_pos = 0 + # This checks wether to use synthetic setting with multiple seeds # or multi network settings with a singlee seed - df = pd.DataFrame(columns=['setting', 'noise_level', 'application_step', 'absolute_loss', 'time_to_vergence']) - for i, fixpoint in enumerate(self.id_functions): #1 / n - row_headers.append(fixpoint.name) - for seed in range(seeds): #n / 1 - for noise_level in range(noise_levels): - self_application_steps = 0 - clone = Net(fixpoint.input_size, fixpoint.hidden_size, fixpoint.out_size, - f"{fixpoint.name}_clone_noise10e-{noise_level}") - clone.load_state_dict(copy.deepcopy(fixpoint.state_dict())) - rand_noise = prng() * pow(10, -noise_level) #n / 1 - clone = self.apply_noise(clone, rand_noise) + df = pd.DataFrame(columns=['setting', 'noise_level', 'steps', 'absolute_loss', 'time_to_vergence', 'time_as_fixpoint']) + with tqdm(total=max(len(self.id_functions), seeds)) as pbar: + for i, fixpoint in enumerate(self.id_functions): # 1 / n + row_headers.append(fixpoint.name) + for seed in range(seeds): # n / 1 + for noise_level in range(noise_levels): + steps = 0 + clone = Net(fixpoint.input_size, fixpoint.hidden_size, fixpoint.out_size, + f"{fixpoint.name}_clone_noise10e-{noise_level}") + clone.load_state_dict(copy.deepcopy(fixpoint.state_dict())) + rand_noise = prng() * pow(10, -noise_level) # n / 1 + clone = self.apply_noise(clone, rand_noise) - while not is_zero_fixpoint(clone) and not is_divergent(clone): - # -> before - clone_weight_pre_application = clone.input_weight_matrix() - target_data_pre_application = clone.create_target_weights(clone_weight_pre_application) + while not is_zero_fixpoint(clone) and not is_divergent(clone): + # -> before + clone_weight_pre_application = clone.input_weight_matrix() + target_data_pre_application = clone.create_target_weights(clone_weight_pre_application) - clone.self_application(1, self.log_step_size) - avg_time_to_vergence[i][noise_level] += 1 - # -> after - clone_weight_post_application = clone.input_weight_matrix() - target_data_post_application = clone.create_target_weights(clone_weight_post_application) + clone.self_application(1, self.log_step_size) + time_to_vergence[i][noise_level] += 1 + # -> after + clone_weight_post_application = clone.input_weight_matrix() + target_data_post_application = clone.create_target_weights(clone_weight_post_application) - absolute_loss = F.l1_loss(target_data_pre_application, target_data_post_application).item() + absolute_loss = F.l1_loss(target_data_pre_application, target_data_post_application).item() - setting = i if is_synthetic else seed + setting = seed if self.is_synthetic else i - if is_identity_function(clone): - avg_time_as_fixpoint[i][noise_level] += 1 - # When this raises a Type Error, we found a second order fixpoint! - self_application_steps += 1 - else: - self_application_steps = pd.NA # Not a Number! + if is_identity_function(clone): + time_as_fixpoint[i][noise_level] += 1 + # When this raises a Type Error, we found a second order fixpoint! + steps += 1 - df.loc[df.shape[0]] = [setting, noise_level, self_application_steps, - absolute_loss, avg_time_to_vergence[i][noise_level]] + df.loc[df.shape[0]] = [setting, noise_level, steps, absolute_loss, + time_to_vergence[i][noise_level], time_as_fixpoint[i][noise_level]] + pbar.update(1) - - # calculate the average: - # df = df.replace([np.inf, -np.inf], np.nan) - # df = df.dropna() - bf = sns.boxplot(data=df, y='self_application_steps', x='noise_level', ) + # Get the measuremts at the highest time_time_to_vergence + df_sorted = df.sort_values('steps', ascending=False).drop_duplicates(['setting', 'noise_level']) + df_melted = df_sorted.reset_index().melt(id_vars=['setting', 'noise_level', 'steps'], + value_vars=['time_to_vergence', 'time_as_fixpoint'], + var_name="Measurement", + value_name="Steps") + # Plotting + sns.set(style='whitegrid') + bf = sns.boxplot(data=df_melted, y='Steps', x='noise_level', hue='Measurement', palette=PALETTE) bf.set_title('Robustness as self application steps per noise level') plt.tight_layout() @@ -158,6 +179,7 @@ class RobustnessComparisonExperiment: # bx = sns.catplot(data=df[df['absolute_loss'] < 1], y='absolute_loss', x='application_step', kind='box', # col='noise_level', col_wrap=3, showfliers=False) directory = Path('output') / 'robustness' + directory.mkdir(parents=True, exist_ok=True) filename = f"absolute_loss_perapplication_boxplot_grid.png" filepath = directory / filename @@ -167,12 +189,12 @@ class RobustnessComparisonExperiment: col_headers = [str(f"10e-{d}") for d in range(noise_levels)] print(f"\nAppplications steps until divergence / zero: ") - print(tabulate(avg_time_to_vergence, showindex=row_headers, headers=col_headers, tablefmt='orgtbl')) + print(tabulate(time_to_vergence, showindex=row_headers, headers=col_headers, tablefmt='orgtbl')) print(f"\nTime as fixpoint: ") - print(tabulate(avg_time_as_fixpoint, showindex=row_headers, headers=col_headers, tablefmt='orgtbl')) + print(tabulate(time_as_fixpoint, showindex=row_headers, headers=col_headers, tablefmt='orgtbl')) - return avg_time_as_fixpoint, avg_time_to_vergence + return time_as_fixpoint, time_to_vergence def count_fixpoints(self): exp_details = f"ST steps: {self.ST_steps}" @@ -198,7 +220,7 @@ if __name__ == "__main__": ST_steps = 1000 ST_epochs = 5 ST_log_step_size = 10 - ST_population_size = 5 + ST_population_size = 100 ST_net_hidden_size = 2 ST_net_learning_rate = 0.04 ST_name_hash = random.getrandbits(32) From 61ae8c2ee5574bb6af40fcffe2ecf1341082a1fe Mon Sep 17 00:00:00 2001 From: steffen-illium Date: Fri, 4 Jun 2021 14:13:38 +0200 Subject: [PATCH 3/4] robustness fixed --- journal_robustness.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/journal_robustness.py b/journal_robustness.py index c6c6ff6..1dda85b 100644 --- a/journal_robustness.py +++ b/journal_robustness.py @@ -131,6 +131,8 @@ class RobustnessComparisonExperiment: for i, fixpoint in enumerate(self.id_functions): # 1 / n row_headers.append(fixpoint.name) for seed in range(seeds): # n / 1 + setting = seed if self.is_synthetic else i + for noise_level in range(noise_levels): steps = 0 clone = Net(fixpoint.input_size, fixpoint.hidden_size, fixpoint.out_size, @@ -145,22 +147,22 @@ class RobustnessComparisonExperiment: target_data_pre_application = clone.create_target_weights(clone_weight_pre_application) clone.self_application(1, self.log_step_size) - time_to_vergence[i][noise_level] += 1 + time_to_vergence[setting][noise_level] += 1 # -> after clone_weight_post_application = clone.input_weight_matrix() target_data_post_application = clone.create_target_weights(clone_weight_post_application) absolute_loss = F.l1_loss(target_data_pre_application, target_data_post_application).item() - setting = seed if self.is_synthetic else i if is_identity_function(clone): - time_as_fixpoint[i][noise_level] += 1 + time_as_fixpoint[setting][noise_level] += 1 # When this raises a Type Error, we found a second order fixpoint! steps += 1 df.loc[df.shape[0]] = [setting, noise_level, steps, absolute_loss, - time_to_vergence[i][noise_level], time_as_fixpoint[i][noise_level]] + time_to_vergence[setting][noise_level], + time_as_fixpoint[setting][noise_level]] pbar.update(1) # Get the measuremts at the highest time_time_to_vergence @@ -189,10 +191,10 @@ class RobustnessComparisonExperiment: col_headers = [str(f"10e-{d}") for d in range(noise_levels)] print(f"\nAppplications steps until divergence / zero: ") - print(tabulate(time_to_vergence, showindex=row_headers, headers=col_headers, tablefmt='orgtbl')) + # print(tabulate(time_to_vergence, showindex=row_headers, headers=col_headers, tablefmt='orgtbl')) print(f"\nTime as fixpoint: ") - print(tabulate(time_as_fixpoint, showindex=row_headers, headers=col_headers, tablefmt='orgtbl')) + # print(tabulate(time_as_fixpoint, showindex=row_headers, headers=col_headers, tablefmt='orgtbl')) return time_as_fixpoint, time_to_vergence From b57d3d32fdedad11c7df8ff47ea266c5fdff5961 Mon Sep 17 00:00:00 2001 From: steffen-illium Date: Fri, 4 Jun 2021 15:01:16 +0200 Subject: [PATCH 4/4] readme updated --- README.md | 12 ++++++++++-- experiments/self_train_exp.py | 3 +-- journal_robustness.py | 1 - 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7be6661..5b2f39d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,11 @@ # self-rep NN paper - ALIFE journal edition -- [x] Plateau / Pillar sizeWhat does happen to the fixpoints after noise introduction and retraining?Options beeing: Same Fixpoint, Similar Fixpoint (Basin), Different Fixpoint? Do they do the clustering thingy? +- [x] Plateau / Pillar sizeWhat does happen to the fixpoints after noise introduction and retraining?Options beeing: Same Fixpoint, Similar Fixpoint (Basin), + - Different Fixpoint? + Yes, we did not found same (10-5) + - Do they do the clustering thingy? + Kind of: Small movement towards (MIM-Distance getting smaller) parent fixpoint. + Small movement for everyone? -> Distribution - see `journal_basins.py` for the "train -> spawn with noise -> train again and see where they end up" functionality. Apply noise follows the `vary` function that was used in the paper robustness test with `+- prng() * eps`. Change if desired. @@ -9,6 +14,9 @@ - [ ] Same Thing with Soup interactionWe would expect the same behaviour...Influence of interaction with near and far away particles. +- [ ] How are basins / "attractor areas" shaped? + - Weired.... tbc... + - [x] Robustness test with a trained NetworkTraining for high quality fixpoints, compare with the "perfect" fixpoint. Average Loss per application step - see `journal_robustness.py` for robustness test modeled after cristians robustness-exp (with the exeption that we put noise on the weights). Has `synthetic` bool to switch to hand-modeled perfect fixpoint instead of naturally trained ones. @@ -19,7 +27,7 @@ - [ ] Adjust Self Training so that it favors second order fixpoints-> Second order test implementation (?) -- [ ] Barplot over clones -> how many become a fixpoint cs how many diverge per noise level +- [x] Barplot over clones -> how many become a fixpoint cs how many diverge per noise level - [ ] Box-Plot of Avg. Distance of clones from parent diff --git a/experiments/self_train_exp.py b/experiments/self_train_exp.py index f422eda..1a467b9 100644 --- a/experiments/self_train_exp.py +++ b/experiments/self_train_exp.py @@ -55,8 +55,6 @@ class SelfTrainExperiment: net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name) for _ in range(self.epochs): - input_data = net.input_weight_matrix() - target_data = net.create_target_weights(input_data) net.self_train(1, self.log_step_size, self.net_learning_rate) print(f"\nLast weight matrix (epoch: {self.epochs}):\n{net.input_weight_matrix()}\nLossHistory: {net.loss_history[-10:]}") @@ -113,5 +111,6 @@ def run_ST_experiment(population_size, batch_size, net_input_size, net_hidden_si summary_fixpoint_experiment(runs, population_size, epochs, experiments, net_learning_rate, summary_directory_name, summary_pre_title) + if __name__ == '__main__': raise NotImplementedError('Test this here!!!') diff --git a/journal_robustness.py b/journal_robustness.py index 1dda85b..4330614 100644 --- a/journal_robustness.py +++ b/journal_robustness.py @@ -195,7 +195,6 @@ class RobustnessComparisonExperiment: print(f"\nTime as fixpoint: ") # print(tabulate(time_as_fixpoint, showindex=row_headers, headers=col_headers, tablefmt='orgtbl')) - return time_as_fixpoint, time_to_vergence def count_fixpoints(self):