Merge branch 'journal' of gitlab.lrz.de:mobile-ifi/bannana-networks into journal

README.md

@@ -1,6 +1,11 @@
 # self-rep NN paper - ALIFE journal edition
 
-- [x] Plateau / Pillar sizeWhat does happen to the fixpoints after noise introduction and retraining?Options beeing: Same Fixpoint, Similar Fixpoint (Basin), Different Fixpoint? Do they do the clustering thingy?
+- [x] Plateau / Pillar sizeWhat does happen to the fixpoints after noise introduction and retraining?Options beeing: Same Fixpoint, Similar Fixpoint (Basin), 
+    - Different Fixpoint?
+      Yes, we did not found same (10-5)
+    - Do they do the clustering thingy?
+      Kind of: Small movement towards (MIM-Distance getting smaller) parent fixpoint.
+      Small movement for everyone? -> Distribution
 
     - see `journal_basins.py` for the "train -> spawn with noise -> train again and see where they end up" functionality. Apply noise follows the `vary` function that was used in the paper robustness test with `+- prng() * eps`. Change if desired.
 
@@ -9,6 +14,9 @@
 
 - [ ] Same Thing with Soup interactionWe would expect the same behaviour...Influence of interaction with near and far away particles.
 
+- [ ] How are basins / "attractor areas" shaped?
+      - Weired.... tbc...
+
 - [x] Robustness test with a trained NetworkTraining for high quality fixpoints, compare with the "perfect" fixpoint. Average Loss per application step
     
     - see `journal_robustness.py` for robustness test modeled after cristians robustness-exp (with the exeption that we put noise on the weights). Has `synthetic` bool to switch to hand-modeled perfect fixpoint instead of naturally trained ones. 
@@ -19,7 +27,7 @@
 
 - [ ] Adjust Self Training so that it favors second order fixpoints-> Second order test implementation (?)
 
-- [ ] Barplot over clones -> how many become a fixpoint cs how many diverge per noise level
+- [x] Barplot over clones -> how many become a fixpoint cs how many diverge per noise level
 
 - [ ] Box-Plot of Avg. Distance of clones from parent
 

experiments/self_train_exp.py

@@ -55,8 +55,6 @@ class SelfTrainExperiment:
             net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name)
 
             for _ in range(self.epochs):
-              input_data = net.input_weight_matrix()
-              target_data = net.create_target_weights(input_data)
               net.self_train(1, self.log_step_size, self.net_learning_rate)
 
             print(f"\nLast weight matrix (epoch: {self.epochs}):\n{net.input_weight_matrix()}\nLossHistory: {net.loss_history[-10:]}")
@@ -113,5 +111,6 @@ def run_ST_experiment(population_size, batch_size, net_input_size, net_hidden_si
     summary_fixpoint_experiment(runs, population_size, epochs, experiments, net_learning_rate, summary_directory_name,
                                 summary_pre_title)
 
+
 if __name__ == '__main__':
     raise NotImplementedError('Test this here!!!')

journal_robustness.py

@@ -4,7 +4,6 @@ import pandas as pd
 import torch
 import random
 import copy
-import numpy as np
 
 from pathlib import Path
 from tqdm import tqdm
@@ -21,6 +20,7 @@ from matplotlib import pyplot as plt
 def prng():
     return random.random()
 
+
 def generate_perfekt_synthetic_fixpoint_weights():
     return torch.tensor([[1.0], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0],
                          [1.0], [0.0], [0.0], [0.0],
@@ -28,15 +28,32 @@ def generate_perfekt_synthetic_fixpoint_weights():
                          ], dtype=torch.float32)
 
 
+PALETTE = 10 * (
+    "#377eb8",
+    "#4daf4a",
+    "#984ea3",
+    "#e41a1c",
+    "#ff7f00",
+    "#a65628",
+    "#f781bf",
+    "#888888",
+    "#a6cee3",
+    "#b2df8a",
+    "#cab2d6",
+    "#fb9a99",
+    "#fdbf6f",
+)
+
+
 class RobustnessComparisonExperiment:
 
     @staticmethod
     def apply_noise(network, noise: int):
-        """ Changing the weights of a network to values + noise """
+        # Changing the weights of a network to values + noise
         for layer_id, layer_name in enumerate(network.state_dict()):
             for line_id, line_values in enumerate(network.state_dict()[layer_name]):
                 for weight_id, weight_value in enumerate(network.state_dict()[layer_name][line_id]):
-                    #network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise
+                    # network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise
                     if prng() < 0.5:
                         network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise
                     else:
@@ -55,7 +72,7 @@ class RobustnessComparisonExperiment:
         self.epochs = epochs
         self.ST_steps = st_steps
         self.loss_history = []
-        self.synthetic = synthetic
+        self.is_synthetic = synthetic
         self.fixpoint_counters = {
             "identity_func": 0,
             "divergent": 0,
@@ -71,14 +88,14 @@ class RobustnessComparisonExperiment:
         self.id_functions = []
         self.nets = self.populate_environment()
         self.count_fixpoints()
-        self.time_to_vergence, self.time_as_fixpoint = self.test_robustness()
+        self.time_to_vergence, self.time_as_fixpoint = self.test_robustness(
+            seeds=population_size if self.is_synthetic else 1)
 
         self.save()
 
     def populate_environment(self):
-        loop_population_size = tqdm(range(self.population_size))
         nets = []
-        if self.synthetic:
+        if self.is_synthetic:
             ''' Either use perfect / hand-constructed fixpoint ... '''
             net_name = f"net_{str(0)}_synthetic"
             net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name)
@@ -86,6 +103,7 @@ class RobustnessComparisonExperiment:
             nets.append(net)
 
         else:
+            loop_population_size = tqdm(range(self.population_size))
             for i in loop_population_size:
                 loop_population_size.set_description("Populating experiment %s" % i)
 
@@ -95,62 +113,75 @@ class RobustnessComparisonExperiment:
                 for _ in range(self.epochs):
                     net.self_train(self.ST_steps, self.log_step_size, self.net_learning_rate)
                 nets.append(net)
-
         return nets
 
     def test_robustness(self, print_it=True, noise_levels=10, seeds=10):
         assert (len(self.id_functions) == 1 and seeds > 1) or (len(self.id_functions) > 1 and seeds == 1)
-        is_synthetic = True if len(self.id_functions) > 1 and seeds == 1 else False
+        time_to_vergence = [[0 for _ in range(noise_levels)] for _ in
-        avg_time_to_vergence = [[0 for _ in range(noise_levels)] for _ in
+                            range(seeds if self.is_synthetic else len(self.id_functions))]
-                                range(seeds if is_synthetic else len(self.id_functions))]
+        time_as_fixpoint = [[0 for _ in range(noise_levels)] for _ in
-        avg_time_as_fixpoint = [[0 for _ in range(noise_levels)] for _ in
+                            range(seeds if self.is_synthetic else len(self.id_functions))]
-                                range(seeds if is_synthetic else len(self.id_functions))]
         row_headers = []
-        data_pos = 0
+
         # This checks wether to use synthetic setting with multiple seeds
         #   or multi network settings with a singlee seed
 
-        df = pd.DataFrame(columns=['seed', 'noise_level', 'application_step', 'absolute_loss'])
+        df = pd.DataFrame(columns=['setting', 'noise_level', 'steps', 'absolute_loss', 'time_to_vergence', 'time_as_fixpoint'])
-        for i, fixpoint in enumerate(self.id_functions): #1 / n
+        with tqdm(total=max(len(self.id_functions), seeds)) as pbar:
-            row_headers.append(fixpoint.name)
+            for i, fixpoint in enumerate(self.id_functions):  # 1 / n
-            for seed in range(seeds): #n / 1
+                row_headers.append(fixpoint.name)
-                for noise_level in range(noise_levels):
+                for seed in range(seeds):  # n / 1
-                    self_application_steps = 1
+                    setting = seed if self.is_synthetic else i
-                    clone = Net(fixpoint.input_size, fixpoint.hidden_size, fixpoint.out_size,
-                                f"{fixpoint.name}_clone_noise10e-{noise_level}")
-                    clone.load_state_dict(copy.deepcopy(fixpoint.state_dict()))
-                    rand_noise = prng() * pow(10, -noise_level) #n / 1
-                    clone = self.apply_noise(clone, rand_noise)
 
-                    while not is_zero_fixpoint(clone) and not is_divergent(clone):
+                    for noise_level in range(noise_levels):
-                        if is_identity_function(clone):
+                        steps = 0
-                            avg_time_as_fixpoint[i][noise_level] += 1
+                        clone = Net(fixpoint.input_size, fixpoint.hidden_size, fixpoint.out_size,
+                                    f"{fixpoint.name}_clone_noise10e-{noise_level}")
+                        clone.load_state_dict(copy.deepcopy(fixpoint.state_dict()))
+                        rand_noise = prng() * pow(10, -noise_level)  # n / 1
+                        clone = self.apply_noise(clone, rand_noise)
 
-                        # -> before
+                        while not is_zero_fixpoint(clone) and not is_divergent(clone):
-                        clone_weight_pre_application = clone.input_weight_matrix()
+                            # -> before
-                        target_data_pre_application = clone.create_target_weights(clone_weight_pre_application)
+                            clone_weight_pre_application = clone.input_weight_matrix()
+                            target_data_pre_application = clone.create_target_weights(clone_weight_pre_application)
 
-                        clone.self_application(1, self.log_step_size)
+                            clone.self_application(1, self.log_step_size)
-                        avg_time_to_vergence[i][noise_level] += 1
+                            time_to_vergence[setting][noise_level] += 1
-                        # -> after
+                            # -> after
-                        clone_weight_post_application = clone.input_weight_matrix()
+                            clone_weight_post_application = clone.input_weight_matrix()
-                        target_data_post_application = clone.create_target_weights(clone_weight_post_application)
+                            target_data_post_application = clone.create_target_weights(clone_weight_post_application)
 
-                        absolute_loss = F.l1_loss(target_data_pre_application, target_data_post_application).item()
+                            absolute_loss = F.l1_loss(target_data_pre_application, target_data_post_application).item()
 
-                        setting = i if is_synthetic else seed
 
-                        df.loc[data_pos] = [setting, noise_level, self_application_steps, absolute_loss]
+                            if is_identity_function(clone):
-                        data_pos += 1
+                                time_as_fixpoint[setting][noise_level] += 1
-                        self_application_steps += 1
+                                # When this raises a Type Error, we found a second order fixpoint!
+                            steps += 1
+
+                            df.loc[df.shape[0]] = [setting, noise_level, steps, absolute_loss,
+                                                   time_to_vergence[setting][noise_level],
+                                                   time_as_fixpoint[setting][noise_level]]
+                    pbar.update(1)
+
+        # Get the measuremts at the highest time_time_to_vergence
+        df_sorted = df.sort_values('steps', ascending=False).drop_duplicates(['setting', 'noise_level'])
+        df_melted = df_sorted.reset_index().melt(id_vars=['setting', 'noise_level', 'steps'],
+                                                 value_vars=['time_to_vergence', 'time_as_fixpoint'],
+                                                 var_name="Measurement",
+                                                 value_name="Steps")
+        # Plotting
+        sns.set(style='whitegrid')
+        bf = sns.boxplot(data=df_melted, y='Steps', x='noise_level', hue='Measurement', palette=PALETTE)
+        bf.set_title('Robustness as self application steps per noise level')
+        plt.tight_layout()
 
-        # calculate the average:
-        df = df.replace([np.inf, -np.inf], np.nan)
-        df = df.dropna()
         # sns.set(rc={'figure.figsize': (10, 50)})
-        bx = sns.catplot(data=df[df['absolute_loss'] < 1], y='absolute_loss', x='application_step', kind='box',
+        # bx = sns.catplot(data=df[df['absolute_loss'] < 1], y='absolute_loss', x='application_step', kind='box',
-                         col='noise_level', col_wrap=3, showfliers=False)
+        #                  col='noise_level', col_wrap=3, showfliers=False)
         directory = Path('output') / 'robustness'
+        directory.mkdir(parents=True, exist_ok=True)
         filename = f"absolute_loss_perapplication_boxplot_grid.png"
         filepath = directory / filename
 
@@ -160,13 +191,11 @@ class RobustnessComparisonExperiment:
             col_headers = [str(f"10e-{d}") for d in range(noise_levels)]
 
             print(f"\nAppplications steps until divergence / zero: ")
-            print(tabulate(avg_time_to_vergence, showindex=row_headers, headers=col_headers, tablefmt='orgtbl'))
+            # print(tabulate(time_to_vergence, showindex=row_headers, headers=col_headers, tablefmt='orgtbl'))
 
             print(f"\nTime as fixpoint: ")
-            print(tabulate(avg_time_as_fixpoint, showindex=row_headers, headers=col_headers, tablefmt='orgtbl'))
+            # print(tabulate(time_as_fixpoint, showindex=row_headers, headers=col_headers, tablefmt='orgtbl'))
-
+        return time_as_fixpoint, time_to_vergence
-        return avg_time_as_fixpoint, avg_time_to_vergence
-
 
     def count_fixpoints(self):
         exp_details = f"ST steps: {self.ST_steps}"
@@ -174,14 +203,12 @@ class RobustnessComparisonExperiment:
         bar_chart_fixpoints(self.fixpoint_counters, self.population_size, self.directory, self.net_learning_rate,
                             exp_details)
 
-
     def visualize_loss(self):
         for i in range(len(self.nets)):
             net_loss_history = self.nets[i].loss_history
             self.loss_history.append(net_loss_history)
         plot_loss(self.loss_history, self.directory)
 
-
     def save(self):
         pickle.dump(self, open(f"{self.directory}/experiment_pickle.p", "wb"))
         print(f"\nSaved experiment to {self.directory}.")
@@ -194,7 +221,7 @@ if __name__ == "__main__":
     ST_steps = 1000
     ST_epochs = 5
     ST_log_step_size = 10
-    ST_population_size = 5
+    ST_population_size = 100
     ST_net_hidden_size = 2
     ST_net_learning_rate = 0.04
     ST_name_hash = random.getrandbits(32)
@@ -211,5 +238,5 @@ if __name__ == "__main__":
         epochs=ST_epochs,
         st_steps=ST_steps,
         synthetic=ST_synthetic,
-        directory=Path('output') / 'robustness' / f'{ST_name_hash}'
+        directory=Path('output') / 'journal_robustness' / f'{ST_name_hash}'
     )