journal_basins.py debugged

experiments/__init__.py

@@ -0,0 +1,5 @@
+from mixed_setting_exp import run_mixed_experiment
+from robustness_exp import run_robustness_experiment
+from self_application_exp import run_SA_experiment
+from self_train_exp import run_ST_experiment
+from soup_exp import run_soup_experiment

experiments/robustness_exp.py

@@ -20,7 +20,6 @@ def add_noise(input_data, epsilon = pow(10, -5)):
     return output
 
 
-
 class RobustnessExperiment:
     def __init__(self, population_size, log_step_size, net_input_size, net_hidden_size, net_out_size, net_learning_rate,
                  ST_steps, directory_name) -> None:

journal_basins.py

@@ -1,4 +1,6 @@
 import os
+from pathlib import Path
+
 from tqdm import tqdm
 import random
 import copy
@@ -14,23 +16,26 @@ from sklearn.metrics import mean_squared_error as MSE
 def prng():
     return random.random()
 
-def l1 (tup):
+
+def l1(tup):
     a, b = tup
     return abs(a-b)
 
-def mean_invariate_manhattan_distance(X,Y):
+
+def mean_invariate_manhattan_distance(x, y):
     # One of these one-liners that might be smart or really dumb. Goal is to find pairwise 
     # distances of ascending values, ie. sum (abs(min1_X-min1_Y), abs(min2_X-min2Y) ...) / mean.  
     # Idea was to find weight sets that have same values but just in different positions, that would
     # make this distance 0.
-    return np.mean(list(map(l1, zip(sorted(X),sorted(Y)))))
+    return np.mean(list(map(l1, zip(sorted(x), sorted(y)))))
+
 
 def distance_matrix(nets, distance="MIM", print_it=True):
     matrix = [[0 for _ in range(len(nets))] for _ in range(len(nets))]
     for net in range(len(nets)):
-      weights = nets[net].input_weight_matrix()[:,0]
+        weights = nets[net].input_weight_matrix()[:, 0]
         for other_net in range(len(nets)):
-        other_weights = nets[other_net].input_weight_matrix()[:,0]
+            other_weights = nets[other_net].input_weight_matrix()[:, 0]
             if distance in ["MSE"]:
                 matrix[net][other_net] = MSE(weights, other_weights)
             elif distance in ["MAE"]:
@@ -43,6 +48,7 @@ def distance_matrix(nets, distance="MIM", print_it=True):
         [print(row) for row in matrix]
     return matrix
 
+
 class SpawnExperiment:
 
     @staticmethod
@@ -52,7 +58,7 @@ class SpawnExperiment:
         for layer_id, layer_name in enumerate(network.state_dict()):
             for line_id, line_values in enumerate(network.state_dict()[layer_name]):
                 for weight_id, weight_value in enumerate(network.state_dict()[layer_name][line_id]):
-                    #network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise
+                    # network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise
                     if prng() < 0.5:
                         network.state_dict()[layer_name][line_id][weight_id] = weight_value + noise
                     else:
@@ -61,7 +67,7 @@ class SpawnExperiment:
         return network
 
     def __init__(self, population_size, log_step_size, net_input_size, net_hidden_size, net_out_size, net_learning_rate,
-                 epochs, ST_steps, noise, directory_name) -> None:
+                 epochs, st_steps, noise, directory_name) -> None:
         self.population_size = population_size
         self.log_step_size = log_step_size
         self.net_input_size = net_input_size
@@ -69,19 +75,19 @@ class SpawnExperiment:
         self.net_out_size = net_out_size
         self.net_learning_rate = net_learning_rate
         self.epochs = epochs
-        self.ST_steps = ST_steps
+        self.ST_steps = st_steps
         self.loss_history = []
         self.nets = []
         self.noise = noise or 10e-5
         print("\nNOISE:", self.noise)
 
-        self.directory_name = directory_name
+        self.directory = Path(directory_name)
-        os.mkdir(self.directory_name)
+        self.directory.mkdir(parents=True, exist_ok=True)
 
         self.populate_environment()
         self.spawn_and_continue()
         self.weights_evolution_3d_experiment()
-        #self.visualize_loss()
+        # self.visualize_loss()
         distance_matrix(self.nets)
 
     def populate_environment(self):
@@ -93,66 +99,67 @@ class SpawnExperiment:
             net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name)
 
             for _ in range(self.ST_steps):
-              input_data = net.input_weight_matrix()
+                net.self_train(1, self.log_step_size, self.net_learning_rate)
-              target_data = net.create_target_weights(input_data)
-              net.self_train(1, self.log_step_size, self.net_learning_rate, input_data, target_data)
 
-            #print(f"\nLast weight matrix (epoch: {self.epochs}):\n{net.input_weight_matrix()}\nLossHistory: {net.loss_history[-10:]}")
+            # print(f"\nLast weight matrix (epoch: {self.epochs}):\n
+            # {net.input_weight_matrix()}\nLossHistory: {net.loss_history[-10:]}")
             self.nets.append(net)
 
-
+    def spawn_and_continue(self, number_spawns: int = 5):
-    def spawn_and_continue(self, number_spawns:int = 5):
         # For every initial net {i} after populating (that is fixpoint after first epoch);
         for i in range(self.population_size):
             net = self.nets[i]
 
             net_input_data = net.input_weight_matrix()
             net_target_data = net.create_target_weights(net_input_data)
-          if is_identity_function(net, net_input_data, net_target_data):
+            if is_identity_function(net):
                 print(f"\nNet {i} is fixpoint")
-              #print("\nNet weights before training\n", target_data)
+                # print("\nNet weights before training\n", target_data)
 
                 # Clone the fixpoint x times and add (+-)self.noise to weight-sets randomly;
                 # To plot clones starting after first epoch (z=ST_steps), set that as start_time!
                 for j in range(number_spawns):
-                clone = Net(net.input_size, net.hidden_size, net.out_size, f"ST_net_{str(i)}_clone_{str(j)}", start_time=self.ST_steps)
+                    clone = Net(net.input_size, net.hidden_size, net.out_size,
+                                f"ST_net_{str(i)}_clone_{str(j)}",
+                                start_time=self.ST_steps)
                     clone.load_state_dict(copy.deepcopy(net.state_dict()))
                     rand_noise = prng() * self.noise
                     clone = self.apply_noise(clone, rand_noise)
 
-                # Then finish training each clone {j} (for remaining epoch-1 * ST_steps) and add to nets for plotting;
+                    # Then finish training each clone {j} (for remaining epoch-1 * ST_steps)
+                    # and add to nets for plotting;
                     for _ in range(self.epochs - 1):
                         for _ in range(self.ST_steps):
+                            clone.self_train(1, self.log_step_size, self.net_learning_rate)
+                    # print(f"clone {j} last weights: {target_data}, noise {noise}")
+                    if is_identity_function(clone):
                         input_data = clone.input_weight_matrix()
                         target_data = clone.create_target_weights(input_data)
-                    clone.self_train(1, self.log_step_size, self.net_learning_rate, input_data, target_data)
+                        print(f"Clone {j} (of net_{i}) is fixpoint. \nMSE(j,i): "
-                #print(f"clone {j} last weights: {target_data}, noise {noise}") 
+                              f"{MSE(net_target_data, target_data)}, \nMAE(j,i): {MAE(net_target_data, target_data)}\n")
-                if is_identity_function(clone, input_data, target_data):
-                  print(f"Clone {j} (of net_{i}) is fixpoint. \nMSE(j,i): {MSE(net_target_data, target_data)}, \nMAE(j,i): {MAE(net_target_data, target_data)}\n")
                     self.nets.append(clone)
 
                 # Finally take parent net {i} and finish it's training for comparison to clone development.
                 for _ in range(self.epochs - 1):
                     for _ in range(self.ST_steps):
-                  input_data = net.input_weight_matrix()
+                        net.self_train(1, self.log_step_size, self.net_learning_rate)
-                  target_data = net.create_target_weights(input_data)
+                # print("\nNet weights after training \n", target_data)
-                  net.self_train(1, self.log_step_size, self.net_learning_rate, input_data, target_data)
-              #print("\nNet weights after training \n", target_data)
 
         else:
             print("No fixpoints found.")
 
     def weights_evolution_3d_experiment(self):
         exp_name = f"ST_{str(len(self.nets))}_nets_3d_weights_PCA"
-        return plot_3d_self_train(self.nets, exp_name, self.directory_name, self.log_step_size)
+        return plot_3d_self_train(self.nets, exp_name, self.directory.name, self.log_step_size)
 
     def visualize_loss(self):
         for i in range(len(self.nets)):
             net_loss_history = self.nets[i].loss_history
             self.loss_history.append(net_loss_history)
-        plot_loss(self.loss_history, self.directory_name)
+        plot_loss(self.loss_history, self.directory.name)
 
-if __name__=="__main__":
+
+if __name__ == "__main__":
 
     NET_INPUT_SIZE = 4
     NET_OUT_SIZE = 1
@@ -180,7 +187,7 @@ if __name__=="__main__":
             net_out_size=NET_OUT_SIZE,
             net_learning_rate=ST_net_learning_rate,
             epochs=ST_epochs,
-            ST_steps=ST_steps,
+            st_steps=ST_steps,
-            noise=pow(10,-noise_factor),
+            noise=pow(10, -noise_factor),
             directory_name=f"./experiments/spawn_basin/{ST_name_hash}_10e-{noise_factor}"
         )

network.py

@@ -1,4 +1,4 @@
-#from __future__ import annotations
+# from __future__ import annotations
 import copy
 from typing import Union
 

visualization.py

@@ -1,7 +1,7 @@
+from pathlib import Path
 from tokenize import String
 from typing import List, Dict
 
-import numpy
 from tqdm import tqdm
 import matplotlib.pyplot as plt
 import matplotlib.patches as mpatches
@@ -120,14 +120,17 @@ def plot_3d(matrices_weights_history, folder_name, population_size, z_axis_legen
     ax.set_ylabel("PCA Y")
     ax.set_zlabel(f"Epochs")
 
-    filepath = f"./{folder_name}"
+    # FIXME: Replace this kind of operation with pathlib.Path() object interactions
-    filename = f"{filepath}/{exp_name}{is_trained}.png"
+    folder = Path(folder_name)
-    if os.path.isfile(filename):
+    folder.mkdir(parents=True, exist_ok=True)
+    filename = f"{exp_name}{is_trained}.png"
+    filepath = folder / filename
+    if filepath.exists():
         letters = string.ascii_lowercase
         random_letters = ''.join(random.choice(letters) for _ in range(5))
-        plt.savefig(f"{filename}_{random_letters}")
+        plt.savefig(f"{filepath.stem}_{random_letters}.png")
     else:
-        plt.savefig(f"{filename}")
+        plt.savefig(str(filepath))
 
     plt.show()
     #plt.clf()