From c1f58f2675e6a67f335e3e4e4626d44a13bb44c2 Mon Sep 17 00:00:00 2001 From: steffen-illium Date: Sun, 16 May 2021 15:34:43 +0200 Subject: [PATCH] All Experiments debugged ToDo: - convert strings in pathlib.Path objects - check usage of fixpoint tests --- experiments/mixed_setting_exp.py | 8 ++------ experiments/robustness_exp.py | 6 ++---- experiments/soup_exp.py | 24 ++++++++++++++---------- functionalities_test.py | 2 +- main.py | 4 ++-- 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/experiments/mixed_setting_exp.py b/experiments/mixed_setting_exp.py index c7fd3dd..a778775 100644 --- a/experiments/mixed_setting_exp.py +++ b/experiments/mixed_setting_exp.py @@ -68,17 +68,13 @@ class MixedSettingExperiment: if self.train_nets == "before_SA": for _ in range(self.ST_steps_between_SA): - input_data = net.input_weight_matrix() - target_data = net.create_target_weights(input_data) - net.self_train(1, self.log_step_size, self.net_learning_rate, input_data, target_data) + net.self_train(1, self.log_step_size, self.net_learning_rate) net.self_application(self.SA_steps, self.log_step_size) elif self.train_nets == "after_SA": net.self_application(self.SA_steps, self.log_step_size) for _ in range(self.ST_steps_between_SA): - input_data = net.input_weight_matrix() - target_data = net.create_target_weights(input_data) - net.self_train(1, self.log_step_size, self.net_learning_rate, input_data, target_data) + net.self_train(1, self.log_step_size, self.net_learning_rate) print( f"\nLast weight matrix (epoch: {j}):\n{net.input_weight_matrix()}\nLossHistory: {net.loss_history[-10:]}") diff --git a/experiments/robustness_exp.py b/experiments/robustness_exp.py index 2f6c09b..41ed347 100644 --- a/experiments/robustness_exp.py +++ b/experiments/robustness_exp.py @@ -11,7 +11,7 @@ from network import Net from visualization import bar_chart_fixpoints, box_plot, write_file -def add_noise(input_data, epsilon = pow(10, -5)): +def add_noise(input_data, epsilon=pow(10, -5)): output = copy.deepcopy(input_data) for k in range(len(input_data)): @@ -63,8 +63,6 @@ class RobustnessExperiment: net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name) for _ in range(self.ST_steps): - input_data = net.input_weight_matrix() - target_data = net.create_target_weights(input_data) net.self_train(1, self.log_step_size, self.net_learning_rate) self.nets.append(net) @@ -86,7 +84,7 @@ class RobustnessExperiment: # Extra safety for the value of the weights original_net_clone.load_state_dict(copy.deepcopy(original_net.state_dict())) - noisy_weights = add_noise(original_net_clone.input_weight_matrix()) + noisy_weights = add_noise(original_net_clone.input_weight_matrix(), epsilon=pow(10, -j)) original_net_clone.apply_weights(noisy_weights) # Testing if the new net is still an identity function after applying noise diff --git a/experiments/soup_exp.py b/experiments/soup_exp.py index bdfcff2..31916f5 100644 --- a/experiments/soup_exp.py +++ b/experiments/soup_exp.py @@ -1,6 +1,8 @@ import random import os.path import pickle +from pathlib import Path +from typing import Union from tqdm import tqdm @@ -12,7 +14,7 @@ from visualization import plot_loss, bar_chart_fixpoints, plot_3d_soup, line_cha class SoupExperiment: def __init__(self, population_size, net_i_size, net_h_size, net_o_size, learning_rate, attack_chance, - train_nets, ST_steps, epochs, log_step_size, directory_name): + train_nets, ST_steps, epochs, log_step_size, directory: Union[str, Path]): super().__init__() self.population_size = population_size @@ -40,8 +42,8 @@ class SoupExperiment: # is used for keeping track of the amount of fixpoints in % self.fixpoint_counters_history = [] - self.directory_name = directory_name - os.mkdir(self.directory_name) + self.directory = Path(directory) + self.directory.mkdir(parents=True, exist_ok=True) self.population = [] self.populate_environment() @@ -69,8 +71,7 @@ class SoupExperiment: loop_epochs.set_description("Evolving soup %s" % i) # A network attacking another network with a given percentage - chance = random.randint(1, 100) - if chance <= self.attack_chance: + if random.randint(1, 100) <= self.attack_chance: random_net1, random_net2 = random.sample(range(self.population_size), 2) random_net1 = self.population[random_net1] random_net2 = self.population[random_net2] @@ -91,24 +92,25 @@ class SoupExperiment: self.fixpoint_counters["fix_sec"]) / self.population_size, 1) self.fixpoint_counters_history.append(fixpoints_percentage) - # Resetting the fixpoint counter. Last iteration not to be reset - it is important for the bar_chart_fixpoints(). + # Resetting the fixpoint counter. Last iteration not to be reset - + # it is important for the bar_chart_fixpoints(). if i < self.epochs: self.reset_fixpoint_counters() def weights_evolution_3d_experiment(self): exp_name = f"soup_{self.population_size}_nets_{self.ST_steps}_training_{self.epochs}_epochs" - return plot_3d_soup(self.population, exp_name, self.directory_name) + return plot_3d_soup(self.population, exp_name, self.directory) def count_fixpoints(self): test_for_fixpoints(self.fixpoint_counters, self.population) exp_details = f"Evolution steps: {self.epochs} epochs" - bar_chart_fixpoints(self.fixpoint_counters, self.population_size, self.directory_name, self.net_learning_rate, + bar_chart_fixpoints(self.fixpoint_counters, self.population_size, self.directory, self.net_learning_rate, exp_details) def fixpoint_percentage(self): runs = self.epochs / self.ST_steps SA_steps = None - line_chart_fixpoints(self.fixpoint_counters_history, runs, self.ST_steps, SA_steps, self.directory_name, + line_chart_fixpoints(self.fixpoint_counters_history, runs, self.ST_steps, SA_steps, self.directory, self.population_size) def visualize_loss(self): @@ -116,7 +118,7 @@ class SoupExperiment: net_loss_history = self.population[i].loss_history self.loss_history.append(net_loss_history) - plot_loss(self.loss_history, self.directory_name) + plot_loss(self.loss_history, self.directory) def reset_fixpoint_counters(self): self.fixpoint_counters = { @@ -138,6 +140,7 @@ def run_soup_experiment(population_size, attack_chance, net_input_size, net_hidd # Running the experiments for i in range(runs): + # FIXME: Make this a pathlib.Path() Operation directory_name = f"experiments/soup/{run_name}_run_{i}_{str(population_size)}_nets_{epochs}_epochs_{str(name_hash)}" soup_experiment = SoupExperiment( @@ -166,6 +169,7 @@ def run_soup_experiment(population_size, attack_chance, net_input_size, net_hidd range(len(fixpoints_percentages))] # Creating a folder for the summary of the current runs + # FIXME: Make this a pathlib.Path() Operation directory_name = f"experiments/soup/summary_{run_name}_{runs}_runs_{str(population_size)}_nets_{epochs}_epochs_{str(name_hash)}" os.mkdir(directory_name) diff --git a/functionalities_test.py b/functionalities_test.py index 54113df..449a89b 100644 --- a/functionalities_test.py +++ b/functionalities_test.py @@ -54,7 +54,7 @@ def is_secondary_fixpoint(network: Net, epsilon: float = pow(10, -5)) -> bool: def test_for_fixpoints(fixpoint_counter: Dict, nets: List, id_functions=None): - id_functions = id_functions or None + id_functions = id_functions or list() for i in range(len(nets)): net = nets[i] diff --git a/main.py b/main.py index 6bf262e..2e711ea 100644 --- a/main.py +++ b/main.py @@ -40,11 +40,11 @@ if __name__ == '__main__': # Constants: NET_INPUT_SIZE = 4 NET_OUT_SIZE = 1 - run_ST_experiment_bool = True + run_ST_experiment_bool = False run_SA_experiment_bool = False run_soup_experiment_bool = False run_mixed_experiment_bool = False - run_robustness_bool = False + run_robustness_bool = True """ ------------------------------------- Self-training (ST) experiment ------------------------------------- """