steffen/self-replicating-neural-networks
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

journal_robustness.py redone, now is sensitive to seeds and plots

Browse Source
This commit is contained in:
steffen-illium
2021-05-25 17:07:24 +02:00
parent 5e5511caf8
commit c9efe0a31b
2 changed files with 26 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
journal_basins.py
View File

@@ -173,6 +173,7 @@ class SpawnExperiment:
# and add to nets for plotting if they are fixpoints themselves; # and add to nets for plotting if they are fixpoints themselves;
for _ in range(self.epochs - 1): for _ in range(self.epochs - 1):
for _ in range(self.ST_steps): for _ in range(self.ST_steps):
# soup Evolve
clone.self_train(1, self.log_step_size, self.net_learning_rate) clone.self_train(1, self.log_step_size, self.net_learning_rate)
if is_identity_function(clone): if is_identity_function(clone):
input_data = clone.input_weight_matrix() input_data = clone.input_weight_matrix()
@@ -212,19 +213,19 @@ if __name__ == "__main__":
# Define number of runs & name: # Define number of runs & name:
ST_runs = 1 ST_runs = 1
ST_runs_name = "test-27" ST_runs_name = "test-27"
ST_steps = 2500 ST_steps = 2000
ST_epochs = 2 ST_epochs = 2
ST_log_step_size = 10 ST_log_step_size = 10
# Define number of networks & their architecture # Define number of networks & their architecture
nr_clones = 10 nr_clones = 50
ST_population_size = 3 ST_population_size = 1
ST_net_hidden_size = 2 ST_net_hidden_size = 2
ST_net_learning_rate = 0.04 ST_net_learning_rate = 0.04
ST_name_hash = random.getrandbits(32) ST_name_hash = random.getrandbits(32)
print(f"Running the Spawn experiment:") print(f"Running the Spawn experiment:")
for noise_factor in [1]: for noise_factor in [9]:
SpawnExperiment( SpawnExperiment(
population_size=ST_population_size, population_size=ST_population_size,
log_step_size=ST_log_step_size, log_step_size=ST_log_step_size,

38
journal_robustness.py
View File

@@ -95,7 +95,6 @@ class RobustnessComparisonExperiment:
for _ in range(self.epochs): for _ in range(self.epochs):
net.self_train(self.ST_steps, self.log_step_size, self.net_learning_rate) net.self_train(self.ST_steps, self.log_step_size, self.net_learning_rate)
nets.append(net) nets.append(net)
return nets return nets
def test_robustness(self, print_it=True, noise_levels=10, seeds=10): def test_robustness(self, print_it=True, noise_levels=10, seeds=10):
@@ -110,12 +109,12 @@ class RobustnessComparisonExperiment:
# This checks wether to use synthetic setting with multiple seeds # This checks wether to use synthetic setting with multiple seeds
# or multi network settings with a singlee seed # or multi network settings with a singlee seed
df = pd.DataFrame(columns=['seed', 'noise_level', 'application_step', 'absolute_loss']) df = pd.DataFrame(columns=['setting', 'noise_level', 'application_step', 'absolute_loss', 'time_to_vergence'])
for i, fixpoint in enumerate(self.id_functions): #1 / n for i, fixpoint in enumerate(self.id_functions): #1 / n
row_headers.append(fixpoint.name) row_headers.append(fixpoint.name)
for seed in range(seeds): #n / 1 for seed in range(seeds): #n / 1
for noise_level in range(noise_levels): for noise_level in range(noise_levels):
self_application_steps = 1 self_application_steps = 0
clone = Net(fixpoint.input_size, fixpoint.hidden_size, fixpoint.out_size, clone = Net(fixpoint.input_size, fixpoint.hidden_size, fixpoint.out_size,
f"{fixpoint.name}_clone_noise10e-{noise_level}") f"{fixpoint.name}_clone_noise10e-{noise_level}")
clone.load_state_dict(copy.deepcopy(fixpoint.state_dict())) clone.load_state_dict(copy.deepcopy(fixpoint.state_dict()))
@@ -123,9 +122,6 @@ class RobustnessComparisonExperiment:
clone = self.apply_noise(clone, rand_noise) clone = self.apply_noise(clone, rand_noise)
while not is_zero_fixpoint(clone) and not is_divergent(clone): while not is_zero_fixpoint(clone) and not is_divergent(clone):
if is_identity_function(clone):
avg_time_as_fixpoint[i][noise_level] += 1
# -> before # -> before
clone_weight_pre_application = clone.input_weight_matrix() clone_weight_pre_application = clone.input_weight_matrix()
target_data_pre_application = clone.create_target_weights(clone_weight_pre_application) target_data_pre_application = clone.create_target_weights(clone_weight_pre_application)
@@ -140,16 +136,27 @@ class RobustnessComparisonExperiment:
setting = i if is_synthetic else seed setting = i if is_synthetic else seed
df.loc[data_pos] = [setting, noise_level, self_application_steps, absolute_loss] if is_identity_function(clone):
data_pos += 1 avg_time_as_fixpoint[i][noise_level] += 1
self_application_steps += 1 # When this raises a Type Error, we found a second order fixpoint!
self_application_steps += 1
else:
self_application_steps = pd.NA # Not a Number!
df.loc[df.shape[0]] = [setting, noise_level, self_application_steps,
absolute_loss, avg_time_to_vergence[i][noise_level]]
# calculate the average: # calculate the average:
df = df.replace([np.inf, -np.inf], np.nan) # df = df.replace([np.inf, -np.inf], np.nan)
df = df.dropna() # df = df.dropna()
bf = sns.boxplot(data=df, y='self_application_steps', x='noise_level', )
bf.set_title('Robustness as self application steps per noise level')
plt.tight_layout()
# sns.set(rc={'figure.figsize': (10, 50)}) # sns.set(rc={'figure.figsize': (10, 50)})
bx = sns.catplot(data=df[df['absolute_loss'] < 1], y='absolute_loss', x='application_step', kind='box', # bx = sns.catplot(data=df[df['absolute_loss'] < 1], y='absolute_loss', x='application_step', kind='box',
col='noise_level', col_wrap=3, showfliers=False) # col='noise_level', col_wrap=3, showfliers=False)
directory = Path('output') / 'robustness' directory = Path('output') / 'robustness'
filename = f"absolute_loss_perapplication_boxplot_grid.png" filename = f"absolute_loss_perapplication_boxplot_grid.png"
filepath = directory / filename filepath = directory / filename
@@ -167,21 +174,18 @@ class RobustnessComparisonExperiment:
return avg_time_as_fixpoint, avg_time_to_vergence return avg_time_as_fixpoint, avg_time_to_vergence
def count_fixpoints(self): def count_fixpoints(self):
exp_details = f"ST steps: {self.ST_steps}" exp_details = f"ST steps: {self.ST_steps}"
self.id_functions = test_for_fixpoints(self.fixpoint_counters, self.nets) self.id_functions = test_for_fixpoints(self.fixpoint_counters, self.nets)
bar_chart_fixpoints(self.fixpoint_counters, self.population_size, self.directory, self.net_learning_rate, bar_chart_fixpoints(self.fixpoint_counters, self.population_size, self.directory, self.net_learning_rate,
exp_details) exp_details)
def visualize_loss(self): def visualize_loss(self):
for i in range(len(self.nets)): for i in range(len(self.nets)):
net_loss_history = self.nets[i].loss_history net_loss_history = self.nets[i].loss_history
self.loss_history.append(net_loss_history) self.loss_history.append(net_loss_history)
plot_loss(self.loss_history, self.directory) plot_loss(self.loss_history, self.directory)
def save(self): def save(self):
pickle.dump(self, open(f"{self.directory}/experiment_pickle.p", "wb")) pickle.dump(self, open(f"{self.directory}/experiment_pickle.p", "wb"))
print(f"\nSaved experiment to {self.directory}.") print(f"\nSaved experiment to {self.directory}.")
@@ -211,5 +215,5 @@ if __name__ == "__main__":
epochs=ST_epochs, epochs=ST_epochs,
st_steps=ST_steps, st_steps=ST_steps,
synthetic=ST_synthetic, synthetic=ST_synthetic,
directory=Path('output') / 'robustness' / f'{ST_name_hash}' directory=Path('output') / 'journal_robustness' / f'{ST_name_hash}'
) )