Delete New Text Document.txt

Step 4 - Aggregating Neural Networks
Step 5 - Training Neural Networks

.gitignore

@@ -1 +1,572 @@
-/output/
+# Created by https://www.gitignore.io/api/data,linux,macos,python,windows,pycharm,database,jupyternotebook
+# Edit at https://www.gitignore.io/?templates=data,linux,macos,python,windows,pycharm,database,jupyternotebook
+
+
+### Local Datasets ###
+/experiments
+/setups/experiments
+
+### Data ###
+*.csv
+*.dat
+*.efx
+*.gbr
+*.key
+*.pps
+*.ppt
+*.pptx
+*.sdf
+*.tax2010
+*.vcf
+*.xml
+
+### Database ###
+*.accdb
+*.db
+*.dbf
+*.mdb
+*.pdb
+*.sqlite3
+
+### JupyterNotebook ###
+.ipynb_checkpoints
+*/.ipynb_checkpoints/*
+
+# Remove previous ipynb_checkpoints
+#   git rm -r .ipynb_checkpoints/
+#
+
+### Linux ###
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### PyCharm Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+.idea/sonarlint
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+### Python Patch ###
+.venv/
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# pycharm
+.idea/
+
+#######################################
+#### Tex related
+## Core latex/pdflatex auxiliary files:
+*.aux
+*.lof
+*.log
+*.lot
+*.fls
+*.out
+*.toc
+*.fmt
+*.fot
+*.cb
+*.cb2
+.*.lb
+
+## Intermediate documents:
+*.dvi
+*.xdv
+*-converted-to.*
+# these rules might exclude image files for figures etc.
+# *.ps
+# *.eps
+# *.pdf
+
+## Generated if empty string is given at "Please type another file name for output:"
+.pdf
+
+## Bibliography auxiliary files (bibtex/biblatex/biber):
+*.bbl
+*.bcf
+*.blg
+*-blx.aux
+*-blx.bib
+*.run.xml
+
+## Build tool auxiliary files:
+*.fdb_latexmk
+*.synctex
+*.synctex(busy)
+*.synctex.gz
+*.synctex.gz(busy)
+*.pdfsync
+
+## Build tool directories for auxiliary files
+# latexrun
+latex.out/
+
+## Auxiliary and intermediate files from other packages:
+# algorithms
+*.alg
+*.loa
+
+# achemso
+acs-*.bib
+
+# amsthm
+*.thm
+
+# beamer
+*.nav
+*.pre
+*.snm
+*.vrb
+
+# changes
+*.soc
+
+# comment
+*.cut
+
+# cprotect
+*.cpt
+
+# elsarticle (documentclass of Elsevier journals)
+*.spl
+
+# endnotes
+*.ent
+
+# fixme
+*.lox
+
+# feynmf/feynmp
+*.mf
+*.mp
+*.t[1-9]
+*.t[1-9][0-9]
+*.tfm
+
+#(r)(e)ledmac/(r)(e)ledpar
+*.end
+*.?end
+*.[1-9]
+*.[1-9][0-9]
+*.[1-9][0-9][0-9]
+*.[1-9]R
+*.[1-9][0-9]R
+*.[1-9][0-9][0-9]R
+*.eledsec[1-9]
+*.eledsec[1-9]R
+*.eledsec[1-9][0-9]
+*.eledsec[1-9][0-9]R
+*.eledsec[1-9][0-9][0-9]
+*.eledsec[1-9][0-9][0-9]R
+
+# glossaries
+*.acn
+*.acr
+*.glg
+*.glo
+*.gls
+*.glsdefs
+
+# gnuplottex
+*-gnuplottex-*
+
+# gregoriotex
+*.gaux
+*.gtex
+
+# htlatex
+*.4ct
+*.4tc
+*.idv
+*.lg
+*.trc
+*.xref
+
+# hyperref
+*.brf
+
+# knitr
+*-concordance.tex
+# TODO Comment the next line if you want to keep your tikz graphics files
+*.tikz
+*-tikzDictionary
+
+# listings
+*.lol
+
+# makeidx
+*.idx
+*.ilg
+*.ind
+*.ist
+
+# minitoc
+*.maf
+*.mlf
+*.mlt
+*.mtc[0-9]*
+*.slf[0-9]*
+*.slt[0-9]*
+*.stc[0-9]*
+
+# minted
+_minted*
+*.pyg
+
+# morewrites
+*.mw
+
+# nomencl
+*.nlg
+*.nlo
+*.nls
+
+# pax
+*.pax
+
+# pdfpcnotes
+*.pdfpc
+
+# sagetex
+*.sagetex.sage
+*.sagetex.py
+*.sagetex.scmd
+
+# scrwfile
+*.wrt
+
+# sympy
+*.sout
+*.sympy
+sympy-plots-for-*.tex/
+
+# pdfcomment
+*.upa
+*.upb
+
+# pythontex
+*.pytxcode
+pythontex-files-*/
+
+# tcolorbox
+*.listing
+
+# thmtools
+*.loe
+
+# TikZ & PGF
+*.dpth
+*.md5
+*.auxlock
+
+# todonotes
+*.tdo
+
+# vhistory
+*.hst
+*.ver
+
+# easy-todo
+*.lod
+
+# xcolor
+*.xcp
+
+# xmpincl
+*.xmpi
+
+# xindy
+*.xdy
+
+# xypic precompiled matrices
+*.xyc
+
+# endfloat
+*.ttt
+*.fff
+
+# Latexian
+TSWLatexianTemp*
+
+## Editors:
+# WinEdt
+*.bak
+*.sav
+
+# Texpad
+.texpadtmp
+
+# LyX
+*.lyx~
+
+# Kile
+*.backup
+
+# KBibTeX
+*~[0-9]*
+
+# auto folder when using emacs and auctex
+./auto/*
+*.el
+
+# expex forward references with \gathertags
+*-tags.tex
+
+# standalone packages
+*.sta
+
+
+# End of https://www.gitignore.io/api/data,linux,macos,python,windows,pycharm,database,jupyternotebook
+

README.md

@@ -1,52 +1,2 @@
-# self-rep NN paper - ALIFE journal edition
-
-- [x] Plateau / Pillar sizeWhat does happen to the fixpoints after noise introduction and retraining?Options beeing: Same Fixpoint, Similar Fixpoint (Basin), 
-    - Different Fixpoint?
-      Yes, we did not found same (10-5)
-    - Do they do the clustering thingy?
-      Kind of: Small movement towards (MIM-Distance getting smaller) parent fixpoint.
-      Small movement for everyone? -> Distribution
-
-    - see `journal_basins.py` for the "train -> spawn with noise -> train again and see where they end up" functionality. Apply noise follows the `vary` function that was used in the paper robustness test with `+- prng() * eps`. Change if desired.
-
-    - there is also a distance matrix for all-to-all particle comparisons (with distance parameter one of: `MSE`, `MAE` (mean absolute error = mean manhattan) and `MIM` (mean position invariant manhattan))
-
-
-- [ ] Same Thing with Soup interaction. We would expect the same behaviour...Influence of interaction with near and far away particles.
-      - 
-      - 
-  
-- [x] Robustness test with a trained NetworkTraining for high quality fixpoints, compare with the "perfect" fixpoint. Average Loss per application step
-
-    - see `journal_robustness.py` for robustness test modeled after cristians robustness-exp (with the exeption that we put noise on the weights). Has `synthetic` bool to switch to hand-modeled perfect fixpoint instead of naturally trained ones. 
-
-    - Also added two difference between the "time-as-fixpoint" and "time-to-verge" (i.e. to divergence / zero).
-
-    - We might need to consult about the "average loss per application step", as I think application loss get gradually higher the worse the weights get. So the average might not tell us much here.
-
-- [x] Adjust Self Training so that it favors second order fixpoints-> Second order test implementation (?)
-
-- [x] Barplot over clones -> how many become a fixpoint cs how many diverge per noise level
-
-- [x] Box-Plot of Avg. Distance of clones from parent
-
-- [x] Search subspace between two fixpoints by linage(10**-5), check were they end up
-
-- [x] How are basins / "attractor areas" shaped?
-
-
-# Future Todos:
-
-- [ ] Find a statistik over weight space that provides a better init function
-- [ ] Test this init function on a mnist classifier - just for the lolz
-
----
-## Notes: 
-
-- In the spawn-experiment we now fit and transform the PCA over *ALL* trajectories, instead of each net-history by its own. This can be toggled by the `plot_pca_together` parameter in `visualisation.py/plot_3d_self_train() & plot_3d()` (default: `False` but set `True` in the spawn-experiment class).
-
-- I have also added a `start_time` property for the nets (default: `1`). This is intended to be set flexibly for e.g., clones (when they are spawned midway through the experiment), such that the PCA can start the plotting trace from this timestep. When we spawn clones we deepcopy their parent's saved weight_history too, so that the PCA transforms same lenght trajectories. With `plot_pca_together` that means that clones and their parents will literally be plotted perfectly overlayed on top, up until the spawn-time, where you can see the offset / noise we apply. By setting the start_time, you can avoid this overlap and avoid hiding the parent's trace color which gets plotted first (because the parent is always added to self.nets first). **But more importantly, you can effectively zoom into the plot, by setting the parents start-time to just shy of the end of first epoch (where they get checked on fixpoint-property and spawn clones) and the start-times of clones to the second epoch. This will make the plot begin at spawn time, cutting off the parents initial trajectory and zoom-in to the action (see. `journal_basins.py/spawn_and_continue()`).**
-
-- Now saving the whole experiment class as pickle dump (`experiment_pickle.p`, just like cristian), hope thats fine.
-
-- Added a `requirement.txt` for quick venv / pip -r installs. Append as necessary.
+# bannana-networks
+

code/experiment.py

@@ -0,0 +1,171 @@
+import os
+import time
+import dill
+from tqdm import tqdm
+import copy
+
+from tensorflow.python.keras import backend as K
+
+from abc import ABC, abstractmethod
+
+
+class Experiment(ABC):
+
+    @staticmethod
+    def from_dill(path):
+        with open(path, "rb") as dill_file:
+            return dill.load(dill_file)
+
+    @staticmethod
+    def reset_model():
+        K.clear_session()
+
+    def __init__(self, name=None, ident=None):
+        self.experiment_id = f'{ident or ""}_{time.time()}'
+        self.experiment_name = name or 'unnamed_experiment'
+        self.next_iteration = 0
+        self.log_messages = list()
+        self.historical_particles = dict()
+
+    def __enter__(self):
+        self.dir = os.path.join('experiments', f'exp-{self.experiment_name}-{self.experiment_id}-{self.next_iteration}')
+        os.makedirs(self.dir)
+        print(f'** created {self.dir} **')
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.save(experiment=self.without_particles())
+        self.save_log()
+        self.next_iteration += 1
+
+    def log(self, message, **kwargs):
+        self.log_messages.append(message)
+        print(message, **kwargs)
+
+    def save_log(self, log_name="log"):
+        with open(os.path.join(self.dir, f"{log_name}.txt"), "w") as log_file:
+            for log_message in self.log_messages:
+                print(str(log_message), file=log_file)
+
+    def __copy__(self):
+        self_copy = self.__class__(name=self.experiment_name,)
+        self_copy.__dict__ = {attr: self.__dict__[attr] for attr in self.__dict__ if
+                              attr not in ['particles', 'historical_particles']}
+        return self_copy
+
+    def without_particles(self):
+        self_copy = copy.copy(self)
+        # self_copy.particles = [particle.states for particle in self.particles]
+        self_copy.historical_particles = {key: val.states for key, val in self.historical_particles.items()}
+        return self_copy
+
+    def save(self, **kwargs):
+        for name, value in kwargs.items():
+            with open(os.path.join(self.dir, f"{name}.dill"), "wb") as dill_file:
+                dill.dump(value, dill_file)
+
+    @abstractmethod
+    def run_net(self, net, trains_per_application=100, step_limit=100, run_id=0, **kwargs):
+        raise NotImplementedError
+        pass
+
+    def run_exp(self, network_generator, exp_iterations, prints=False, **kwargs):
+        # INFO Run_ID needs to be more than 0, so that exp stores the trajectories!
+        for run_id in range(exp_iterations):
+            network = network_generator()
+            self.run_net(network, 100, run_id=run_id + 1, **kwargs)
+            self.historical_particles[run_id] = network
+            if prints:
+                print("Fixpoint? " + str(network.is_fixpoint()))
+        self.reset_model()
+
+    def reset_all(self):
+        self.reset_model()
+
+
+class FixpointExperiment(Experiment):
+    if kwargs.get('logging', False):
+        self.log(self.counters)
+
+    def __init__(self, **kwargs):
+        kwargs['name'] = self.__class__.__name__ if 'name' not in kwargs else kwargs['name']
+        super().__init__(**kwargs)
+        self.counters = dict(divergent=0, fix_zero=0, fix_other=0, fix_sec=0, other=0)
+        self.interesting_fixpoints = []
+
+    def run_net(self, net, step_limit=100, run_id=0, **kwargs):
+        i = 0
+        while i < step_limit and not net.is_diverged() and not net.is_fixpoint():
+            net.self_attack()
+            i += 1
+            if run_id:
+                net.save_state(time=i)
+        self.count(net)
+
+    def count(self, net):
+        if net.is_diverged():
+            self.counters['divergent'] += 1
+        elif net.is_fixpoint():
+            if net.is_zero():
+                self.counters['fix_zero'] += 1
+            else:
+                self.counters['fix_other'] += 1
+                self.interesting_fixpoints.append(net.get_weights())
+        elif net.is_fixpoint(2):
+            self.counters['fix_sec'] += 1
+        else:
+            self.counters['other'] += 1
+
+    def reset_counters(self):
+        for key in self.counters.keys():
+            self.counters[key] = 0
+        return True
+
+    def reset_all(self):
+        super(FixpointExperiment, self).reset_all()
+        self.reset_counters()
+
+
+class MixedFixpointExperiment(FixpointExperiment):
+
+    def run_net(self, net, trains_per_application=100, step_limit=100, run_id=0, **kwargs):
+        for i in range(step_limit):
+            if net.is_diverged() or net.is_fixpoint():
+                break
+            net.self_attack()
+            with tqdm(postfix=["Loss", dict(value=0)]) as bar:
+                for _ in range(trains_per_application):
+                    loss = net.compiled().train()
+                    bar.postfix[1]["value"] = loss
+                    bar.update()
+            if run_id:
+                net.save_state()
+        self.count(net)
+
+
+class SoupExperiment(Experiment):
+
+    def __init__(self, **kwargs):
+        super(SoupExperiment, self).__init__(name=kwargs.get('name', self.__class__.__name__))
+
+    def run_exp(self, network_generator, exp_iterations, soup_generator=None, soup_iterations=0, prints=False):
+        for i in range(soup_iterations):
+            soup = soup_generator()
+            soup.seed()
+            for _ in tqdm(exp_iterations):
+                soup.evolve()
+            self.log(soup.count())
+            self.save(soup=soup.without_particles())
+
+    def run_net(self, net, trains_per_application=100, step_limit=100, run_id=0, **kwargs):
+        raise NotImplementedError
+        pass
+
+
+class IdentLearningExperiment(Experiment):
+
+    def __init__(self):
+        super(IdentLearningExperiment, self).__init__(name=self.__class__.__name__)
+
+    def run_net(self, net, trains_per_application=100, step_limit=100, run_id=0, **kwargs):
+        pass

code/network.py

@@ -0,0 +1,618 @@
+import numpy as np
+from abc import abstractmethod, ABC
+from typing import List, Union
+from types import FunctionType
+
+from tensorflow.python.keras.models import Sequential
+from tensorflow.python.keras.callbacks import Callback
+from tensorflow.python.keras.layers import SimpleRNN, Dense
+from tensorflow.python.keras import backend as K
+
+from experiment import *
+
+# Supress warnings and info messages
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
+
+
+class SaveStateCallback(Callback):
+    def __init__(self, network, epoch=0):
+        super(SaveStateCallback, self).__init__()
+        self.net = network
+        self.init_epoch = epoch
+
+    def on_epoch_end(self, epoch, logs=None):
+        description = dict(time=epoch+self.init_epoch)
+        description['action'] = 'train_self'
+        description['counterpart'] = None
+        self.net.save_state(**description)
+        return
+
+
+class Weights:
+
+    @staticmethod
+    def __reshape_flat_array__(array, shapes):
+        sizes: List[int] = [int(np.prod(shape)) for shape in shapes]
+        # Split the incoming array into slices for layers
+        slices = [array[x: y] for x, y in zip(np.cumsum([0]+sizes), np.cumsum([0]+sizes)[1:])]
+        # reshape them in accordance to the given shapes
+        weights = [np.reshape(weight_slice, shape) for weight_slice, shape in zip(slices, shapes)]
+        return weights
+
+    def __init__(self, weight_vector: Union[List[np.ndarray], np.ndarray], flat_array_shape=None):
+        """
+        Weight class, for easy manipulation of weight vectors from Keras models
+
+        :param weight_vector: A numpy array holding weights
+        :type weight_vector: List[np.ndarray]
+        """
+        self.__iter_idx = [0, 0]
+        if flat_array_shape:
+            weight_vector = self.__reshape_flat_array__(weight_vector, flat_array_shape)
+
+        self.layers = weight_vector
+
+        # TODO: implement a way to access the cells directly
+        # self.cells = len(self)
+        # TODO: implement a way to access the weights directly
+        # self.weights = self.to_flat_array() ?
+
+    def __iter__(self):
+        self.__iter_idx = [0, 0]
+        return self
+
+    def __getitem__(self, item):
+        return self.layers[item]
+
+    def max(self):
+        np.max(self.layers)
+
+    def avg(self):
+        return np.average(self.layers)
+
+    def __len__(self):
+        return sum([x.size for x in self.layers])
+
+    def shapes(self):
+        return [x.shape for x in self.layers]
+
+    def num_layers(self):
+        return len(self.layers)
+
+    def __copy__(self):
+        return copy.deepcopy(self)
+
+    def __next__(self):
+        # ToDo: Check iteration progress over layers
+        # ToDo: There is still a problem interation, currently only cell level is the last loop stage.
+        # Do we need this?
+        if self.__iter_idx[0] >= len(self.layers):
+            if self.__iter_idx[1] >= len(self.layers[self.__iter_idx[0]]):
+                raise StopIteration
+        result = self.layers[self.__iter_idx[0]][self.__iter_idx[1]]
+
+        if self.__iter_idx[1] >= len(self.layers[self.__iter_idx[0]]):
+            self.__iter_idx[0] += 1
+            self.__iter_idx[1] = 0
+        else:
+            self.__iter_idx[1] += 1
+        return result
+
+    def __repr__(self):
+        return f'Weights({self.to_flat_array().tolist()})'
+
+    def to_flat_array(self) -> np.ndarray:
+        return np.hstack([weight.flatten() for weight in self.layers])
+
+    def from_flat_array(self, array):
+        new_weights = self.__reshape_flat_array__(array, self.shapes())
+        return new_weights
+
+    def shuffle(self):
+        flat = self.to_flat_array()
+        np.random.shuffle(flat)
+        self.from_flat_array(flat)
+        return True
+
+    def are_diverged(self):
+        return any([np.isnan(x).any() for x in self.layers]) or any([np.isinf(x).any() for x in self.layers])
+
+    def are_within_bounds(self, lower_bound: float, upper_bound: float):
+        return bool(sum([((lower_bound < x) & (x > upper_bound)).size for x in self.layers]))
+
+    def aggregate_by(self, func: FunctionType, num_aggregates):
+        collection_sizes = len(self) // num_aggregates
+        weights = self.to_flat_array()[:collection_sizes * num_aggregates].reshape((num_aggregates, -1))
+        aggregated_weights = func(weights, num_aggregates)
+        left_overs = self.to_flat_array()[collection_sizes * num_aggregates:]
+        return aggregated_weights, left_overs
+
+
+class NeuralNetwork(ABC):
+    """
+    This is the Base Network Class, including abstract functions that must be implemented.
+    """
+
+    def __init__(self, **params):
+        super().__init__()
+        self.params = dict(epsilon=0.00000000000001)
+        self.params.update(params)
+        self.keras_params = dict(activation='linear', use_bias=False)
+        self.states = []
+        self.model: Sequential
+
+    def get_params(self) -> dict:
+        return self.params
+
+    def get_keras_params(self) -> dict:
+        return self.keras_params
+
+    def with_params(self, **kwargs):
+        self.params.update(kwargs)
+        return self
+
+    def with_keras_params(self, **kwargs):
+        self.keras_params.update(kwargs)
+        return self
+
+    def get_weights(self) -> Weights:
+        return Weights(self.model.get_weights())
+
+    def get_weights_flat(self) -> np.ndarray:
+        return self.get_weights().to_flat_array()
+
+    def set_weights(self, new_weights: Weights):
+        return self.model.set_weights(new_weights.layers)
+
+    @abstractmethod
+    def get_samples(self):
+        # TODO: add a dogstring, telling the user what this does, e.g. what is a sample?
+        raise NotImplementedError
+
+    @abstractmethod
+    def apply_to_weights(self, old_weights) -> Weights:
+        # TODO: add a dogstring, telling the user what this does, e.g. what is applied?
+        raise NotImplementedError
+
+    def apply_to_network(self, other_network) -> Weights:
+        # TODO: add a dogstring, telling the user what this does, e.g. what is applied?
+        new_weights = self.apply_to_weights(other_network.get_weights())
+        return new_weights
+
+    def attack(self, other_network):
+        # TODO: add a dogstring, telling the user what this does, e.g. what is an attack?
+        other_network.set_weights(self.apply_to_network(other_network))
+        return self
+
+    def fuck(self, other_network):
+        # TODO: add a dogstring, telling the user what this does, e.g. what is fucking?
+        self.set_weights(self.apply_to_network(other_network))
+        return self
+
+    def self_attack(self, iterations=1):
+        # TODO: add a dogstring, telling the user what this does, e.g. what is self attack?
+        for _ in range(iterations):
+            self.attack(self)
+        return self
+
+    def meet(self, other_network):
+        # TODO: add a dogstring, telling the user what this does, e.g. what is meeting?
+        new_other_network = copy.deepcopy(other_network)
+        return self.attack(new_other_network)
+
+    def is_diverged(self):
+        return self.get_weights().are_diverged()
+
+    def is_zero(self, epsilon=None):
+        epsilon = epsilon or self.get_params().get('epsilon')
+        return self.get_weights().are_within_bounds(-epsilon, epsilon)
+
+    def is_fixpoint(self, degree: int = 1, epsilon: float = None) -> bool:
+        assert degree >= 1, "degree must be >= 1"
+        epsilon = epsilon or self.get_params().get('epsilon')
+
+        new_weights = copy.deepcopy(self.get_weights())
+
+        for _ in range(degree):
+            new_weights = self.apply_to_weights(new_weights)
+            if new_weights.are_diverged():
+                return False
+
+        biggerEpsilon = (np.abs(new_weights.to_flat_array() - self.get_weights().to_flat_array()) >= epsilon).any()
+
+        # Boolean Value needs to be flipped to answer "is_fixpoint"
+        return not biggerEpsilon
+
+    def print_weights(self, weights=None):
+        print(weights or self.get_weights())
+
+
+class ParticleDecorator:
+    next_uid = 0
+
+    def __init__(self, network):
+
+        # ToDo: Add DocString, What does it do?
+
+        self.uid = self.__class__.next_uid
+        self.__class__.next_uid += 1
+        self.network = network
+        self.states = []
+        self.save_state(time=0, action='init', counterpart=None)
+
+    def __getattr__(self, name):
+        return getattr(self.network, name)
+
+    def get_uid(self):
+        return self.uid
+
+    def make_state(self, **kwargs):
+        if self.network.is_diverged():
+            return None
+        state = {'class': self.network.__class__.__name__, 'weights': self.network.get_weights_flat()}
+        state.update(kwargs)
+        return state
+
+    def save_state(self, **kwargs):
+        state = self.make_state(**kwargs)
+        if state is not None:
+            self.states += [state]
+        else:
+            pass
+        return True
+
+    def update_state(self, number, **kwargs):
+        raise NotImplementedError('Result is vague')
+        # if number < len(self.states):
+        #     self.states[number] = self.make_state(**kwargs)
+        # else:
+        #     for i in range(len(self.states), number):
+        #         self.states += [None]
+        #     self.states += self.make_state(**kwargs)
+
+    def get_states(self):
+        return self.states
+
+
+class WeightwiseNeuralNetwork(NeuralNetwork):
+
+    def __init__(self, width, depth, **kwargs):
+        # ToDo: Insert Docstring
+        super().__init__(**kwargs)
+        self.width: int = width
+        self.depth: int = depth
+        self.model = Sequential()
+        self.model.add(Dense(units=self.width, input_dim=4, **self.keras_params))
+        for _ in range(self.depth-1):
+            self.model.add(Dense(units=self.width, **self.keras_params))
+        self.model.add(Dense(units=1, **self.keras_params))
+
+    def apply(self, inputs):
+        # TODO: Write about it... What does it do?
+        return self.model.predict(inputs)
+
+    def get_samples(self):
+        weights = self.get_weights()
+        sample = np.asarray([
+            [weight, idx, *x] for idx, layer in enumerate(weights.layers) for x, weight in np.ndenumerate(layer)
+        ])
+        # normalize [layer, cell, position]
+        for idx in range(1, sample.shape[1]):
+            sample[:, idx] = sample[:, idx] / np.max(sample[:, idx])
+        return sample, sample
+
+    def apply_to_weights(self, weights) -> Weights:
+        # ToDo: Insert DocString
+        # Transform the weight matrix in an horizontal stack as: array([[weight, layer, cell, position], ...])
+        transformed_weights = self.get_samples()[0]
+        new_weights = self.apply(transformed_weights)
+        # use the original weight shape to transform the new tensor
+        return Weights(new_weights, flat_array_shape=weights.shapes())
+
+
+class AggregatingNeuralNetwork(NeuralNetwork):
+
+    @staticmethod
+    def aggregate_fft(array: np.ndarray, aggregates: int):
+        flat = array.flatten()
+        # noinspection PyTypeChecker
+        fft_reduction = np.fft.fftn(flat, aggregates)
+        return fft_reduction
+
+    @staticmethod
+    def aggregate_average(array, _):
+        return np.average(array, axis=1)
+
+    @staticmethod
+    def aggregate_max(array, _):
+        return np.max(array, axis=1)
+
+    @staticmethod
+    def deaggregate_identically(aggregate, amount):
+        # ToDo: Find a better way than using the a hardcoded [0]
+        return np.hstack([aggregate for _ in range(amount)])[0]
+
+    @staticmethod
+    def shuffle_not(weights: Weights):
+        """
+        Doesn't do a thing. f(x)
+
+        :param weights: A List of Weights
+        :type weights: Weights
+        :return: The same old weights.
+        :rtype: Weights
+        """
+        return weights
+
+    @staticmethod
+    def shuffle_random(weights: Weights):
+        assert weights.shuffle()
+        return weights
+
+    def __init__(self, aggregates, width, depth, **kwargs):
+        super().__init__(**kwargs)
+        self.aggregates = aggregates
+        self.width = width
+        self.depth = depth
+        self.model = Sequential()
+        self.model.add(Dense(units=width, input_dim=self.aggregates, **self.keras_params))
+        for _ in range(depth-1):
+            self.model.add(Dense(units=width, **self.keras_params))
+        self.model.add(Dense(units=self.aggregates, **self.keras_params))
+
+    def get_aggregator(self):
+        return self.params.get('aggregator', self.aggregate_average)
+
+    def get_deaggregator(self):
+        return self.params.get('deaggregator', self.deaggregate_identically)
+
+    def get_shuffler(self):
+        return self.params.get('shuffler', self.shuffle_not)
+
+    def get_amount_of_weights(self):
+        return len(self.get_weights())
+
+    def apply(self, inputs):
+        # You need to add an dimension here... "..." copies array values
+        return self.model.predict(inputs[None, ...])
+
+    def get_aggregated_weights(self):
+        return self.get_weights().aggregate_by(self.get_aggregator(), self.aggregates)
+
+    def apply_to_weights(self, old_weights) -> Weights:
+
+        # build aggregations of old_weights
+        old_aggregations, leftovers = self.get_aggregated_weights()
+
+        # call network
+        new_aggregations = self.apply(old_aggregations)
+        collection_sizes = self.get_amount_of_weights() // self.aggregates
+        new_aggregations = self.deaggregate_identically(new_aggregations, collection_sizes)
+        # generate new weights
+        # only include leftovers if there are some then coonvert them to Weight on base of th old shape
+        new_weights = Weights(new_aggregations if not leftovers.shape[0] else np.hstack((new_aggregations, leftovers)),
+                              flat_array_shape=old_weights.shapes())
+
+        # maybe shuffle
+        new_weights = self.get_shuffler()(new_weights)
+        return new_weights
+
+    def get_samples(self):
+        aggregations, _ = self.get_aggregated_weights()
+        # What did that do?
+        # sample = np.transpose(np.array([[aggregations[i]] for i in range(self.aggregates)]))
+        return aggregations, aggregations
+
+    def is_fixpoint_after_aggregation(self, degree=1, epsilon=None):
+        assert degree >= 1, "degree must be >= 1"
+        epsilon = epsilon or self.get_params().get('epsilon')
+
+        old_aggregations, _ = self.get_aggregated_weights()
+        new_weights = copy.deepcopy(self.get_weights())
+
+        for _ in range(degree):
+            new_weights = self.apply_to_weights(new_weights)
+            if new_weights.are_diverged():
+                return False
+
+        new_aggregations, leftovers = self.get_aggregated_weights()
+
+        # ToDo: Explain This, why are you additionally checking tolerances of aggregated weights?
+        biggerEpsilon = (np.abs(np.asarray(old_aggregations) - np.asarray(new_aggregations)) >= epsilon).any()
+
+        # Boolean value has to be flipped to answer the question.
+        return True, not biggerEpsilon
+
+
+class RecurrentNeuralNetwork(NeuralNetwork):
+
+    def __init__(self, width, depth, **kwargs):
+        super().__init__(**kwargs)
+        self.features = 1
+        self.width = width
+        self.depth = depth
+        self.model = Sequential()
+        self.model.add(SimpleRNN(units=width, input_dim=self.features, return_sequences=True, **self.keras_params))
+        for _ in range(depth-1):
+            self.model.add(SimpleRNN(units=width, return_sequences=True, **self.keras_params))
+        self.model.add(SimpleRNN(units=self.features, return_sequences=True, **self.keras_params))
+
+    def apply(self, *inputs):
+        stuff = np.transpose(np.array([[[inputs[i]] for i in range(len(inputs))]]))
+        return self.model.predict(stuff)[0].flatten()
+
+    def apply_to_weights(self, old_weights):
+        # build list from old weights
+        new_weights = copy.deepcopy(old_weights)
+        old_weights_list = []
+        for layer_id, layer in enumerate(old_weights):
+            for cell_id, cell in enumerate(layer):
+                for weight_id, weight in enumerate(cell):
+                    old_weights_list += [weight]
+
+        # call network
+        new_weights_list = self.apply(*old_weights_list)
+
+        # write back new weights from list of rnn returns
+        current_weight_id = 0
+        for layer_id, layer in enumerate(new_weights):
+            for cell_id, cell in enumerate(layer):
+                for weight_id, weight in enumerate(cell):
+                    new_weight = new_weights_list[current_weight_id]
+                    new_weights[layer_id][cell_id][weight_id] = new_weight
+                    current_weight_id += 1
+        return new_weights
+
+    def compute_samples(self):
+        # build list from old weights
+        old_weights_list = []
+        for layer_id, layer in enumerate(self.get_weights()):
+            for cell_id, cell in enumerate(layer):
+                for weight_id, weight in enumerate(cell):
+                    old_weights_list += [weight]
+        sample = np.asarray(old_weights_list)[None, ..., None]
+        return sample, sample
+
+
+class TrainingNeuralNetworkDecorator:
+
+    def __init__(self, network):
+        self.network = network
+        self.compile_params = dict(loss='mse', optimizer='sgd')
+        self.model_compiled = False
+
+    def __getattr__(self, name):
+        return getattr(self.network, name)
+
+    def with_params(self, **kwargs):
+        self.network.with_params(**kwargs)
+        return self
+
+    def with_keras_params(self, **kwargs):
+        self.network.with_keras_params(**kwargs)
+        return self
+
+    def get_compile_params(self):
+        return self.compile_params
+
+    def with_compile_params(self, **kwargs):
+        self.compile_params.update(kwargs)
+        return self
+
+    def compile_model(self, **kwargs):
+        compile_params = copy.deepcopy(self.compile_params)
+        compile_params.update(kwargs)
+        return self.network.model.compile(**compile_params)
+
+    def compiled(self, **kwargs):
+        if not self.model_compiled:
+            self.compile_model(**kwargs)
+            self.model_compiled = True
+        return self
+
+    def train(self, batchsize=1, store_states=True, epoch=0):
+        self.compiled()
+        x, y = self.network.get_samples()
+        savestatecallback = [SaveStateCallback(network=self, epoch=epoch)] if store_states else None
+        history = self.network.model.fit(x=x, y=y, epochs=epoch+1, verbose=0,
+                                         batch_size=batchsize, callbacks=savestatecallback,
+                                         initial_epoch=epoch)
+        return history.history['loss'][-1]
+
+    def learn_from(self, other_network, batchsize=1):
+        self.compiled()
+        other_network.compiled()
+        x, y = other_network.network.get_samples()
+        history = self.network.model.fit(x=x, y=y, verbose=0, batch_size=batchsize)
+
+        return history.history['loss'][-1]
+
+
+if __name__ == '__main__':
+
+    if True:
+        # WeightWise Neural Network
+        net_generator = ParticleDecorator(WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation='linear'))
+        with FixpointExperiment() as exp:
+            exp.run_exp(net_generator, 10, logging=True)
+            exp.reset_all()
+
+    if False:
+        # Aggregating Neural Network
+        net_generator = ParticleDecorator(AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params())
+        with FixpointExperiment() as exp:
+            exp.run_exp(net_generator, 10, logging=True)
+
+            exp.reset_all()
+
+    if False:
+        # FFT Aggregation
+        net_generator = lambda: ParticleDecorator(
+            AggregatingNeuralNetwork(
+                aggregates=4, width=2, depth=2, aggregator=AggregatingNeuralNetwork.aggregate_fft
+            ).with_keras_params(activation='linear'))
+        with FixpointExperiment() as exp:
+            for run_id in tqdm(range(10)):
+                exp.run_exp(net_generator, 1)
+                exp.log(exp.counters)
+                exp.reset_model()
+            exp.reset_all()
+
+    if True:
+        # ok so this works quite realiably
+        run_count = 10000
+        net_generator = TrainingNeuralNetworkDecorator(
+            ParticleDecorator(WeightwiseNeuralNetwork(width=2, depth=2))
+        ).with_params(epsilon=0.0001).with_keras_params(optimizer='sgd')
+        with MixedFixpointExperiment() as exp:
+            for run_id in tqdm(range(run_count+1)):
+                exp.run_exp(net_generator, 1)
+                if run_id % 100 == 0:
+                    exp.run_net(net_generator, 1)
+            K.clear_session()
+
+    if False:
+        with FixpointExperiment() as exp:
+            run_count = 1000
+            net = TrainingNeuralNetworkDecorator(AggregatingNeuralNetwork(4, width=2, depth=2)).with_params(epsilon=0.1e-6)
+            for run_id in tqdm(range(run_count+1)):
+                loss = net.compiled().train()
+                if run_id % 100 == 0:
+                    net.print_weights()
+                    old_aggs, _ = net.net.get_aggregated_weights()
+                    print("old weights agg: " + str(old_aggs))
+                    fp, new_aggs = net.net.is_fixpoint_after_aggregation(epsilon=0.0001)
+                    print("new weights agg: " + str(new_aggs))
+                    print("Fixpoint? " + str(net.is_fixpoint()))
+                    print("Fixpoint after Agg? " + str(fp))
+                    print("Loss " + str(loss))
+                    print()
+
+    if False:
+        # this explodes in our faces completely... NAN everywhere
+        # TODO: Wtf is happening here?
+        with FixpointExperiment() as exp:
+            run_count = 10000
+            net = TrainingNeuralNetworkDecorator(RecurrentNeuralNetwork(width=2, depth=2))\
+                .with_params(epsilon=0.1e-2).with_keras_params(optimizer='sgd', activation='linear')
+            for run_id in tqdm(range(run_count+1)):
+                loss = net.compiled().train()
+                if run_id % 500 == 0:
+                    net.print_weights()
+                    # print(net.apply_to_network(net))
+                    print("Fixpoint? " + str(net.is_fixpoint()))
+                    print("Loss " + str(loss))
+                    print()
+    if False:
+        # and this gets somewhat interesting... we can still achieve non-trivial fixpoints
+        # over multiple applications when training enough in-between
+        with MixedFixpointExperiment() as exp:
+            for run_id in range(10):
+                net = TrainingNeuralNetworkDecorator(FFTNeuralNetwork(2, width=2, depth=2))\
+                    .with_params(epsilon=0.0001, activation='sigmoid')
+                exp.run_net(net, 500, 10)
+
+                net.print_weights()
+
+                print("Fixpoint? " + str(net.is_fixpoint()))
+            exp.log(exp.counters)

code/results/Soup/log.txt

@@ -0,0 +1 @@
+{'divergent': 0, 'fix_zero': 0, 'fix_other': 13, 'fix_sec': 0, 'other': 7}

code/results/Soup/weights.txt

@@ -0,0 +1,30 @@
+[-0.15321673  1.0428386  -0.7245892  -0.04343993  0.42338863  0.02538261
+ -0.40465942 -0.0242596  -1.226809   -0.8168446   0.26588777 -1.0929432
+  0.5383322  -0.73875046]
+[-0.03072096 -1.369665   -0.357126   -0.21180922  0.3853204   0.22853081
+ -0.3705557  -0.21977347 -0.6684716   0.12849599  1.0226644  -0.0922638
+ -0.7828449  -0.6572327 ]
+[-1.2444692   0.61213857  0.07965802  0.12361202  0.62641835  0.9720597
+  0.3863232   0.59948945  1.0857513   0.49231085 -0.5319295   0.29433587
+ -0.64177823  0.17603302]
+[-0.9938292  -0.4438207  -0.03172896  0.06261964 -0.3870194   0.7637992
+  0.0244509  -0.04825407  0.91551745 -0.78740424  0.29226422 -0.52767307
+ -0.41744384  0.5567152 ]
+[-0.39049304  0.8842579  -0.8447943  -0.19669186  0.7207061   0.16780053
+  0.3728221   0.08680353  0.7535456  -0.1000197   0.02029054  0.8640245
+ -0.15881588  1.1905665 ]
+[ 1.0482084   0.9248296  -0.26946014  0.57047915 -0.32660747  0.6914731
+ -0.18025818  0.3816289  -0.69358927  0.21312684 -0.39932403 -0.02991759
+ -0.83068466  0.45619962]
+[ 0.75814664  0.10328437  0.07867077 -0.0743314  -0.53440267  0.50492585
+ -0.54172474  0.51184535  0.3462249   1.0527638  -0.9503541   0.9235086
+ -0.1665241   1.1497779 ]
+[-0.77187353  1.1105504   0.24265823  0.53782856 -0.34098852 -0.75576884
+ -0.25396293 -0.56288165  0.3851537  -0.67497945  0.14336896  0.763481
+ -0.9224985   0.6374753 ]
+[-0.79123825  0.68166596 -0.30061013 -0.19360289  0.5632736   0.36276665
+  0.7470975   0.48115698  0.10046808 -0.8064349  -1.036736   -0.68296516
+ -1.156437    0.52633154]
+[ 0.1788832  -1.5321186  -0.62001514 -0.3870902   0.97524184  0.6088638
+ -0.08297889 -0.05180515 -0.29096788  0.7519439   0.8803648   0.82771575
+ -0.854887    0.1742936 ]

code/results/exp-applying_fixpoint-_1552681870.3570378-0/log.txt

@@ -0,0 +1,12 @@
+WeightwiseNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 23, 'fix_zero': 27, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+AggregatingNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 4, 'fix_zero': 46, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+RecurrentNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 46, 'fix_zero': 4, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+

code/results/exp-learn-from-soup-_1552658566.5572753-0/log.txt

@@ -0,0 +1,4 @@
+TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
+{'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], 'zs': [0.0, 1.2, 5.2, 7.4, 8.1, 9.1, 9.6, 9.8, 10.0, 9.9, 9.9]}
+
+

code/results/exp-mixed-self-fixpoints-_1552666977.5858653-0/log.txt

@@ -0,0 +1,12 @@
+WeightwiseNeuralNetwork activiation='linear' use_bias=False
+{'xs': [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500], 'ys': [0.2, 0.3, 0.15, 0.55, 0.7, 0.85, 0.8, 0.95, 0.9, 1.0, 1.0]}
+
+
+AggregatingNeuralNetwork activiation='linear' use_bias=False
+{'xs': [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500], 'ys': [1.0, 0.95, 1.0, 1.0, 0.95, 0.9, 0.8, 1.0, 0.85, 1.0, 0.9]}
+
+
+RecurrentNeuralNetwork activiation='linear' use_bias=False
+{'xs': [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500], 'ys': [0.05, 0.0, 0.05, 0.0, 0.0, 0.1, 0.1, 0.05, 0.1, 0.0, 0.0]}
+
+

code/results/exp-mixed-soup-_1552674483.9866457-0/log.txt

@@ -0,0 +1,8 @@
+TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
+{'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.0, 0.0, 0.1, 0.0, 0.0, 0.0, 0.0, 0.1, 0.0, 0.0, 0.0], 'zs': [0.0, 0.0, 0.7, 1.9, 3.6, 4.3, 6.0, 6.1, 8.3, 7.7, 8.8]}
+
+
+TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
+{'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.8, 0.4, 0.4, 0.3, 0.2, 0.2, 0.2, 0.2, 0.2, 0.4, 0.3], 'zs': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}
+
+

code/results/exp-training_fixpoint-_1552658296.0913951-0/log.txt

@@ -0,0 +1,12 @@
+WeightwiseNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 0, 'fix_zero': 0, 'fix_other': 50, 'fix_sec': 0, 'other': 0}
+
+
+AggregatingNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 50}
+
+
+RecurrentNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 38, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 12}
+
+

code/results/known_fixpoint_variation/log.txt

@@ -0,0 +1,30 @@
+variation 10e-0
+avg time to vergence 3.63
+avg time as fixpoint 0
+variation 10e-1
+avg time to vergence 5.02
+avg time as fixpoint 0
+variation 10e-2
+avg time to vergence 6.46
+avg time as fixpoint 0
+variation 10e-3
+avg time to vergence 8.04
+avg time as fixpoint 0
+variation 10e-4
+avg time to vergence 9.61
+avg time as fixpoint 0.04
+variation 10e-5
+avg time to vergence 11.23
+avg time as fixpoint 1.38
+variation 10e-6
+avg time to vergence 12.99
+avg time as fixpoint 3.23
+variation 10e-7
+avg time to vergence 14.58
+avg time as fixpoint 4.84
+variation 10e-8
+avg time to vergence 21.95
+avg time as fixpoint 11.91
+variation 10e-9
+avg time to vergence 26.45
+avg time as fixpoint 16.47

code/results/self_application_aggregation_network/log.txt

@@ -0,0 +1 @@
+{'divergent': 0, 'fix_zero': 10, 'fix_other': 0, 'fix_sec': 0, 'other': 0}

code/setups/applying-fixpoints.py

@@ -0,0 +1,69 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from experiment import *
+from network import *
+
+import keras.backend as K
+
+def generate_counters():
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+def count(counters, net, notable_nets=[]):
+    if net.is_diverged():
+        counters['divergent'] += 1
+    elif net.is_fixpoint():
+        if net.is_zero():
+            counters['fix_zero'] += 1
+        else:
+            counters['fix_other'] += 1
+            notable_nets += [net]
+    elif net.is_fixpoint(2):
+        counters['fix_sec'] += 1
+        notable_nets += [net]
+    else:
+        counters['other'] += 1
+    return counters, notable_nets
+
+
+if __name__ == '__main__':
+
+    with Experiment('applying_fixpoint') as exp:
+        exp.trials = 50
+        exp.run_count = 100
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:  # , 'sigmoid', 'relu']:
+            for use_bias in [False]:
+                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+        all_counters = []
+        all_notable_nets = []
+        all_names = []
+        for net_generator_id, net_generator in enumerate(net_generators):
+            counters = generate_counters()
+            notable_nets = []
+            for _ in tqdm(range(exp.trials)):
+                net = ParticleDecorator(net_generator())
+                net.with_params(epsilon=exp.epsilon)
+                name = str(net.net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias=" + str(net.get_keras_params().get('use_bias'))
+                for run_id in range(exp.run_count):
+                    loss = net.self_attack()
+                count(counters, net, notable_nets)
+            all_counters += [counters]
+            all_notable_nets += [notable_nets]
+            all_names += [name]
+            K.clear_session()
+        exp.save(all_counters=all_counters)
+        exp.save(trajectorys=exp.without_particles())
+        # net types reached in the end
+        # exp.save(all_notable_nets=all_notable_nets)
+        exp.save(all_names=all_names) #experiment setups
+        for exp_id, counter in enumerate(all_counters):
+            exp.log(all_names[exp_id])
+            exp.log(all_counters[exp_id])
+            exp.log('\n')

code/setups/experiments/exp-learn-from-soup-_1552658566.5572753-0/log.txt

@@ -0,0 +1,4 @@
+TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
+{'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], 'zs': [0.0, 1.2, 5.2, 7.4, 8.1, 9.1, 9.6, 9.8, 10.0, 9.9, 9.9]}
+
+

code/setups/experiments/exp-training_fixpoint-_1552658296.0913951-0/log.txt

@@ -0,0 +1,12 @@
+WeightwiseNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 0, 'fix_zero': 0, 'fix_other': 50, 'fix_sec': 0, 'other': 0}
+
+
+AggregatingNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 50}
+
+
+RecurrentNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 38, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 12}
+
+

code/setups/experiments/exp-weightwise_self_application-_1552664922.4501734-0/log.txt

@@ -0,0 +1 @@
+{'divergent': 11, 'fix_zero': 9, 'fix_other': 0, 'fix_sec': 0, 'other': 0}

code/setups/fixpoint-density.py

@@ -0,0 +1,69 @@
+import sys
+import os
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from experiment import *
+from network import *
+
+import tensorflow.python.keras.backend as K
+
+
+def generate_counters():
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+def count(counters, net, notable_nets=None):
+    notable_nets = notable_nets or []
+    if net.is_diverged():
+        counters['divergent'] += 1
+    elif net.is_fixpoint():
+        if net.is_zero():
+            counters['fix_zero'] += 1
+        else:
+            counters['fix_other'] += 1
+            notable_nets += [net]
+    elif net.is_fixpoint(2):
+        counters['fix_sec'] += 1
+        notable_nets += [net]
+    else:
+        counters['other'] += 1
+    return counters, notable_nets
+
+
+if __name__ == '__main__':
+    with Experiment('fixpoint-density') as exp:
+        #NOTE: settings could/should stay this way
+        #FFT doesn't work though
+        exp.trials = 100000
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:
+            net_generators += [lambda activation=activation: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
+            net_generators += [lambda activation=activation: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
+            # net_generators += [lambda activation=activation: FFTNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
+            # net_generators += [lambda activation=activation: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
+        all_counters = []
+        all_notable_nets = []
+        all_names = []
+        for net_generator_id, net_generator in enumerate(net_generators):
+            counters = generate_counters()
+            notable_nets = []
+            for _ in tqdm(range(exp.trials)):
+                net = net_generator().with_params(epsilon=exp.epsilon)
+                net = ParticleDecorator(net)
+                name = str(net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias='" + str(net.get_keras_params().get('use_bias')) + "'"
+                count(counters, net, notable_nets)
+                K.clear_session()
+            all_counters += [counters]
+            # all_notable_nets += [notable_nets]
+            all_names += [name]
+        exp.save(all_counters=all_counters)
+        exp.save(all_notable_nets=all_notable_nets)
+        exp.save(all_names=all_names)
+        for exp_id, counter in enumerate(all_counters):
+            exp.log(all_names[exp_id])
+            exp.log(all_counters[exp_id])
+            exp.log('\n')
+
+    print('Done')

code/setups/known-fixpoint-variation.py

@@ -0,0 +1,92 @@
+import sys
+
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from experiment import *
+from network import *
+from soup import prng
+
+import tensorflow.python.keras.backend as K
+
+
+from statistics import mean
+avg = mean
+
+
+def generate_fixpoint_weights():
+    return [
+        np.array([[1.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0]], dtype=np.float32),
+        np.array([[1.0, 0.0], [0.0, 0.0]], dtype=np.float32),
+        np.array([[1.0], [0.0]], dtype=np.float32)
+    ]
+
+
+def generate_fixpoint_net():
+    #NOTE: Weightwise only is all we can do right now IMO
+    net = WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation='sigmoid')
+    # I don't know if this work for aggregaeting. We don't actually need it, though.
+    # net = AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation='sigmoid')
+    net.set_weights(generate_fixpoint_weights())
+    return net
+
+
+def vary(old_weights, e=1.0):
+    new_weights = copy.deepcopy(old_weights)
+    for layer_id, layer in enumerate(new_weights):
+        for cell_id, cell in enumerate(layer):
+            for weight_id, weight in enumerate(cell):
+                if prng() < 0.5:
+                    new_weights[layer_id][cell_id][weight_id] = weight + prng() * e
+                else:
+                    new_weights[layer_id][cell_id][weight_id] = weight - prng() * e
+    return new_weights
+
+
+if __name__ == '__main__':
+    with Experiment('known-fixpoint-variation') as exp:
+        exp.depth = 10
+        exp.trials = 100
+        exp.max_steps = 100
+        exp.epsilon = 1e-4
+        exp.xs = []
+        exp.ys = []
+        exp.zs = []
+        exp.notable_nets = []
+        current_scale = 1.0
+        for _ in range(exp.depth):
+            print('variation scale ' + str(current_scale))
+            for _ in tqdm(range(exp.trials)):
+                net = generate_fixpoint_net().with_params(epsilon=exp.epsilon)
+                net = ParticleDecorator(net)
+                net.set_weights(vary(net.get_weights(), current_scale))
+                time_to_something = 0
+                time_as_fixpoint = 0
+                still_fixpoint = True
+                for _ in range(exp.max_steps):
+                    net.self_attack()
+                    if net.is_zero() or net.is_diverged():
+                        break
+                    if net.is_fixpoint():
+                        if still_fixpoint:
+                            time_as_fixpoint += 1
+                        else:
+                            print('remarkable')
+                            exp.notable_nets += [net.get_weights()]
+                            still_fixpoint = True
+                    else:
+                        still_fixpoint = False
+                    time_to_something += 1
+                exp.xs += [current_scale]
+                # time steps taken to reach divergence or zero (reaching another fix-point is basically never happening)
+                exp.ys += [time_to_something]
+                # time steps still regarded as sthe initial fix-point
+                exp.zs += [time_as_fixpoint]
+                K.backend.clear_session()
+            current_scale /= 10.0
+        for d in range(exp.depth):
+            exp.log('variation 10e-' + str(d))
+            exp.log('avg time to vergence ' + str(avg(exp.ys[d*exp.trials:(d+1) * exp.trials])))
+            exp.log('avg time as fixpoint ' + str(avg(exp.zs[d*exp.trials:(d+1) * exp.trials])))

code/setups/learn_from_soup.py

@@ -0,0 +1,110 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from typing import Tuple
+
+from experiment import *
+from network import *
+from soup import *
+
+
+import tensorflow.python.keras.backend as K
+
+from statistics import mean
+avg = mean
+
+
+def generate_counters():
+    """
+    Initial build of the counter dict, to store counts.
+
+    :rtype: dict
+    :return: dictionary holding counter for: 'divergent', 'fix_zero', 'fix_sec', 'other'
+    """
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+def count(counters, soup, notable_nets=None):
+    """
+    Count the occurences ot the types of weight trajectories.
+
+    :param counters:      A counter dictionary.
+    :param soup:          A Soup
+    :param notable_nets:  A list to store and save intersting candidates
+
+    :rtype      Tuple[dict, list]
+    :return:    Both the counter dictionary and the list of interessting nets.
+    """
+
+    notable_nets = notable_nets or list()
+    for net in soup.particles:
+        if net.is_diverged():
+            counters['divergent'] += 1
+        elif net.is_fixpoint():
+            if net.is_zero():
+                counters['fix_zero'] += 1
+            else:
+                counters['fix_other'] += 1
+                # notable_nets += [net]
+        # elif net.is_fixpoint(2):
+        #     counters['fix_sec'] += 1
+        #     notable_nets += [net]
+        else:
+            counters['other'] += 1
+    return counters, notable_nets
+
+
+if __name__ == '__main__':
+
+    with SoupExperiment('learn-from-soup') as exp:
+        exp.soup_size = 10
+        exp.soup_life = 100
+        exp.trials = 10
+        exp.learn_from_severity_values = [10 * i for i in range(11)]
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:  # ['sigmoid', 'linear', 'relu']:
+            for use_bias in [False]:
+                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                # net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                # net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+
+        all_names = []
+        all_data = []
+        for net_generator_id, net_generator in enumerate(net_generators):
+            xs = []
+            ys = []
+            zs = []
+            notable_nets = []
+            for learn_from_severity in exp.learn_from_severity_values:
+                counters = generate_counters()
+                results = []
+                for _ in tqdm(range(exp.trials)):
+                    soup = Soup(exp.soup_size, lambda net_generator=net_generator,exp=exp: TrainingNeuralNetworkDecorator(net_generator()).with_params(epsilon=exp.epsilon))
+                    soup.with_params(attacking_rate=-1, learn_from_rate=0.1, train=0, learn_from_severity=learn_from_severity)
+                    soup.seed()
+                    name = str(soup.particles[0].net.__class__.__name__) + " activiation='" + str(soup.particles[0].get_keras_params().get('activation')) + "' use_bias=" + str(soup.particles[0].get_keras_params().get('use_bias'))
+                    for time in range(exp.soup_life):
+                        soup.evolve()
+                    count(counters, soup, notable_nets)
+                    K.clear_session()
+
+                xs += [learn_from_severity]
+                ys += [float(counters['fix_zero']) / float(exp.trials)]
+                zs += [float(counters['fix_other']) / float(exp.trials)]
+            all_names += [name]
+            # xs: learn_from_intensity according to exp.learn_from_intensity_values
+            # ys: zero-fixpoints after life time
+            # zs: non-zero-fixpoints after life time
+            all_data += [{'xs':xs, 'ys':ys, 'zs':zs}]
+
+        exp.save(all_names=all_names)
+        exp.save(all_data=all_data)
+        exp.save(soup=soup.without_particles())
+        for exp_id, name in enumerate(all_names):
+            exp.log(all_names[exp_id])
+            exp.log(all_data[exp_id])
+            exp.log('\n')

code/setups/mixed-self-fixpoints.py

@@ -0,0 +1,100 @@
+import sys
+import os
+
+from typing import Tuple
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from experiment import *
+from network import *
+
+import tensorflow.python.keras.backend as K
+
+
+def generate_counters():
+    """
+    Initial build of the counter dict, to store counts.
+
+    :rtype: dict
+    :return: dictionary holding counter for: 'divergent', 'fix_zero', 'fix_sec', 'other'
+    """
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+def count(counters, net, notable_nets=None):
+    """
+    Count the occurences ot the types of weight trajectories.
+
+    :param counters:      A counter dictionary.
+    :param net:           A Neural Network
+    :param notable_nets:  A list to store and save intersting candidates
+
+    :rtype      Tuple[dict, list]
+    :return:    Both the counter dictionary and the list of interessting nets.
+    """
+    notable_nets = notable_nets or list()
+    if net.is_diverged():
+        counters['divergent'] += 1
+    elif net.is_fixpoint():
+        if net.is_zero():
+            counters['fix_zero'] += 1
+        else:
+            counters['fix_other'] += 1
+            notable_nets += [net]
+    elif net.is_fixpoint(2):
+        counters['fix_sec'] += 1
+        notable_nets += [net]
+    else:
+        counters['other'] += 1
+    return counters, notable_nets
+
+if __name__ == '__main__':
+
+    with Experiment('mixed-self-fixpoints') as exp:
+        exp.trials = 20
+        exp.selfattacks = 4
+        exp.trains_per_selfattack_values = [50 * i for i in range(11)]
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:  # , 'sigmoid', 'relu']:
+            for use_bias in [False]:
+                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                # net_generators += [lambda activation=activation, use_bias=use_bias: FFTNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+
+        all_names = []
+        all_data = []
+
+        for net_generator_id, net_generator in enumerate(net_generators):
+            xs = []
+            ys = []
+            for trains_per_selfattack in exp.trains_per_selfattack_values:
+                counters = generate_counters()
+                notable_nets = []
+                for _ in tqdm(range(exp.trials)):
+                    net = ParticleDecorator(net_generator())
+                    net = TrainingNeuralNetworkDecorator(net).with_params(epsilon=exp.epsilon)
+                    name = str(net.net.net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias=" + str(net.get_keras_params().get('use_bias'))
+                    for selfattack_id in range(exp.selfattacks):
+                        net.self_attack()
+                        for train_id in range(trains_per_selfattack):
+                            loss = net.compiled().train(epoch=selfattack_id*trains_per_selfattack+train_id)
+                        if net.is_diverged() or net.is_fixpoint():
+                            break
+                    count(counters, net, notable_nets)
+                    keras.backend.clear_session()
+                xs += [trains_per_selfattack]
+                ys += [float(counters['fix_zero'] + counters['fix_other']) / float(exp.trials)]
+            all_names += [name]
+            # xs: how many trains per self-attack from exp.trains_per_selfattack_values
+            # ys: average amount of fixpoints found
+            all_data += [{'xs': xs, 'ys': ys}]
+
+        exp.save(all_names=all_names)
+        exp.save(all_data=all_data)
+        for exp_id, name in enumerate(all_names):
+            exp.log(all_names[exp_id])
+            exp.log(all_data[exp_id])
+            exp.log('\n')

code/setups/mixed-soup.py

@@ -0,0 +1,108 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from typing import Tuple
+
+from experiment import *
+from network import *
+from soup import *
+
+import tensorflow.python.keras.backend as K
+
+
+def generate_counters():
+    """
+    Initial build of the counter dict, to store counts.
+
+    :rtype: dict
+    :return: dictionary holding counter for: 'divergent', 'fix_zero', 'fix_sec', 'other'
+    """
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+def count(counters, soup, notable_nets=None):
+    """
+    Count the occurences ot the types of weight trajectories.
+
+    :param counters:      A counter dictionary.
+    :param soup:          A Soup
+    :param notable_nets:  A list to store and save intersting candidates
+
+    :rtype      Tuple[dict, list]
+    :return:    Both the counter dictionary and the list of interessting nets.
+    """
+
+    notable_nets = notable_nets or list()
+    for net in soup.particles:
+        if net.is_diverged():
+            counters['divergent'] += 1
+        elif net.is_fixpoint():
+            if net.is_zero():
+                counters['fix_zero'] += 1
+            else:
+                counters['fix_other'] += 1
+                # notable_nets += [net]
+        # elif net.is_fixpoint(2):
+        #     counters['fix_sec'] += 1
+        #     notable_nets += [net]
+        else:
+            counters['other'] += 1
+    return counters, notable_nets
+
+if __name__ == '__main__':
+
+    with Experiment('mixed-soup') as exp:
+        exp.trials = 10
+        exp.soup_size = 10
+        exp.soup_life = 5
+        exp.trains_per_selfattack_values = [10 * i for i in range(11)]
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:  # ['linear', 'sigmoid', 'relu']:
+            for use_bias in [False]:
+                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                # net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+
+        all_names = []
+        all_data = []
+        for net_generator_id, net_generator in enumerate(net_generators):
+            xs = []
+            ys = []
+            zs = []
+            for trains_per_selfattack in exp.trains_per_selfattack_values:
+                counters = generate_counters()
+                notable_nets = []
+                for soup_idx in tqdm(range(exp.trials)):
+                    soup = Soup(exp.soup_size,
+                                lambda net_generator=net_generator, exp=exp: TrainingNeuralNetworkDecorator(
+                                    net_generator()).with_params(epsilon=exp.epsilon))
+                    soup.with_params(attacking_rate=0.1, learn_from_rate=-1, train=trains_per_selfattack,
+                                     learn_from_severity=-1)
+                    soup.seed()
+                    name = str(soup.particles[0].net.__class__.__name__) + " activiation='" + str(
+                        soup.particles[0].get_keras_params().get('activation')) + "' use_bias=" + str(
+                        soup.particles[0].get_keras_params().get('use_bias'))
+                    for _ in range(exp.soup_life):
+                        soup.evolve()
+                    count(counters, soup, notable_nets)
+                    K.clear_session()
+
+                xs += [trains_per_selfattack]
+                ys += [float(counters['fix_zero']) / float(exp.trials)]
+                zs += [float(counters['fix_other']) / float(exp.trials)]
+            all_names += [name]
+            # xs: how many trains per self-attack from exp.trains_per_selfattack_values
+            # ys: average amount of zero-fixpoints found
+            # zs: average amount of non-zero fixpoints
+            all_data += [{'xs': xs, 'ys': ys, 'zs': zs}]
+
+        exp.save(all_names=all_names)
+        exp.save(all_data=all_data)
+        for exp_id, name in enumerate(all_names):
+            exp.log(all_names[exp_id])
+            exp.log(all_data[exp_id])
+            exp.log('\n')

code/setups/network_trajectorys.py

@@ -0,0 +1,112 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from soup import *
+from experiment import *
+
+
+if __name__ == '__main__':
+    def run_exp(net, prints=False):
+        # INFO Run_ID needs to be more than 0, so that exp stores the trajectories!
+        exp.run_net(net, 100, run_id=run_id + 1)
+        exp.historical_particles[run_id] = net
+        if prints:
+            print("Fixpoint? " + str(net.is_fixpoint()))
+            print("Loss " + str(loss))
+
+    if True:
+        # WeightWise Neural Network
+        with FixpointExperiment(name="weightwise_self_application") as exp:
+            for run_id in tqdm(range(20)):
+                net = ParticleDecorator(WeightwiseNeuralNetwork(width=2, depth=2)
+                                        .with_keras_params(activation='linear'))
+                run_exp(net)
+                K.clear_session()
+            exp.log(exp.counters)
+            exp.save(trajectorys=exp.without_particles())
+
+    if False:
+        # Aggregating Neural Network
+        with FixpointExperiment(name="aggregating_self_application") as exp:
+            for run_id in tqdm(range(10)):
+                net = ParticleDecorator(AggregatingNeuralNetwork(aggregates=4, width=2, depth=2)
+                                        .with_keras_params(activation='linear'))
+                run_exp(net)
+                K.clear_session()
+            exp.log(exp.counters)
+            exp.save(trajectorys=exp.without_particles())
+
+    if False:
+        #FFT Neural Network
+        with FixpointExperiment() as exp:
+            for run_id in tqdm(range(10)):
+                net = ParticleDecorator(FFTNeuralNetwork(aggregates=4, width=2, depth=2)
+                                        .with_keras_params(activation='linear'))
+                run_exp(net)
+                K.clear_session()
+            exp.log(exp.counters)
+            exp.save(trajectorys=exp.without_particles())
+
+    if False:
+        # ok so this works quite realiably
+        with FixpointExperiment(name="weightwise_learning") as exp:
+            for i in range(10):
+                run_count = 100
+                net = TrainingNeuralNetworkDecorator(ParticleDecorator(WeightwiseNeuralNetwork(width=2, depth=2)))
+                net.with_params(epsilon=0.0001).with_keras_params(activation='linear')
+                exp.historical_particles[net.get_uid()] = net
+                for run_id in tqdm(range(run_count+1)):
+                    net.compiled()
+                    loss = net.train(epoch=run_id)
+                    # run_exp(net)
+                    # net.save_state(time=run_id)
+                K.clear_session()
+            exp.save(trajectorys=exp.without_particles())
+
+    if False:
+        # ok so this works quite realiably
+        with FixpointExperiment(name="aggregating_learning") as exp:
+            for i in range(10):
+                run_count = 100
+                net = TrainingNeuralNetworkDecorator(ParticleDecorator(AggregatingNeuralNetwork(4, width=2, depth=2)))
+                net.with_params(epsilon=0.0001).with_keras_params(activation='linear')
+                exp.historical_particles[net.get_uid()] = net
+                for run_id in tqdm(range(run_count+1)):
+                    net.compiled()
+                    loss = net.train(epoch=run_id)
+                    # run_exp(net)
+                    # net.save_state(time=run_id)
+                K.clear_session()
+            exp.save(trajectorys=exp.without_particles())
+
+    if False:
+        # this explodes in our faces completely... NAN everywhere
+        # TODO: Wtf is happening here?
+        with FixpointExperiment() as exp:
+            run_count = 10000
+            net = TrainingNeuralNetworkDecorator(RecurrentNeuralNetwork(width=2, depth=2))\
+                .with_params(epsilon=0.1e-2).with_keras_params(optimizer='sgd', activation='linear')
+            for run_id in tqdm(range(run_count+1)):
+                loss = net.compiled().train()
+                if run_id % 500 == 0:
+                    net.print_weights()
+                    # print(net.apply_to_network(net))
+                    print("Fixpoint? " + str(net.is_fixpoint()))
+                    print("Loss " + str(loss))
+                    print()
+    if False:
+        # and this gets somewhat interesting... we can still achieve non-trivial fixpoints
+        # over multiple applications when training enough in-between
+        with MixedFixpointExperiment() as exp:
+            for run_id in range(10):
+                net = TrainingNeuralNetworkDecorator(FFTNeuralNetwork(2, width=2, depth=2))\
+                    .with_params(epsilon=0.0001, activation='sigmoid')
+                exp.run_net(net, 500, 10)
+
+                net.print_weights()
+
+                print("Fixpoint? " + str(net.is_fixpoint()))
+            exp.log(exp.counters)

code/setups/soup_trajectorys.py

@@ -0,0 +1,32 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from soup import *
+from experiment import *
+
+
+if __name__ == '__main__':
+    if True:
+        with SoupExperiment("soup") as exp:
+            for run_id in range(1):
+                net_generator = lambda: TrainingNeuralNetworkDecorator(WeightwiseNeuralNetwork(2, 2)) \
+                     .with_keras_params(activation='linear').with_params(epsilon=0.0001)
+                # net_generator = lambda: TrainingNeuralNetworkDecorator(AggregatingNeuralNetwork(4, 2, 2))\
+                #     .with_keras_params(activation='linear')
+                # net_generator = lambda: TrainingNeuralNetworkDecorator(FFTNeuralNetwork(4, 2, 2))\
+                #    .with_keras_params(activation='linear')
+                # net_generator = lambda: RecurrentNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
+                soup = Soup(20, net_generator).with_params(remove_divergent=True, remove_zero=True,
+                                                           train=30,
+                                                           learn_from_rate=-1)
+                soup.seed()
+                for _ in tqdm(range(100)):
+                    soup.evolve()
+                exp.log(soup.count())
+                # you can access soup.historical_particles[particle_uid].states[time_step]['loss']
+                #             or soup.historical_particles[particle_uid].states[time_step]['weights']
+                # from soup.dill
+                exp.save(soup=soup.without_particles())

code/setups/training-fixpoints.py

@@ -0,0 +1,70 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from experiment import *
+from network import *
+
+import tensorflow.python.keras.backend as K
+
+def generate_counters():
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+def count(counters, net, notable_nets=None):
+    notable_nets = notable_nets or list()
+    if net.is_diverged():
+        counters['divergent'] += 1
+    elif net.is_fixpoint():
+        if net.is_zero():
+            counters['fix_zero'] += 1
+        else:
+            counters['fix_other'] += 1
+            notable_nets += [net]
+    elif net.is_fixpoint(2):
+        counters['fix_sec'] += 1
+        notable_nets += [net]
+    else:
+        counters['other'] += 1
+    return counters, notable_nets
+
+
+if __name__ == '__main__':
+
+    with Experiment('training_fixpoint') as exp:
+        exp.trials = 50
+        exp.run_count = 1000
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:  # , 'sigmoid', 'relu']:
+            for use_bias in [False]:
+                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+        all_counters = []
+        all_notable_nets = []
+        all_names = []
+        for net_generator_id, net_generator in enumerate(net_generators):
+            counters = generate_counters()
+            notable_nets = []
+            for _ in tqdm(range(exp.trials)):
+                net = ParticleDecorator(net_generator())
+                net = TrainingNeuralNetworkDecorator(net).with_params(epsilon=exp.epsilon)
+                name = str(net.net.net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias=" + str(net.get_keras_params().get('use_bias'))
+                for run_id in range(exp.run_count):
+                    loss = net.compiled().train(epoch=run_id+1)
+                count(counters, net, notable_nets)
+            all_counters += [counters]
+            all_notable_nets += [notable_nets]
+            all_names += [name]
+            K.clear_session()
+        exp.save(all_counters=all_counters)
+        exp.save(trajectorys=exp.without_particles())
+        # net types reached in the end
+        # exp.save(all_notable_nets=all_notable_nets)
+        exp.save(all_names=all_names) #experiment setups
+        for exp_id, counter in enumerate(all_counters):
+            exp.log(all_names[exp_id])
+            exp.log(all_counters[exp_id])
+            exp.log('\n')

code/soup.py

@@ -0,0 +1,137 @@
+import random
+
+from network import *
+
+
+def prng():
+    return random.random()
+
+
+class Soup(object):
+    
+    def __init__(self, size, generator, **kwargs):
+        self.size = size
+        self.generator = generator
+        self.particles = []
+        self.historical_particles = {}
+        self.params = dict(attacking_rate=0.1, learn_from_rate=0.1, train=0, learn_from_severity=1)
+        self.params.update(kwargs)
+        self.time = 0
+
+    def __copy__(self):
+        copy_ = Soup(self.size, self.generator, **self.params)
+        copy_.__dict__ = {attr: self.__dict__[attr] for attr in self.__dict__ if
+                          attr not in ['particles', 'historical_particles']}
+        return copy_
+
+    def without_particles(self):
+        self_copy = copy.copy(self)
+        # self_copy.particles = [particle.states for particle in self.particles]
+        self_copy.historical_particles = {key: val.states for key, val in self.historical_particles.items()}
+        return self_copy
+
+    def with_params(self, **kwargs):
+        self.params.update(kwargs)
+        return self
+    
+    def generate_particle(self):
+        new_particle = ParticleDecorator(self.generator())
+        self.historical_particles[new_particle.get_uid()] = new_particle
+        return new_particle
+        
+    def get_particle(self, uid, otherwise=None):
+        return self.historical_particles.get(uid, otherwise)
+    
+    def seed(self):
+        self.particles = []
+        for _ in range(self.size):
+            self.particles += [self.generate_particle()]
+        return self       
+    
+    def evolve(self, iterations=1):
+        for _ in range(iterations):
+            self.time += 1
+            for particle_id, particle in enumerate(self.particles):
+                description = {'time': self.time}
+                if prng() < self.params.get('attacking_rate'):
+                    other_particle_id = int(prng() * len(self.particles))
+                    other_particle = self.particles[other_particle_id]
+                    particle.attack(other_particle)
+                    description['action'] = 'attacking'
+                    description['counterpart'] = other_particle.get_uid()
+                if prng() < self.params.get('learn_from_rate'):
+                    other_particle_id = int(prng() * len(self.particles))
+                    other_particle = self.particles[other_particle_id]
+                    for _ in range(self.params.get('learn_from_severity', 1)):
+                        particle.learn_from(other_particle)
+                    description['action'] = 'learn_from'
+                    description['counterpart'] = other_particle.get_uid()
+                for _ in range(self.params.get('train', 0)):
+                    particle.compiled()
+                    # callbacks on save_state are broken for TrainingNeuralNetwork
+                    loss = particle.train(store_states=False)
+                    description['fitted'] = self.params.get('train', 0)
+                    description['loss'] = loss
+                    description['action'] = 'train_self'
+                    description['counterpart'] = None
+                if self.params.get('remove_divergent') and particle.is_diverged():
+                    new_particle = self.generate_particle()
+                    self.particles[particle_id] = new_particle
+                    description['action'] = 'divergent_dead'
+                    description['counterpart'] = new_particle.get_uid()
+                if self.params.get('remove_zero') and particle.is_zero():
+                    new_particle = self.generate_particle()
+                    self.particles[particle_id] = new_particle
+                    description['action'] = 'zweo_dead'
+                    description['counterpart'] = new_particle.get_uid()
+                particle.save_state(**description)
+
+    def count(self):
+        counters = dict(divergent=0, fix_zero=0, fix_other=0, fix_sec=0, other=0)
+        for particle in self.particles:
+            if particle.is_diverged():
+                counters['divergent'] += 1
+            elif particle.is_fixpoint():
+                if particle.is_zero():
+                    counters['fix_zero'] += 1
+                else:
+                    counters['fix_other'] += 1
+            elif particle.is_fixpoint(2):
+                counters['fix_sec'] += 1
+            else:
+                counters['other'] += 1
+        return counters
+        
+    def print_all(self):
+        for particle in self.particles:
+            particle.print_weights()
+            print(particle.is_fixpoint())
+
+
+if __name__ == '__main__':
+    if True:
+        net_generator = lambda: WeightwiseNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
+        soup_generator = Soup(100, net_generator).with_params(remove_divergent=True, remove_zero=True)
+        exp = SoupExperiment()
+        exp.run_exp(net_generator, 1000, soup_generator, 1, False)
+
+        # net_generator = lambda: FFTNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
+        # net_generator = lambda: AggregatingNeuralNetwork(4, 2, 2).with_keras_params(activation='sigmoid')\
+        # .with_params(shuffler=AggregatingNeuralNetwork.shuffle_random)
+        # net_generator = lambda: RecurrentNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
+
+    if True:
+        net_generator = lambda: TrainingNeuralNetworkDecorator(WeightwiseNeuralNetwork(2, 2)) \
+            .with_keras_params(activation='linear').with_params(epsilon=0.0001)
+        soup_generator = lambda: Soup(100, net_generator).with_params(remove_divergent=True, remove_zero=True, train=20)
+        exp = SoupExperiment(name="soup")
+
+        exp.run_exp(net_generator, 100, soup_generator, 1, False)
+
+        # net_generator = lambda: TrainingNeuralNetworkDecorator(AggregatingNeuralNetwork(4, 2, 2))
+        # .with_keras_params(activation='linear')\
+        # .with_params(shuffler=AggregatingNeuralNetwork.shuffle_random)
+        # net_generator = lambda: TrainingNeuralNetworkDecorator(FFTNeuralNetwork(4, 2, 2))\
+        #     .with_keras_params(activation='linear')\
+        # .with_params(shuffler=AggregatingNeuralNetwork.shuffle_random)
+        # net_generator = lambda: RecurrentNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()

experiments/__init__.py

@@ -1,6 +0,0 @@
-from .mixed_setting_exp import run_mixed_experiment
-from .robustness_exp import run_robustness_experiment
-from .self_application_exp import run_SA_experiment
-from .self_train_exp import run_ST_experiment
-from .soup_exp import run_soup_experiment
-import functionalities_test

experiments/helpers.py

@@ -1,59 +0,0 @@
-""" -------------------------------- Methods for summarizing the experiments --------------------------------- """
-from pathlib import Path
-
-from visualization import line_chart_fixpoints, bar_chart_fixpoints
-
-
-def summary_fixpoint_experiment(runs, population_size, epochs, experiments, net_learning_rate, directory,
-                                summary_pre_title):
-    avg_fixpoint_counters = {
-        "avg_identity_func": 0,
-        "avg_divergent": 0,
-        "avg_fix_zero": 0,
-        "avg_fix_weak": 0,
-        "avg_fix_sec": 0,
-        "avg_other_func": 0
-    }
-
-    for i in range(len(experiments)):
-        fixpoint_counters = experiments[i].fixpoint_counters
-
-        avg_fixpoint_counters["avg_identity_func"] += fixpoint_counters["identity_func"]
-        avg_fixpoint_counters["avg_divergent"] += fixpoint_counters["divergent"]
-        avg_fixpoint_counters["avg_fix_zero"] += fixpoint_counters["fix_zero"]
-        avg_fixpoint_counters["avg_fix_weak"] += fixpoint_counters["fix_weak"]
-        avg_fixpoint_counters["avg_fix_sec"] += fixpoint_counters["fix_sec"]
-        avg_fixpoint_counters["avg_other_func"] += fixpoint_counters["other_func"]
-
-    # Calculating the average for each fixpoint
-    avg_fixpoint_counters.update((x, y / len(experiments)) for x, y in avg_fixpoint_counters.items())
-
-    # Checking where the data is coming from to have a relevant title in the plot.
-    if summary_pre_title not in ["ST", "SA", "soup", "mixed", "robustness"]:
-        summary_pre_title = ""
-
-    # Plotting the summary
-    source_checker = "summary"
-    exp_details = f"{summary_pre_title}: {runs} runs & {epochs} epochs each."
-    bar_chart_fixpoints(avg_fixpoint_counters, population_size, directory, net_learning_rate, exp_details,
-                        source_checker)
-
-
-def summary_fixpoint_percentage(runs, epochs, fixpoints_percentages, ST_steps, SA_steps, directory_name,
-                                population_size):
-    fixpoints_percentages = [round(fixpoints_percentages[i] / runs, 1) for i in range(len(fixpoints_percentages))]
-
-    # Plotting summary
-    if "soup" in directory_name:
-        line_chart_fixpoints(fixpoints_percentages, epochs / ST_steps, ST_steps, SA_steps, directory_name,
-                             population_size)
-    else:
-        line_chart_fixpoints(fixpoints_percentages, epochs, ST_steps, SA_steps, directory_name, population_size)
-
-
-""" -------------------------------------------- Miscellaneous --------------------------------------------------- """
-
-
-def check_folder(experiment_folder: str):
-    exp_path = Path('experiments') / experiment_folder
-    exp_path.mkdir(parents=True, exist_ok=True)

experiments/meta_task_exp.py

@@ -1,271 +0,0 @@
-import pickle
-from collections import defaultdict
-from pathlib import Path
-import sys
-import platform
-
-import pandas as pd
-import torchmetrics
-import numpy as np
-import torch
-from matplotlib import pyplot as plt
-import seaborn as sns
-from torch import nn
-from torch.nn import Flatten
-from torch.utils.data import Dataset, DataLoader
-from torchvision.datasets import MNIST
-from torchvision.transforms import ToTensor, Compose, Resize
-from tqdm import tqdm
-
-if platform.node() == 'CarbonX':
-    debug = True
-    print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
-    print("@ Warning, Debugging Config@!!!!!! @")
-    print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
-else:
-    debug = False
-    try:
-        # noinspection PyUnboundLocalVariable
-        if __package__ is None:
-            DIR = Path(__file__).resolve().parent
-            sys.path.insert(0, str(DIR.parent))
-            __package__ = DIR.name
-        else:
-            DIR = None
-    except NameError:
-        DIR = None
-        pass
-
-from network import MetaNet
-from functionalities_test import test_for_fixpoints
-
-WORKER = 10 if not debug else 2
-BATCHSIZE = 500 if not debug else 50
-EPOCH = 100 if not debug else 3
-VALIDATION_FRQ = 5 if not debug else 1
-SELF_TRAIN_FRQ = 1 if not debug else 1
-DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-
-if debug:
-    torch.autograd.set_detect_anomaly(True)
-
-
-class ToFloat:
-
-    def __call__(self, x):
-        return x.to(torch.float32)
-
-
-class AddTaskDataset(Dataset):
-    def __init__(self, length=int(5e5)):
-        super().__init__()
-        self.length = length
-        self.prng = np.random.default_rng()
-
-    def __len__(self):
-        return self.length
-
-    def __getitem__(self, _):
-        ab = self.prng.normal(size=(2,)).astype(np.float32)
-        return ab, ab.sum(axis=-1, keepdims=True)
-
-
-def set_checkpoint(model, out_path, epoch_n, final_model=False):
-    epoch_n = str(epoch_n)
-    if not final_model:
-        ckpt_path = Path(out_path) / 'ckpt' / f'{epoch_n.zfill(4)}_model_ckpt.tp'
-    else:
-        ckpt_path = Path(out_path) / f'trained_model_ckpt_e{epoch_n}.tp'
-    ckpt_path.parent.mkdir(exist_ok=True, parents=True)
-
-    torch.save(model, ckpt_path, pickle_protocol=pickle.HIGHEST_PROTOCOL)
-    return ckpt_path
-
-
-def validate(checkpoint_path, ratio=0.1):
-    checkpoint_path = Path(checkpoint_path)
-    import torchmetrics
-
-    # initialize metric
-    validmetric = torchmetrics.Accuracy()
-    ut = Compose([ToTensor(), ToFloat(), Resize((15, 15)), Flatten(start_dim=0)])
-
-    try:
-        datas = MNIST(str(data_path), transform=ut, train=False)
-    except RuntimeError:
-        datas = MNIST(str(data_path), transform=ut, train=False, download=True)
-    valid_d = DataLoader(datas, batch_size=BATCHSIZE, shuffle=True, drop_last=True, num_workers=WORKER)
-
-    model = torch.load(checkpoint_path, map_location=DEVICE).eval()
-    n_samples = int(len(valid_d) * ratio)
-
-    with tqdm(total=n_samples, desc='Validation Run: ') as pbar:
-        for idx, (valid_batch_x, valid_batch_y) in enumerate(valid_d):
-            valid_batch_x, valid_batch_y = valid_batch_x.to(DEVICE), valid_batch_y.to(DEVICE)
-            y_valid = model(valid_batch_x)
-
-            # metric on current batch
-            acc = validmetric(y_valid.cpu(), valid_batch_y.cpu())
-            pbar.set_postfix_str(f'Acc: {acc}')
-            pbar.update()
-            if idx == n_samples:
-                break
-
-    # metric on all batches using custom accumulation
-    acc = validmetric.compute()
-    tqdm.write(f"Avg. accuracy on all data: {acc}")
-    return acc
-
-
-def new_train_storage_df():
-    return pd.DataFrame(columns=['Epoch', 'Batch', 'Metric', 'Score'])
-
-
-def checkpoint_and_validate(model, out_path, epoch_n, final_model=False):
-    out_path = Path(out_path)
-    ckpt_path = set_checkpoint(model, out_path, epoch_n, final_model=final_model)
-    result = validate(ckpt_path)
-    return result
-
-
-def plot_training_result(path_to_dataframe):
-    # load from Drive
-    df = pd.read_csv(path_to_dataframe, index_col=0)
-
-    # Set up figure
-    fig, ax1 = plt.subplots()  # initializes figure and plots
-    ax2 = ax1.twinx()  # applies twinx to ax2, which is the second y-axis.
-
-    # plots the first set of data
-    data = df[(df['Metric'] == 'Task Loss') | (df['Metric'] == 'Self Train Loss')].groupby(['Epoch', 'Metric']).mean()
-    palette = sns.color_palette()[0:data.reset_index()['Metric'].unique().shape[0]]
-    sns.lineplot(data=data.groupby(['Epoch', 'Metric']).mean(), x='Epoch', y='Score', hue='Metric',
-                 palette=palette, ax=ax1)
-
-    # plots the second set of data
-    data = df[(df['Metric'] == 'Test Accuracy') | (df['Metric'] == 'Train Accuracy')]
-    palette = sns.color_palette()[len(palette):data.reset_index()['Metric'].unique().shape[0] + len(palette)]
-    sns.lineplot(data=data, x='Epoch', y='Score', marker='o', hue='Metric', palette=palette)
-
-    ax1.set(yscale='log', ylabel='Losses')
-    ax1.set_title('Training Lineplot')
-    ax2.set(ylabel='Accuracy')
-
-    fig.legend(loc="center right", title='Metric', bbox_to_anchor=(0.85, 0.5))
-    ax1.get_legend().remove()
-    ax2.get_legend().remove()
-    plt.tight_layout()
-    if debug:
-        plt.show()
-    else:
-        plt.savefig(Path(path_to_dataframe.parent / 'training_lineplot.png'), dpi=300)
-
-
-if __name__ == '__main__':
-
-    self_train = False
-    training = False
-    plotting = False
-    particle_analysis = True
-    as_sparse_network_test = True
-
-    data_path = Path('data')
-    data_path.mkdir(exist_ok=True, parents=True)
-
-    run_path = Path('output') / 'mnist_self_train_100_NEW_STYLE'
-    model_path = run_path / '0000_trained_model.zip'
-    df_store_path = run_path / 'train_store.csv'
-
-    if training:
-        utility_transforms = Compose([ToTensor(), ToFloat(), Resize((15, 15)), Flatten(start_dim=0)])
-        try:
-            dataset = MNIST(str(data_path), transform=utility_transforms)
-        except RuntimeError:
-            dataset = MNIST(str(data_path), transform=utility_transforms, download=True)
-        d = DataLoader(dataset, batch_size=BATCHSIZE, shuffle=True, drop_last=True, num_workers=WORKER)
-
-        interface = np.prod(dataset[0][0].shape)
-        metanet = MetaNet(interface, depth=4, width=6, out=10).to(DEVICE).train()
-
-        loss_fn = nn.CrossEntropyLoss()
-        optimizer = torch.optim.SGD(metanet.parameters(), lr=0.004, momentum=0.9)
-
-        train_store = new_train_storage_df()
-        for epoch in tqdm(range(EPOCH), desc='MetaNet Train - Epochs'):
-            is_validation_epoch = epoch % VALIDATION_FRQ == 0 if not debug else True
-            is_self_train_epoch = epoch % SELF_TRAIN_FRQ == 0 if not debug else True
-            if is_validation_epoch:
-                metric = torchmetrics.Accuracy()
-            else:
-                metric = None
-            for batch, (batch_x, batch_y) in tqdm(enumerate(d), total=len(d), desc='MetaNet Train - Batch'):
-                if self_train and is_self_train_epoch:
-                    self_train_loss = metanet.combined_self_train(optimizer)
-                    step_log = dict(Epoch=epoch, Batch=batch, Metric='Self Train Loss', Score=self_train_loss.item())
-                    train_store.loc[train_store.shape[0]] = step_log
-
-                # Zero your gradients for every batch!
-                optimizer.zero_grad()
-                batch_x, batch_y = batch_x.to(DEVICE), batch_y.to(DEVICE)
-                y = metanet(batch_x)
-                # loss = loss_fn(y, batch_y.unsqueeze(-1).to(torch.float32))
-                loss = loss_fn(y, batch_y.to(torch.long))
-                loss.backward()
-
-                # Adjust learning weights
-                optimizer.step()
-
-                step_log = dict(Epoch=epoch, Batch=batch,
-                                Metric='Task Loss', Score=loss.item())
-                train_store.loc[train_store.shape[0]] = step_log
-                if is_validation_epoch:
-                    metric(y.cpu(), batch_y.cpu())
-
-                if batch >= 3 and debug:
-                    break
-
-            if is_validation_epoch:
-                validation_log = dict(Epoch=int(epoch), Batch=BATCHSIZE,
-                                      Metric='Train Accuracy', Score=metric.compute().item())
-                train_store.loc[train_store.shape[0]] = validation_log
-
-                accuracy = checkpoint_and_validate(metanet, run_path, epoch)
-                validation_log = dict(Epoch=int(epoch), Batch=BATCHSIZE,
-                                      Metric='Test Accuracy', Score=accuracy.item())
-                train_store.loc[train_store.shape[0]] = validation_log
-                if particle_analysis:
-                    counter_dict = defaultdict(lambda: 0)
-                    # This returns ID-functions
-                    _ = test_for_fixpoints(counter_dict, list(metanet.particles))
-                    for key, value in dict(counter_dict).items():
-                        step_log = dict(Epoch=int(epoch), Batch=BATCHSIZE, Metric=key, Score=value)
-                        train_store.loc[train_store.shape[0]] = step_log
-                train_store.to_csv(df_store_path, mode='a', header=not df_store_path.exists())
-                train_store = new_train_storage_df()
-
-        accuracy = checkpoint_and_validate(metanet, run_path, EPOCH, final_model=True)
-        validation_log = dict(Epoch=EPOCH, Batch=BATCHSIZE,
-                              Metric='Test Accuracy', Score=accuracy.item())
-
-        train_store.loc[train_store.shape[0]] = validation_log
-        train_store.to_csv(df_store_path)
-
-    if plotting:
-        plot_training_result(df_store_path)
-
-    if particle_analysis:
-        model_path = next(run_path.glob('*ckpt.tp'))
-        latest_model = torch.load(model_path, map_location=DEVICE).eval()
-        counter_dict = defaultdict(lambda: 0)
-        _ = test_for_fixpoints(counter_dict, list(latest_model.particles))
-        tqdm.write(str(dict(counter_dict)))
-        zero_ident = torch.load(model_path, map_location=DEVICE).eval().replace_with_zero('identity_func')
-        zero_other = torch.load(model_path, map_location=DEVICE).eval().replace_with_zero('other_func')
-        if as_sparse_network_test:
-            acc_pre = validate(model_path, ratio=1)
-            ident_ckpt = set_checkpoint(zero_ident, model_path.parent, -1, final_model=True)
-            ident_acc_post = validate(ident_ckpt, ratio=1)
-            tqdm.write(f'Zero_ident diff = {abs(ident_acc_post-acc_pre)}')
-            other_ckpt = set_checkpoint(zero_other, model_path.parent, -2, final_model=True)
-            other_acc_post = validate(other_ckpt, ratio=1)
-            tqdm.write(f'Zero_other diff = {abs(other_acc_post - acc_pre)}')

experiments/mixed_setting_exp.py

@@ -1,177 +0,0 @@
-import os.path
-import pickle
-
-from tqdm import tqdm
-
-from experiments.helpers import check_folder, summary_fixpoint_experiment, summary_fixpoint_percentage
-from functionalities_test import test_for_fixpoints
-from network import Net
-from visualization import plot_loss, bar_chart_fixpoints, line_chart_fixpoints
-from visualization import plot_3d_self_train
-
-
-class MixedSettingExperiment:
-    def __init__(self, population_size, net_i_size, net_h_size, net_o_size, learning_rate, train_nets,
-                 epochs, SA_steps, ST_steps_between_SA, log_step_size, directory_name):
-        super().__init__()
-        self.population_size = population_size
-
-        self.net_input_size = net_i_size
-        self.net_hidden_size = net_h_size
-        self.net_out_size = net_o_size
-        self.net_learning_rate = learning_rate
-        self.train_nets = train_nets
-        self.epochs = epochs
-        self.SA_steps = SA_steps
-        self.ST_steps_between_SA = ST_steps_between_SA
-        self.log_step_size = log_step_size
-
-        self.fixpoint_counters = {
-            "identity_func": 0,
-            "divergent": 0,
-            "fix_zero": 0,
-            "fix_weak": 0,
-            "fix_sec": 0,
-            "other_func": 0
-        }
-
-        self.loss_history = []
-
-        self.fixpoint_counters_history = []
-
-        self.directory_name = directory_name
-        os.mkdir(self.directory_name)
-
-        self.nets = []
-        self.populate_environment()
-
-        self.fixpoint_percentage()
-        self.weights_evolution_3d_experiment()
-        self.count_fixpoints()
-        self.visualize_loss()
-
-    def populate_environment(self):
-        loop_population_size = tqdm(range(self.population_size))
-        for i in loop_population_size:
-            loop_population_size.set_description("Populating mixed experiment %s" % i)
-
-            net_name = f"mixed_net_{str(i)}"
-            net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name)
-            self.nets.append(net)
-
-        loop_epochs = tqdm(range(self.epochs))
-        for j in loop_epochs:
-            loop_epochs.set_description("Running mixed experiment %s" % j)
-
-            for i in loop_population_size:
-                net = self.nets[i]
-
-                if self.train_nets == "before_SA":
-                    for _ in range(self.ST_steps_between_SA):
-                        net.self_train(1, self.log_step_size, self.net_learning_rate)
-                    net.self_application(self.SA_steps, self.log_step_size)
-
-                elif self.train_nets == "after_SA":
-                    net.self_application(self.SA_steps, self.log_step_size)
-                    for _ in range(self.ST_steps_between_SA):
-                        net.self_train(1, self.log_step_size, self.net_learning_rate)
-
-                print(
-                    f"\nLast weight matrix (epoch: {j}):\n{net.input_weight_matrix()}\nLossHistory: {net.loss_history[-10:]}")
-            test_for_fixpoints(self.fixpoint_counters, self.nets)
-            # Rounding the result not to run into other problems later regarding the exact representation of floating number
-            fixpoints_percentage = round((self.fixpoint_counters["fix_zero"] + self.fixpoint_counters[
-                "fix_sec"]) / self.population_size, 1)
-            self.fixpoint_counters_history.append(fixpoints_percentage)
-
-            # Resetting the fixpoint counter. Last iteration not to be reset - it is important for the bar_chart_fixpoints().
-            if j < self.epochs:
-                self.reset_fixpoint_counters()
-
-    def weights_evolution_3d_experiment(self):
-        exp_name = f"Mixed {str(len(self.nets))}"
-
-        # This batch size is not relevant for mixed settings because during an epoch there are more steps of SA & ST happening
-        # and only they need the batch size. To not affect the number of epochs shown in the 3D plot, will send
-        # forward the number "1" for batch size with the variable <irrelevant_batch_size>
-        irrelevant_batch_size = 1
-        plot_3d_self_train(self.nets, exp_name, self.directory_name, irrelevant_batch_size, True)
-
-    def count_fixpoints(self):
-        exp_details = f"SA steps: {self.SA_steps}; ST steps: {self.ST_steps_between_SA}"
-
-        test_for_fixpoints(self.fixpoint_counters, self.nets)
-        bar_chart_fixpoints(self.fixpoint_counters, self.population_size, self.directory_name, self.net_learning_rate,
-                            exp_details)
-
-    def fixpoint_percentage(self):
-        line_chart_fixpoints(self.fixpoint_counters_history, self.epochs, self.ST_steps_between_SA,
-                             self.SA_steps, self.directory_name, self.population_size)
-
-    def visualize_loss(self):
-        for i in range(len(self.nets)):
-            net_loss_history = self.nets[i].loss_history
-            self.loss_history.append(net_loss_history)
-
-        plot_loss(self.loss_history, self.directory_name)
-
-    def reset_fixpoint_counters(self):
-        self.fixpoint_counters = {
-            "identity_func": 0,
-            "divergent": 0,
-            "fix_zero": 0,
-            "fix_weak": 0,
-            "fix_sec": 0,
-            "other_func": 0
-        }
-
-
-def run_mixed_experiment(population_size, net_input_size, net_hidden_size, net_out_size, net_learning_rate, train_nets,
-                         epochs, SA_steps, ST_steps_between_SA, batch_size, name_hash, runs, run_name):
-    experiments = {}
-    fixpoints_percentages = []
-
-    check_folder("mixed")
-
-    # Running the experiments
-    for i in range(runs):
-        directory_name = f"experiments/mixed/{run_name}_run_{i}_{str(population_size)}_nets_{SA_steps}_SA_{ST_steps_between_SA}_ST_{str(name_hash)}"
-
-        mixed_experiment = MixedSettingExperiment(
-            population_size,
-            net_input_size,
-            net_hidden_size,
-            net_out_size,
-            net_learning_rate,
-            train_nets,
-            epochs,
-            SA_steps,
-            ST_steps_between_SA,
-            batch_size,
-            directory_name
-        )
-        pickle.dump(mixed_experiment, open(f"{directory_name}/full_experiment_pickle.p", "wb"))
-        experiments[i] = mixed_experiment
-
-        # Building history of fixpoint percentages for summary
-        fixpoint_counters_history = mixed_experiment.fixpoint_counters_history
-        if not fixpoints_percentages:
-            fixpoints_percentages = mixed_experiment.fixpoint_counters_history
-        else:
-            # Using list comprehension to make the sum of all the percentages
-            fixpoints_percentages = [fixpoints_percentages[i] + fixpoint_counters_history[i] for i in
-                                     range(len(fixpoints_percentages))]
-
-    # Building a summary of all the runs
-    directory_name = f"experiments/mixed/summary_{run_name}_{runs}_runs_{str(population_size)}_nets_{str(name_hash)}"
-    os.mkdir(directory_name)
-
-    summary_pre_title = "mixed"
-    summary_fixpoint_experiment(runs, population_size, epochs, experiments, net_learning_rate, directory_name,
-                                summary_pre_title)
-    summary_fixpoint_percentage(runs, epochs, fixpoints_percentages, ST_steps_between_SA, SA_steps, directory_name,
-                                population_size)
-
-
-if __name__ == '__main__':
-    raise NotImplementedError('Test this here!!!')

experiments/robustness_exp.py

@@ -1,151 +0,0 @@
-import copy
-import os.path
-import pickle
-import random
-
-from tqdm import tqdm
-
-from experiments.helpers import check_folder, summary_fixpoint_experiment
-from functionalities_test import test_for_fixpoints, is_identity_function
-from network import Net
-from visualization import bar_chart_fixpoints, box_plot, write_file
-
-
-def add_noise(input_data, epsilon=pow(10, -5)):
-
-    output = copy.deepcopy(input_data)
-    for k in range(len(input_data)):
-        output[k][0] += random.random() * epsilon
-
-    return output
-
-
-class RobustnessExperiment:
-    def __init__(self, population_size, log_step_size, net_input_size, net_hidden_size, net_out_size, net_learning_rate,
-                 ST_steps, directory_name) -> None:
-        self.population_size = population_size
-        self.log_step_size = log_step_size
-        self.net_input_size = net_input_size
-        self.net_hidden_size = net_hidden_size
-        self.net_out_size = net_out_size
-
-        self.net_learning_rate = net_learning_rate
-
-        self.ST_steps = ST_steps
-        self.fixpoint_counters = {
-            "identity_func": 0,
-            "divergent": 0,
-            "fix_zero": 0,
-            "fix_weak": 0,
-            "fix_sec": 0,
-            "other_func": 0
-        }
-        self.id_functions = []
-
-        self.directory_name = directory_name
-        os.mkdir(self.directory_name)
-
-        self.nets = []
-        # Create population:
-        self.populate_environment()
-        print("Nets:\n", self.nets)
-
-        self.count_fixpoints()
-        [print(net.is_fixpoint) for net in self.nets]
-        self.test_robustness()
-
-    def populate_environment(self):
-        loop_population_size = tqdm(range(self.population_size))
-        for i in loop_population_size:
-            loop_population_size.set_description("Populating robustness experiment %s" % i)
-
-            net_name = f"net_{str(i)}"
-            net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name)
-
-            for _ in range(self.ST_steps):
-                net.self_train(1, self.log_step_size, self.net_learning_rate)
-
-            self.nets.append(net)
-
-    def test_robustness(self):
-        # test_for_fixpoints(self.fixpoint_counters, self.nets, self.id_functions)
-
-        zero_epsilon = pow(10, -5)
-        data = [[0 for _ in range(10)] for _ in range(len(self.id_functions))]
-
-        for i in range(len(self.id_functions)):
-            for j in range(10):
-                original_net = self.id_functions[i]
-
-                # Creating a clone of the network. Not by copying it, but by creating a completely new network
-                # and changing its weights to the original ones.
-                original_net_clone = Net(original_net.input_size, original_net.hidden_size, original_net.out_size,
-                                         original_net.name)
-                # Extra safety for the value of the weights
-                original_net_clone.load_state_dict(copy.deepcopy(original_net.state_dict()))
-
-                noisy_weights = add_noise(original_net_clone.input_weight_matrix(), epsilon=pow(10, -j))
-                original_net_clone.apply_weights(noisy_weights)
-
-                # Testing if the new net is still an identity function after applying noise
-                still_id_func = is_identity_function(original_net_clone, zero_epsilon)
-
-                # If the net is still an id. func. after applying the first run of noise, continue to apply it until otherwise
-                while still_id_func and data[i][j] <= 1000:
-                    data[i][j] += 1
-
-                    original_net_clone = original_net_clone.self_application(1, self.log_step_size)
-
-                    still_id_func = is_identity_function(original_net_clone, zero_epsilon)
-
-        print(f"Data {data}")
-
-        if data.count(0) == 10:
-            print(f"There is no network resisting the robustness test.")
-            text = f"For this population of \n {self.population_size} networks \n there is no" \
-                   f" network resisting the robustness test."
-            write_file(text, self.directory_name)
-        else:
-            box_plot(data, self.directory_name, self.population_size)
-
-    def count_fixpoints(self):
-        exp_details = f"ST steps: {self.ST_steps}"
-
-        self.id_functions = test_for_fixpoints(self.fixpoint_counters, self.nets)
-        bar_chart_fixpoints(self.fixpoint_counters, self.population_size, self.directory_name, self.net_learning_rate,
-                            exp_details)
-
-
-def run_robustness_experiment(population_size, batch_size, net_input_size, net_hidden_size, net_out_size,
-                              net_learning_rate, epochs, runs, run_name, name_hash):
-    experiments = {}
-
-    check_folder("robustness")
-
-    # Running the experiments
-    for i in range(runs):
-        ST_directory_name = f"experiments/robustness/{run_name}_run_{i}_{str(population_size)}_nets_{epochs}_epochs_{str(name_hash)}"
-
-        robustness_experiment = RobustnessExperiment(
-            population_size,
-            batch_size,
-            net_input_size,
-            net_hidden_size,
-            net_out_size,
-            net_learning_rate,
-            epochs,
-            ST_directory_name
-        )
-        pickle.dump(robustness_experiment, open(f"{ST_directory_name}/full_experiment_pickle.p", "wb"))
-        experiments[i] = robustness_experiment
-
-    # Building a summary of all the runs
-    directory_name = f"experiments/robustness/summary_{run_name}_{runs}_runs_{str(population_size)}_nets_{str(name_hash)}"
-    os.mkdir(directory_name)
-
-    summary_pre_title = "robustness"
-    summary_fixpoint_experiment(runs, population_size, epochs, experiments, net_learning_rate, directory_name,
-                                summary_pre_title)
-
-if __name__ == '__main__':
-    raise NotImplementedError('Test this here!!!')

experiments/self_application_exp.py

@@ -1,120 +0,0 @@
-import os.path
-import pickle
-
-from tqdm import tqdm
-
-from experiments.helpers import check_folder, summary_fixpoint_experiment
-from functionalities_test import test_for_fixpoints
-from network import Net
-from visualization import bar_chart_fixpoints
-from visualization import plot_3d_self_application
-
-
-class SelfApplicationExperiment:
-    def __init__(self, population_size, log_step_size, net_input_size, net_hidden_size, net_out_size,
-                 net_learning_rate, application_steps, train_nets, directory_name, training_steps
-                 ) -> None:
-        self.population_size = population_size
-        self.log_step_size = log_step_size
-        self.net_input_size = net_input_size
-        self.net_hidden_size = net_hidden_size
-        self.net_out_size = net_out_size
-
-        self.net_learning_rate = net_learning_rate
-        self.SA_steps = application_steps  #
-
-        self.train_nets = train_nets
-        self.ST_steps = training_steps
-
-        self.directory_name = directory_name
-        os.mkdir(self.directory_name)
-
-        """ Creating the nets & making the SA steps & (maybe) also training the networks. """
-        self.nets = []
-        # Create population:
-        self.populate_environment()
-
-        self.fixpoint_counters = {
-            "identity_func": 0,
-            "divergent": 0,
-            "fix_zero": 0,
-            "fix_weak": 0,
-            "fix_sec": 0,
-            "other_func": 0
-        }
-
-        self.weights_evolution_3d_experiment()
-        self.count_fixpoints()
-
-    def populate_environment(self):
-        loop_population_size = tqdm(range(self.population_size))
-        for i in loop_population_size:
-            loop_population_size.set_description("Populating SA experiment %s" % i)
-
-            net_name = f"SA_net_{str(i)}"
-
-            net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name
-                      )
-            for _ in range(self.SA_steps):
-                input_data = net.input_weight_matrix()
-                target_data = net.create_target_weights(input_data)
-
-                if self.train_nets == "before_SA":
-                    net.self_train(1, self.log_step_size, self.net_learning_rate)
-                    net.self_application(self.SA_steps, self.log_step_size)
-                elif self.train_nets == "after_SA":
-                    net.self_application(self.SA_steps, self.log_step_size)
-                    net.self_train(1, self.log_step_size, self.net_learning_rate)
-                else:
-                    net.self_application(self.SA_steps, self.log_step_size)
-
-            self.nets.append(net)
-
-    def weights_evolution_3d_experiment(self):
-        exp_name = f"SA_{str(len(self.nets))}_nets_3d_weights_PCA"
-        plot_3d_self_application(self.nets, exp_name, self.directory_name, self.log_step_size)
-
-    def count_fixpoints(self):
-        test_for_fixpoints(self.fixpoint_counters, self.nets)
-        exp_details = f"{self.SA_steps} SA steps"
-        bar_chart_fixpoints(self.fixpoint_counters, self.population_size, self.directory_name, self.net_learning_rate,
-                            exp_details)
-
-
-def run_SA_experiment(population_size, batch_size, net_input_size, net_hidden_size, net_out_size,
-                      net_learning_rate, runs, run_name, name_hash, application_steps, train_nets, training_steps):
-    experiments = {}
-
-    check_folder("self_application")
-
-    # Running the experiments
-    for i in range(runs):
-        directory_name = f"experiments/self_application/{run_name}_run_{i}_{str(population_size)}_nets_{application_steps}_SA_{str(name_hash)}"
-
-        SA_experiment = SelfApplicationExperiment(
-            population_size,
-            batch_size,
-            net_input_size,
-            net_hidden_size,
-            net_out_size,
-            net_learning_rate,
-            application_steps,
-            train_nets,
-            directory_name,
-            training_steps
-        )
-        pickle.dump(SA_experiment, open(f"{directory_name}/full_experiment_pickle.p", "wb"))
-        experiments[i] = SA_experiment
-
-    # Building a summary of all the runs
-    directory_name = f"experiments/self_application/summary_{run_name}_{runs}_runs_{str(population_size)}_nets_{application_steps}_SA_{str(name_hash)}"
-    os.mkdir(directory_name)
-
-    summary_pre_title = "SA"
-    summary_fixpoint_experiment(runs, population_size, application_steps, experiments, net_learning_rate,
-                                directory_name,
-                                summary_pre_title)
-
-
-if __name__ == '__main__':
-    raise NotImplementedError('Test this here!!!')

experiments/self_train_exp.py

@@ -1,116 +0,0 @@
-import os.path
-import pickle
-from pathlib import Path
-
-from tqdm import tqdm
-
-from experiments.helpers import check_folder, summary_fixpoint_experiment
-from functionalities_test import test_for_fixpoints
-from network import Net
-from visualization import plot_loss, bar_chart_fixpoints
-from visualization import plot_3d_self_train
-
-
-
-class SelfTrainExperiment:
-    def __init__(self, population_size, log_step_size, net_input_size, net_hidden_size, net_out_size, net_learning_rate,
-                 epochs, directory_name) -> None:
-        self.population_size = population_size
-        self.log_step_size = log_step_size
-        self.net_input_size = net_input_size
-        self.net_hidden_size = net_hidden_size
-        self.net_out_size = net_out_size
-
-        self.net_learning_rate = net_learning_rate
-        self.epochs = epochs
-
-        self.loss_history = []
-
-        self.fixpoint_counters = {
-            "identity_func": 0,
-            "divergent": 0,
-            "fix_zero": 0,
-            "fix_weak": 0,
-            "fix_sec": 0,
-            "other_func": 0
-        }
-
-        self.directory_name = directory_name
-        os.mkdir(self.directory_name)
-
-        self.nets = []
-        # Create population:
-        self.populate_environment()
-
-        self.weights_evolution_3d_experiment()
-        self.count_fixpoints()
-        self.visualize_loss()
-
-    def populate_environment(self):
-        loop_population_size = tqdm(range(self.population_size))
-        for i in loop_population_size:
-            loop_population_size.set_description("Populating ST experiment %s" % i)
-
-            net_name = f"ST_net_{str(i)}"
-            net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name)
-
-            for _ in range(self.epochs):
-              net.self_train(1, self.log_step_size, self.net_learning_rate)
-
-            print(f"\nLast weight matrix (epoch: {self.epochs}):\n{net.input_weight_matrix()}\nLossHistory: {net.loss_history[-10:]}")
-            self.nets.append(net)
-
-    def weights_evolution_3d_experiment(self):
-        exp_name = f"ST_{str(len(self.nets))}_nets_3d_weights_PCA"
-        return plot_3d_self_train(self.nets, exp_name, self.directory_name, self.log_step_size)
-
-    def count_fixpoints(self):
-        test_for_fixpoints(self.fixpoint_counters, self.nets)
-        exp_details = f"Self-train for {self.epochs} epochs"
-        bar_chart_fixpoints(self.fixpoint_counters, self.population_size, self.directory_name, self.net_learning_rate,
-                            exp_details)
-
-    def visualize_loss(self):
-        for i in range(len(self.nets)):
-            net_loss_history = self.nets[i].loss_history
-            self.loss_history.append(net_loss_history)
-
-        plot_loss(self.loss_history, self.directory_name)
-
-
-def run_ST_experiment(population_size, batch_size, net_input_size, net_hidden_size, net_out_size, net_learning_rate,
-                      epochs, runs, run_name, name_hash):
-    experiments = {}
-    logging_directory = Path('output') / 'self_training'
-    logging_directory.mkdir(parents=True, exist_ok=True)
-
-    # Running the experiments
-    for i in range(runs):
-        experiment_name = f"{run_name}_run_{i}_{str(population_size)}_nets_{epochs}_epochs_{str(name_hash)}"
-        this_exp_directory = logging_directory / experiment_name
-        ST_experiment = SelfTrainExperiment(
-            population_size,
-            batch_size,
-            net_input_size,
-            net_hidden_size,
-            net_out_size,
-            net_learning_rate,
-            epochs,
-            this_exp_directory
-        )
-        with (this_exp_directory / 'full_experiment_pickle.p').open('wb') as f:
-            pickle.dump(ST_experiment, f)
-        experiments[i] = ST_experiment
-
-    # Building a summary of all the runs
-    summary_name = f"/summary_{run_name}_{runs}_runs_{str(population_size)}_nets_{epochs}_epochs_{str(name_hash)}"
-    summary_directory_name = logging_directory / summary_name
-    summary_directory_name.mkdir(parents=True, exist_ok=True)
-
-    summary_pre_title = "ST"
-    summary_fixpoint_experiment(runs, population_size, epochs, experiments, net_learning_rate, summary_directory_name,
-                                summary_pre_title)
-
-
-if __name__ == '__main__':
-    raise NotImplementedError('Test this here!!!')