Update README.md

# Conflicts:
#	sanity_check_weights.py

.gitignore

@@ -1,572 +1 @@
-# Created by https://www.gitignore.io/api/data,linux,macos,python,windows,pycharm,database,jupyternotebook
-# Edit at https://www.gitignore.io/?templates=data,linux,macos,python,windows,pycharm,database,jupyternotebook
-
-
-### Local Datasets ###
-/experiments
-/setups/experiments
-
-### Data ###
-*.csv
-*.dat
-*.efx
-*.gbr
-*.key
-*.pps
-*.ppt
-*.pptx
-*.sdf
-*.tax2010
-*.vcf
-*.xml
-
-### Database ###
-*.accdb
-*.db
-*.dbf
-*.mdb
-*.pdb
-*.sqlite3
-
-### JupyterNotebook ###
-.ipynb_checkpoints
-*/.ipynb_checkpoints/*
-
-# Remove previous ipynb_checkpoints
-#   git rm -r .ipynb_checkpoints/
-#
-
-### Linux ###
-*~
-
-# temporary files which can be created if a process still has a handle open of a deleted file
-.fuse_hidden*
-
-# KDE directory preferences
-.directory
-
-# Linux trash folder which might appear on any partition or disk
-.Trash-*
-
-# .nfs files are created when an open file is removed but is still being accessed
-.nfs*
-
-### macOS ###
-# General
-.DS_Store
-.AppleDouble
-.LSOverride
-
-# Icon must end with two \r
-Icon
-
-# Thumbnails
-._*
-
-# Files that might appear in the root of a volume
-.DocumentRevisions-V100
-.fseventsd
-.Spotlight-V100
-.TemporaryItems
-.Trashes
-.VolumeIcon.icns
-.com.apple.timemachine.donotpresent
-
-# Directories potentially created on remote AFP share
-.AppleDB
-.AppleDesktop
-Network Trash Folder
-Temporary Items
-.apdisk
-
-### PyCharm ###
-# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
-# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
-
-# User-specific stuff
-.idea/**/workspace.xml
-.idea/**/tasks.xml
-.idea/**/usage.statistics.xml
-.idea/**/dictionaries
-.idea/**/shelf
-
-# Generated files
-.idea/**/contentModel.xml
-
-# Sensitive or high-churn files
-.idea/**/dataSources/
-.idea/**/dataSources.ids
-.idea/**/dataSources.local.xml
-.idea/**/sqlDataSources.xml
-.idea/**/dynamic.xml
-.idea/**/uiDesigner.xml
-.idea/**/dbnavigator.xml
-
-# Gradle
-.idea/**/gradle.xml
-.idea/**/libraries
-
-# Gradle and Maven with auto-import
-# When using Gradle or Maven with auto-import, you should exclude module files,
-# since they will be recreated, and may cause churn.  Uncomment if using
-# auto-import.
-# .idea/modules.xml
-# .idea/*.iml
-# .idea/modules
-
-# CMake
-cmake-build-*/
-
-# Mongo Explorer plugin
-.idea/**/mongoSettings.xml
-
-# File-based project format
-*.iws
-
-# IntelliJ
-out/
-
-# mpeltonen/sbt-idea plugin
-.idea_modules/
-
-# JIRA plugin
-atlassian-ide-plugin.xml
-
-# Cursive Clojure plugin
-.idea/replstate.xml
-
-# Crashlytics plugin (for Android Studio and IntelliJ)
-com_crashlytics_export_strings.xml
-crashlytics.properties
-crashlytics-build.properties
-fabric.properties
-
-# Editor-based Rest Client
-.idea/httpRequests
-
-# Android studio 3.1+ serialized cache file
-.idea/caches/build_file_checksums.ser
-
-### PyCharm Patch ###
-# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
-
-# *.iml
-# modules.xml
-# .idea/misc.xml
-# *.ipr
-
-# Sonarlint plugin
-.idea/sonarlint
-
-### Python ###
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-### Python Patch ###
-.venv/
-
-### Windows ###
-# Windows thumbnail cache files
-Thumbs.db
-ehthumbs.db
-ehthumbs_vista.db
-
-# Dump file
-*.stackdump
-
-# Folder config file
-[Dd]esktop.ini
-
-# Recycle Bin used on file shares
-$RECYCLE.BIN/
-
-# Windows Installer files
-*.cab
-*.msi
-*.msix
-*.msm
-*.msp
-
-# Windows shortcuts
-*.lnk
-
-# pycharm
-.idea/
-
-#######################################
-#### Tex related
-## Core latex/pdflatex auxiliary files:
-*.aux
-*.lof
-*.log
-*.lot
-*.fls
-*.out
-*.toc
-*.fmt
-*.fot
-*.cb
-*.cb2
-.*.lb
-
-## Intermediate documents:
-*.dvi
-*.xdv
-*-converted-to.*
-# these rules might exclude image files for figures etc.
-# *.ps
-# *.eps
-# *.pdf
-
-## Generated if empty string is given at "Please type another file name for output:"
-.pdf
-
-## Bibliography auxiliary files (bibtex/biblatex/biber):
-*.bbl
-*.bcf
-*.blg
-*-blx.aux
-*-blx.bib
-*.run.xml
-
-## Build tool auxiliary files:
-*.fdb_latexmk
-*.synctex
-*.synctex(busy)
-*.synctex.gz
-*.synctex.gz(busy)
-*.pdfsync
-
-## Build tool directories for auxiliary files
-# latexrun
-latex.out/
-
-## Auxiliary and intermediate files from other packages:
-# algorithms
-*.alg
-*.loa
-
-# achemso
-acs-*.bib
-
-# amsthm
-*.thm
-
-# beamer
-*.nav
-*.pre
-*.snm
-*.vrb
-
-# changes
-*.soc
-
-# comment
-*.cut
-
-# cprotect
-*.cpt
-
-# elsarticle (documentclass of Elsevier journals)
-*.spl
-
-# endnotes
-*.ent
-
-# fixme
-*.lox
-
-# feynmf/feynmp
-*.mf
-*.mp
-*.t[1-9]
-*.t[1-9][0-9]
-*.tfm
-
-#(r)(e)ledmac/(r)(e)ledpar
-*.end
-*.?end
-*.[1-9]
-*.[1-9][0-9]
-*.[1-9][0-9][0-9]
-*.[1-9]R
-*.[1-9][0-9]R
-*.[1-9][0-9][0-9]R
-*.eledsec[1-9]
-*.eledsec[1-9]R
-*.eledsec[1-9][0-9]
-*.eledsec[1-9][0-9]R
-*.eledsec[1-9][0-9][0-9]
-*.eledsec[1-9][0-9][0-9]R
-
-# glossaries
-*.acn
-*.acr
-*.glg
-*.glo
-*.gls
-*.glsdefs
-
-# gnuplottex
-*-gnuplottex-*
-
-# gregoriotex
-*.gaux
-*.gtex
-
-# htlatex
-*.4ct
-*.4tc
-*.idv
-*.lg
-*.trc
-*.xref
-
-# hyperref
-*.brf
-
-# knitr
-*-concordance.tex
-# TODO Comment the next line if you want to keep your tikz graphics files
-*.tikz
-*-tikzDictionary
-
-# listings
-*.lol
-
-# makeidx
-*.idx
-*.ilg
-*.ind
-*.ist
-
-# minitoc
-*.maf
-*.mlf
-*.mlt
-*.mtc[0-9]*
-*.slf[0-9]*
-*.slt[0-9]*
-*.stc[0-9]*
-
-# minted
-_minted*
-*.pyg
-
-# morewrites
-*.mw
-
-# nomencl
-*.nlg
-*.nlo
-*.nls
-
-# pax
-*.pax
-
-# pdfpcnotes
-*.pdfpc
-
-# sagetex
-*.sagetex.sage
-*.sagetex.py
-*.sagetex.scmd
-
-# scrwfile
-*.wrt
-
-# sympy
-*.sout
-*.sympy
-sympy-plots-for-*.tex/
-
-# pdfcomment
-*.upa
-*.upb
-
-# pythontex
-*.pytxcode
-pythontex-files-*/
-
-# tcolorbox
-*.listing
-
-# thmtools
-*.loe
-
-# TikZ & PGF
-*.dpth
-*.md5
-*.auxlock
-
-# todonotes
-*.tdo
-
-# vhistory
-*.hst
-*.ver
-
-# easy-todo
-*.lod
-
-# xcolor
-*.xcp
-
-# xmpincl
-*.xmpi
-
-# xindy
-*.xdy
-
-# xypic precompiled matrices
-*.xyc
-
-# endfloat
-*.ttt
-*.fff
-
-# Latexian
-TSWLatexianTemp*
-
-## Editors:
-# WinEdt
-*.bak
-*.sav
-
-# Texpad
-.texpadtmp
-
-# LyX
-*.lyx~
-
-# Kile
-*.backup
-
-# KBibTeX
-*~[0-9]*
-
-# auto folder when using emacs and auctex
-./auto/*
-*.el
-
-# expex forward references with \gathertags
-*-tags.tex
-
-# standalone packages
-*.sta
-
-
-# End of https://www.gitignore.io/api/data,linux,macos,python,windows,pycharm,database,jupyternotebook
-
+/output/

README.md

@@ -1,2 +1,84 @@
-# bannana-networks
-
+# Bureaucratic Cohort Swarms
+### Pruning Networks by SRNN 
+###### Deadline: 28.02.22
+
+## Experimente
+
+### Fixpoint Tests:
+    
+- [X] Dropout Test 
+  - (Macht das Partikel beim Goal mit oder ist es nur SRN)
+  - Zero_ident diff = -00.04999637603759766 %
+	   
+- [ ] gnf(1) -> Aprox. Weight
+  - Übersetung in ein Gewichtsskalar
+    - Einbettung in ein Reguläres Netz
+	
+- [ ] Übersetzung in ein Explainable AI Framework
+  - Rückschlüsse auf Mikro Netze
+	
+- [ ] Visualiserung
+  - Der Zugehörigkeit 
+  - Der Vernetzung
+	
+- [ ] PCA()
+  - Dataframe Epoch, Weight, dim_1, ..., dim_n
+  - Visualisierung als Trajectory Cube
+	
+- [ ] Recherche zu Makro Mikro Netze Strukturen 
+  - gits das schon?
+  - Hypernetwork?
+  - arxiv: 1905.02898
+  - Sparse Networks
+  - Pruning 
+
+---
+
+### Tasks für Steffen:
+- [x] Sanity Check:
+
+  - [x] Neuronen können lernen einen Eingabewert mit x zu multiplizieren?
+
+    | SRNN x*n    3 Neurons            Identity_Func     | SRNN x*n    4 Neurons              Identity_Func  |
+    |---------------------------------------------------|----------------------------------------------------|
+    | ![](./figures/sanity/sanity_3hidden_xtimesn.png)   | ![](./figures/sanity/sanity_4hidden_xtimesn.png)  |
+    | SRNN x*n    6 Neurons              Other_Func      | SRNN x*n    10 Neurons      Other_Func            |
+    | ![](./figures/sanity/sanity_6hidden_xtimesn.png)   | ![](./figures/sanity/sanity_10hidden_xtimesn.png) |
+  
+- [ ] Connectivity
+  - Das Netz dünnt sich wirklich aus.
+    
+    |||
+    |---------------------------------------------------|----------------------------------------------------|
+    | 200 Epochs - 4 Neurons - \alpha 100     RES     |                                                           |
+    | ![](./figures/connectivity/training_lineplot.png) | ![](./figures/connectivity/training_particle_type_lp.png) |
+    | OTHER FUNTIONS                                    |                  IDENTITY FUNCTIONS                       |
+    | ![](./figures/connectivity/other.png)             | ![](./figures/connectivity/identity.png)                  |
+
+- [ ] Training mit kleineren GNs
+  
+    
+- [ ] Weiter Trainieren -> 500 Epochs?
+- [x] Training ohne Residual Skip Connection
+  - Ist anders:
+     Self Training wird zunächst priorisiert, dann kommt langsam der eigentliche Task durch:
+      
+      | No Residual Skip connections 8 Neurons in SRNN  Alpha=100                                    | Residual Skip connections 8 Neurons in SRNN    Alpha=100                                 |
+      |------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------|
+      | ![LinePlot](./figures/res_no_res/mn_st_200_8_alpha_100_no_res_training_particle_type_lp.png) | ![image info](./figures/res_no_res/mn_st_200_8_alpha_100_training_particle_type_lp.png)  |
+      | ![image info](./figures/res_no_res/mn_st_200_8_alpha_100_no_res_training_lineplot.png)       | ![image info](./figures/res_no_res/mn_st_200_8_alpha_100_training_lineplot.png)          |
+
+- [ ] Test mit Baseline Dense Network 
+  - [ ] mit vergleichbaren Neuron Count
+  - [ ] mit gesamt Weight Count
+
+- [ ] Task/Goal statt SRNN-Task
+
+---
+
+### Für Menschen mit zu viel Zeit:
+- [ ] Sparse Network Training der Self Replication
+  - Just for the lulz and speeeeeeed)
+  - (Spaß bei Seite, wäre wichtig für schnellere Forschung)
+    <https://pytorch.org/docs/stable/sparse.html>
+

code/experiment.py

@@ -1,171 +0,0 @@
-import os
-import time
-import dill
-from tqdm import tqdm
-import copy
-
-from tensorflow.python.keras import backend as K
-
-from abc import ABC, abstractmethod
-
-
-class Experiment(ABC):
-
-    @staticmethod
-    def from_dill(path):
-        with open(path, "rb") as dill_file:
-            return dill.load(dill_file)
-
-    @staticmethod
-    def reset_model():
-        K.clear_session()
-
-    def __init__(self, name=None, ident=None):
-        self.experiment_id = f'{ident or ""}_{time.time()}'
-        self.experiment_name = name or 'unnamed_experiment'
-        self.next_iteration = 0
-        self.log_messages = list()
-        self.historical_particles = dict()
-
-    def __enter__(self):
-        self.dir = os.path.join('experiments', f'exp-{self.experiment_name}-{self.experiment_id}-{self.next_iteration}')
-        os.makedirs(self.dir)
-        print(f'** created {self.dir} **')
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.save(experiment=self.without_particles())
-        self.save_log()
-        self.next_iteration += 1
-
-    def log(self, message, **kwargs):
-        self.log_messages.append(message)
-        print(message, **kwargs)
-
-    def save_log(self, log_name="log"):
-        with open(os.path.join(self.dir, f"{log_name}.txt"), "w") as log_file:
-            for log_message in self.log_messages:
-                print(str(log_message), file=log_file)
-
-    def __copy__(self):
-        self_copy = self.__class__(name=self.experiment_name,)
-        self_copy.__dict__ = {attr: self.__dict__[attr] for attr in self.__dict__ if
-                              attr not in ['particles', 'historical_particles']}
-        return self_copy
-
-    def without_particles(self):
-        self_copy = copy.copy(self)
-        # self_copy.particles = [particle.states for particle in self.particles]
-        self_copy.historical_particles = {key: val.states for key, val in self.historical_particles.items()}
-        return self_copy
-
-    def save(self, **kwargs):
-        for name, value in kwargs.items():
-            with open(os.path.join(self.dir, f"{name}.dill"), "wb") as dill_file:
-                dill.dump(value, dill_file)
-
-    @abstractmethod
-    def run_net(self, net, trains_per_application=100, step_limit=100, run_id=0, **kwargs):
-        raise NotImplementedError
-        pass
-
-    def run_exp(self, network_generator, exp_iterations, prints=False, **kwargs):
-        # INFO Run_ID needs to be more than 0, so that exp stores the trajectories!
-        for run_id in range(exp_iterations):
-            network = network_generator()
-            self.run_net(network, 100, run_id=run_id + 1, **kwargs)
-            self.historical_particles[run_id] = network
-            if prints:
-                print("Fixpoint? " + str(network.is_fixpoint()))
-        self.reset_model()
-
-    def reset_all(self):
-        self.reset_model()
-
-
-class FixpointExperiment(Experiment):
-    if kwargs.get('logging', False):
-        self.log(self.counters)
-
-    def __init__(self, **kwargs):
-        kwargs['name'] = self.__class__.__name__ if 'name' not in kwargs else kwargs['name']
-        super().__init__(**kwargs)
-        self.counters = dict(divergent=0, fix_zero=0, fix_other=0, fix_sec=0, other=0)
-        self.interesting_fixpoints = []
-
-    def run_net(self, net, step_limit=100, run_id=0, **kwargs):
-        i = 0
-        while i < step_limit and not net.is_diverged() and not net.is_fixpoint():
-            net.self_attack()
-            i += 1
-            if run_id:
-                net.save_state(time=i)
-        self.count(net)
-
-    def count(self, net):
-        if net.is_diverged():
-            self.counters['divergent'] += 1
-        elif net.is_fixpoint():
-            if net.is_zero():
-                self.counters['fix_zero'] += 1
-            else:
-                self.counters['fix_other'] += 1
-                self.interesting_fixpoints.append(net.get_weights())
-        elif net.is_fixpoint(2):
-            self.counters['fix_sec'] += 1
-        else:
-            self.counters['other'] += 1
-
-    def reset_counters(self):
-        for key in self.counters.keys():
-            self.counters[key] = 0
-        return True
-
-    def reset_all(self):
-        super(FixpointExperiment, self).reset_all()
-        self.reset_counters()
-
-
-class MixedFixpointExperiment(FixpointExperiment):
-
-    def run_net(self, net, trains_per_application=100, step_limit=100, run_id=0, **kwargs):
-        for i in range(step_limit):
-            if net.is_diverged() or net.is_fixpoint():
-                break
-            net.self_attack()
-            with tqdm(postfix=["Loss", dict(value=0)]) as bar:
-                for _ in range(trains_per_application):
-                    loss = net.compiled().train()
-                    bar.postfix[1]["value"] = loss
-                    bar.update()
-            if run_id:
-                net.save_state()
-        self.count(net)
-
-
-class SoupExperiment(Experiment):
-
-    def __init__(self, **kwargs):
-        super(SoupExperiment, self).__init__(name=kwargs.get('name', self.__class__.__name__))
-
-    def run_exp(self, network_generator, exp_iterations, soup_generator=None, soup_iterations=0, prints=False):
-        for i in range(soup_iterations):
-            soup = soup_generator()
-            soup.seed()
-            for _ in tqdm(exp_iterations):
-                soup.evolve()
-            self.log(soup.count())
-            self.save(soup=soup.without_particles())
-
-    def run_net(self, net, trains_per_application=100, step_limit=100, run_id=0, **kwargs):
-        raise NotImplementedError
-        pass
-
-
-class IdentLearningExperiment(Experiment):
-
-    def __init__(self):
-        super(IdentLearningExperiment, self).__init__(name=self.__class__.__name__)
-
-    def run_net(self, net, trains_per_application=100, step_limit=100, run_id=0, **kwargs):
-        pass

code/network.py

@@ -1,618 +0,0 @@
-import numpy as np
-from abc import abstractmethod, ABC
-from typing import List, Union
-from types import FunctionType
-
-from tensorflow.python.keras.models import Sequential
-from tensorflow.python.keras.callbacks import Callback
-from tensorflow.python.keras.layers import SimpleRNN, Dense
-from tensorflow.python.keras import backend as K
-
-from experiment import *
-
-# Supress warnings and info messages
-os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
-
-
-class SaveStateCallback(Callback):
-    def __init__(self, network, epoch=0):
-        super(SaveStateCallback, self).__init__()
-        self.net = network
-        self.init_epoch = epoch
-
-    def on_epoch_end(self, epoch, logs=None):
-        description = dict(time=epoch+self.init_epoch)
-        description['action'] = 'train_self'
-        description['counterpart'] = None
-        self.net.save_state(**description)
-        return
-
-
-class Weights:
-
-    @staticmethod
-    def __reshape_flat_array__(array, shapes):
-        sizes: List[int] = [int(np.prod(shape)) for shape in shapes]
-        # Split the incoming array into slices for layers
-        slices = [array[x: y] for x, y in zip(np.cumsum([0]+sizes), np.cumsum([0]+sizes)[1:])]
-        # reshape them in accordance to the given shapes
-        weights = [np.reshape(weight_slice, shape) for weight_slice, shape in zip(slices, shapes)]
-        return weights
-
-    def __init__(self, weight_vector: Union[List[np.ndarray], np.ndarray], flat_array_shape=None):
-        """
-        Weight class, for easy manipulation of weight vectors from Keras models
-
-        :param weight_vector: A numpy array holding weights
-        :type weight_vector: List[np.ndarray]
-        """
-        self.__iter_idx = [0, 0]
-        if flat_array_shape:
-            weight_vector = self.__reshape_flat_array__(weight_vector, flat_array_shape)
-
-        self.layers = weight_vector
-
-        # TODO: implement a way to access the cells directly
-        # self.cells = len(self)
-        # TODO: implement a way to access the weights directly
-        # self.weights = self.to_flat_array() ?
-
-    def __iter__(self):
-        self.__iter_idx = [0, 0]
-        return self
-
-    def __getitem__(self, item):
-        return self.layers[item]
-
-    def max(self):
-        np.max(self.layers)
-
-    def avg(self):
-        return np.average(self.layers)
-
-    def __len__(self):
-        return sum([x.size for x in self.layers])
-
-    def shapes(self):
-        return [x.shape for x in self.layers]
-
-    def num_layers(self):
-        return len(self.layers)
-
-    def __copy__(self):
-        return copy.deepcopy(self)
-
-    def __next__(self):
-        # ToDo: Check iteration progress over layers
-        # ToDo: There is still a problem interation, currently only cell level is the last loop stage.
-        # Do we need this?
-        if self.__iter_idx[0] >= len(self.layers):
-            if self.__iter_idx[1] >= len(self.layers[self.__iter_idx[0]]):
-                raise StopIteration
-        result = self.layers[self.__iter_idx[0]][self.__iter_idx[1]]
-
-        if self.__iter_idx[1] >= len(self.layers[self.__iter_idx[0]]):
-            self.__iter_idx[0] += 1
-            self.__iter_idx[1] = 0
-        else:
-            self.__iter_idx[1] += 1
-        return result
-
-    def __repr__(self):
-        return f'Weights({self.to_flat_array().tolist()})'
-
-    def to_flat_array(self) -> np.ndarray:
-        return np.hstack([weight.flatten() for weight in self.layers])
-
-    def from_flat_array(self, array):
-        new_weights = self.__reshape_flat_array__(array, self.shapes())
-        return new_weights
-
-    def shuffle(self):
-        flat = self.to_flat_array()
-        np.random.shuffle(flat)
-        self.from_flat_array(flat)
-        return True
-
-    def are_diverged(self):
-        return any([np.isnan(x).any() for x in self.layers]) or any([np.isinf(x).any() for x in self.layers])
-
-    def are_within_bounds(self, lower_bound: float, upper_bound: float):
-        return bool(sum([((lower_bound < x) & (x > upper_bound)).size for x in self.layers]))
-
-    def aggregate_by(self, func: FunctionType, num_aggregates):
-        collection_sizes = len(self) // num_aggregates
-        weights = self.to_flat_array()[:collection_sizes * num_aggregates].reshape((num_aggregates, -1))
-        aggregated_weights = func(weights, num_aggregates)
-        left_overs = self.to_flat_array()[collection_sizes * num_aggregates:]
-        return aggregated_weights, left_overs
-
-
-class NeuralNetwork(ABC):
-    """
-    This is the Base Network Class, including abstract functions that must be implemented.
-    """
-
-    def __init__(self, **params):
-        super().__init__()
-        self.params = dict(epsilon=0.00000000000001)
-        self.params.update(params)
-        self.keras_params = dict(activation='linear', use_bias=False)
-        self.states = []
-        self.model: Sequential
-
-    def get_params(self) -> dict:
-        return self.params
-
-    def get_keras_params(self) -> dict:
-        return self.keras_params
-
-    def with_params(self, **kwargs):
-        self.params.update(kwargs)
-        return self
-
-    def with_keras_params(self, **kwargs):
-        self.keras_params.update(kwargs)
-        return self
-
-    def get_weights(self) -> Weights:
-        return Weights(self.model.get_weights())
-
-    def get_weights_flat(self) -> np.ndarray:
-        return self.get_weights().to_flat_array()
-
-    def set_weights(self, new_weights: Weights):
-        return self.model.set_weights(new_weights.layers)
-
-    @abstractmethod
-    def get_samples(self):
-        # TODO: add a dogstring, telling the user what this does, e.g. what is a sample?
-        raise NotImplementedError
-
-    @abstractmethod
-    def apply_to_weights(self, old_weights) -> Weights:
-        # TODO: add a dogstring, telling the user what this does, e.g. what is applied?
-        raise NotImplementedError
-
-    def apply_to_network(self, other_network) -> Weights:
-        # TODO: add a dogstring, telling the user what this does, e.g. what is applied?
-        new_weights = self.apply_to_weights(other_network.get_weights())
-        return new_weights
-
-    def attack(self, other_network):
-        # TODO: add a dogstring, telling the user what this does, e.g. what is an attack?
-        other_network.set_weights(self.apply_to_network(other_network))
-        return self
-
-    def fuck(self, other_network):
-        # TODO: add a dogstring, telling the user what this does, e.g. what is fucking?
-        self.set_weights(self.apply_to_network(other_network))
-        return self
-
-    def self_attack(self, iterations=1):
-        # TODO: add a dogstring, telling the user what this does, e.g. what is self attack?
-        for _ in range(iterations):
-            self.attack(self)
-        return self
-
-    def meet(self, other_network):
-        # TODO: add a dogstring, telling the user what this does, e.g. what is meeting?
-        new_other_network = copy.deepcopy(other_network)
-        return self.attack(new_other_network)
-
-    def is_diverged(self):
-        return self.get_weights().are_diverged()
-
-    def is_zero(self, epsilon=None):
-        epsilon = epsilon or self.get_params().get('epsilon')
-        return self.get_weights().are_within_bounds(-epsilon, epsilon)
-
-    def is_fixpoint(self, degree: int = 1, epsilon: float = None) -> bool:
-        assert degree >= 1, "degree must be >= 1"
-        epsilon = epsilon or self.get_params().get('epsilon')
-
-        new_weights = copy.deepcopy(self.get_weights())
-
-        for _ in range(degree):
-            new_weights = self.apply_to_weights(new_weights)
-            if new_weights.are_diverged():
-                return False
-
-        biggerEpsilon = (np.abs(new_weights.to_flat_array() - self.get_weights().to_flat_array()) >= epsilon).any()
-
-        # Boolean Value needs to be flipped to answer "is_fixpoint"
-        return not biggerEpsilon
-
-    def print_weights(self, weights=None):
-        print(weights or self.get_weights())
-
-
-class ParticleDecorator:
-    next_uid = 0
-
-    def __init__(self, network):
-
-        # ToDo: Add DocString, What does it do?
-
-        self.uid = self.__class__.next_uid
-        self.__class__.next_uid += 1
-        self.network = network
-        self.states = []
-        self.save_state(time=0, action='init', counterpart=None)
-
-    def __getattr__(self, name):
-        return getattr(self.network, name)
-
-    def get_uid(self):
-        return self.uid
-
-    def make_state(self, **kwargs):
-        if self.network.is_diverged():
-            return None
-        state = {'class': self.network.__class__.__name__, 'weights': self.network.get_weights_flat()}
-        state.update(kwargs)
-        return state
-
-    def save_state(self, **kwargs):
-        state = self.make_state(**kwargs)
-        if state is not None:
-            self.states += [state]
-        else:
-            pass
-        return True
-
-    def update_state(self, number, **kwargs):
-        raise NotImplementedError('Result is vague')
-        # if number < len(self.states):
-        #     self.states[number] = self.make_state(**kwargs)
-        # else:
-        #     for i in range(len(self.states), number):
-        #         self.states += [None]
-        #     self.states += self.make_state(**kwargs)
-
-    def get_states(self):
-        return self.states
-
-
-class WeightwiseNeuralNetwork(NeuralNetwork):
-
-    def __init__(self, width, depth, **kwargs):
-        # ToDo: Insert Docstring
-        super().__init__(**kwargs)
-        self.width: int = width
-        self.depth: int = depth
-        self.model = Sequential()
-        self.model.add(Dense(units=self.width, input_dim=4, **self.keras_params))
-        for _ in range(self.depth-1):
-            self.model.add(Dense(units=self.width, **self.keras_params))
-        self.model.add(Dense(units=1, **self.keras_params))
-
-    def apply(self, inputs):
-        # TODO: Write about it... What does it do?
-        return self.model.predict(inputs)
-
-    def get_samples(self):
-        weights = self.get_weights()
-        sample = np.asarray([
-            [weight, idx, *x] for idx, layer in enumerate(weights.layers) for x, weight in np.ndenumerate(layer)
-        ])
-        # normalize [layer, cell, position]
-        for idx in range(1, sample.shape[1]):
-            sample[:, idx] = sample[:, idx] / np.max(sample[:, idx])
-        return sample, sample
-
-    def apply_to_weights(self, weights) -> Weights:
-        # ToDo: Insert DocString
-        # Transform the weight matrix in an horizontal stack as: array([[weight, layer, cell, position], ...])
-        transformed_weights = self.get_samples()[0]
-        new_weights = self.apply(transformed_weights)
-        # use the original weight shape to transform the new tensor
-        return Weights(new_weights, flat_array_shape=weights.shapes())
-
-
-class AggregatingNeuralNetwork(NeuralNetwork):
-
-    @staticmethod
-    def aggregate_fft(array: np.ndarray, aggregates: int):
-        flat = array.flatten()
-        # noinspection PyTypeChecker
-        fft_reduction = np.fft.fftn(flat, aggregates)
-        return fft_reduction
-
-    @staticmethod
-    def aggregate_average(array, _):
-        return np.average(array, axis=1)
-
-    @staticmethod
-    def aggregate_max(array, _):
-        return np.max(array, axis=1)
-
-    @staticmethod
-    def deaggregate_identically(aggregate, amount):
-        # ToDo: Find a better way than using the a hardcoded [0]
-        return np.hstack([aggregate for _ in range(amount)])[0]
-
-    @staticmethod
-    def shuffle_not(weights: Weights):
-        """
-        Doesn't do a thing. f(x)
-
-        :param weights: A List of Weights
-        :type weights: Weights
-        :return: The same old weights.
-        :rtype: Weights
-        """
-        return weights
-
-    @staticmethod
-    def shuffle_random(weights: Weights):
-        assert weights.shuffle()
-        return weights
-
-    def __init__(self, aggregates, width, depth, **kwargs):
-        super().__init__(**kwargs)
-        self.aggregates = aggregates
-        self.width = width
-        self.depth = depth
-        self.model = Sequential()
-        self.model.add(Dense(units=width, input_dim=self.aggregates, **self.keras_params))
-        for _ in range(depth-1):
-            self.model.add(Dense(units=width, **self.keras_params))
-        self.model.add(Dense(units=self.aggregates, **self.keras_params))
-
-    def get_aggregator(self):
-        return self.params.get('aggregator', self.aggregate_average)
-
-    def get_deaggregator(self):
-        return self.params.get('deaggregator', self.deaggregate_identically)
-
-    def get_shuffler(self):
-        return self.params.get('shuffler', self.shuffle_not)
-
-    def get_amount_of_weights(self):
-        return len(self.get_weights())
-
-    def apply(self, inputs):
-        # You need to add an dimension here... "..." copies array values
-        return self.model.predict(inputs[None, ...])
-
-    def get_aggregated_weights(self):
-        return self.get_weights().aggregate_by(self.get_aggregator(), self.aggregates)
-
-    def apply_to_weights(self, old_weights) -> Weights:
-
-        # build aggregations of old_weights
-        old_aggregations, leftovers = self.get_aggregated_weights()
-
-        # call network
-        new_aggregations = self.apply(old_aggregations)
-        collection_sizes = self.get_amount_of_weights() // self.aggregates
-        new_aggregations = self.deaggregate_identically(new_aggregations, collection_sizes)
-        # generate new weights
-        # only include leftovers if there are some then coonvert them to Weight on base of th old shape
-        new_weights = Weights(new_aggregations if not leftovers.shape[0] else np.hstack((new_aggregations, leftovers)),
-                              flat_array_shape=old_weights.shapes())
-
-        # maybe shuffle
-        new_weights = self.get_shuffler()(new_weights)
-        return new_weights
-
-    def get_samples(self):
-        aggregations, _ = self.get_aggregated_weights()
-        # What did that do?
-        # sample = np.transpose(np.array([[aggregations[i]] for i in range(self.aggregates)]))
-        return aggregations, aggregations
-
-    def is_fixpoint_after_aggregation(self, degree=1, epsilon=None):
-        assert degree >= 1, "degree must be >= 1"
-        epsilon = epsilon or self.get_params().get('epsilon')
-
-        old_aggregations, _ = self.get_aggregated_weights()
-        new_weights = copy.deepcopy(self.get_weights())
-
-        for _ in range(degree):
-            new_weights = self.apply_to_weights(new_weights)
-            if new_weights.are_diverged():
-                return False
-
-        new_aggregations, leftovers = self.get_aggregated_weights()
-
-        # ToDo: Explain This, why are you additionally checking tolerances of aggregated weights?
-        biggerEpsilon = (np.abs(np.asarray(old_aggregations) - np.asarray(new_aggregations)) >= epsilon).any()
-
-        # Boolean value has to be flipped to answer the question.
-        return True, not biggerEpsilon
-
-
-class RecurrentNeuralNetwork(NeuralNetwork):
-
-    def __init__(self, width, depth, **kwargs):
-        super().__init__(**kwargs)
-        self.features = 1
-        self.width = width
-        self.depth = depth
-        self.model = Sequential()
-        self.model.add(SimpleRNN(units=width, input_dim=self.features, return_sequences=True, **self.keras_params))
-        for _ in range(depth-1):
-            self.model.add(SimpleRNN(units=width, return_sequences=True, **self.keras_params))
-        self.model.add(SimpleRNN(units=self.features, return_sequences=True, **self.keras_params))
-
-    def apply(self, *inputs):
-        stuff = np.transpose(np.array([[[inputs[i]] for i in range(len(inputs))]]))
-        return self.model.predict(stuff)[0].flatten()
-
-    def apply_to_weights(self, old_weights):
-        # build list from old weights
-        new_weights = copy.deepcopy(old_weights)
-        old_weights_list = []
-        for layer_id, layer in enumerate(old_weights):
-            for cell_id, cell in enumerate(layer):
-                for weight_id, weight in enumerate(cell):
-                    old_weights_list += [weight]
-
-        # call network
-        new_weights_list = self.apply(*old_weights_list)
-
-        # write back new weights from list of rnn returns
-        current_weight_id = 0
-        for layer_id, layer in enumerate(new_weights):
-            for cell_id, cell in enumerate(layer):
-                for weight_id, weight in enumerate(cell):
-                    new_weight = new_weights_list[current_weight_id]
-                    new_weights[layer_id][cell_id][weight_id] = new_weight
-                    current_weight_id += 1
-        return new_weights
-
-    def compute_samples(self):
-        # build list from old weights
-        old_weights_list = []
-        for layer_id, layer in enumerate(self.get_weights()):
-            for cell_id, cell in enumerate(layer):
-                for weight_id, weight in enumerate(cell):
-                    old_weights_list += [weight]
-        sample = np.asarray(old_weights_list)[None, ..., None]
-        return sample, sample
-
-
-class TrainingNeuralNetworkDecorator:
-
-    def __init__(self, network):
-        self.network = network
-        self.compile_params = dict(loss='mse', optimizer='sgd')
-        self.model_compiled = False
-
-    def __getattr__(self, name):
-        return getattr(self.network, name)
-
-    def with_params(self, **kwargs):
-        self.network.with_params(**kwargs)
-        return self
-
-    def with_keras_params(self, **kwargs):
-        self.network.with_keras_params(**kwargs)
-        return self
-
-    def get_compile_params(self):
-        return self.compile_params
-
-    def with_compile_params(self, **kwargs):
-        self.compile_params.update(kwargs)
-        return self
-
-    def compile_model(self, **kwargs):
-        compile_params = copy.deepcopy(self.compile_params)
-        compile_params.update(kwargs)
-        return self.network.model.compile(**compile_params)
-
-    def compiled(self, **kwargs):
-        if not self.model_compiled:
-            self.compile_model(**kwargs)
-            self.model_compiled = True
-        return self
-
-    def train(self, batchsize=1, store_states=True, epoch=0):
-        self.compiled()
-        x, y = self.network.get_samples()
-        savestatecallback = [SaveStateCallback(network=self, epoch=epoch)] if store_states else None
-        history = self.network.model.fit(x=x, y=y, epochs=epoch+1, verbose=0,
-                                         batch_size=batchsize, callbacks=savestatecallback,
-                                         initial_epoch=epoch)
-        return history.history['loss'][-1]
-
-    def learn_from(self, other_network, batchsize=1):
-        self.compiled()
-        other_network.compiled()
-        x, y = other_network.network.get_samples()
-        history = self.network.model.fit(x=x, y=y, verbose=0, batch_size=batchsize)
-
-        return history.history['loss'][-1]
-
-
-if __name__ == '__main__':
-
-    if True:
-        # WeightWise Neural Network
-        net_generator = ParticleDecorator(WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation='linear'))
-        with FixpointExperiment() as exp:
-            exp.run_exp(net_generator, 10, logging=True)
-            exp.reset_all()
-
-    if False:
-        # Aggregating Neural Network
-        net_generator = ParticleDecorator(AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params())
-        with FixpointExperiment() as exp:
-            exp.run_exp(net_generator, 10, logging=True)
-
-            exp.reset_all()
-
-    if False:
-        # FFT Aggregation
-        net_generator = lambda: ParticleDecorator(
-            AggregatingNeuralNetwork(
-                aggregates=4, width=2, depth=2, aggregator=AggregatingNeuralNetwork.aggregate_fft
-            ).with_keras_params(activation='linear'))
-        with FixpointExperiment() as exp:
-            for run_id in tqdm(range(10)):
-                exp.run_exp(net_generator, 1)
-                exp.log(exp.counters)
-                exp.reset_model()
-            exp.reset_all()
-
-    if True:
-        # ok so this works quite realiably
-        run_count = 10000
-        net_generator = TrainingNeuralNetworkDecorator(
-            ParticleDecorator(WeightwiseNeuralNetwork(width=2, depth=2))
-        ).with_params(epsilon=0.0001).with_keras_params(optimizer='sgd')
-        with MixedFixpointExperiment() as exp:
-            for run_id in tqdm(range(run_count+1)):
-                exp.run_exp(net_generator, 1)
-                if run_id % 100 == 0:
-                    exp.run_net(net_generator, 1)
-            K.clear_session()
-
-    if False:
-        with FixpointExperiment() as exp:
-            run_count = 1000
-            net = TrainingNeuralNetworkDecorator(AggregatingNeuralNetwork(4, width=2, depth=2)).with_params(epsilon=0.1e-6)
-            for run_id in tqdm(range(run_count+1)):
-                loss = net.compiled().train()
-                if run_id % 100 == 0:
-                    net.print_weights()
-                    old_aggs, _ = net.net.get_aggregated_weights()
-                    print("old weights agg: " + str(old_aggs))
-                    fp, new_aggs = net.net.is_fixpoint_after_aggregation(epsilon=0.0001)
-                    print("new weights agg: " + str(new_aggs))
-                    print("Fixpoint? " + str(net.is_fixpoint()))
-                    print("Fixpoint after Agg? " + str(fp))
-                    print("Loss " + str(loss))
-                    print()
-
-    if False:
-        # this explodes in our faces completely... NAN everywhere
-        # TODO: Wtf is happening here?
-        with FixpointExperiment() as exp:
-            run_count = 10000
-            net = TrainingNeuralNetworkDecorator(RecurrentNeuralNetwork(width=2, depth=2))\
-                .with_params(epsilon=0.1e-2).with_keras_params(optimizer='sgd', activation='linear')
-            for run_id in tqdm(range(run_count+1)):
-                loss = net.compiled().train()
-                if run_id % 500 == 0:
-                    net.print_weights()
-                    # print(net.apply_to_network(net))
-                    print("Fixpoint? " + str(net.is_fixpoint()))
-                    print("Loss " + str(loss))
-                    print()
-    if False:
-        # and this gets somewhat interesting... we can still achieve non-trivial fixpoints
-        # over multiple applications when training enough in-between
-        with MixedFixpointExperiment() as exp:
-            for run_id in range(10):
-                net = TrainingNeuralNetworkDecorator(FFTNeuralNetwork(2, width=2, depth=2))\
-                    .with_params(epsilon=0.0001, activation='sigmoid')
-                exp.run_net(net, 500, 10)
-
-                net.print_weights()
-
-                print("Fixpoint? " + str(net.is_fixpoint()))
-            exp.log(exp.counters)

code/results/Soup/log.txt

@@ -1 +0,0 @@
-{'divergent': 0, 'fix_zero': 0, 'fix_other': 13, 'fix_sec': 0, 'other': 7}

code/results/Soup/weights.txt

@@ -1,30 +0,0 @@
-[-0.15321673  1.0428386  -0.7245892  -0.04343993  0.42338863  0.02538261
- -0.40465942 -0.0242596  -1.226809   -0.8168446   0.26588777 -1.0929432
-  0.5383322  -0.73875046]
-[-0.03072096 -1.369665   -0.357126   -0.21180922  0.3853204   0.22853081
- -0.3705557  -0.21977347 -0.6684716   0.12849599  1.0226644  -0.0922638
- -0.7828449  -0.6572327 ]
-[-1.2444692   0.61213857  0.07965802  0.12361202  0.62641835  0.9720597
-  0.3863232   0.59948945  1.0857513   0.49231085 -0.5319295   0.29433587
- -0.64177823  0.17603302]
-[-0.9938292  -0.4438207  -0.03172896  0.06261964 -0.3870194   0.7637992
-  0.0244509  -0.04825407  0.91551745 -0.78740424  0.29226422 -0.52767307
- -0.41744384  0.5567152 ]
-[-0.39049304  0.8842579  -0.8447943  -0.19669186  0.7207061   0.16780053
-  0.3728221   0.08680353  0.7535456  -0.1000197   0.02029054  0.8640245
- -0.15881588  1.1905665 ]
-[ 1.0482084   0.9248296  -0.26946014  0.57047915 -0.32660747  0.6914731
- -0.18025818  0.3816289  -0.69358927  0.21312684 -0.39932403 -0.02991759
- -0.83068466  0.45619962]
-[ 0.75814664  0.10328437  0.07867077 -0.0743314  -0.53440267  0.50492585
- -0.54172474  0.51184535  0.3462249   1.0527638  -0.9503541   0.9235086
- -0.1665241   1.1497779 ]
-[-0.77187353  1.1105504   0.24265823  0.53782856 -0.34098852 -0.75576884
- -0.25396293 -0.56288165  0.3851537  -0.67497945  0.14336896  0.763481
- -0.9224985   0.6374753 ]
-[-0.79123825  0.68166596 -0.30061013 -0.19360289  0.5632736   0.36276665
-  0.7470975   0.48115698  0.10046808 -0.8064349  -1.036736   -0.68296516
- -1.156437    0.52633154]
-[ 0.1788832  -1.5321186  -0.62001514 -0.3870902   0.97524184  0.6088638
- -0.08297889 -0.05180515 -0.29096788  0.7519439   0.8803648   0.82771575
- -0.854887    0.1742936 ]

code/results/exp-applying_fixpoint-_1552681870.3570378-0/log.txt

@@ -1,12 +0,0 @@
-WeightwiseNeuralNetwork activiation='linear' use_bias=False
-{'divergent': 23, 'fix_zero': 27, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
-
-
-AggregatingNeuralNetwork activiation='linear' use_bias=False
-{'divergent': 4, 'fix_zero': 46, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
-
-
-RecurrentNeuralNetwork activiation='linear' use_bias=False
-{'divergent': 46, 'fix_zero': 4, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
-
-

code/results/exp-learn-from-soup-_1552658566.5572753-0/log.txt

@@ -1,4 +0,0 @@
-TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
-{'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], 'zs': [0.0, 1.2, 5.2, 7.4, 8.1, 9.1, 9.6, 9.8, 10.0, 9.9, 9.9]}
-
-

code/results/exp-mixed-self-fixpoints-_1552666977.5858653-0/log.txt

@@ -1,12 +0,0 @@
-WeightwiseNeuralNetwork activiation='linear' use_bias=False
-{'xs': [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500], 'ys': [0.2, 0.3, 0.15, 0.55, 0.7, 0.85, 0.8, 0.95, 0.9, 1.0, 1.0]}
-
-
-AggregatingNeuralNetwork activiation='linear' use_bias=False
-{'xs': [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500], 'ys': [1.0, 0.95, 1.0, 1.0, 0.95, 0.9, 0.8, 1.0, 0.85, 1.0, 0.9]}
-
-
-RecurrentNeuralNetwork activiation='linear' use_bias=False
-{'xs': [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500], 'ys': [0.05, 0.0, 0.05, 0.0, 0.0, 0.1, 0.1, 0.05, 0.1, 0.0, 0.0]}
-
-

code/results/exp-mixed-soup-_1552674483.9866457-0/log.txt

@@ -1,8 +0,0 @@
-TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
-{'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.0, 0.0, 0.1, 0.0, 0.0, 0.0, 0.0, 0.1, 0.0, 0.0, 0.0], 'zs': [0.0, 0.0, 0.7, 1.9, 3.6, 4.3, 6.0, 6.1, 8.3, 7.7, 8.8]}
-
-
-TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
-{'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.8, 0.4, 0.4, 0.3, 0.2, 0.2, 0.2, 0.2, 0.2, 0.4, 0.3], 'zs': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}
-
-

code/results/exp-training_fixpoint-_1552658296.0913951-0/log.txt

@@ -1,12 +0,0 @@
-WeightwiseNeuralNetwork activiation='linear' use_bias=False
-{'divergent': 0, 'fix_zero': 0, 'fix_other': 50, 'fix_sec': 0, 'other': 0}
-
-
-AggregatingNeuralNetwork activiation='linear' use_bias=False
-{'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 50}
-
-
-RecurrentNeuralNetwork activiation='linear' use_bias=False
-{'divergent': 38, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 12}
-
-

code/results/known_fixpoint_variation/log.txt

@@ -1,30 +0,0 @@
-variation 10e-0
-avg time to vergence 3.63
-avg time as fixpoint 0
-variation 10e-1
-avg time to vergence 5.02
-avg time as fixpoint 0
-variation 10e-2
-avg time to vergence 6.46
-avg time as fixpoint 0
-variation 10e-3
-avg time to vergence 8.04
-avg time as fixpoint 0
-variation 10e-4
-avg time to vergence 9.61
-avg time as fixpoint 0.04
-variation 10e-5
-avg time to vergence 11.23
-avg time as fixpoint 1.38
-variation 10e-6
-avg time to vergence 12.99
-avg time as fixpoint 3.23
-variation 10e-7
-avg time to vergence 14.58
-avg time as fixpoint 4.84
-variation 10e-8
-avg time to vergence 21.95
-avg time as fixpoint 11.91
-variation 10e-9
-avg time to vergence 26.45
-avg time as fixpoint 16.47

code/results/self_application_aggregation_network/log.txt

@@ -1 +0,0 @@
-{'divergent': 0, 'fix_zero': 10, 'fix_other': 0, 'fix_sec': 0, 'other': 0}

code/setups/applying-fixpoints.py

@@ -1,69 +0,0 @@
-import sys
-import os
-
-# Concat top Level dir to system environmental variables
-sys.path += os.path.join('..', '.')
-
-from experiment import *
-from network import *
-
-import keras.backend as K
-
-def generate_counters():
-    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
-
-def count(counters, net, notable_nets=[]):
-    if net.is_diverged():
-        counters['divergent'] += 1
-    elif net.is_fixpoint():
-        if net.is_zero():
-            counters['fix_zero'] += 1
-        else:
-            counters['fix_other'] += 1
-            notable_nets += [net]
-    elif net.is_fixpoint(2):
-        counters['fix_sec'] += 1
-        notable_nets += [net]
-    else:
-        counters['other'] += 1
-    return counters, notable_nets
-
-
-if __name__ == '__main__':
-
-    with Experiment('applying_fixpoint') as exp:
-        exp.trials = 50
-        exp.run_count = 100
-        exp.epsilon = 1e-4
-        net_generators = []
-        for activation in ['linear']:  # , 'sigmoid', 'relu']:
-            for use_bias in [False]:
-                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
-                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
-                net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
-        all_counters = []
-        all_notable_nets = []
-        all_names = []
-        for net_generator_id, net_generator in enumerate(net_generators):
-            counters = generate_counters()
-            notable_nets = []
-            for _ in tqdm(range(exp.trials)):
-                net = ParticleDecorator(net_generator())
-                net.with_params(epsilon=exp.epsilon)
-                name = str(net.net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias=" + str(net.get_keras_params().get('use_bias'))
-                for run_id in range(exp.run_count):
-                    loss = net.self_attack()
-                count(counters, net, notable_nets)
-            all_counters += [counters]
-            all_notable_nets += [notable_nets]
-            all_names += [name]
-            K.clear_session()
-        exp.save(all_counters=all_counters)
-        exp.save(trajectorys=exp.without_particles())
-        # net types reached in the end
-        # exp.save(all_notable_nets=all_notable_nets)
-        exp.save(all_names=all_names) #experiment setups
-        for exp_id, counter in enumerate(all_counters):
-            exp.log(all_names[exp_id])
-            exp.log(all_counters[exp_id])
-            exp.log('\n')

code/setups/experiments/exp-learn-from-soup-_1552658566.5572753-0/log.txt

@@ -1,4 +0,0 @@
-TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
-{'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], 'zs': [0.0, 1.2, 5.2, 7.4, 8.1, 9.1, 9.6, 9.8, 10.0, 9.9, 9.9]}
-
-

code/setups/experiments/exp-training_fixpoint-_1552658296.0913951-0/log.txt

@@ -1,12 +0,0 @@
-WeightwiseNeuralNetwork activiation='linear' use_bias=False
-{'divergent': 0, 'fix_zero': 0, 'fix_other': 50, 'fix_sec': 0, 'other': 0}
-
-
-AggregatingNeuralNetwork activiation='linear' use_bias=False
-{'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 50}
-
-
-RecurrentNeuralNetwork activiation='linear' use_bias=False
-{'divergent': 38, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 12}
-
-

code/setups/experiments/exp-weightwise_self_application-_1552664922.4501734-0/log.txt

@@ -1 +0,0 @@
-{'divergent': 11, 'fix_zero': 9, 'fix_other': 0, 'fix_sec': 0, 'other': 0}

code/setups/fixpoint-density.py

@@ -1,69 +0,0 @@
-import sys
-import os
-# Concat top Level dir to system environmental variables
-sys.path += os.path.join('..', '.')
-
-from experiment import *
-from network import *
-
-import tensorflow.python.keras.backend as K
-
-
-def generate_counters():
-    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
-
-
-def count(counters, net, notable_nets=None):
-    notable_nets = notable_nets or []
-    if net.is_diverged():
-        counters['divergent'] += 1
-    elif net.is_fixpoint():
-        if net.is_zero():
-            counters['fix_zero'] += 1
-        else:
-            counters['fix_other'] += 1
-            notable_nets += [net]
-    elif net.is_fixpoint(2):
-        counters['fix_sec'] += 1
-        notable_nets += [net]
-    else:
-        counters['other'] += 1
-    return counters, notable_nets
-
-
-if __name__ == '__main__':
-    with Experiment('fixpoint-density') as exp:
-        #NOTE: settings could/should stay this way
-        #FFT doesn't work though
-        exp.trials = 100000
-        exp.epsilon = 1e-4
-        net_generators = []
-        for activation in ['linear']:
-            net_generators += [lambda activation=activation: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
-            net_generators += [lambda activation=activation: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
-            # net_generators += [lambda activation=activation: FFTNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
-            # net_generators += [lambda activation=activation: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
-        all_counters = []
-        all_notable_nets = []
-        all_names = []
-        for net_generator_id, net_generator in enumerate(net_generators):
-            counters = generate_counters()
-            notable_nets = []
-            for _ in tqdm(range(exp.trials)):
-                net = net_generator().with_params(epsilon=exp.epsilon)
-                net = ParticleDecorator(net)
-                name = str(net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias='" + str(net.get_keras_params().get('use_bias')) + "'"
-                count(counters, net, notable_nets)
-                K.clear_session()
-            all_counters += [counters]
-            # all_notable_nets += [notable_nets]
-            all_names += [name]
-        exp.save(all_counters=all_counters)
-        exp.save(all_notable_nets=all_notable_nets)
-        exp.save(all_names=all_names)
-        for exp_id, counter in enumerate(all_counters):
-            exp.log(all_names[exp_id])
-            exp.log(all_counters[exp_id])
-            exp.log('\n')
-
-    print('Done')

code/setups/known-fixpoint-variation.py

@@ -1,92 +0,0 @@
-import sys
-
-import os
-
-# Concat top Level dir to system environmental variables
-sys.path += os.path.join('..', '.')
-
-from experiment import *
-from network import *
-from soup import prng
-
-import tensorflow.python.keras.backend as K
-
-
-from statistics import mean
-avg = mean
-
-
-def generate_fixpoint_weights():
-    return [
-        np.array([[1.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0]], dtype=np.float32),
-        np.array([[1.0, 0.0], [0.0, 0.0]], dtype=np.float32),
-        np.array([[1.0], [0.0]], dtype=np.float32)
-    ]
-
-
-def generate_fixpoint_net():
-    #NOTE: Weightwise only is all we can do right now IMO
-    net = WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation='sigmoid')
-    # I don't know if this work for aggregaeting. We don't actually need it, though.
-    # net = AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation='sigmoid')
-    net.set_weights(generate_fixpoint_weights())
-    return net
-
-
-def vary(old_weights, e=1.0):
-    new_weights = copy.deepcopy(old_weights)
-    for layer_id, layer in enumerate(new_weights):
-        for cell_id, cell in enumerate(layer):
-            for weight_id, weight in enumerate(cell):
-                if prng() < 0.5:
-                    new_weights[layer_id][cell_id][weight_id] = weight + prng() * e
-                else:
-                    new_weights[layer_id][cell_id][weight_id] = weight - prng() * e
-    return new_weights
-
-
-if __name__ == '__main__':
-    with Experiment('known-fixpoint-variation') as exp:
-        exp.depth = 10
-        exp.trials = 100
-        exp.max_steps = 100
-        exp.epsilon = 1e-4
-        exp.xs = []
-        exp.ys = []
-        exp.zs = []
-        exp.notable_nets = []
-        current_scale = 1.0
-        for _ in range(exp.depth):
-            print('variation scale ' + str(current_scale))
-            for _ in tqdm(range(exp.trials)):
-                net = generate_fixpoint_net().with_params(epsilon=exp.epsilon)
-                net = ParticleDecorator(net)
-                net.set_weights(vary(net.get_weights(), current_scale))
-                time_to_something = 0
-                time_as_fixpoint = 0
-                still_fixpoint = True
-                for _ in range(exp.max_steps):
-                    net.self_attack()
-                    if net.is_zero() or net.is_diverged():
-                        break
-                    if net.is_fixpoint():
-                        if still_fixpoint:
-                            time_as_fixpoint += 1
-                        else:
-                            print('remarkable')
-                            exp.notable_nets += [net.get_weights()]
-                            still_fixpoint = True
-                    else:
-                        still_fixpoint = False
-                    time_to_something += 1
-                exp.xs += [current_scale]
-                # time steps taken to reach divergence or zero (reaching another fix-point is basically never happening)
-                exp.ys += [time_to_something]
-                # time steps still regarded as sthe initial fix-point
-                exp.zs += [time_as_fixpoint]
-                K.backend.clear_session()
-            current_scale /= 10.0
-        for d in range(exp.depth):
-            exp.log('variation 10e-' + str(d))
-            exp.log('avg time to vergence ' + str(avg(exp.ys[d*exp.trials:(d+1) * exp.trials])))
-            exp.log('avg time as fixpoint ' + str(avg(exp.zs[d*exp.trials:(d+1) * exp.trials])))

code/setups/learn_from_soup.py

@@ -1,110 +0,0 @@
-import sys
-import os
-
-# Concat top Level dir to system environmental variables
-sys.path += os.path.join('..', '.')
-
-from typing import Tuple
-
-from experiment import *
-from network import *
-from soup import *
-
-
-import tensorflow.python.keras.backend as K
-
-from statistics import mean
-avg = mean
-
-
-def generate_counters():
-    """
-    Initial build of the counter dict, to store counts.
-
-    :rtype: dict
-    :return: dictionary holding counter for: 'divergent', 'fix_zero', 'fix_sec', 'other'
-    """
-    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
-
-
-def count(counters, soup, notable_nets=None):
-    """
-    Count the occurences ot the types of weight trajectories.
-
-    :param counters:      A counter dictionary.
-    :param soup:          A Soup
-    :param notable_nets:  A list to store and save intersting candidates
-
-    :rtype      Tuple[dict, list]
-    :return:    Both the counter dictionary and the list of interessting nets.
-    """
-
-    notable_nets = notable_nets or list()
-    for net in soup.particles:
-        if net.is_diverged():
-            counters['divergent'] += 1
-        elif net.is_fixpoint():
-            if net.is_zero():
-                counters['fix_zero'] += 1
-            else:
-                counters['fix_other'] += 1
-                # notable_nets += [net]
-        # elif net.is_fixpoint(2):
-        #     counters['fix_sec'] += 1
-        #     notable_nets += [net]
-        else:
-            counters['other'] += 1
-    return counters, notable_nets
-
-
-if __name__ == '__main__':
-
-    with SoupExperiment('learn-from-soup') as exp:
-        exp.soup_size = 10
-        exp.soup_life = 100
-        exp.trials = 10
-        exp.learn_from_severity_values = [10 * i for i in range(11)]
-        exp.epsilon = 1e-4
-        net_generators = []
-        for activation in ['linear']:  # ['sigmoid', 'linear', 'relu']:
-            for use_bias in [False]:
-                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
-                # net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
-                # net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
-
-        all_names = []
-        all_data = []
-        for net_generator_id, net_generator in enumerate(net_generators):
-            xs = []
-            ys = []
-            zs = []
-            notable_nets = []
-            for learn_from_severity in exp.learn_from_severity_values:
-                counters = generate_counters()
-                results = []
-                for _ in tqdm(range(exp.trials)):
-                    soup = Soup(exp.soup_size, lambda net_generator=net_generator,exp=exp: TrainingNeuralNetworkDecorator(net_generator()).with_params(epsilon=exp.epsilon))
-                    soup.with_params(attacking_rate=-1, learn_from_rate=0.1, train=0, learn_from_severity=learn_from_severity)
-                    soup.seed()
-                    name = str(soup.particles[0].net.__class__.__name__) + " activiation='" + str(soup.particles[0].get_keras_params().get('activation')) + "' use_bias=" + str(soup.particles[0].get_keras_params().get('use_bias'))
-                    for time in range(exp.soup_life):
-                        soup.evolve()
-                    count(counters, soup, notable_nets)
-                    K.clear_session()
-
-                xs += [learn_from_severity]
-                ys += [float(counters['fix_zero']) / float(exp.trials)]
-                zs += [float(counters['fix_other']) / float(exp.trials)]
-            all_names += [name]
-            # xs: learn_from_intensity according to exp.learn_from_intensity_values
-            # ys: zero-fixpoints after life time
-            # zs: non-zero-fixpoints after life time
-            all_data += [{'xs':xs, 'ys':ys, 'zs':zs}]
-
-        exp.save(all_names=all_names)
-        exp.save(all_data=all_data)
-        exp.save(soup=soup.without_particles())
-        for exp_id, name in enumerate(all_names):
-            exp.log(all_names[exp_id])
-            exp.log(all_data[exp_id])
-            exp.log('\n')

code/setups/mixed-self-fixpoints.py

@@ -1,100 +0,0 @@
-import sys
-import os
-
-from typing import Tuple
-
-# Concat top Level dir to system environmental variables
-sys.path += os.path.join('..', '.')
-
-from experiment import *
-from network import *
-
-import tensorflow.python.keras.backend as K
-
-
-def generate_counters():
-    """
-    Initial build of the counter dict, to store counts.
-
-    :rtype: dict
-    :return: dictionary holding counter for: 'divergent', 'fix_zero', 'fix_sec', 'other'
-    """
-    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
-
-
-def count(counters, net, notable_nets=None):
-    """
-    Count the occurences ot the types of weight trajectories.
-
-    :param counters:      A counter dictionary.
-    :param net:           A Neural Network
-    :param notable_nets:  A list to store and save intersting candidates
-
-    :rtype      Tuple[dict, list]
-    :return:    Both the counter dictionary and the list of interessting nets.
-    """
-    notable_nets = notable_nets or list()
-    if net.is_diverged():
-        counters['divergent'] += 1
-    elif net.is_fixpoint():
-        if net.is_zero():
-            counters['fix_zero'] += 1
-        else:
-            counters['fix_other'] += 1
-            notable_nets += [net]
-    elif net.is_fixpoint(2):
-        counters['fix_sec'] += 1
-        notable_nets += [net]
-    else:
-        counters['other'] += 1
-    return counters, notable_nets
-
-if __name__ == '__main__':
-
-    with Experiment('mixed-self-fixpoints') as exp:
-        exp.trials = 20
-        exp.selfattacks = 4
-        exp.trains_per_selfattack_values = [50 * i for i in range(11)]
-        exp.epsilon = 1e-4
-        net_generators = []
-        for activation in ['linear']:  # , 'sigmoid', 'relu']:
-            for use_bias in [False]:
-                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
-                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
-                # net_generators += [lambda activation=activation, use_bias=use_bias: FFTNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
-                net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
-
-        all_names = []
-        all_data = []
-
-        for net_generator_id, net_generator in enumerate(net_generators):
-            xs = []
-            ys = []
-            for trains_per_selfattack in exp.trains_per_selfattack_values:
-                counters = generate_counters()
-                notable_nets = []
-                for _ in tqdm(range(exp.trials)):
-                    net = ParticleDecorator(net_generator())
-                    net = TrainingNeuralNetworkDecorator(net).with_params(epsilon=exp.epsilon)
-                    name = str(net.net.net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias=" + str(net.get_keras_params().get('use_bias'))
-                    for selfattack_id in range(exp.selfattacks):
-                        net.self_attack()
-                        for train_id in range(trains_per_selfattack):
-                            loss = net.compiled().train(epoch=selfattack_id*trains_per_selfattack+train_id)
-                        if net.is_diverged() or net.is_fixpoint():
-                            break
-                    count(counters, net, notable_nets)
-                    keras.backend.clear_session()
-                xs += [trains_per_selfattack]
-                ys += [float(counters['fix_zero'] + counters['fix_other']) / float(exp.trials)]
-            all_names += [name]
-            # xs: how many trains per self-attack from exp.trains_per_selfattack_values
-            # ys: average amount of fixpoints found
-            all_data += [{'xs': xs, 'ys': ys}]
-
-        exp.save(all_names=all_names)
-        exp.save(all_data=all_data)
-        for exp_id, name in enumerate(all_names):
-            exp.log(all_names[exp_id])
-            exp.log(all_data[exp_id])
-            exp.log('\n')

code/setups/mixed-soup.py

@@ -1,108 +0,0 @@
-import sys
-import os
-
-# Concat top Level dir to system environmental variables
-sys.path += os.path.join('..', '.')
-
-from typing import Tuple
-
-from experiment import *
-from network import *
-from soup import *
-
-import tensorflow.python.keras.backend as K
-
-
-def generate_counters():
-    """
-    Initial build of the counter dict, to store counts.
-
-    :rtype: dict
-    :return: dictionary holding counter for: 'divergent', 'fix_zero', 'fix_sec', 'other'
-    """
-    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
-
-
-def count(counters, soup, notable_nets=None):
-    """
-    Count the occurences ot the types of weight trajectories.
-
-    :param counters:      A counter dictionary.
-    :param soup:          A Soup
-    :param notable_nets:  A list to store and save intersting candidates
-
-    :rtype      Tuple[dict, list]
-    :return:    Both the counter dictionary and the list of interessting nets.
-    """
-
-    notable_nets = notable_nets or list()
-    for net in soup.particles:
-        if net.is_diverged():
-            counters['divergent'] += 1
-        elif net.is_fixpoint():
-            if net.is_zero():
-                counters['fix_zero'] += 1
-            else:
-                counters['fix_other'] += 1
-                # notable_nets += [net]
-        # elif net.is_fixpoint(2):
-        #     counters['fix_sec'] += 1
-        #     notable_nets += [net]
-        else:
-            counters['other'] += 1
-    return counters, notable_nets
-
-if __name__ == '__main__':
-
-    with Experiment('mixed-soup') as exp:
-        exp.trials = 10
-        exp.soup_size = 10
-        exp.soup_life = 5
-        exp.trains_per_selfattack_values = [10 * i for i in range(11)]
-        exp.epsilon = 1e-4
-        net_generators = []
-        for activation in ['linear']:  # ['linear', 'sigmoid', 'relu']:
-            for use_bias in [False]:
-                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
-                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
-                # net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
-
-        all_names = []
-        all_data = []
-        for net_generator_id, net_generator in enumerate(net_generators):
-            xs = []
-            ys = []
-            zs = []
-            for trains_per_selfattack in exp.trains_per_selfattack_values:
-                counters = generate_counters()
-                notable_nets = []
-                for soup_idx in tqdm(range(exp.trials)):
-                    soup = Soup(exp.soup_size,
-                                lambda net_generator=net_generator, exp=exp: TrainingNeuralNetworkDecorator(
-                                    net_generator()).with_params(epsilon=exp.epsilon))
-                    soup.with_params(attacking_rate=0.1, learn_from_rate=-1, train=trains_per_selfattack,
-                                     learn_from_severity=-1)
-                    soup.seed()
-                    name = str(soup.particles[0].net.__class__.__name__) + " activiation='" + str(
-                        soup.particles[0].get_keras_params().get('activation')) + "' use_bias=" + str(
-                        soup.particles[0].get_keras_params().get('use_bias'))
-                    for _ in range(exp.soup_life):
-                        soup.evolve()
-                    count(counters, soup, notable_nets)
-                    K.clear_session()
-
-                xs += [trains_per_selfattack]
-                ys += [float(counters['fix_zero']) / float(exp.trials)]
-                zs += [float(counters['fix_other']) / float(exp.trials)]
-            all_names += [name]
-            # xs: how many trains per self-attack from exp.trains_per_selfattack_values
-            # ys: average amount of zero-fixpoints found
-            # zs: average amount of non-zero fixpoints
-            all_data += [{'xs': xs, 'ys': ys, 'zs': zs}]
-
-        exp.save(all_names=all_names)
-        exp.save(all_data=all_data)
-        for exp_id, name in enumerate(all_names):
-            exp.log(all_names[exp_id])
-            exp.log(all_data[exp_id])
-            exp.log('\n')

code/setups/network_trajectorys.py

@@ -1,112 +0,0 @@
-import sys
-import os
-
-# Concat top Level dir to system environmental variables
-sys.path += os.path.join('..', '.')
-
-from soup import *
-from experiment import *
-
-
-if __name__ == '__main__':
-    def run_exp(net, prints=False):
-        # INFO Run_ID needs to be more than 0, so that exp stores the trajectories!
-        exp.run_net(net, 100, run_id=run_id + 1)
-        exp.historical_particles[run_id] = net
-        if prints:
-            print("Fixpoint? " + str(net.is_fixpoint()))
-            print("Loss " + str(loss))
-
-    if True:
-        # WeightWise Neural Network
-        with FixpointExperiment(name="weightwise_self_application") as exp:
-            for run_id in tqdm(range(20)):
-                net = ParticleDecorator(WeightwiseNeuralNetwork(width=2, depth=2)
-                                        .with_keras_params(activation='linear'))
-                run_exp(net)
-                K.clear_session()
-            exp.log(exp.counters)
-            exp.save(trajectorys=exp.without_particles())
-
-    if False:
-        # Aggregating Neural Network
-        with FixpointExperiment(name="aggregating_self_application") as exp:
-            for run_id in tqdm(range(10)):
-                net = ParticleDecorator(AggregatingNeuralNetwork(aggregates=4, width=2, depth=2)
-                                        .with_keras_params(activation='linear'))
-                run_exp(net)
-                K.clear_session()
-            exp.log(exp.counters)
-            exp.save(trajectorys=exp.without_particles())
-
-    if False:
-        #FFT Neural Network
-        with FixpointExperiment() as exp:
-            for run_id in tqdm(range(10)):
-                net = ParticleDecorator(FFTNeuralNetwork(aggregates=4, width=2, depth=2)
-                                        .with_keras_params(activation='linear'))
-                run_exp(net)
-                K.clear_session()
-            exp.log(exp.counters)
-            exp.save(trajectorys=exp.without_particles())
-
-    if False:
-        # ok so this works quite realiably
-        with FixpointExperiment(name="weightwise_learning") as exp:
-            for i in range(10):
-                run_count = 100
-                net = TrainingNeuralNetworkDecorator(ParticleDecorator(WeightwiseNeuralNetwork(width=2, depth=2)))
-                net.with_params(epsilon=0.0001).with_keras_params(activation='linear')
-                exp.historical_particles[net.get_uid()] = net
-                for run_id in tqdm(range(run_count+1)):
-                    net.compiled()
-                    loss = net.train(epoch=run_id)
-                    # run_exp(net)
-                    # net.save_state(time=run_id)
-                K.clear_session()
-            exp.save(trajectorys=exp.without_particles())
-
-    if False:
-        # ok so this works quite realiably
-        with FixpointExperiment(name="aggregating_learning") as exp:
-            for i in range(10):
-                run_count = 100
-                net = TrainingNeuralNetworkDecorator(ParticleDecorator(AggregatingNeuralNetwork(4, width=2, depth=2)))
-                net.with_params(epsilon=0.0001).with_keras_params(activation='linear')
-                exp.historical_particles[net.get_uid()] = net
-                for run_id in tqdm(range(run_count+1)):
-                    net.compiled()
-                    loss = net.train(epoch=run_id)
-                    # run_exp(net)
-                    # net.save_state(time=run_id)
-                K.clear_session()
-            exp.save(trajectorys=exp.without_particles())
-
-    if False:
-        # this explodes in our faces completely... NAN everywhere
-        # TODO: Wtf is happening here?
-        with FixpointExperiment() as exp:
-            run_count = 10000
-            net = TrainingNeuralNetworkDecorator(RecurrentNeuralNetwork(width=2, depth=2))\
-                .with_params(epsilon=0.1e-2).with_keras_params(optimizer='sgd', activation='linear')
-            for run_id in tqdm(range(run_count+1)):
-                loss = net.compiled().train()
-                if run_id % 500 == 0:
-                    net.print_weights()
-                    # print(net.apply_to_network(net))
-                    print("Fixpoint? " + str(net.is_fixpoint()))
-                    print("Loss " + str(loss))
-                    print()
-    if False:
-        # and this gets somewhat interesting... we can still achieve non-trivial fixpoints
-        # over multiple applications when training enough in-between
-        with MixedFixpointExperiment() as exp:
-            for run_id in range(10):
-                net = TrainingNeuralNetworkDecorator(FFTNeuralNetwork(2, width=2, depth=2))\
-                    .with_params(epsilon=0.0001, activation='sigmoid')
-                exp.run_net(net, 500, 10)
-
-                net.print_weights()
-
-                print("Fixpoint? " + str(net.is_fixpoint()))
-            exp.log(exp.counters)

code/setups/soup_trajectorys.py

@@ -1,32 +0,0 @@
-import sys
-import os
-
-# Concat top Level dir to system environmental variables
-sys.path += os.path.join('..', '.')
-
-from soup import *
-from experiment import *
-
-
-if __name__ == '__main__':
-    if True:
-        with SoupExperiment("soup") as exp:
-            for run_id in range(1):
-                net_generator = lambda: TrainingNeuralNetworkDecorator(WeightwiseNeuralNetwork(2, 2)) \
-                     .with_keras_params(activation='linear').with_params(epsilon=0.0001)
-                # net_generator = lambda: TrainingNeuralNetworkDecorator(AggregatingNeuralNetwork(4, 2, 2))\
-                #     .with_keras_params(activation='linear')
-                # net_generator = lambda: TrainingNeuralNetworkDecorator(FFTNeuralNetwork(4, 2, 2))\
-                #    .with_keras_params(activation='linear')
-                # net_generator = lambda: RecurrentNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
-                soup = Soup(20, net_generator).with_params(remove_divergent=True, remove_zero=True,
-                                                           train=30,
-                                                           learn_from_rate=-1)
-                soup.seed()
-                for _ in tqdm(range(100)):
-                    soup.evolve()
-                exp.log(soup.count())
-                # you can access soup.historical_particles[particle_uid].states[time_step]['loss']
-                #             or soup.historical_particles[particle_uid].states[time_step]['weights']
-                # from soup.dill
-                exp.save(soup=soup.without_particles())

code/setups/training-fixpoints.py

@@ -1,70 +0,0 @@
-import sys
-import os
-
-# Concat top Level dir to system environmental variables
-sys.path += os.path.join('..', '.')
-
-from experiment import *
-from network import *
-
-import tensorflow.python.keras.backend as K
-
-def generate_counters():
-    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
-
-def count(counters, net, notable_nets=None):
-    notable_nets = notable_nets or list()
-    if net.is_diverged():
-        counters['divergent'] += 1
-    elif net.is_fixpoint():
-        if net.is_zero():
-            counters['fix_zero'] += 1
-        else:
-            counters['fix_other'] += 1
-            notable_nets += [net]
-    elif net.is_fixpoint(2):
-        counters['fix_sec'] += 1
-        notable_nets += [net]
-    else:
-        counters['other'] += 1
-    return counters, notable_nets
-
-
-if __name__ == '__main__':
-
-    with Experiment('training_fixpoint') as exp:
-        exp.trials = 50
-        exp.run_count = 1000
-        exp.epsilon = 1e-4
-        net_generators = []
-        for activation in ['linear']:  # , 'sigmoid', 'relu']:
-            for use_bias in [False]:
-                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
-                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
-                net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
-        all_counters = []
-        all_notable_nets = []
-        all_names = []
-        for net_generator_id, net_generator in enumerate(net_generators):
-            counters = generate_counters()
-            notable_nets = []
-            for _ in tqdm(range(exp.trials)):
-                net = ParticleDecorator(net_generator())
-                net = TrainingNeuralNetworkDecorator(net).with_params(epsilon=exp.epsilon)
-                name = str(net.net.net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias=" + str(net.get_keras_params().get('use_bias'))
-                for run_id in range(exp.run_count):
-                    loss = net.compiled().train(epoch=run_id+1)
-                count(counters, net, notable_nets)
-            all_counters += [counters]
-            all_notable_nets += [notable_nets]
-            all_names += [name]
-            K.clear_session()
-        exp.save(all_counters=all_counters)
-        exp.save(trajectorys=exp.without_particles())
-        # net types reached in the end
-        # exp.save(all_notable_nets=all_notable_nets)
-        exp.save(all_names=all_names) #experiment setups
-        for exp_id, counter in enumerate(all_counters):
-            exp.log(all_names[exp_id])
-            exp.log(all_counters[exp_id])
-            exp.log('\n')

code/soup.py

@@ -1,137 +0,0 @@
-import random
-
-from network import *
-
-
-def prng():
-    return random.random()
-
-
-class Soup(object):
-    
-    def __init__(self, size, generator, **kwargs):
-        self.size = size
-        self.generator = generator
-        self.particles = []
-        self.historical_particles = {}
-        self.params = dict(attacking_rate=0.1, learn_from_rate=0.1, train=0, learn_from_severity=1)
-        self.params.update(kwargs)
-        self.time = 0
-
-    def __copy__(self):
-        copy_ = Soup(self.size, self.generator, **self.params)
-        copy_.__dict__ = {attr: self.__dict__[attr] for attr in self.__dict__ if
-                          attr not in ['particles', 'historical_particles']}
-        return copy_
-
-    def without_particles(self):
-        self_copy = copy.copy(self)
-        # self_copy.particles = [particle.states for particle in self.particles]
-        self_copy.historical_particles = {key: val.states for key, val in self.historical_particles.items()}
-        return self_copy
-
-    def with_params(self, **kwargs):
-        self.params.update(kwargs)
-        return self
-    
-    def generate_particle(self):
-        new_particle = ParticleDecorator(self.generator())
-        self.historical_particles[new_particle.get_uid()] = new_particle
-        return new_particle
-        
-    def get_particle(self, uid, otherwise=None):
-        return self.historical_particles.get(uid, otherwise)
-    
-    def seed(self):
-        self.particles = []
-        for _ in range(self.size):
-            self.particles += [self.generate_particle()]
-        return self       
-    
-    def evolve(self, iterations=1):
-        for _ in range(iterations):
-            self.time += 1
-            for particle_id, particle in enumerate(self.particles):
-                description = {'time': self.time}
-                if prng() < self.params.get('attacking_rate'):
-                    other_particle_id = int(prng() * len(self.particles))
-                    other_particle = self.particles[other_particle_id]
-                    particle.attack(other_particle)
-                    description['action'] = 'attacking'
-                    description['counterpart'] = other_particle.get_uid()
-                if prng() < self.params.get('learn_from_rate'):
-                    other_particle_id = int(prng() * len(self.particles))
-                    other_particle = self.particles[other_particle_id]
-                    for _ in range(self.params.get('learn_from_severity', 1)):
-                        particle.learn_from(other_particle)
-                    description['action'] = 'learn_from'
-                    description['counterpart'] = other_particle.get_uid()
-                for _ in range(self.params.get('train', 0)):
-                    particle.compiled()
-                    # callbacks on save_state are broken for TrainingNeuralNetwork
-                    loss = particle.train(store_states=False)
-                    description['fitted'] = self.params.get('train', 0)
-                    description['loss'] = loss
-                    description['action'] = 'train_self'
-                    description['counterpart'] = None
-                if self.params.get('remove_divergent') and particle.is_diverged():
-                    new_particle = self.generate_particle()
-                    self.particles[particle_id] = new_particle
-                    description['action'] = 'divergent_dead'
-                    description['counterpart'] = new_particle.get_uid()
-                if self.params.get('remove_zero') and particle.is_zero():
-                    new_particle = self.generate_particle()
-                    self.particles[particle_id] = new_particle
-                    description['action'] = 'zweo_dead'
-                    description['counterpart'] = new_particle.get_uid()
-                particle.save_state(**description)
-
-    def count(self):
-        counters = dict(divergent=0, fix_zero=0, fix_other=0, fix_sec=0, other=0)
-        for particle in self.particles:
-            if particle.is_diverged():
-                counters['divergent'] += 1
-            elif particle.is_fixpoint():
-                if particle.is_zero():
-                    counters['fix_zero'] += 1
-                else:
-                    counters['fix_other'] += 1
-            elif particle.is_fixpoint(2):
-                counters['fix_sec'] += 1
-            else:
-                counters['other'] += 1
-        return counters
-        
-    def print_all(self):
-        for particle in self.particles:
-            particle.print_weights()
-            print(particle.is_fixpoint())
-
-
-if __name__ == '__main__':
-    if True:
-        net_generator = lambda: WeightwiseNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
-        soup_generator = Soup(100, net_generator).with_params(remove_divergent=True, remove_zero=True)
-        exp = SoupExperiment()
-        exp.run_exp(net_generator, 1000, soup_generator, 1, False)
-
-        # net_generator = lambda: FFTNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
-        # net_generator = lambda: AggregatingNeuralNetwork(4, 2, 2).with_keras_params(activation='sigmoid')\
-        # .with_params(shuffler=AggregatingNeuralNetwork.shuffle_random)
-        # net_generator = lambda: RecurrentNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
-
-    if True:
-        net_generator = lambda: TrainingNeuralNetworkDecorator(WeightwiseNeuralNetwork(2, 2)) \
-            .with_keras_params(activation='linear').with_params(epsilon=0.0001)
-        soup_generator = lambda: Soup(100, net_generator).with_params(remove_divergent=True, remove_zero=True, train=20)
-        exp = SoupExperiment(name="soup")
-
-        exp.run_exp(net_generator, 100, soup_generator, 1, False)
-
-        # net_generator = lambda: TrainingNeuralNetworkDecorator(AggregatingNeuralNetwork(4, 2, 2))
-        # .with_keras_params(activation='linear')\
-        # .with_params(shuffler=AggregatingNeuralNetwork.shuffle_random)
-        # net_generator = lambda: TrainingNeuralNetworkDecorator(FFTNeuralNetwork(4, 2, 2))\
-        #     .with_keras_params(activation='linear')\
-        # .with_params(shuffler=AggregatingNeuralNetwork.shuffle_random)
-        # net_generator = lambda: RecurrentNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()

experiments/helpers.py

@@ -0,0 +1,59 @@
+""" -------------------------------- Methods for summarizing the experiments --------------------------------- """
+from pathlib import Path
+
+from visualization import line_chart_fixpoints, bar_chart_fixpoints
+
+
+def summary_fixpoint_experiment(runs, population_size, epochs, experiments, net_learning_rate, directory,
+                                summary_pre_title):
+    avg_fixpoint_counters = {
+        "avg_identity_func": 0,
+        "avg_divergent": 0,
+        "avg_fix_zero": 0,
+        "avg_fix_weak": 0,
+        "avg_fix_sec": 0,
+        "avg_other_func": 0
+    }
+
+    for i in range(len(experiments)):
+        fixpoint_counters = experiments[i].fixpoint_counters
+
+        avg_fixpoint_counters["avg_identity_func"] += fixpoint_counters["identity_func"]
+        avg_fixpoint_counters["avg_divergent"] += fixpoint_counters["divergent"]
+        avg_fixpoint_counters["avg_fix_zero"] += fixpoint_counters["fix_zero"]
+        avg_fixpoint_counters["avg_fix_weak"] += fixpoint_counters["fix_weak"]
+        avg_fixpoint_counters["avg_fix_sec"] += fixpoint_counters["fix_sec"]
+        avg_fixpoint_counters["avg_other_func"] += fixpoint_counters["other_func"]
+
+    # Calculating the average for each fixpoint
+    avg_fixpoint_counters.update((x, y / len(experiments)) for x, y in avg_fixpoint_counters.items())
+
+    # Checking where the data is coming from to have a relevant title in the plot.
+    if summary_pre_title not in ["ST", "SA", "soup", "mixed", "robustness"]:
+        summary_pre_title = ""
+
+    # Plotting the summary
+    source_checker = "summary"
+    exp_details = f"{summary_pre_title}: {runs} runs & {epochs} epochs each."
+    bar_chart_fixpoints(avg_fixpoint_counters, population_size, directory, net_learning_rate, exp_details,
+                        source_checker)
+
+
+def summary_fixpoint_percentage(runs, epochs, fixpoints_percentages, ST_steps, SA_steps, directory_name,
+                                population_size):
+    fixpoints_percentages = [round(fixpoints_percentages[i] / runs, 1) for i in range(len(fixpoints_percentages))]
+
+    # Plotting summary
+    if "soup" in directory_name:
+        line_chart_fixpoints(fixpoints_percentages, epochs / ST_steps, ST_steps, SA_steps, directory_name,
+                             population_size)
+    else:
+        line_chart_fixpoints(fixpoints_percentages, epochs, ST_steps, SA_steps, directory_name, population_size)
+
+
+""" -------------------------------------------- Miscellaneous --------------------------------------------------- """
+
+
+def check_folder(experiment_folder: str):
+    exp_path = Path('experiments') / experiment_folder
+    exp_path.mkdir(parents=True, exist_ok=True)

experiments/meta_task_small_utility.py

@@ -0,0 +1,38 @@
+import torch
+from torch.utils.data import Dataset
+
+DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+
+class AddTaskDataset(Dataset):
+    def __init__(self, length=int(1e3)):
+        super().__init__()
+        self.length = length
+
+    def __len__(self):
+        return self.length
+
+    def __getitem__(self, _):
+        ab = torch.randn(size=(2,)).to(torch.float32)
+        return ab, ab.sum(axis=-1, keepdims=True)
+
+
+def train_task(model, optimizer, loss_func, btch_x, btch_y) -> (dict, torch.Tensor):
+    # Zero your gradients for every batch!
+    optimizer.zero_grad()
+    btch_x, btch_y = btch_x.to(DEVICE), btch_y.to(DEVICE)
+    y_prd = model(btch_x)
+
+    loss = loss_func(y_prd, btch_y.to(torch.float))
+    loss.backward()
+
+    # Adjust learning weights
+    optimizer.step()
+
+    stp_log = dict(Metric='Task Loss', Score=loss.item())
+
+    return stp_log, y_prd
+
+
+if __name__ == '__main__':
+    raise(NotImplementedError('Get out of here'))

experiments/meta_task_utility.py

@@ -0,0 +1,405 @@
+import pickle
+import re
+import shutil
+from collections import defaultdict
+from pathlib import Path
+import pandas as pd
+import numpy as np
+import torch
+import torchmetrics
+from matplotlib import pyplot as plt
+import seaborn as sns
+from torch.utils.data import Dataset
+
+from tqdm import tqdm
+
+from functionalities_test import test_for_fixpoints, FixTypes as ft
+from sanity_check_weights import test_weights_as_model, extract_weights_from_model
+
+WORKER = 10
+BATCHSIZE = 500
+
+DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+DATA_PATH = Path('data')
+DATA_PATH.mkdir(exist_ok=True, parents=True)
+
+PALETTE = sns.color_palette()
+PALETTE.insert(0, PALETTE.pop(1))  # Orange First
+
+FINAL_CHECKPOINT_NAME = f'trained_model_ckpt_FINAL.tp'
+
+
+class AddGaussianNoise(object):
+    def __init__(self, ratio=1e-4):
+        self.ratio = ratio
+
+    def __call__(self, tensor: torch.Tensor):
+        return tensor + (torch.randn_like(tensor, device=tensor.device) * self.ratio)
+
+    def __repr__(self):
+        return self.__class__.__name__ + f'(ratio={self.ratio}'
+
+
+class ToFloat:
+
+    def __init__(self):
+        pass
+
+    def __call__(self, x):
+        return x.to(torch.float32)
+
+
+class AddTaskDataset(Dataset):
+    def __init__(self, length=int(5e5)):
+        super().__init__()
+        self.length = length
+        self.prng = np.random.default_rng()
+
+    def __len__(self):
+        return self.length
+
+    def __getitem__(self, _):
+        ab = self.prng.normal(size=(2,)).astype(np.float32)
+        return ab, ab.sum(axis=-1, keepdims=True)
+
+
+def set_checkpoint(model, out_path, epoch_n, final_model=False):
+    if not final_model:
+        epoch_n = str(epoch_n)
+        ckpt_path = Path(out_path) / 'ckpt' / f'{epoch_n.zfill(4)}_model_ckpt.tp'
+    else:
+        if isinstance(epoch_n, str):
+            ckpt_path = Path(out_path) / f'{Path(FINAL_CHECKPOINT_NAME).stem}_{epoch_n}.tp'
+        else:
+            ckpt_path = Path(out_path) / FINAL_CHECKPOINT_NAME
+    ckpt_path.parent.mkdir(exist_ok=True, parents=True)
+
+    torch.save(model, ckpt_path, pickle_protocol=pickle.HIGHEST_PROTOCOL)
+    py_store_path = Path(out_path) / 'exp_py.txt'
+    if not py_store_path.exists():
+        shutil.copy(__file__, py_store_path)
+    return ckpt_path
+
+
+# noinspection PyProtectedMember
+def validate(checkpoint_path, valid_loader, metric_class=torchmetrics.Accuracy):
+    checkpoint_path = Path(checkpoint_path)
+
+    # initialize metric
+    validmetric = metric_class()
+    model = torch.load(checkpoint_path, map_location=DEVICE).eval()
+
+    with tqdm(total=len(valid_loader), desc='Validation Run: ') as pbar:
+        for idx, (valid_batch_x, valid_batch_y) in enumerate(valid_loader):
+            valid_batch_x, valid_batch_y = valid_batch_x.to(DEVICE), valid_batch_y.to(DEVICE)
+            y_valid = model(valid_batch_x)
+
+            # metric on current batch
+            measure = validmetric(y_valid.cpu(), valid_batch_y.cpu())
+            pbar.set_postfix_str(f'Measure: {measure}')
+            pbar.update()
+
+    # metric on all batches using custom accumulation
+    measure = validmetric.compute()
+    tqdm.write(f"Avg. {validmetric._get_name()} on all data: {measure}")
+    return measure
+
+
+def new_storage_df(identifier, weight_count):
+    if identifier == 'train':
+        return pd.DataFrame(columns=['Epoch', 'Batch', 'Metric', 'Score'])
+    elif identifier == 'weights':
+        return pd.DataFrame(columns=['Epoch', 'Weight', *(f'weight_{x}' for x in range(weight_count))])
+
+
+def checkpoint_and_validate(model, valid_loader, out_path, epoch_n, keep_n=5, final_model=False,
+                            validation_metric=torchmetrics.Accuracy):
+    out_path = Path(out_path)
+    ckpt_path = set_checkpoint(model, out_path, epoch_n, final_model=final_model)
+    # Clean up Checkpoints
+    if keep_n > 0:
+        all_ckpts = sorted(list(ckpt_path.parent.iterdir()))
+        while len(all_ckpts) > keep_n:
+            all_ckpts.pop(0).unlink()
+    elif keep_n == 0:
+        pass
+    else:
+        raise ValueError(f'"keep_n" cannot be negative, but was: {keep_n}')
+
+    result = validate(ckpt_path, valid_loader, metric_class=validation_metric)
+    return result
+
+
+def plot_training_particle_types(path_to_dataframe):
+    plt.close('all')
+    plt.clf()
+    # load from Drive
+    df = pd.read_csv(path_to_dataframe, index_col=False).sort_values('Metric')
+    # Set up figure
+    fig, ax = plt.subplots()  # initializes figure and plots
+    data = df.loc[df['Metric'].isin(ft.all_types())]
+    fix_types = data['Metric'].unique()
+    data = data.pivot(index='Epoch', columns='Metric', values='Score').reset_index().fillna(0)
+    _ = plt.stackplot(data['Epoch'], *[data[fixtype] for fixtype in fix_types],
+                      labels=fix_types.tolist(), colors=PALETTE)
+
+    ax.set(ylabel='Particle Count', xlabel='Epoch')
+    ax.yaxis.get_major_locator().set_params(integer=True)
+    # ax.set_title('Particle Type Count')
+
+    fig.legend(loc="center right", title='Particle Type', bbox_to_anchor=(0.85, 0.5))
+    plt.tight_layout()
+    plt.savefig(Path(path_to_dataframe.parent / 'training_particle_type_lp.png'), dpi=300)
+
+
+def plot_training_result(path_to_dataframe, metric_name='Accuracy', plot_name=None):
+    plt.clf()
+    # load from Drive
+    df = pd.read_csv(path_to_dataframe, index_col=False).sort_values('Metric')
+
+    # Check if this is a single lineplot or if aggregated
+    group = ['Epoch', 'Metric']
+    if 'Seed' in df.columns:
+        group.append('Seed')
+
+    # Set up figure
+    fig, ax1 = plt.subplots()  # initializes figure and plots
+    ax2 = ax1.twinx()  # applies twinx to ax2, which is the second y-axis.
+
+    # plots the first set of data
+    data = df[(df['Metric'] == 'Task Loss') | (df['Metric'] == 'Self Train Loss')].groupby(['Epoch', 'Metric']).mean()
+    grouped_for_lineplot = data.groupby(group).mean()
+    palette_len_1 = len(grouped_for_lineplot.droplevel(0).reset_index().Metric.unique())
+
+    sns.lineplot(data=grouped_for_lineplot, x='Epoch', y='Score', hue='Metric',
+                 palette=PALETTE[:palette_len_1], ax=ax1, ci='sd')
+
+    # plots the second set of data
+    data = df[(df['Metric'] == f'Test {metric_name}') | (df['Metric'] == f'Train {metric_name}')]
+    palette_len_2 = len(data.Metric.unique())
+    sns.lineplot(data=data, x='Epoch', y='Score', hue='Metric',
+                 palette=PALETTE[palette_len_1:palette_len_2+palette_len_1], ci='sd')
+
+    ax1.set(yscale='log', ylabel='Losses')
+    # ax1.set_title('Training Lineplot')
+    ax2.set(ylabel=metric_name)
+    if metric_name != 'Accuracy':
+        ax2.set(yscale='log')
+
+    fig.legend(loc="center right", title='Metric', bbox_to_anchor=(0.85, 0.5))
+    for ax in [ax1, ax2]:
+        if legend := ax.get_legend():
+            legend.remove()
+    plt.tight_layout()
+    plt.savefig(Path(path_to_dataframe.parent / ('training_lineplot.png' if plot_name is None else plot_name)), dpi=300)
+
+
+def plot_network_connectivity_by_fixtype(path_to_trained_model):
+    m = torch.load(path_to_trained_model, map_location=DEVICE).eval()
+    # noinspection PyProtectedMember
+    particles = list(m.particles)
+    df = pd.DataFrame(columns=['Type', 'Layer', 'Neuron', 'Name'])
+
+    for prtcl in particles:
+        l, c, w = [float(x) for x in re.sub("[^0-9|_]", "", prtcl.name).split('_')]
+        df.loc[df.shape[0]] = (prtcl.is_fixpoint, l-1, w, prtcl.name)
+        df.loc[df.shape[0]] = (prtcl.is_fixpoint, l, c, prtcl.name)
+    for layer in list(df['Layer'].unique()):
+        # Rescale
+        divisor = df.loc[(df['Layer'] == layer), 'Neuron'].max()
+        df.loc[(df['Layer'] == layer), 'Neuron'] /= divisor
+
+    tqdm.write(f'Connectivity Data gathered')
+    df = df.sort_values('Type')
+    n = 0
+    for fixtype in ft.all_types():
+        if df[df['Type'] == fixtype].shape[0] > 0:
+            plt.clf()
+            ax = sns.lineplot(y='Neuron', x='Layer', hue='Name', data=df[df['Type'] == fixtype],
+                              legend=False, estimator=None, lw=1)
+            _ = sns.lineplot(y=[0, 1], x=[-1, df['Layer'].max()], legend=False, estimator=None, lw=0)
+            ax.set_title(fixtype)
+            ax.yaxis.get_major_locator().set_params(integer=True)
+            ax.xaxis.get_major_locator().set_params(integer=True)
+            ax.set_ylabel('Normalized Neuron Position (1/n)')  # XAXIS Label
+            lines = ax.get_lines()
+            for line in lines:
+                line.set_color(PALETTE[n])
+            plt.savefig(Path(path_to_trained_model.parent / f'net_connectivity_{fixtype}.png'), dpi=300)
+            tqdm.write(f'Connectivity plottet: {fixtype} - n = {df[df["Type"] == fixtype].shape[0] // 2}')
+            n += 1
+        else:
+            # tqdm.write(f'No Connectivity {fixtype}')
+            pass
+
+
+# noinspection PyProtectedMember
+def run_particle_dropout_test(model_path, valid_loader, metric_class=torchmetrics.Accuracy):
+    diff_store_path = model_path.parent / 'diff_store.csv'
+    latest_model = torch.load(model_path, map_location=DEVICE).eval()
+    prtcl_dict = defaultdict(lambda: 0)
+    _ = test_for_fixpoints(prtcl_dict, list(latest_model.particles))
+    tqdm.write(str(dict(prtcl_dict)))
+    diff_df = pd.DataFrame(columns=['Particle Type', metric_class()._get_name(), 'Diff'])
+
+    acc_pre = validate(model_path, valid_loader, metric_class=metric_class).item()
+    diff_df.loc[diff_df.shape[0]] = ('All Organism', acc_pre, 0)
+
+    for fixpoint_type in ft.all_types():
+        new_model = torch.load(model_path, map_location=DEVICE).eval().replace_with_zero(fixpoint_type)
+        if [x for x in new_model.particles if x.is_fixpoint == fixpoint_type]:
+            new_ckpt = set_checkpoint(new_model, model_path.parent, fixpoint_type, final_model=True)
+            acc_post = validate(new_ckpt, valid_loader, metric_class=metric_class).item()
+            acc_diff = abs(acc_post - acc_pre)
+            tqdm.write(f'Zero_ident diff = {acc_diff}')
+            diff_df.loc[diff_df.shape[0]] = (fixpoint_type, acc_post, acc_diff)
+
+    diff_df.to_csv(diff_store_path, mode='w', header=True, index=False)
+    return diff_store_path
+
+
+# noinspection PyProtectedMember
+def plot_dropout_stacked_barplot(mdl_path, diff_store_path, metric_class=torchmetrics.Accuracy):
+    metric_name = metric_class()._get_name()
+    diff_df = pd.read_csv(diff_store_path).sort_values('Particle Type')
+    particle_dict = defaultdict(lambda: 0)
+    latest_model = torch.load(mdl_path, map_location=DEVICE).eval()
+    _ = test_for_fixpoints(particle_dict, list(latest_model.particles))
+    particle_dict = dict(particle_dict)
+    sorted_particle_dict = dict(sorted(particle_dict.items()))
+    tqdm.write(str(sorted_particle_dict))
+    plt.clf()
+    fig, ax = plt.subplots(ncols=2)
+    colors = PALETTE.copy()
+    colors.insert(0, colors.pop(-1))
+    palette_len = len(diff_df['Particle Type'].unique())
+    _ = sns.barplot(data=diff_df, y=metric_name, x='Particle Type', ax=ax[0], palette=colors[:palette_len], ci=None)
+
+    ax[0].set_title(f'{metric_name} after particle dropout')
+    # ax[0].set_xlabel('Particle Type')  # XAXIS Label
+    ax[0].set_xticklabels(ax[0].get_xticklabels(), rotation=30)
+
+    ax[1].pie(sorted_particle_dict.values(), labels=sorted_particle_dict.keys(),
+              colors=PALETTE[:len(sorted_particle_dict)])
+    ax[1].set_title('Particle Count')
+
+    plt.tight_layout()
+    plt.savefig(Path(diff_store_path.parent / 'dropout_stacked_barplot.png'), dpi=300)
+
+
+def run_particle_dropout_and_plot(model_path, valid_loader, metric_class=torchmetrics.Accuracy):
+    diff_store_path = run_particle_dropout_test(model_path, valid_loader=valid_loader, metric_class=metric_class)
+    plot_dropout_stacked_barplot(model_path, diff_store_path, metric_class=metric_class)
+
+
+def flat_for_store(parameters):
+    return (x.item() for y in parameters for x in y.detach().flatten())
+
+
+def train_self_replication(model, st_stps, **kwargs) -> dict:
+    self_train_loss = model.combined_self_train(st_stps, **kwargs)
+    # noinspection PyUnboundLocalVariable
+    stp_log = dict(Metric='Self Train Loss', Score=self_train_loss.item())
+    return stp_log
+
+
+def train_task(model, optimizer, loss_func, btch_x, btch_y) -> (dict, torch.Tensor):
+    # Zero your gradients for every batch!
+    optimizer.zero_grad()
+    btch_x, btch_y = btch_x.to(DEVICE), btch_y.to(DEVICE)
+    y_prd = model(btch_x)
+
+    loss = loss_func(y_prd, btch_y.to(torch.long))
+    loss.backward()
+
+    # Adjust learning weights
+    optimizer.step()
+
+    stp_log = dict(Metric='Task Loss', Score=loss.item())
+
+    return stp_log, y_prd
+
+
+def highlight_fixpoints_vs_mnist_mean(mdl_path, dataloader):
+    latest_model = torch.load(mdl_path, map_location=DEVICE).eval()
+    activation_vector = torch.as_tensor([[0, 0, 0, 0, 1]], dtype=torch.float32, device=DEVICE)
+    binary_images = []
+    real_images = []
+    with torch.no_grad():
+        # noinspection PyProtectedMember
+        for cell in latest_model._meta_layer_first.meta_cell_list:
+            cell_image_binary = torch.zeros((len(cell.meta_weight_list)), device=DEVICE)
+            cell_image_real = torch.zeros((len(cell.meta_weight_list)), device=DEVICE)
+            for idx, particle in enumerate(cell.particles):
+                if particle.is_fixpoint == ft.identity_func:
+                    cell_image_binary[idx] += 1
+                    cell_image_real[idx] = particle(activation_vector).abs().squeeze().item()
+            binary_images.append(cell_image_binary.reshape((15, 15)))
+            real_images.append(cell_image_real.reshape((15, 15)))
+
+        binary_images = torch.stack(binary_images)
+        real_images = torch.stack(real_images)
+
+        binary_image = torch.sum(binary_images, keepdim=True, dim=0)
+        real_image = torch.sum(real_images, keepdim=True, dim=0)
+
+    mnist_images = [x for x, _ in dataloader]
+    mnist_mean = torch.cat(mnist_images).reshape(10000, 15, 15).abs().sum(dim=0)
+
+    fig, axs = plt.subplots(1, 3)
+
+    for idx, (image, title) in enumerate(zip([binary_image, real_image, mnist_mean],
+                                           ["Particle Count", "Particle Value", "MNIST mean"])):
+        img = axs[idx].imshow(image.squeeze().detach().cpu())
+        img.axes.axis('off')
+        img.axes.set_title('Random Noise')
+
+    plt.tight_layout()
+    plt.savefig(mdl_path.parent / 'heatmap.png', dpi=300)
+    plt.clf()
+    plt.close('all')
+
+
+def plot_training_results_over_n_seeds(exp_path, df_train_store_name='train_store.csv', metric_name='Accuracy'):
+    combined_df_store_path = exp_path / f'comb_train_{exp_path.stem[:-1]}n.csv'
+    # noinspection PyUnboundLocalVariable
+    found_train_stores = exp_path.rglob(df_train_store_name)
+    train_dfs = []
+    for found_train_store in found_train_stores:
+        train_store_df = pd.read_csv(found_train_store, index_col=False)
+        train_store_df['Seed'] = int(found_train_store.parent.name)
+        train_dfs.append(train_store_df)
+    combined_train_df = pd.concat(train_dfs)
+    combined_train_df.to_csv(combined_df_store_path, index=False)
+    plot_training_result(combined_df_store_path, metric_name=metric_name,
+                         plot_name=f"{combined_df_store_path.stem}.png"
+                         )
+    plt.clf()
+    plt.close('all')
+
+
+def sanity_weight_swap(exp_path, dataloader, metric_class=torchmetrics.Accuracy):
+    # noinspection PyProtectedMember
+    metric_name = metric_class()._get_name()
+    found_models = exp_path.rglob(f'*{FINAL_CHECKPOINT_NAME}')
+    df = pd.DataFrame(columns=['Seed', 'Model', metric_name])
+    for model_idx, found_model in enumerate(found_models):
+        model = torch.load(found_model, map_location=DEVICE).eval()
+        weights = extract_weights_from_model(model)
+
+        results = test_weights_as_model(model, weights, dataloader, metric_class=metric_class)
+        for model_name, measurement in results.items():
+            df.loc[df.shape[0]] = (model_idx, model_name, measurement)
+        df.loc[df.shape[0]] = (model_idx, 'Difference', np.abs(np.subtract(*results.values())))
+
+    df.to_csv(exp_path / 'sanity_weight_swap.csv', index=False)
+    _ = sns.boxplot(data=df, x='Model', y=metric_name)
+    plt.tight_layout()
+    plt.savefig(exp_path / 'sanity_weight_swap.png', dpi=300)
+    plt.clf()
+    plt.close('all')
+
+
+if __name__ == '__main__':
+    raise NotImplementedError('Test this here!!!')

experiments/robustness_tester.py

@@ -0,0 +1,118 @@
+import pandas as pd
+import torch
+import random
+import copy
+
+from tqdm import tqdm
+from functionalities_test import (is_identity_function, is_zero_fixpoint, test_for_fixpoints, is_divergent,
+                                  FixTypes as FT)
+from network import Net
+from torch.nn import functional as F
+import seaborn as sns
+from matplotlib import pyplot as plt
+
+
+def prng():
+    return random.random()
+
+
+def generate_perfekt_synthetic_fixpoint_weights():
+    return torch.tensor([[1.0], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0],
+                         [1.0], [0.0], [0.0], [0.0],
+                         [1.0], [0.0]
+                         ], dtype=torch.float32)
+
+PALETTE = 10 * (
+    "#377eb8",
+    "#4daf4a",
+    "#984ea3",
+    "#e41a1c",
+    "#ff7f00",
+    "#a65628",
+    "#f781bf",
+    "#888888",
+    "#a6cee3",
+    "#b2df8a",
+    "#cab2d6",
+    "#fb9a99",
+    "#fdbf6f",
+)
+
+
+def test_robustness(model_path, noise_levels=10, seeds=10, log_step_size=10):
+    model = torch.load(model_path, map_location='cpu')
+    networks = [x for x in model.particles if x.is_fixpoint == FT.identity_func]
+    time_to_vergence = [[0 for _ in range(noise_levels)] for _ in range(len(networks))]
+    time_as_fixpoint = [[0 for _ in range(noise_levels)] for _ in range(len(networks))]
+    row_headers = []
+    df = pd.DataFrame(columns=['setting', 'Noise Level', 'Self Train Steps', 'absolute_loss',
+                               'Time to convergence', 'Time as fixpoint'])
+    with tqdm(total=(seeds * noise_levels * len(networks)), desc='Per Particle Robustness') as pbar:
+        for setting, fixpoint in enumerate(networks):  # 1 / n
+            row_headers.append(fixpoint.name)
+            for seed in range(seeds):  # n / 1
+                for noise_level in range(noise_levels):
+                    steps = 0
+                    clone = Net(fixpoint.input_size, fixpoint.hidden_size, fixpoint.out_size,
+                                f"{fixpoint.name}_clone_noise_1e-{noise_level}")
+                    clone.load_state_dict(copy.deepcopy(fixpoint.state_dict()))
+                    clone = clone.apply_noise(pow(10, -noise_level))
+
+                    while not is_zero_fixpoint(clone) and not is_divergent(clone):
+                        # -> before
+                        clone_weight_pre_application = clone.input_weight_matrix()
+                        target_data_pre_application = clone.create_target_weights(clone_weight_pre_application)
+
+                        clone.self_application(1, log_step_size)
+                        time_to_vergence[setting][noise_level] += 1
+                        # -> after
+                        clone_weight_post_application = clone.input_weight_matrix()
+                        target_data_post_application = clone.create_target_weights(clone_weight_post_application)
+
+                        absolute_loss = F.l1_loss(target_data_pre_application, target_data_post_application).item()
+
+                        if is_identity_function(clone):
+                            time_as_fixpoint[setting][noise_level] += 1
+                            # When this raises a Type Error, we found a second order fixpoint!
+                        steps += 1
+
+                        df.loc[df.shape[0]] = [f'{setting}_{seed}', fr'$\mathregular{{10^{{-{noise_level}}}}}$',
+                                               steps, absolute_loss,
+                                               time_to_vergence[setting][noise_level],
+                                               time_as_fixpoint[setting][noise_level]]
+                    pbar.update(1)
+
+    # Get the measuremts at the highest time_time_to_vergence
+    df_sorted = df.sort_values('Self Train Steps', ascending=False).drop_duplicates(['setting', 'Noise Level'])
+    df_melted = df_sorted.reset_index().melt(id_vars=['setting', 'Noise Level', 'Self Train Steps'],
+                                             value_vars=['Time to convergence', 'Time as fixpoint'],
+                                             var_name="Measurement",
+                                             value_name="Steps").sort_values('Noise Level')
+
+    df_melted.to_csv(model_path.parent / 'robustness_boxplot.csv', index=False)
+
+    # Plotting
+    # plt.rcParams.update({
+    #    "text.usetex": True,
+    #    "font.family": "sans-serif",
+    #    "font.size": 12,
+    #    "font.weight": 'bold',
+    #    "font.sans-serif": ["Helvetica"]})
+    plt.clf()
+    sns.set(style='whitegrid', font_scale=1)
+    _ = sns.boxplot(data=df_melted, y='Steps', x='Noise Level', hue='Measurement', palette=PALETTE)
+    plt.tight_layout()
+
+    # sns.set(rc={'figure.figsize': (10, 50)})
+    # bx = sns.catplot(data=df[df['absolute_loss'] < 1], y='absolute_loss', x='application_step', kind='box',
+    #                  col='noise_level', col_wrap=3, showfliers=False)
+
+    filename = f"robustness_boxplot.png"
+    filepath = model_path.parent / filename
+    plt.savefig(str(filepath))
+    plt.close('all')
+    return time_as_fixpoint, time_to_vergence
+
+
+if __name__ == "__main__":
+    raise NotImplementedError('Get out of here!')