Delete New Text Document.txt

Step 4 - Aggregating Neural Networks
Step 5 - Training Neural Networks

.gitignore

@@ -1 +1,572 @@
-/output/
+# Created by https://www.gitignore.io/api/data,linux,macos,python,windows,pycharm,database,jupyternotebook
+# Edit at https://www.gitignore.io/?templates=data,linux,macos,python,windows,pycharm,database,jupyternotebook
+
+
+### Local Datasets ###
+/experiments
+/setups/experiments
+
+### Data ###
+*.csv
+*.dat
+*.efx
+*.gbr
+*.key
+*.pps
+*.ppt
+*.pptx
+*.sdf
+*.tax2010
+*.vcf
+*.xml
+
+### Database ###
+*.accdb
+*.db
+*.dbf
+*.mdb
+*.pdb
+*.sqlite3
+
+### JupyterNotebook ###
+.ipynb_checkpoints
+*/.ipynb_checkpoints/*
+
+# Remove previous ipynb_checkpoints
+#   git rm -r .ipynb_checkpoints/
+#
+
+### Linux ###
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### PyCharm Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+.idea/sonarlint
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+### Python Patch ###
+.venv/
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# pycharm
+.idea/
+
+#######################################
+#### Tex related
+## Core latex/pdflatex auxiliary files:
+*.aux
+*.lof
+*.log
+*.lot
+*.fls
+*.out
+*.toc
+*.fmt
+*.fot
+*.cb
+*.cb2
+.*.lb
+
+## Intermediate documents:
+*.dvi
+*.xdv
+*-converted-to.*
+# these rules might exclude image files for figures etc.
+# *.ps
+# *.eps
+# *.pdf
+
+## Generated if empty string is given at "Please type another file name for output:"
+.pdf
+
+## Bibliography auxiliary files (bibtex/biblatex/biber):
+*.bbl
+*.bcf
+*.blg
+*-blx.aux
+*-blx.bib
+*.run.xml
+
+## Build tool auxiliary files:
+*.fdb_latexmk
+*.synctex
+*.synctex(busy)
+*.synctex.gz
+*.synctex.gz(busy)
+*.pdfsync
+
+## Build tool directories for auxiliary files
+# latexrun
+latex.out/
+
+## Auxiliary and intermediate files from other packages:
+# algorithms
+*.alg
+*.loa
+
+# achemso
+acs-*.bib
+
+# amsthm
+*.thm
+
+# beamer
+*.nav
+*.pre
+*.snm
+*.vrb
+
+# changes
+*.soc
+
+# comment
+*.cut
+
+# cprotect
+*.cpt
+
+# elsarticle (documentclass of Elsevier journals)
+*.spl
+
+# endnotes
+*.ent
+
+# fixme
+*.lox
+
+# feynmf/feynmp
+*.mf
+*.mp
+*.t[1-9]
+*.t[1-9][0-9]
+*.tfm
+
+#(r)(e)ledmac/(r)(e)ledpar
+*.end
+*.?end
+*.[1-9]
+*.[1-9][0-9]
+*.[1-9][0-9][0-9]
+*.[1-9]R
+*.[1-9][0-9]R
+*.[1-9][0-9][0-9]R
+*.eledsec[1-9]
+*.eledsec[1-9]R
+*.eledsec[1-9][0-9]
+*.eledsec[1-9][0-9]R
+*.eledsec[1-9][0-9][0-9]
+*.eledsec[1-9][0-9][0-9]R
+
+# glossaries
+*.acn
+*.acr
+*.glg
+*.glo
+*.gls
+*.glsdefs
+
+# gnuplottex
+*-gnuplottex-*
+
+# gregoriotex
+*.gaux
+*.gtex
+
+# htlatex
+*.4ct
+*.4tc
+*.idv
+*.lg
+*.trc
+*.xref
+
+# hyperref
+*.brf
+
+# knitr
+*-concordance.tex
+# TODO Comment the next line if you want to keep your tikz graphics files
+*.tikz
+*-tikzDictionary
+
+# listings
+*.lol
+
+# makeidx
+*.idx
+*.ilg
+*.ind
+*.ist
+
+# minitoc
+*.maf
+*.mlf
+*.mlt
+*.mtc[0-9]*
+*.slf[0-9]*
+*.slt[0-9]*
+*.stc[0-9]*
+
+# minted
+_minted*
+*.pyg
+
+# morewrites
+*.mw
+
+# nomencl
+*.nlg
+*.nlo
+*.nls
+
+# pax
+*.pax
+
+# pdfpcnotes
+*.pdfpc
+
+# sagetex
+*.sagetex.sage
+*.sagetex.py
+*.sagetex.scmd
+
+# scrwfile
+*.wrt
+
+# sympy
+*.sout
+*.sympy
+sympy-plots-for-*.tex/
+
+# pdfcomment
+*.upa
+*.upb
+
+# pythontex
+*.pytxcode
+pythontex-files-*/
+
+# tcolorbox
+*.listing
+
+# thmtools
+*.loe
+
+# TikZ & PGF
+*.dpth
+*.md5
+*.auxlock
+
+# todonotes
+*.tdo
+
+# vhistory
+*.hst
+*.ver
+
+# easy-todo
+*.lod
+
+# xcolor
+*.xcp
+
+# xmpincl
+*.xmpi
+
+# xindy
+*.xdy
+
+# xypic precompiled matrices
+*.xyc
+
+# endfloat
+*.ttt
+*.fff
+
+# Latexian
+TSWLatexianTemp*
+
+## Editors:
+# WinEdt
+*.bak
+*.sav
+
+# Texpad
+.texpadtmp
+
+# LyX
+*.lyx~
+
+# Kile
+*.backup
+
+# KBibTeX
+*~[0-9]*
+
+# auto folder when using emacs and auctex
+./auto/*
+*.el
+
+# expex forward references with \gathertags
+*-tags.tex
+
+# standalone packages
+*.sta
+
+
+# End of https://www.gitignore.io/api/data,linux,macos,python,windows,pycharm,database,jupyternotebook
+

README.md

@@ -1,84 +1,2 @@
-# Bureaucratic Cohort Swarms
-### Pruning Networks by SRNN 
-###### Deadline: 28.02.22
-
-## Experimente
-
-### Fixpoint Tests:
-    
-- [X] Dropout Test 
-  - (Macht das Partikel beim Goal mit oder ist es nur SRN)
-  - Zero_ident diff = -00.04999637603759766 %
-	   
-- [ ] gnf(1) -> Aprox. Weight
-  - Übersetung in ein Gewichtsskalar
-    - Einbettung in ein Reguläres Netz
-	
-- [ ] Übersetzung in ein Explainable AI Framework
-  - Rückschlüsse auf Mikro Netze
-	
-- [ ] Visualiserung
-  - Der Zugehörigkeit 
-  - Der Vernetzung
-	
-- [ ] PCA()
-  - Dataframe Epoch, Weight, dim_1, ..., dim_n
-  - Visualisierung als Trajectory Cube
-	
-- [ ] Recherche zu Makro Mikro Netze Strukturen 
-  - gits das schon?
-  - Hypernetwork?
-  - arxiv: 1905.02898
-  - Sparse Networks
-  - Pruning 
-
----
-
-### Tasks für Steffen:
-- [x] Sanity Check:
-
-  - [x] Neuronen können lernen einen Eingabewert mit x zu multiplizieren?
-
-    | SRNN x*n    3 Neurons            Identity_Func     | SRNN x*n    4 Neurons              Identity_Func  |
-    |---------------------------------------------------|----------------------------------------------------|
-    | ![](./figures/sanity/sanity_3hidden_xtimesn.png)   | ![](./figures/sanity/sanity_4hidden_xtimesn.png)  |
-    | SRNN x*n    6 Neurons              Other_Func      | SRNN x*n    10 Neurons      Other_Func            |
-    | ![](./figures/sanity/sanity_6hidden_xtimesn.png)   | ![](./figures/sanity/sanity_10hidden_xtimesn.png) |
-  
-- [ ] Connectivity
-  - Das Netz dünnt sich wirklich aus.
-    
-    |||
-    |---------------------------------------------------|----------------------------------------------------|
-    | 200 Epochs - 4 Neurons - \alpha 100     RES     |                                                           |
-    | ![](./figures/connectivity/training_lineplot.png) | ![](./figures/connectivity/training_particle_type_lp.png) |
-    | OTHER FUNTIONS                                    |                  IDENTITY FUNCTIONS                       |
-    | ![](./figures/connectivity/other.png)             | ![](./figures/connectivity/identity.png)                  |
-
-- [ ] Training mit kleineren GNs
-  
-    
-- [ ] Weiter Trainieren -> 500 Epochs?
-- [x] Training ohne Residual Skip Connection
-  - Ist anders:
-     Self Training wird zunächst priorisiert, dann kommt langsam der eigentliche Task durch:
-      
-      | No Residual Skip connections 8 Neurons in SRNN  Alpha=100                                    | Residual Skip connections 8 Neurons in SRNN    Alpha=100                                 |
-      |------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------|
-      | ![LinePlot](./figures/res_no_res/mn_st_200_8_alpha_100_no_res_training_particle_type_lp.png) | ![image info](./figures/res_no_res/mn_st_200_8_alpha_100_training_particle_type_lp.png)  |
-      | ![image info](./figures/res_no_res/mn_st_200_8_alpha_100_no_res_training_lineplot.png)       | ![image info](./figures/res_no_res/mn_st_200_8_alpha_100_training_lineplot.png)          |
-
-- [ ] Test mit Baseline Dense Network 
-  - [ ] mit vergleichbaren Neuron Count
-  - [ ] mit gesamt Weight Count
-
-- [ ] Task/Goal statt SRNN-Task
-
----
-
-### Für Menschen mit zu viel Zeit:
-- [ ] Sparse Network Training der Self Replication
-  - Just for the lulz and speeeeeeed)
-  - (Spaß bei Seite, wäre wichtig für schnellere Forschung)
-    <https://pytorch.org/docs/stable/sparse.html>
-
+# bannana-networks
+

code/experiment.py

@@ -0,0 +1,171 @@
+import os
+import time
+import dill
+from tqdm import tqdm
+import copy
+
+from tensorflow.python.keras import backend as K
+
+from abc import ABC, abstractmethod
+
+
+class Experiment(ABC):
+
+    @staticmethod
+    def from_dill(path):
+        with open(path, "rb") as dill_file:
+            return dill.load(dill_file)
+
+    @staticmethod
+    def reset_model():
+        K.clear_session()
+
+    def __init__(self, name=None, ident=None):
+        self.experiment_id = f'{ident or ""}_{time.time()}'
+        self.experiment_name = name or 'unnamed_experiment'
+        self.next_iteration = 0
+        self.log_messages = list()
+        self.historical_particles = dict()
+
+    def __enter__(self):
+        self.dir = os.path.join('experiments', f'exp-{self.experiment_name}-{self.experiment_id}-{self.next_iteration}')
+        os.makedirs(self.dir)
+        print(f'** created {self.dir} **')
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.save(experiment=self.without_particles())
+        self.save_log()
+        self.next_iteration += 1
+
+    def log(self, message, **kwargs):
+        self.log_messages.append(message)
+        print(message, **kwargs)
+
+    def save_log(self, log_name="log"):
+        with open(os.path.join(self.dir, f"{log_name}.txt"), "w") as log_file:
+            for log_message in self.log_messages:
+                print(str(log_message), file=log_file)
+
+    def __copy__(self):
+        self_copy = self.__class__(name=self.experiment_name,)
+        self_copy.__dict__ = {attr: self.__dict__[attr] for attr in self.__dict__ if
+                              attr not in ['particles', 'historical_particles']}
+        return self_copy
+
+    def without_particles(self):
+        self_copy = copy.copy(self)
+        # self_copy.particles = [particle.states for particle in self.particles]
+        self_copy.historical_particles = {key: val.states for key, val in self.historical_particles.items()}
+        return self_copy
+
+    def save(self, **kwargs):
+        for name, value in kwargs.items():
+            with open(os.path.join(self.dir, f"{name}.dill"), "wb") as dill_file:
+                dill.dump(value, dill_file)
+
+    @abstractmethod
+    def run_net(self, net, trains_per_application=100, step_limit=100, run_id=0, **kwargs):
+        raise NotImplementedError
+        pass
+
+    def run_exp(self, network_generator, exp_iterations, prints=False, **kwargs):
+        # INFO Run_ID needs to be more than 0, so that exp stores the trajectories!
+        for run_id in range(exp_iterations):
+            network = network_generator()
+            self.run_net(network, 100, run_id=run_id + 1, **kwargs)
+            self.historical_particles[run_id] = network
+            if prints:
+                print("Fixpoint? " + str(network.is_fixpoint()))
+        self.reset_model()
+
+    def reset_all(self):
+        self.reset_model()
+
+
+class FixpointExperiment(Experiment):
+    if kwargs.get('logging', False):
+        self.log(self.counters)
+
+    def __init__(self, **kwargs):
+        kwargs['name'] = self.__class__.__name__ if 'name' not in kwargs else kwargs['name']
+        super().__init__(**kwargs)
+        self.counters = dict(divergent=0, fix_zero=0, fix_other=0, fix_sec=0, other=0)
+        self.interesting_fixpoints = []
+
+    def run_net(self, net, step_limit=100, run_id=0, **kwargs):
+        i = 0
+        while i < step_limit and not net.is_diverged() and not net.is_fixpoint():
+            net.self_attack()
+            i += 1
+            if run_id:
+                net.save_state(time=i)
+        self.count(net)
+
+    def count(self, net):
+        if net.is_diverged():
+            self.counters['divergent'] += 1
+        elif net.is_fixpoint():
+            if net.is_zero():
+                self.counters['fix_zero'] += 1
+            else:
+                self.counters['fix_other'] += 1
+                self.interesting_fixpoints.append(net.get_weights())
+        elif net.is_fixpoint(2):
+            self.counters['fix_sec'] += 1
+        else:
+            self.counters['other'] += 1
+
+    def reset_counters(self):
+        for key in self.counters.keys():
+            self.counters[key] = 0
+        return True
+
+    def reset_all(self):
+        super(FixpointExperiment, self).reset_all()
+        self.reset_counters()
+
+
+class MixedFixpointExperiment(FixpointExperiment):
+
+    def run_net(self, net, trains_per_application=100, step_limit=100, run_id=0, **kwargs):
+        for i in range(step_limit):
+            if net.is_diverged() or net.is_fixpoint():
+                break
+            net.self_attack()
+            with tqdm(postfix=["Loss", dict(value=0)]) as bar:
+                for _ in range(trains_per_application):
+                    loss = net.compiled().train()
+                    bar.postfix[1]["value"] = loss
+                    bar.update()
+            if run_id:
+                net.save_state()
+        self.count(net)
+
+
+class SoupExperiment(Experiment):
+
+    def __init__(self, **kwargs):
+        super(SoupExperiment, self).__init__(name=kwargs.get('name', self.__class__.__name__))
+
+    def run_exp(self, network_generator, exp_iterations, soup_generator=None, soup_iterations=0, prints=False):
+        for i in range(soup_iterations):
+            soup = soup_generator()
+            soup.seed()
+            for _ in tqdm(exp_iterations):
+                soup.evolve()
+            self.log(soup.count())
+            self.save(soup=soup.without_particles())
+
+    def run_net(self, net, trains_per_application=100, step_limit=100, run_id=0, **kwargs):
+        raise NotImplementedError
+        pass
+
+
+class IdentLearningExperiment(Experiment):
+
+    def __init__(self):
+        super(IdentLearningExperiment, self).__init__(name=self.__class__.__name__)
+
+    def run_net(self, net, trains_per_application=100, step_limit=100, run_id=0, **kwargs):
+        pass

code/network.py

@@ -0,0 +1,618 @@
+import numpy as np
+from abc import abstractmethod, ABC
+from typing import List, Union
+from types import FunctionType
+
+from tensorflow.python.keras.models import Sequential
+from tensorflow.python.keras.callbacks import Callback
+from tensorflow.python.keras.layers import SimpleRNN, Dense
+from tensorflow.python.keras import backend as K
+
+from experiment import *
+
+# Supress warnings and info messages
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
+
+
+class SaveStateCallback(Callback):
+    def __init__(self, network, epoch=0):
+        super(SaveStateCallback, self).__init__()
+        self.net = network
+        self.init_epoch = epoch
+
+    def on_epoch_end(self, epoch, logs=None):
+        description = dict(time=epoch+self.init_epoch)
+        description['action'] = 'train_self'
+        description['counterpart'] = None
+        self.net.save_state(**description)
+        return
+
+
+class Weights:
+
+    @staticmethod
+    def __reshape_flat_array__(array, shapes):
+        sizes: List[int] = [int(np.prod(shape)) for shape in shapes]
+        # Split the incoming array into slices for layers
+        slices = [array[x: y] for x, y in zip(np.cumsum([0]+sizes), np.cumsum([0]+sizes)[1:])]
+        # reshape them in accordance to the given shapes
+        weights = [np.reshape(weight_slice, shape) for weight_slice, shape in zip(slices, shapes)]
+        return weights
+
+    def __init__(self, weight_vector: Union[List[np.ndarray], np.ndarray], flat_array_shape=None):
+        """
+        Weight class, for easy manipulation of weight vectors from Keras models
+
+        :param weight_vector: A numpy array holding weights
+        :type weight_vector: List[np.ndarray]
+        """
+        self.__iter_idx = [0, 0]
+        if flat_array_shape:
+            weight_vector = self.__reshape_flat_array__(weight_vector, flat_array_shape)
+
+        self.layers = weight_vector
+
+        # TODO: implement a way to access the cells directly
+        # self.cells = len(self)
+        # TODO: implement a way to access the weights directly
+        # self.weights = self.to_flat_array() ?
+
+    def __iter__(self):
+        self.__iter_idx = [0, 0]
+        return self
+
+    def __getitem__(self, item):
+        return self.layers[item]
+
+    def max(self):
+        np.max(self.layers)
+
+    def avg(self):
+        return np.average(self.layers)
+
+    def __len__(self):
+        return sum([x.size for x in self.layers])
+
+    def shapes(self):
+        return [x.shape for x in self.layers]
+
+    def num_layers(self):
+        return len(self.layers)
+
+    def __copy__(self):
+        return copy.deepcopy(self)
+
+    def __next__(self):
+        # ToDo: Check iteration progress over layers
+        # ToDo: There is still a problem interation, currently only cell level is the last loop stage.
+        # Do we need this?
+        if self.__iter_idx[0] >= len(self.layers):
+            if self.__iter_idx[1] >= len(self.layers[self.__iter_idx[0]]):
+                raise StopIteration
+        result = self.layers[self.__iter_idx[0]][self.__iter_idx[1]]
+
+        if self.__iter_idx[1] >= len(self.layers[self.__iter_idx[0]]):
+            self.__iter_idx[0] += 1
+            self.__iter_idx[1] = 0
+        else:
+            self.__iter_idx[1] += 1
+        return result
+
+    def __repr__(self):
+        return f'Weights({self.to_flat_array().tolist()})'
+
+    def to_flat_array(self) -> np.ndarray:
+        return np.hstack([weight.flatten() for weight in self.layers])
+
+    def from_flat_array(self, array):
+        new_weights = self.__reshape_flat_array__(array, self.shapes())
+        return new_weights
+
+    def shuffle(self):
+        flat = self.to_flat_array()
+        np.random.shuffle(flat)
+        self.from_flat_array(flat)
+        return True
+
+    def are_diverged(self):
+        return any([np.isnan(x).any() for x in self.layers]) or any([np.isinf(x).any() for x in self.layers])
+
+    def are_within_bounds(self, lower_bound: float, upper_bound: float):
+        return bool(sum([((lower_bound < x) & (x > upper_bound)).size for x in self.layers]))
+
+    def aggregate_by(self, func: FunctionType, num_aggregates):
+        collection_sizes = len(self) // num_aggregates
+        weights = self.to_flat_array()[:collection_sizes * num_aggregates].reshape((num_aggregates, -1))
+        aggregated_weights = func(weights, num_aggregates)
+        left_overs = self.to_flat_array()[collection_sizes * num_aggregates:]
+        return aggregated_weights, left_overs
+
+
+class NeuralNetwork(ABC):
+    """
+    This is the Base Network Class, including abstract functions that must be implemented.
+    """
+
+    def __init__(self, **params):
+        super().__init__()
+        self.params = dict(epsilon=0.00000000000001)
+        self.params.update(params)
+        self.keras_params = dict(activation='linear', use_bias=False)
+        self.states = []
+        self.model: Sequential
+
+    def get_params(self) -> dict:
+        return self.params
+
+    def get_keras_params(self) -> dict:
+        return self.keras_params
+
+    def with_params(self, **kwargs):
+        self.params.update(kwargs)
+        return self
+
+    def with_keras_params(self, **kwargs):
+        self.keras_params.update(kwargs)
+        return self
+
+    def get_weights(self) -> Weights:
+        return Weights(self.model.get_weights())
+
+    def get_weights_flat(self) -> np.ndarray:
+        return self.get_weights().to_flat_array()
+
+    def set_weights(self, new_weights: Weights):
+        return self.model.set_weights(new_weights.layers)
+
+    @abstractmethod
+    def get_samples(self):
+        # TODO: add a dogstring, telling the user what this does, e.g. what is a sample?
+        raise NotImplementedError
+
+    @abstractmethod
+    def apply_to_weights(self, old_weights) -> Weights:
+        # TODO: add a dogstring, telling the user what this does, e.g. what is applied?
+        raise NotImplementedError
+
+    def apply_to_network(self, other_network) -> Weights:
+        # TODO: add a dogstring, telling the user what this does, e.g. what is applied?
+        new_weights = self.apply_to_weights(other_network.get_weights())
+        return new_weights
+
+    def attack(self, other_network):
+        # TODO: add a dogstring, telling the user what this does, e.g. what is an attack?
+        other_network.set_weights(self.apply_to_network(other_network))
+        return self
+
+    def fuck(self, other_network):
+        # TODO: add a dogstring, telling the user what this does, e.g. what is fucking?
+        self.set_weights(self.apply_to_network(other_network))
+        return self
+
+    def self_attack(self, iterations=1):
+        # TODO: add a dogstring, telling the user what this does, e.g. what is self attack?
+        for _ in range(iterations):
+            self.attack(self)
+        return self
+
+    def meet(self, other_network):
+        # TODO: add a dogstring, telling the user what this does, e.g. what is meeting?
+        new_other_network = copy.deepcopy(other_network)
+        return self.attack(new_other_network)
+
+    def is_diverged(self):
+        return self.get_weights().are_diverged()
+
+    def is_zero(self, epsilon=None):
+        epsilon = epsilon or self.get_params().get('epsilon')
+        return self.get_weights().are_within_bounds(-epsilon, epsilon)
+
+    def is_fixpoint(self, degree: int = 1, epsilon: float = None) -> bool:
+        assert degree >= 1, "degree must be >= 1"
+        epsilon = epsilon or self.get_params().get('epsilon')
+
+        new_weights = copy.deepcopy(self.get_weights())
+
+        for _ in range(degree):
+            new_weights = self.apply_to_weights(new_weights)
+            if new_weights.are_diverged():
+                return False
+
+        biggerEpsilon = (np.abs(new_weights.to_flat_array() - self.get_weights().to_flat_array()) >= epsilon).any()
+
+        # Boolean Value needs to be flipped to answer "is_fixpoint"
+        return not biggerEpsilon
+
+    def print_weights(self, weights=None):
+        print(weights or self.get_weights())
+
+
+class ParticleDecorator:
+    next_uid = 0
+
+    def __init__(self, network):
+
+        # ToDo: Add DocString, What does it do?
+
+        self.uid = self.__class__.next_uid
+        self.__class__.next_uid += 1
+        self.network = network
+        self.states = []
+        self.save_state(time=0, action='init', counterpart=None)
+
+    def __getattr__(self, name):
+        return getattr(self.network, name)
+
+    def get_uid(self):
+        return self.uid
+
+    def make_state(self, **kwargs):
+        if self.network.is_diverged():
+            return None
+        state = {'class': self.network.__class__.__name__, 'weights': self.network.get_weights_flat()}
+        state.update(kwargs)
+        return state
+
+    def save_state(self, **kwargs):
+        state = self.make_state(**kwargs)
+        if state is not None:
+            self.states += [state]
+        else:
+            pass
+        return True
+
+    def update_state(self, number, **kwargs):
+        raise NotImplementedError('Result is vague')
+        # if number < len(self.states):
+        #     self.states[number] = self.make_state(**kwargs)
+        # else:
+        #     for i in range(len(self.states), number):
+        #         self.states += [None]
+        #     self.states += self.make_state(**kwargs)
+
+    def get_states(self):
+        return self.states
+
+
+class WeightwiseNeuralNetwork(NeuralNetwork):
+
+    def __init__(self, width, depth, **kwargs):
+        # ToDo: Insert Docstring
+        super().__init__(**kwargs)
+        self.width: int = width
+        self.depth: int = depth
+        self.model = Sequential()
+        self.model.add(Dense(units=self.width, input_dim=4, **self.keras_params))
+        for _ in range(self.depth-1):
+            self.model.add(Dense(units=self.width, **self.keras_params))
+        self.model.add(Dense(units=1, **self.keras_params))
+
+    def apply(self, inputs):
+        # TODO: Write about it... What does it do?
+        return self.model.predict(inputs)
+
+    def get_samples(self):
+        weights = self.get_weights()
+        sample = np.asarray([
+            [weight, idx, *x] for idx, layer in enumerate(weights.layers) for x, weight in np.ndenumerate(layer)
+        ])
+        # normalize [layer, cell, position]
+        for idx in range(1, sample.shape[1]):
+            sample[:, idx] = sample[:, idx] / np.max(sample[:, idx])
+        return sample, sample
+
+    def apply_to_weights(self, weights) -> Weights:
+        # ToDo: Insert DocString
+        # Transform the weight matrix in an horizontal stack as: array([[weight, layer, cell, position], ...])
+        transformed_weights = self.get_samples()[0]
+        new_weights = self.apply(transformed_weights)
+        # use the original weight shape to transform the new tensor
+        return Weights(new_weights, flat_array_shape=weights.shapes())
+
+
+class AggregatingNeuralNetwork(NeuralNetwork):
+
+    @staticmethod
+    def aggregate_fft(array: np.ndarray, aggregates: int):
+        flat = array.flatten()
+        # noinspection PyTypeChecker
+        fft_reduction = np.fft.fftn(flat, aggregates)
+        return fft_reduction
+
+    @staticmethod
+    def aggregate_average(array, _):
+        return np.average(array, axis=1)
+
+    @staticmethod
+    def aggregate_max(array, _):
+        return np.max(array, axis=1)
+
+    @staticmethod
+    def deaggregate_identically(aggregate, amount):
+        # ToDo: Find a better way than using the a hardcoded [0]
+        return np.hstack([aggregate for _ in range(amount)])[0]
+
+    @staticmethod
+    def shuffle_not(weights: Weights):
+        """
+        Doesn't do a thing. f(x)
+
+        :param weights: A List of Weights
+        :type weights: Weights
+        :return: The same old weights.
+        :rtype: Weights
+        """
+        return weights
+
+    @staticmethod
+    def shuffle_random(weights: Weights):
+        assert weights.shuffle()
+        return weights
+
+    def __init__(self, aggregates, width, depth, **kwargs):
+        super().__init__(**kwargs)
+        self.aggregates = aggregates
+        self.width = width
+        self.depth = depth
+        self.model = Sequential()
+        self.model.add(Dense(units=width, input_dim=self.aggregates, **self.keras_params))
+        for _ in range(depth-1):
+            self.model.add(Dense(units=width, **self.keras_params))
+        self.model.add(Dense(units=self.aggregates, **self.keras_params))
+
+    def get_aggregator(self):
+        return self.params.get('aggregator', self.aggregate_average)
+
+    def get_deaggregator(self):
+        return self.params.get('deaggregator', self.deaggregate_identically)
+
+    def get_shuffler(self):
+        return self.params.get('shuffler', self.shuffle_not)
+
+    def get_amount_of_weights(self):
+        return len(self.get_weights())
+
+    def apply(self, inputs):
+        # You need to add an dimension here... "..." copies array values
+        return self.model.predict(inputs[None, ...])
+
+    def get_aggregated_weights(self):
+        return self.get_weights().aggregate_by(self.get_aggregator(), self.aggregates)
+
+    def apply_to_weights(self, old_weights) -> Weights:
+
+        # build aggregations of old_weights
+        old_aggregations, leftovers = self.get_aggregated_weights()
+
+        # call network
+        new_aggregations = self.apply(old_aggregations)
+        collection_sizes = self.get_amount_of_weights() // self.aggregates
+        new_aggregations = self.deaggregate_identically(new_aggregations, collection_sizes)
+        # generate new weights
+        # only include leftovers if there are some then coonvert them to Weight on base of th old shape
+        new_weights = Weights(new_aggregations if not leftovers.shape[0] else np.hstack((new_aggregations, leftovers)),
+                              flat_array_shape=old_weights.shapes())
+
+        # maybe shuffle
+        new_weights = self.get_shuffler()(new_weights)
+        return new_weights
+
+    def get_samples(self):
+        aggregations, _ = self.get_aggregated_weights()
+        # What did that do?
+        # sample = np.transpose(np.array([[aggregations[i]] for i in range(self.aggregates)]))
+        return aggregations, aggregations
+
+    def is_fixpoint_after_aggregation(self, degree=1, epsilon=None):
+        assert degree >= 1, "degree must be >= 1"
+        epsilon = epsilon or self.get_params().get('epsilon')
+
+        old_aggregations, _ = self.get_aggregated_weights()
+        new_weights = copy.deepcopy(self.get_weights())
+
+        for _ in range(degree):
+            new_weights = self.apply_to_weights(new_weights)
+            if new_weights.are_diverged():
+                return False
+
+        new_aggregations, leftovers = self.get_aggregated_weights()
+
+        # ToDo: Explain This, why are you additionally checking tolerances of aggregated weights?
+        biggerEpsilon = (np.abs(np.asarray(old_aggregations) - np.asarray(new_aggregations)) >= epsilon).any()
+
+        # Boolean value has to be flipped to answer the question.
+        return True, not biggerEpsilon
+
+
+class RecurrentNeuralNetwork(NeuralNetwork):
+
+    def __init__(self, width, depth, **kwargs):
+        super().__init__(**kwargs)
+        self.features = 1
+        self.width = width
+        self.depth = depth
+        self.model = Sequential()
+        self.model.add(SimpleRNN(units=width, input_dim=self.features, return_sequences=True, **self.keras_params))
+        for _ in range(depth-1):
+            self.model.add(SimpleRNN(units=width, return_sequences=True, **self.keras_params))
+        self.model.add(SimpleRNN(units=self.features, return_sequences=True, **self.keras_params))
+
+    def apply(self, *inputs):
+        stuff = np.transpose(np.array([[[inputs[i]] for i in range(len(inputs))]]))
+        return self.model.predict(stuff)[0].flatten()
+
+    def apply_to_weights(self, old_weights):
+        # build list from old weights
+        new_weights = copy.deepcopy(old_weights)
+        old_weights_list = []
+        for layer_id, layer in enumerate(old_weights):
+            for cell_id, cell in enumerate(layer):
+                for weight_id, weight in enumerate(cell):
+                    old_weights_list += [weight]
+
+        # call network
+        new_weights_list = self.apply(*old_weights_list)
+
+        # write back new weights from list of rnn returns
+        current_weight_id = 0
+        for layer_id, layer in enumerate(new_weights):
+            for cell_id, cell in enumerate(layer):
+                for weight_id, weight in enumerate(cell):
+                    new_weight = new_weights_list[current_weight_id]
+                    new_weights[layer_id][cell_id][weight_id] = new_weight
+                    current_weight_id += 1
+        return new_weights
+
+    def compute_samples(self):
+        # build list from old weights
+        old_weights_list = []
+        for layer_id, layer in enumerate(self.get_weights()):
+            for cell_id, cell in enumerate(layer):
+                for weight_id, weight in enumerate(cell):
+                    old_weights_list += [weight]
+        sample = np.asarray(old_weights_list)[None, ..., None]
+        return sample, sample
+
+
+class TrainingNeuralNetworkDecorator:
+
+    def __init__(self, network):
+        self.network = network
+        self.compile_params = dict(loss='mse', optimizer='sgd')
+        self.model_compiled = False
+
+    def __getattr__(self, name):
+        return getattr(self.network, name)
+
+    def with_params(self, **kwargs):
+        self.network.with_params(**kwargs)
+        return self
+
+    def with_keras_params(self, **kwargs):
+        self.network.with_keras_params(**kwargs)
+        return self
+
+    def get_compile_params(self):
+        return self.compile_params
+
+    def with_compile_params(self, **kwargs):
+        self.compile_params.update(kwargs)
+        return self
+
+    def compile_model(self, **kwargs):
+        compile_params = copy.deepcopy(self.compile_params)
+        compile_params.update(kwargs)
+        return self.network.model.compile(**compile_params)
+
+    def compiled(self, **kwargs):
+        if not self.model_compiled:
+            self.compile_model(**kwargs)
+            self.model_compiled = True
+        return self
+
+    def train(self, batchsize=1, store_states=True, epoch=0):
+        self.compiled()
+        x, y = self.network.get_samples()
+        savestatecallback = [SaveStateCallback(network=self, epoch=epoch)] if store_states else None
+        history = self.network.model.fit(x=x, y=y, epochs=epoch+1, verbose=0,
+                                         batch_size=batchsize, callbacks=savestatecallback,
+                                         initial_epoch=epoch)
+        return history.history['loss'][-1]
+
+    def learn_from(self, other_network, batchsize=1):
+        self.compiled()
+        other_network.compiled()
+        x, y = other_network.network.get_samples()
+        history = self.network.model.fit(x=x, y=y, verbose=0, batch_size=batchsize)
+
+        return history.history['loss'][-1]
+
+
+if __name__ == '__main__':
+
+    if True:
+        # WeightWise Neural Network
+        net_generator = ParticleDecorator(WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation='linear'))
+        with FixpointExperiment() as exp:
+            exp.run_exp(net_generator, 10, logging=True)
+            exp.reset_all()
+
+    if False:
+        # Aggregating Neural Network
+        net_generator = ParticleDecorator(AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params())
+        with FixpointExperiment() as exp:
+            exp.run_exp(net_generator, 10, logging=True)
+
+            exp.reset_all()
+
+    if False:
+        # FFT Aggregation
+        net_generator = lambda: ParticleDecorator(
+            AggregatingNeuralNetwork(
+                aggregates=4, width=2, depth=2, aggregator=AggregatingNeuralNetwork.aggregate_fft
+            ).with_keras_params(activation='linear'))
+        with FixpointExperiment() as exp:
+            for run_id in tqdm(range(10)):
+                exp.run_exp(net_generator, 1)
+                exp.log(exp.counters)
+                exp.reset_model()
+            exp.reset_all()
+
+    if True:
+        # ok so this works quite realiably
+        run_count = 10000
+        net_generator = TrainingNeuralNetworkDecorator(
+            ParticleDecorator(WeightwiseNeuralNetwork(width=2, depth=2))
+        ).with_params(epsilon=0.0001).with_keras_params(optimizer='sgd')
+        with MixedFixpointExperiment() as exp:
+            for run_id in tqdm(range(run_count+1)):
+                exp.run_exp(net_generator, 1)
+                if run_id % 100 == 0:
+                    exp.run_net(net_generator, 1)
+            K.clear_session()
+
+    if False:
+        with FixpointExperiment() as exp:
+            run_count = 1000
+            net = TrainingNeuralNetworkDecorator(AggregatingNeuralNetwork(4, width=2, depth=2)).with_params(epsilon=0.1e-6)
+            for run_id in tqdm(range(run_count+1)):
+                loss = net.compiled().train()
+                if run_id % 100 == 0:
+                    net.print_weights()
+                    old_aggs, _ = net.net.get_aggregated_weights()
+                    print("old weights agg: " + str(old_aggs))
+                    fp, new_aggs = net.net.is_fixpoint_after_aggregation(epsilon=0.0001)
+                    print("new weights agg: " + str(new_aggs))
+                    print("Fixpoint? " + str(net.is_fixpoint()))
+                    print("Fixpoint after Agg? " + str(fp))
+                    print("Loss " + str(loss))
+                    print()
+
+    if False:
+        # this explodes in our faces completely... NAN everywhere
+        # TODO: Wtf is happening here?
+        with FixpointExperiment() as exp:
+            run_count = 10000
+            net = TrainingNeuralNetworkDecorator(RecurrentNeuralNetwork(width=2, depth=2))\
+                .with_params(epsilon=0.1e-2).with_keras_params(optimizer='sgd', activation='linear')
+            for run_id in tqdm(range(run_count+1)):
+                loss = net.compiled().train()
+                if run_id % 500 == 0:
+                    net.print_weights()
+                    # print(net.apply_to_network(net))
+                    print("Fixpoint? " + str(net.is_fixpoint()))
+                    print("Loss " + str(loss))
+                    print()
+    if False:
+        # and this gets somewhat interesting... we can still achieve non-trivial fixpoints
+        # over multiple applications when training enough in-between
+        with MixedFixpointExperiment() as exp:
+            for run_id in range(10):
+                net = TrainingNeuralNetworkDecorator(FFTNeuralNetwork(2, width=2, depth=2))\
+                    .with_params(epsilon=0.0001, activation='sigmoid')
+                exp.run_net(net, 500, 10)
+
+                net.print_weights()
+
+                print("Fixpoint? " + str(net.is_fixpoint()))
+            exp.log(exp.counters)

code/results/Soup/log.txt

@@ -0,0 +1 @@
+{'divergent': 0, 'fix_zero': 0, 'fix_other': 13, 'fix_sec': 0, 'other': 7}

code/results/Soup/weights.txt

@@ -0,0 +1,30 @@
+[-0.15321673  1.0428386  -0.7245892  -0.04343993  0.42338863  0.02538261
+ -0.40465942 -0.0242596  -1.226809   -0.8168446   0.26588777 -1.0929432
+  0.5383322  -0.73875046]
+[-0.03072096 -1.369665   -0.357126   -0.21180922  0.3853204   0.22853081
+ -0.3705557  -0.21977347 -0.6684716   0.12849599  1.0226644  -0.0922638
+ -0.7828449  -0.6572327 ]
+[-1.2444692   0.61213857  0.07965802  0.12361202  0.62641835  0.9720597
+  0.3863232   0.59948945  1.0857513   0.49231085 -0.5319295   0.29433587
+ -0.64177823  0.17603302]
+[-0.9938292  -0.4438207  -0.03172896  0.06261964 -0.3870194   0.7637992
+  0.0244509  -0.04825407  0.91551745 -0.78740424  0.29226422 -0.52767307
+ -0.41744384  0.5567152 ]
+[-0.39049304  0.8842579  -0.8447943  -0.19669186  0.7207061   0.16780053
+  0.3728221   0.08680353  0.7535456  -0.1000197   0.02029054  0.8640245
+ -0.15881588  1.1905665 ]
+[ 1.0482084   0.9248296  -0.26946014  0.57047915 -0.32660747  0.6914731
+ -0.18025818  0.3816289  -0.69358927  0.21312684 -0.39932403 -0.02991759
+ -0.83068466  0.45619962]
+[ 0.75814664  0.10328437  0.07867077 -0.0743314  -0.53440267  0.50492585
+ -0.54172474  0.51184535  0.3462249   1.0527638  -0.9503541   0.9235086
+ -0.1665241   1.1497779 ]
+[-0.77187353  1.1105504   0.24265823  0.53782856 -0.34098852 -0.75576884
+ -0.25396293 -0.56288165  0.3851537  -0.67497945  0.14336896  0.763481
+ -0.9224985   0.6374753 ]
+[-0.79123825  0.68166596 -0.30061013 -0.19360289  0.5632736   0.36276665
+  0.7470975   0.48115698  0.10046808 -0.8064349  -1.036736   -0.68296516
+ -1.156437    0.52633154]
+[ 0.1788832  -1.5321186  -0.62001514 -0.3870902   0.97524184  0.6088638
+ -0.08297889 -0.05180515 -0.29096788  0.7519439   0.8803648   0.82771575
+ -0.854887    0.1742936 ]

code/results/exp-applying_fixpoint-_1552681870.3570378-0/log.txt

@@ -0,0 +1,12 @@
+WeightwiseNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 23, 'fix_zero': 27, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+AggregatingNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 4, 'fix_zero': 46, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+RecurrentNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 46, 'fix_zero': 4, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+

code/results/exp-learn-from-soup-_1552658566.5572753-0/log.txt

@@ -0,0 +1,4 @@
+TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
+{'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], 'zs': [0.0, 1.2, 5.2, 7.4, 8.1, 9.1, 9.6, 9.8, 10.0, 9.9, 9.9]}
+
+

code/results/exp-mixed-self-fixpoints-_1552666977.5858653-0/log.txt

@@ -0,0 +1,12 @@
+WeightwiseNeuralNetwork activiation='linear' use_bias=False
+{'xs': [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500], 'ys': [0.2, 0.3, 0.15, 0.55, 0.7, 0.85, 0.8, 0.95, 0.9, 1.0, 1.0]}
+
+
+AggregatingNeuralNetwork activiation='linear' use_bias=False
+{'xs': [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500], 'ys': [1.0, 0.95, 1.0, 1.0, 0.95, 0.9, 0.8, 1.0, 0.85, 1.0, 0.9]}
+
+
+RecurrentNeuralNetwork activiation='linear' use_bias=False
+{'xs': [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500], 'ys': [0.05, 0.0, 0.05, 0.0, 0.0, 0.1, 0.1, 0.05, 0.1, 0.0, 0.0]}
+
+

code/results/exp-mixed-soup-_1552674483.9866457-0/log.txt

@@ -0,0 +1,8 @@
+TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
+{'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.0, 0.0, 0.1, 0.0, 0.0, 0.0, 0.0, 0.1, 0.0, 0.0, 0.0], 'zs': [0.0, 0.0, 0.7, 1.9, 3.6, 4.3, 6.0, 6.1, 8.3, 7.7, 8.8]}
+
+
+TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
+{'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.8, 0.4, 0.4, 0.3, 0.2, 0.2, 0.2, 0.2, 0.2, 0.4, 0.3], 'zs': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}
+
+

code/results/exp-training_fixpoint-_1552658296.0913951-0/log.txt

@@ -0,0 +1,12 @@
+WeightwiseNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 0, 'fix_zero': 0, 'fix_other': 50, 'fix_sec': 0, 'other': 0}
+
+
+AggregatingNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 50}
+
+
+RecurrentNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 38, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 12}
+
+

code/results/known_fixpoint_variation/log.txt

@@ -0,0 +1,30 @@
+variation 10e-0
+avg time to vergence 3.63
+avg time as fixpoint 0
+variation 10e-1
+avg time to vergence 5.02
+avg time as fixpoint 0
+variation 10e-2
+avg time to vergence 6.46
+avg time as fixpoint 0
+variation 10e-3
+avg time to vergence 8.04
+avg time as fixpoint 0
+variation 10e-4
+avg time to vergence 9.61
+avg time as fixpoint 0.04
+variation 10e-5
+avg time to vergence 11.23
+avg time as fixpoint 1.38
+variation 10e-6
+avg time to vergence 12.99
+avg time as fixpoint 3.23
+variation 10e-7
+avg time to vergence 14.58
+avg time as fixpoint 4.84
+variation 10e-8
+avg time to vergence 21.95
+avg time as fixpoint 11.91
+variation 10e-9
+avg time to vergence 26.45
+avg time as fixpoint 16.47

code/results/self_application_aggregation_network/log.txt

@@ -0,0 +1 @@
+{'divergent': 0, 'fix_zero': 10, 'fix_other': 0, 'fix_sec': 0, 'other': 0}

code/setups/applying-fixpoints.py

@@ -0,0 +1,69 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from experiment import *
+from network import *
+
+import keras.backend as K
+
+def generate_counters():
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+def count(counters, net, notable_nets=[]):
+    if net.is_diverged():
+        counters['divergent'] += 1
+    elif net.is_fixpoint():
+        if net.is_zero():
+            counters['fix_zero'] += 1
+        else:
+            counters['fix_other'] += 1
+            notable_nets += [net]
+    elif net.is_fixpoint(2):
+        counters['fix_sec'] += 1
+        notable_nets += [net]
+    else:
+        counters['other'] += 1
+    return counters, notable_nets
+
+
+if __name__ == '__main__':
+
+    with Experiment('applying_fixpoint') as exp:
+        exp.trials = 50
+        exp.run_count = 100
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:  # , 'sigmoid', 'relu']:
+            for use_bias in [False]:
+                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+        all_counters = []
+        all_notable_nets = []
+        all_names = []
+        for net_generator_id, net_generator in enumerate(net_generators):
+            counters = generate_counters()
+            notable_nets = []
+            for _ in tqdm(range(exp.trials)):
+                net = ParticleDecorator(net_generator())
+                net.with_params(epsilon=exp.epsilon)
+                name = str(net.net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias=" + str(net.get_keras_params().get('use_bias'))
+                for run_id in range(exp.run_count):
+                    loss = net.self_attack()
+                count(counters, net, notable_nets)
+            all_counters += [counters]
+            all_notable_nets += [notable_nets]
+            all_names += [name]
+            K.clear_session()
+        exp.save(all_counters=all_counters)
+        exp.save(trajectorys=exp.without_particles())
+        # net types reached in the end
+        # exp.save(all_notable_nets=all_notable_nets)
+        exp.save(all_names=all_names) #experiment setups
+        for exp_id, counter in enumerate(all_counters):
+            exp.log(all_names[exp_id])
+            exp.log(all_counters[exp_id])
+            exp.log('\n')

code/setups/experiments/exp-learn-from-soup-_1552658566.5572753-0/log.txt

@@ -0,0 +1,4 @@
+TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
+{'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], 'zs': [0.0, 1.2, 5.2, 7.4, 8.1, 9.1, 9.6, 9.8, 10.0, 9.9, 9.9]}
+
+

code/setups/experiments/exp-training_fixpoint-_1552658296.0913951-0/log.txt

@@ -0,0 +1,12 @@
+WeightwiseNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 0, 'fix_zero': 0, 'fix_other': 50, 'fix_sec': 0, 'other': 0}
+
+
+AggregatingNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 50}
+
+
+RecurrentNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 38, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 12}
+
+

code/setups/experiments/exp-weightwise_self_application-_1552664922.4501734-0/log.txt

@@ -0,0 +1 @@
+{'divergent': 11, 'fix_zero': 9, 'fix_other': 0, 'fix_sec': 0, 'other': 0}

code/setups/fixpoint-density.py

@@ -0,0 +1,69 @@
+import sys
+import os
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from experiment import *
+from network import *
+
+import tensorflow.python.keras.backend as K
+
+
+def generate_counters():
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+def count(counters, net, notable_nets=None):
+    notable_nets = notable_nets or []
+    if net.is_diverged():
+        counters['divergent'] += 1
+    elif net.is_fixpoint():
+        if net.is_zero():
+            counters['fix_zero'] += 1
+        else:
+            counters['fix_other'] += 1
+            notable_nets += [net]
+    elif net.is_fixpoint(2):
+        counters['fix_sec'] += 1
+        notable_nets += [net]
+    else:
+        counters['other'] += 1
+    return counters, notable_nets
+
+
+if __name__ == '__main__':
+    with Experiment('fixpoint-density') as exp:
+        #NOTE: settings could/should stay this way
+        #FFT doesn't work though
+        exp.trials = 100000
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:
+            net_generators += [lambda activation=activation: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
+            net_generators += [lambda activation=activation: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
+            # net_generators += [lambda activation=activation: FFTNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
+            # net_generators += [lambda activation=activation: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
+        all_counters = []
+        all_notable_nets = []
+        all_names = []
+        for net_generator_id, net_generator in enumerate(net_generators):
+            counters = generate_counters()
+            notable_nets = []
+            for _ in tqdm(range(exp.trials)):
+                net = net_generator().with_params(epsilon=exp.epsilon)
+                net = ParticleDecorator(net)
+                name = str(net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias='" + str(net.get_keras_params().get('use_bias')) + "'"
+                count(counters, net, notable_nets)
+                K.clear_session()
+            all_counters += [counters]
+            # all_notable_nets += [notable_nets]
+            all_names += [name]
+        exp.save(all_counters=all_counters)
+        exp.save(all_notable_nets=all_notable_nets)
+        exp.save(all_names=all_names)
+        for exp_id, counter in enumerate(all_counters):
+            exp.log(all_names[exp_id])
+            exp.log(all_counters[exp_id])
+            exp.log('\n')
+
+    print('Done')

code/setups/known-fixpoint-variation.py

@@ -0,0 +1,92 @@
+import sys
+
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from experiment import *
+from network import *
+from soup import prng
+
+import tensorflow.python.keras.backend as K
+
+
+from statistics import mean
+avg = mean
+
+
+def generate_fixpoint_weights():
+    return [
+        np.array([[1.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0]], dtype=np.float32),
+        np.array([[1.0, 0.0], [0.0, 0.0]], dtype=np.float32),
+        np.array([[1.0], [0.0]], dtype=np.float32)
+    ]
+
+
+def generate_fixpoint_net():
+    #NOTE: Weightwise only is all we can do right now IMO
+    net = WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation='sigmoid')
+    # I don't know if this work for aggregaeting. We don't actually need it, though.
+    # net = AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation='sigmoid')
+    net.set_weights(generate_fixpoint_weights())
+    return net
+
+
+def vary(old_weights, e=1.0):
+    new_weights = copy.deepcopy(old_weights)
+    for layer_id, layer in enumerate(new_weights):
+        for cell_id, cell in enumerate(layer):
+            for weight_id, weight in enumerate(cell):
+                if prng() < 0.5:
+                    new_weights[layer_id][cell_id][weight_id] = weight + prng() * e
+                else:
+                    new_weights[layer_id][cell_id][weight_id] = weight - prng() * e
+    return new_weights
+
+
+if __name__ == '__main__':
+    with Experiment('known-fixpoint-variation') as exp:
+        exp.depth = 10
+        exp.trials = 100
+        exp.max_steps = 100
+        exp.epsilon = 1e-4
+        exp.xs = []
+        exp.ys = []
+        exp.zs = []
+        exp.notable_nets = []
+        current_scale = 1.0
+        for _ in range(exp.depth):
+            print('variation scale ' + str(current_scale))
+            for _ in tqdm(range(exp.trials)):
+                net = generate_fixpoint_net().with_params(epsilon=exp.epsilon)
+                net = ParticleDecorator(net)
+                net.set_weights(vary(net.get_weights(), current_scale))
+                time_to_something = 0
+                time_as_fixpoint = 0
+                still_fixpoint = True
+                for _ in range(exp.max_steps):
+                    net.self_attack()
+                    if net.is_zero() or net.is_diverged():
+                        break
+                    if net.is_fixpoint():
+                        if still_fixpoint:
+                            time_as_fixpoint += 1
+                        else:
+                            print('remarkable')
+                            exp.notable_nets += [net.get_weights()]
+                            still_fixpoint = True
+                    else:
+                        still_fixpoint = False
+                    time_to_something += 1
+                exp.xs += [current_scale]
+                # time steps taken to reach divergence or zero (reaching another fix-point is basically never happening)
+                exp.ys += [time_to_something]
+                # time steps still regarded as sthe initial fix-point
+                exp.zs += [time_as_fixpoint]
+                K.backend.clear_session()
+            current_scale /= 10.0
+        for d in range(exp.depth):
+            exp.log('variation 10e-' + str(d))
+            exp.log('avg time to vergence ' + str(avg(exp.ys[d*exp.trials:(d+1) * exp.trials])))
+            exp.log('avg time as fixpoint ' + str(avg(exp.zs[d*exp.trials:(d+1) * exp.trials])))

code/setups/learn_from_soup.py

@@ -0,0 +1,110 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from typing import Tuple
+
+from experiment import *
+from network import *
+from soup import *
+
+
+import tensorflow.python.keras.backend as K
+
+from statistics import mean
+avg = mean
+
+
+def generate_counters():
+    """
+    Initial build of the counter dict, to store counts.
+
+    :rtype: dict
+    :return: dictionary holding counter for: 'divergent', 'fix_zero', 'fix_sec', 'other'
+    """
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+def count(counters, soup, notable_nets=None):
+    """
+    Count the occurences ot the types of weight trajectories.
+
+    :param counters:      A counter dictionary.
+    :param soup:          A Soup
+    :param notable_nets:  A list to store and save intersting candidates
+
+    :rtype      Tuple[dict, list]
+    :return:    Both the counter dictionary and the list of interessting nets.
+    """
+
+    notable_nets = notable_nets or list()
+    for net in soup.particles:
+        if net.is_diverged():
+            counters['divergent'] += 1
+        elif net.is_fixpoint():
+            if net.is_zero():
+                counters['fix_zero'] += 1
+            else:
+                counters['fix_other'] += 1
+                # notable_nets += [net]
+        # elif net.is_fixpoint(2):
+        #     counters['fix_sec'] += 1
+        #     notable_nets += [net]
+        else:
+            counters['other'] += 1
+    return counters, notable_nets
+
+
+if __name__ == '__main__':
+
+    with SoupExperiment('learn-from-soup') as exp:
+        exp.soup_size = 10
+        exp.soup_life = 100
+        exp.trials = 10
+        exp.learn_from_severity_values = [10 * i for i in range(11)]
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:  # ['sigmoid', 'linear', 'relu']:
+            for use_bias in [False]:
+                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                # net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                # net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+
+        all_names = []
+        all_data = []
+        for net_generator_id, net_generator in enumerate(net_generators):
+            xs = []
+            ys = []
+            zs = []
+            notable_nets = []
+            for learn_from_severity in exp.learn_from_severity_values:
+                counters = generate_counters()
+                results = []
+                for _ in tqdm(range(exp.trials)):
+                    soup = Soup(exp.soup_size, lambda net_generator=net_generator,exp=exp: TrainingNeuralNetworkDecorator(net_generator()).with_params(epsilon=exp.epsilon))
+                    soup.with_params(attacking_rate=-1, learn_from_rate=0.1, train=0, learn_from_severity=learn_from_severity)
+                    soup.seed()
+                    name = str(soup.particles[0].net.__class__.__name__) + " activiation='" + str(soup.particles[0].get_keras_params().get('activation')) + "' use_bias=" + str(soup.particles[0].get_keras_params().get('use_bias'))
+                    for time in range(exp.soup_life):
+                        soup.evolve()
+                    count(counters, soup, notable_nets)
+                    K.clear_session()
+
+                xs += [learn_from_severity]
+                ys += [float(counters['fix_zero']) / float(exp.trials)]
+                zs += [float(counters['fix_other']) / float(exp.trials)]
+            all_names += [name]
+            # xs: learn_from_intensity according to exp.learn_from_intensity_values
+            # ys: zero-fixpoints after life time
+            # zs: non-zero-fixpoints after life time
+            all_data += [{'xs':xs, 'ys':ys, 'zs':zs}]
+
+        exp.save(all_names=all_names)
+        exp.save(all_data=all_data)
+        exp.save(soup=soup.without_particles())
+        for exp_id, name in enumerate(all_names):
+            exp.log(all_names[exp_id])
+            exp.log(all_data[exp_id])
+            exp.log('\n')

code/setups/mixed-self-fixpoints.py

@@ -0,0 +1,100 @@
+import sys
+import os
+
+from typing import Tuple
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from experiment import *
+from network import *
+
+import tensorflow.python.keras.backend as K
+
+
+def generate_counters():
+    """
+    Initial build of the counter dict, to store counts.
+
+    :rtype: dict
+    :return: dictionary holding counter for: 'divergent', 'fix_zero', 'fix_sec', 'other'
+    """
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+def count(counters, net, notable_nets=None):
+    """
+    Count the occurences ot the types of weight trajectories.
+
+    :param counters:      A counter dictionary.
+    :param net:           A Neural Network
+    :param notable_nets:  A list to store and save intersting candidates
+
+    :rtype      Tuple[dict, list]
+    :return:    Both the counter dictionary and the list of interessting nets.
+    """
+    notable_nets = notable_nets or list()
+    if net.is_diverged():
+        counters['divergent'] += 1
+    elif net.is_fixpoint():
+        if net.is_zero():
+            counters['fix_zero'] += 1
+        else:
+            counters['fix_other'] += 1
+            notable_nets += [net]
+    elif net.is_fixpoint(2):
+        counters['fix_sec'] += 1
+        notable_nets += [net]
+    else:
+        counters['other'] += 1
+    return counters, notable_nets
+
+if __name__ == '__main__':
+
+    with Experiment('mixed-self-fixpoints') as exp:
+        exp.trials = 20
+        exp.selfattacks = 4
+        exp.trains_per_selfattack_values = [50 * i for i in range(11)]
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:  # , 'sigmoid', 'relu']:
+            for use_bias in [False]:
+                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                # net_generators += [lambda activation=activation, use_bias=use_bias: FFTNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+
+        all_names = []
+        all_data = []
+
+        for net_generator_id, net_generator in enumerate(net_generators):
+            xs = []
+            ys = []
+            for trains_per_selfattack in exp.trains_per_selfattack_values:
+                counters = generate_counters()
+                notable_nets = []
+                for _ in tqdm(range(exp.trials)):
+                    net = ParticleDecorator(net_generator())
+                    net = TrainingNeuralNetworkDecorator(net).with_params(epsilon=exp.epsilon)
+                    name = str(net.net.net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias=" + str(net.get_keras_params().get('use_bias'))
+                    for selfattack_id in range(exp.selfattacks):
+                        net.self_attack()
+                        for train_id in range(trains_per_selfattack):
+                            loss = net.compiled().train(epoch=selfattack_id*trains_per_selfattack+train_id)
+                        if net.is_diverged() or net.is_fixpoint():
+                            break
+                    count(counters, net, notable_nets)
+                    keras.backend.clear_session()
+                xs += [trains_per_selfattack]
+                ys += [float(counters['fix_zero'] + counters['fix_other']) / float(exp.trials)]
+            all_names += [name]
+            # xs: how many trains per self-attack from exp.trains_per_selfattack_values
+            # ys: average amount of fixpoints found
+            all_data += [{'xs': xs, 'ys': ys}]
+
+        exp.save(all_names=all_names)
+        exp.save(all_data=all_data)
+        for exp_id, name in enumerate(all_names):
+            exp.log(all_names[exp_id])
+            exp.log(all_data[exp_id])
+            exp.log('\n')

code/setups/mixed-soup.py

@@ -0,0 +1,108 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from typing import Tuple
+
+from experiment import *
+from network import *
+from soup import *
+
+import tensorflow.python.keras.backend as K
+
+
+def generate_counters():
+    """
+    Initial build of the counter dict, to store counts.
+
+    :rtype: dict
+    :return: dictionary holding counter for: 'divergent', 'fix_zero', 'fix_sec', 'other'
+    """
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+def count(counters, soup, notable_nets=None):
+    """
+    Count the occurences ot the types of weight trajectories.
+
+    :param counters:      A counter dictionary.
+    :param soup:          A Soup
+    :param notable_nets:  A list to store and save intersting candidates
+
+    :rtype      Tuple[dict, list]
+    :return:    Both the counter dictionary and the list of interessting nets.
+    """
+
+    notable_nets = notable_nets or list()
+    for net in soup.particles:
+        if net.is_diverged():
+            counters['divergent'] += 1
+        elif net.is_fixpoint():
+            if net.is_zero():
+                counters['fix_zero'] += 1
+            else:
+                counters['fix_other'] += 1
+                # notable_nets += [net]
+        # elif net.is_fixpoint(2):
+        #     counters['fix_sec'] += 1
+        #     notable_nets += [net]
+        else:
+            counters['other'] += 1
+    return counters, notable_nets
+
+if __name__ == '__main__':
+
+    with Experiment('mixed-soup') as exp:
+        exp.trials = 10
+        exp.soup_size = 10
+        exp.soup_life = 5
+        exp.trains_per_selfattack_values = [10 * i for i in range(11)]
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:  # ['linear', 'sigmoid', 'relu']:
+            for use_bias in [False]:
+                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                # net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+
+        all_names = []
+        all_data = []
+        for net_generator_id, net_generator in enumerate(net_generators):
+            xs = []
+            ys = []
+            zs = []
+            for trains_per_selfattack in exp.trains_per_selfattack_values:
+                counters = generate_counters()
+                notable_nets = []
+                for soup_idx in tqdm(range(exp.trials)):
+                    soup = Soup(exp.soup_size,
+                                lambda net_generator=net_generator, exp=exp: TrainingNeuralNetworkDecorator(
+                                    net_generator()).with_params(epsilon=exp.epsilon))
+                    soup.with_params(attacking_rate=0.1, learn_from_rate=-1, train=trains_per_selfattack,
+                                     learn_from_severity=-1)
+                    soup.seed()
+                    name = str(soup.particles[0].net.__class__.__name__) + " activiation='" + str(
+                        soup.particles[0].get_keras_params().get('activation')) + "' use_bias=" + str(
+                        soup.particles[0].get_keras_params().get('use_bias'))
+                    for _ in range(exp.soup_life):
+                        soup.evolve()
+                    count(counters, soup, notable_nets)
+                    K.clear_session()
+
+                xs += [trains_per_selfattack]
+                ys += [float(counters['fix_zero']) / float(exp.trials)]
+                zs += [float(counters['fix_other']) / float(exp.trials)]
+            all_names += [name]
+            # xs: how many trains per self-attack from exp.trains_per_selfattack_values
+            # ys: average amount of zero-fixpoints found
+            # zs: average amount of non-zero fixpoints
+            all_data += [{'xs': xs, 'ys': ys, 'zs': zs}]
+
+        exp.save(all_names=all_names)
+        exp.save(all_data=all_data)
+        for exp_id, name in enumerate(all_names):
+            exp.log(all_names[exp_id])
+            exp.log(all_data[exp_id])
+            exp.log('\n')

code/setups/network_trajectorys.py

@@ -0,0 +1,112 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from soup import *
+from experiment import *
+
+
+if __name__ == '__main__':
+    def run_exp(net, prints=False):
+        # INFO Run_ID needs to be more than 0, so that exp stores the trajectories!
+        exp.run_net(net, 100, run_id=run_id + 1)
+        exp.historical_particles[run_id] = net
+        if prints:
+            print("Fixpoint? " + str(net.is_fixpoint()))
+            print("Loss " + str(loss))
+
+    if True:
+        # WeightWise Neural Network
+        with FixpointExperiment(name="weightwise_self_application") as exp:
+            for run_id in tqdm(range(20)):
+                net = ParticleDecorator(WeightwiseNeuralNetwork(width=2, depth=2)
+                                        .with_keras_params(activation='linear'))
+                run_exp(net)
+                K.clear_session()
+            exp.log(exp.counters)
+            exp.save(trajectorys=exp.without_particles())
+
+    if False:
+        # Aggregating Neural Network
+        with FixpointExperiment(name="aggregating_self_application") as exp:
+            for run_id in tqdm(range(10)):
+                net = ParticleDecorator(AggregatingNeuralNetwork(aggregates=4, width=2, depth=2)
+                                        .with_keras_params(activation='linear'))
+                run_exp(net)
+                K.clear_session()
+            exp.log(exp.counters)
+            exp.save(trajectorys=exp.without_particles())
+
+    if False:
+        #FFT Neural Network
+        with FixpointExperiment() as exp:
+            for run_id in tqdm(range(10)):
+                net = ParticleDecorator(FFTNeuralNetwork(aggregates=4, width=2, depth=2)
+                                        .with_keras_params(activation='linear'))
+                run_exp(net)
+                K.clear_session()
+            exp.log(exp.counters)
+            exp.save(trajectorys=exp.without_particles())
+
+    if False:
+        # ok so this works quite realiably
+        with FixpointExperiment(name="weightwise_learning") as exp:
+            for i in range(10):
+                run_count = 100
+                net = TrainingNeuralNetworkDecorator(ParticleDecorator(WeightwiseNeuralNetwork(width=2, depth=2)))
+                net.with_params(epsilon=0.0001).with_keras_params(activation='linear')
+                exp.historical_particles[net.get_uid()] = net
+                for run_id in tqdm(range(run_count+1)):
+                    net.compiled()
+                    loss = net.train(epoch=run_id)
+                    # run_exp(net)
+                    # net.save_state(time=run_id)
+                K.clear_session()
+            exp.save(trajectorys=exp.without_particles())
+
+    if False:
+        # ok so this works quite realiably
+        with FixpointExperiment(name="aggregating_learning") as exp:
+            for i in range(10):
+                run_count = 100
+                net = TrainingNeuralNetworkDecorator(ParticleDecorator(AggregatingNeuralNetwork(4, width=2, depth=2)))
+                net.with_params(epsilon=0.0001).with_keras_params(activation='linear')
+                exp.historical_particles[net.get_uid()] = net
+                for run_id in tqdm(range(run_count+1)):
+                    net.compiled()
+                    loss = net.train(epoch=run_id)
+                    # run_exp(net)
+                    # net.save_state(time=run_id)
+                K.clear_session()
+            exp.save(trajectorys=exp.without_particles())
+
+    if False:
+        # this explodes in our faces completely... NAN everywhere
+        # TODO: Wtf is happening here?
+        with FixpointExperiment() as exp:
+            run_count = 10000
+            net = TrainingNeuralNetworkDecorator(RecurrentNeuralNetwork(width=2, depth=2))\
+                .with_params(epsilon=0.1e-2).with_keras_params(optimizer='sgd', activation='linear')
+            for run_id in tqdm(range(run_count+1)):
+                loss = net.compiled().train()
+                if run_id % 500 == 0:
+                    net.print_weights()
+                    # print(net.apply_to_network(net))
+                    print("Fixpoint? " + str(net.is_fixpoint()))
+                    print("Loss " + str(loss))
+                    print()
+    if False:
+        # and this gets somewhat interesting... we can still achieve non-trivial fixpoints
+        # over multiple applications when training enough in-between
+        with MixedFixpointExperiment() as exp:
+            for run_id in range(10):
+                net = TrainingNeuralNetworkDecorator(FFTNeuralNetwork(2, width=2, depth=2))\
+                    .with_params(epsilon=0.0001, activation='sigmoid')
+                exp.run_net(net, 500, 10)
+
+                net.print_weights()
+
+                print("Fixpoint? " + str(net.is_fixpoint()))
+            exp.log(exp.counters)

code/setups/soup_trajectorys.py

@@ -0,0 +1,32 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from soup import *
+from experiment import *
+
+
+if __name__ == '__main__':
+    if True:
+        with SoupExperiment("soup") as exp:
+            for run_id in range(1):
+                net_generator = lambda: TrainingNeuralNetworkDecorator(WeightwiseNeuralNetwork(2, 2)) \
+                     .with_keras_params(activation='linear').with_params(epsilon=0.0001)
+                # net_generator = lambda: TrainingNeuralNetworkDecorator(AggregatingNeuralNetwork(4, 2, 2))\
+                #     .with_keras_params(activation='linear')
+                # net_generator = lambda: TrainingNeuralNetworkDecorator(FFTNeuralNetwork(4, 2, 2))\
+                #    .with_keras_params(activation='linear')
+                # net_generator = lambda: RecurrentNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
+                soup = Soup(20, net_generator).with_params(remove_divergent=True, remove_zero=True,
+                                                           train=30,
+                                                           learn_from_rate=-1)
+                soup.seed()
+                for _ in tqdm(range(100)):
+                    soup.evolve()
+                exp.log(soup.count())
+                # you can access soup.historical_particles[particle_uid].states[time_step]['loss']
+                #             or soup.historical_particles[particle_uid].states[time_step]['weights']
+                # from soup.dill
+                exp.save(soup=soup.without_particles())

code/setups/training-fixpoints.py

@@ -0,0 +1,70 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from experiment import *
+from network import *
+
+import tensorflow.python.keras.backend as K
+
+def generate_counters():
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+def count(counters, net, notable_nets=None):
+    notable_nets = notable_nets or list()
+    if net.is_diverged():
+        counters['divergent'] += 1
+    elif net.is_fixpoint():
+        if net.is_zero():
+            counters['fix_zero'] += 1
+        else:
+            counters['fix_other'] += 1
+            notable_nets += [net]
+    elif net.is_fixpoint(2):
+        counters['fix_sec'] += 1
+        notable_nets += [net]
+    else:
+        counters['other'] += 1
+    return counters, notable_nets
+
+
+if __name__ == '__main__':
+
+    with Experiment('training_fixpoint') as exp:
+        exp.trials = 50
+        exp.run_count = 1000
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:  # , 'sigmoid', 'relu']:
+            for use_bias in [False]:
+                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+        all_counters = []
+        all_notable_nets = []
+        all_names = []
+        for net_generator_id, net_generator in enumerate(net_generators):
+            counters = generate_counters()
+            notable_nets = []
+            for _ in tqdm(range(exp.trials)):
+                net = ParticleDecorator(net_generator())
+                net = TrainingNeuralNetworkDecorator(net).with_params(epsilon=exp.epsilon)
+                name = str(net.net.net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias=" + str(net.get_keras_params().get('use_bias'))
+                for run_id in range(exp.run_count):
+                    loss = net.compiled().train(epoch=run_id+1)
+                count(counters, net, notable_nets)
+            all_counters += [counters]
+            all_notable_nets += [notable_nets]
+            all_names += [name]
+            K.clear_session()
+        exp.save(all_counters=all_counters)
+        exp.save(trajectorys=exp.without_particles())
+        # net types reached in the end
+        # exp.save(all_notable_nets=all_notable_nets)
+        exp.save(all_names=all_names) #experiment setups
+        for exp_id, counter in enumerate(all_counters):
+            exp.log(all_names[exp_id])
+            exp.log(all_counters[exp_id])
+            exp.log('\n')

code/soup.py

@@ -0,0 +1,137 @@
+import random
+
+from network import *
+
+
+def prng():
+    return random.random()
+
+
+class Soup(object):
+    
+    def __init__(self, size, generator, **kwargs):
+        self.size = size
+        self.generator = generator
+        self.particles = []
+        self.historical_particles = {}
+        self.params = dict(attacking_rate=0.1, learn_from_rate=0.1, train=0, learn_from_severity=1)
+        self.params.update(kwargs)
+        self.time = 0
+
+    def __copy__(self):
+        copy_ = Soup(self.size, self.generator, **self.params)
+        copy_.__dict__ = {attr: self.__dict__[attr] for attr in self.__dict__ if
+                          attr not in ['particles', 'historical_particles']}
+        return copy_
+
+    def without_particles(self):
+        self_copy = copy.copy(self)
+        # self_copy.particles = [particle.states for particle in self.particles]
+        self_copy.historical_particles = {key: val.states for key, val in self.historical_particles.items()}
+        return self_copy
+
+    def with_params(self, **kwargs):
+        self.params.update(kwargs)
+        return self
+    
+    def generate_particle(self):
+        new_particle = ParticleDecorator(self.generator())
+        self.historical_particles[new_particle.get_uid()] = new_particle
+        return new_particle
+        
+    def get_particle(self, uid, otherwise=None):
+        return self.historical_particles.get(uid, otherwise)
+    
+    def seed(self):
+        self.particles = []
+        for _ in range(self.size):
+            self.particles += [self.generate_particle()]
+        return self       
+    
+    def evolve(self, iterations=1):
+        for _ in range(iterations):
+            self.time += 1
+            for particle_id, particle in enumerate(self.particles):
+                description = {'time': self.time}
+                if prng() < self.params.get('attacking_rate'):
+                    other_particle_id = int(prng() * len(self.particles))
+                    other_particle = self.particles[other_particle_id]
+                    particle.attack(other_particle)
+                    description['action'] = 'attacking'
+                    description['counterpart'] = other_particle.get_uid()
+                if prng() < self.params.get('learn_from_rate'):
+                    other_particle_id = int(prng() * len(self.particles))
+                    other_particle = self.particles[other_particle_id]
+                    for _ in range(self.params.get('learn_from_severity', 1)):
+                        particle.learn_from(other_particle)
+                    description['action'] = 'learn_from'
+                    description['counterpart'] = other_particle.get_uid()
+                for _ in range(self.params.get('train', 0)):
+                    particle.compiled()
+                    # callbacks on save_state are broken for TrainingNeuralNetwork
+                    loss = particle.train(store_states=False)
+                    description['fitted'] = self.params.get('train', 0)
+                    description['loss'] = loss
+                    description['action'] = 'train_self'
+                    description['counterpart'] = None
+                if self.params.get('remove_divergent') and particle.is_diverged():
+                    new_particle = self.generate_particle()
+                    self.particles[particle_id] = new_particle
+                    description['action'] = 'divergent_dead'
+                    description['counterpart'] = new_particle.get_uid()
+                if self.params.get('remove_zero') and particle.is_zero():
+                    new_particle = self.generate_particle()
+                    self.particles[particle_id] = new_particle
+                    description['action'] = 'zweo_dead'
+                    description['counterpart'] = new_particle.get_uid()
+                particle.save_state(**description)
+
+    def count(self):
+        counters = dict(divergent=0, fix_zero=0, fix_other=0, fix_sec=0, other=0)
+        for particle in self.particles:
+            if particle.is_diverged():
+                counters['divergent'] += 1
+            elif particle.is_fixpoint():
+                if particle.is_zero():
+                    counters['fix_zero'] += 1
+                else:
+                    counters['fix_other'] += 1
+            elif particle.is_fixpoint(2):
+                counters['fix_sec'] += 1
+            else:
+                counters['other'] += 1
+        return counters
+        
+    def print_all(self):
+        for particle in self.particles:
+            particle.print_weights()
+            print(particle.is_fixpoint())
+
+
+if __name__ == '__main__':
+    if True:
+        net_generator = lambda: WeightwiseNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
+        soup_generator = Soup(100, net_generator).with_params(remove_divergent=True, remove_zero=True)
+        exp = SoupExperiment()
+        exp.run_exp(net_generator, 1000, soup_generator, 1, False)
+
+        # net_generator = lambda: FFTNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
+        # net_generator = lambda: AggregatingNeuralNetwork(4, 2, 2).with_keras_params(activation='sigmoid')\
+        # .with_params(shuffler=AggregatingNeuralNetwork.shuffle_random)
+        # net_generator = lambda: RecurrentNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
+
+    if True:
+        net_generator = lambda: TrainingNeuralNetworkDecorator(WeightwiseNeuralNetwork(2, 2)) \
+            .with_keras_params(activation='linear').with_params(epsilon=0.0001)
+        soup_generator = lambda: Soup(100, net_generator).with_params(remove_divergent=True, remove_zero=True, train=20)
+        exp = SoupExperiment(name="soup")
+
+        exp.run_exp(net_generator, 100, soup_generator, 1, False)
+
+        # net_generator = lambda: TrainingNeuralNetworkDecorator(AggregatingNeuralNetwork(4, 2, 2))
+        # .with_keras_params(activation='linear')\
+        # .with_params(shuffler=AggregatingNeuralNetwork.shuffle_random)
+        # net_generator = lambda: TrainingNeuralNetworkDecorator(FFTNeuralNetwork(4, 2, 2))\
+        #     .with_keras_params(activation='linear')\
+        # .with_params(shuffler=AggregatingNeuralNetwork.shuffle_random)
+        # net_generator = lambda: RecurrentNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()

experiments/helpers.py

@@ -1,59 +0,0 @@
-""" -------------------------------- Methods for summarizing the experiments --------------------------------- """
-from pathlib import Path
-
-from visualization import line_chart_fixpoints, bar_chart_fixpoints
-
-
-def summary_fixpoint_experiment(runs, population_size, epochs, experiments, net_learning_rate, directory,
-                                summary_pre_title):
-    avg_fixpoint_counters = {
-        "avg_identity_func": 0,
-        "avg_divergent": 0,
-        "avg_fix_zero": 0,
-        "avg_fix_weak": 0,
-        "avg_fix_sec": 0,
-        "avg_other_func": 0
-    }
-
-    for i in range(len(experiments)):
-        fixpoint_counters = experiments[i].fixpoint_counters
-
-        avg_fixpoint_counters["avg_identity_func"] += fixpoint_counters["identity_func"]
-        avg_fixpoint_counters["avg_divergent"] += fixpoint_counters["divergent"]
-        avg_fixpoint_counters["avg_fix_zero"] += fixpoint_counters["fix_zero"]
-        avg_fixpoint_counters["avg_fix_weak"] += fixpoint_counters["fix_weak"]
-        avg_fixpoint_counters["avg_fix_sec"] += fixpoint_counters["fix_sec"]
-        avg_fixpoint_counters["avg_other_func"] += fixpoint_counters["other_func"]
-
-    # Calculating the average for each fixpoint
-    avg_fixpoint_counters.update((x, y / len(experiments)) for x, y in avg_fixpoint_counters.items())
-
-    # Checking where the data is coming from to have a relevant title in the plot.
-    if summary_pre_title not in ["ST", "SA", "soup", "mixed", "robustness"]:
-        summary_pre_title = ""
-
-    # Plotting the summary
-    source_checker = "summary"
-    exp_details = f"{summary_pre_title}: {runs} runs & {epochs} epochs each."
-    bar_chart_fixpoints(avg_fixpoint_counters, population_size, directory, net_learning_rate, exp_details,
-                        source_checker)
-
-
-def summary_fixpoint_percentage(runs, epochs, fixpoints_percentages, ST_steps, SA_steps, directory_name,
-                                population_size):
-    fixpoints_percentages = [round(fixpoints_percentages[i] / runs, 1) for i in range(len(fixpoints_percentages))]
-
-    # Plotting summary
-    if "soup" in directory_name:
-        line_chart_fixpoints(fixpoints_percentages, epochs / ST_steps, ST_steps, SA_steps, directory_name,
-                             population_size)
-    else:
-        line_chart_fixpoints(fixpoints_percentages, epochs, ST_steps, SA_steps, directory_name, population_size)
-
-
-""" -------------------------------------------- Miscellaneous --------------------------------------------------- """
-
-
-def check_folder(experiment_folder: str):
-    exp_path = Path('experiments') / experiment_folder
-    exp_path.mkdir(parents=True, exist_ok=True)

experiments/meta_task_small_utility.py

@@ -1,38 +0,0 @@
-import torch
-from torch.utils.data import Dataset
-
-DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-
-
-class AddTaskDataset(Dataset):
-    def __init__(self, length=int(1e3)):
-        super().__init__()
-        self.length = length
-
-    def __len__(self):
-        return self.length
-
-    def __getitem__(self, _):
-        ab = torch.randn(size=(2,)).to(torch.float32)
-        return ab, ab.sum(axis=-1, keepdims=True)
-
-
-def train_task(model, optimizer, loss_func, btch_x, btch_y) -> (dict, torch.Tensor):
-    # Zero your gradients for every batch!
-    optimizer.zero_grad()
-    btch_x, btch_y = btch_x.to(DEVICE), btch_y.to(DEVICE)
-    y_prd = model(btch_x)
-
-    loss = loss_func(y_prd, btch_y.to(torch.float))
-    loss.backward()
-
-    # Adjust learning weights
-    optimizer.step()
-
-    stp_log = dict(Metric='Task Loss', Score=loss.item())
-
-    return stp_log, y_prd
-
-
-if __name__ == '__main__':
-    raise(NotImplementedError('Get out of here'))

experiments/meta_task_utility.py

@@ -1,405 +0,0 @@
-import pickle
-import re
-import shutil
-from collections import defaultdict
-from pathlib import Path
-import pandas as pd
-import numpy as np
-import torch
-import torchmetrics
-from matplotlib import pyplot as plt
-import seaborn as sns
-from torch.utils.data import Dataset
-
-from tqdm import tqdm
-
-from functionalities_test import test_for_fixpoints, FixTypes as ft
-from sanity_check_weights import test_weights_as_model, extract_weights_from_model
-
-WORKER = 10
-BATCHSIZE = 500
-
-DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-
-DATA_PATH = Path('data')
-DATA_PATH.mkdir(exist_ok=True, parents=True)
-
-PALETTE = sns.color_palette()
-PALETTE.insert(0, PALETTE.pop(1))  # Orange First
-
-FINAL_CHECKPOINT_NAME = f'trained_model_ckpt_FINAL.tp'
-
-
-class AddGaussianNoise(object):
-    def __init__(self, ratio=1e-4):
-        self.ratio = ratio
-
-    def __call__(self, tensor: torch.Tensor):
-        return tensor + (torch.randn_like(tensor, device=tensor.device) * self.ratio)
-
-    def __repr__(self):
-        return self.__class__.__name__ + f'(ratio={self.ratio}'
-
-
-class ToFloat:
-
-    def __init__(self):
-        pass
-
-    def __call__(self, x):
-        return x.to(torch.float32)
-
-
-class AddTaskDataset(Dataset):
-    def __init__(self, length=int(5e5)):
-        super().__init__()
-        self.length = length
-        self.prng = np.random.default_rng()
-
-    def __len__(self):
-        return self.length
-
-    def __getitem__(self, _):
-        ab = self.prng.normal(size=(2,)).astype(np.float32)
-        return ab, ab.sum(axis=-1, keepdims=True)
-
-
-def set_checkpoint(model, out_path, epoch_n, final_model=False):
-    if not final_model:
-        epoch_n = str(epoch_n)
-        ckpt_path = Path(out_path) / 'ckpt' / f'{epoch_n.zfill(4)}_model_ckpt.tp'
-    else:
-        if isinstance(epoch_n, str):
-            ckpt_path = Path(out_path) / f'{Path(FINAL_CHECKPOINT_NAME).stem}_{epoch_n}.tp'
-        else:
-            ckpt_path = Path(out_path) / FINAL_CHECKPOINT_NAME
-    ckpt_path.parent.mkdir(exist_ok=True, parents=True)
-
-    torch.save(model, ckpt_path, pickle_protocol=pickle.HIGHEST_PROTOCOL)
-    py_store_path = Path(out_path) / 'exp_py.txt'
-    if not py_store_path.exists():
-        shutil.copy(__file__, py_store_path)
-    return ckpt_path
-
-
-# noinspection PyProtectedMember
-def validate(checkpoint_path, valid_loader, metric_class=torchmetrics.Accuracy):
-    checkpoint_path = Path(checkpoint_path)
-
-    # initialize metric
-    validmetric = metric_class()
-    model = torch.load(checkpoint_path, map_location=DEVICE).eval()
-
-    with tqdm(total=len(valid_loader), desc='Validation Run: ') as pbar:
-        for idx, (valid_batch_x, valid_batch_y) in enumerate(valid_loader):
-            valid_batch_x, valid_batch_y = valid_batch_x.to(DEVICE), valid_batch_y.to(DEVICE)
-            y_valid = model(valid_batch_x)
-
-            # metric on current batch
-            measure = validmetric(y_valid.cpu(), valid_batch_y.cpu())
-            pbar.set_postfix_str(f'Measure: {measure}')
-            pbar.update()
-
-    # metric on all batches using custom accumulation
-    measure = validmetric.compute()
-    tqdm.write(f"Avg. {validmetric._get_name()} on all data: {measure}")
-    return measure
-
-
-def new_storage_df(identifier, weight_count):
-    if identifier == 'train':
-        return pd.DataFrame(columns=['Epoch', 'Batch', 'Metric', 'Score'])
-    elif identifier == 'weights':
-        return pd.DataFrame(columns=['Epoch', 'Weight', *(f'weight_{x}' for x in range(weight_count))])
-
-
-def checkpoint_and_validate(model, valid_loader, out_path, epoch_n, keep_n=5, final_model=False,
-                            validation_metric=torchmetrics.Accuracy):
-    out_path = Path(out_path)
-    ckpt_path = set_checkpoint(model, out_path, epoch_n, final_model=final_model)
-    # Clean up Checkpoints
-    if keep_n > 0:
-        all_ckpts = sorted(list(ckpt_path.parent.iterdir()))
-        while len(all_ckpts) > keep_n:
-            all_ckpts.pop(0).unlink()
-    elif keep_n == 0:
-        pass
-    else:
-        raise ValueError(f'"keep_n" cannot be negative, but was: {keep_n}')
-
-    result = validate(ckpt_path, valid_loader, metric_class=validation_metric)
-    return result
-
-
-def plot_training_particle_types(path_to_dataframe):
-    plt.close('all')
-    plt.clf()
-    # load from Drive
-    df = pd.read_csv(path_to_dataframe, index_col=False).sort_values('Metric')
-    # Set up figure
-    fig, ax = plt.subplots()  # initializes figure and plots
-    data = df.loc[df['Metric'].isin(ft.all_types())]
-    fix_types = data['Metric'].unique()
-    data = data.pivot(index='Epoch', columns='Metric', values='Score').reset_index().fillna(0)
-    _ = plt.stackplot(data['Epoch'], *[data[fixtype] for fixtype in fix_types],
-                      labels=fix_types.tolist(), colors=PALETTE)
-
-    ax.set(ylabel='Particle Count', xlabel='Epoch')
-    ax.yaxis.get_major_locator().set_params(integer=True)
-    # ax.set_title('Particle Type Count')
-
-    fig.legend(loc="center right", title='Particle Type', bbox_to_anchor=(0.85, 0.5))
-    plt.tight_layout()
-    plt.savefig(Path(path_to_dataframe.parent / 'training_particle_type_lp.png'), dpi=300)
-
-
-def plot_training_result(path_to_dataframe, metric_name='Accuracy', plot_name=None):
-    plt.clf()
-    # load from Drive
-    df = pd.read_csv(path_to_dataframe, index_col=False).sort_values('Metric')
-
-    # Check if this is a single lineplot or if aggregated
-    group = ['Epoch', 'Metric']
-    if 'Seed' in df.columns:
-        group.append('Seed')
-
-    # Set up figure
-    fig, ax1 = plt.subplots()  # initializes figure and plots
-    ax2 = ax1.twinx()  # applies twinx to ax2, which is the second y-axis.
-
-    # plots the first set of data
-    data = df[(df['Metric'] == 'Task Loss') | (df['Metric'] == 'Self Train Loss')].groupby(['Epoch', 'Metric']).mean()
-    grouped_for_lineplot = data.groupby(group).mean()
-    palette_len_1 = len(grouped_for_lineplot.droplevel(0).reset_index().Metric.unique())
-
-    sns.lineplot(data=grouped_for_lineplot, x='Epoch', y='Score', hue='Metric',
-                 palette=PALETTE[:palette_len_1], ax=ax1, ci='sd')
-
-    # plots the second set of data
-    data = df[(df['Metric'] == f'Test {metric_name}') | (df['Metric'] == f'Train {metric_name}')]
-    palette_len_2 = len(data.Metric.unique())
-    sns.lineplot(data=data, x='Epoch', y='Score', hue='Metric',
-                 palette=PALETTE[palette_len_1:palette_len_2+palette_len_1], ci='sd')
-
-    ax1.set(yscale='log', ylabel='Losses')
-    # ax1.set_title('Training Lineplot')
-    ax2.set(ylabel=metric_name)
-    if metric_name != 'Accuracy':
-        ax2.set(yscale='log')
-
-    fig.legend(loc="center right", title='Metric', bbox_to_anchor=(0.85, 0.5))
-    for ax in [ax1, ax2]:
-        if legend := ax.get_legend():
-            legend.remove()
-    plt.tight_layout()
-    plt.savefig(Path(path_to_dataframe.parent / ('training_lineplot.png' if plot_name is None else plot_name)), dpi=300)
-
-
-def plot_network_connectivity_by_fixtype(path_to_trained_model):
-    m = torch.load(path_to_trained_model, map_location=DEVICE).eval()
-    # noinspection PyProtectedMember
-    particles = list(m.particles)
-    df = pd.DataFrame(columns=['Type', 'Layer', 'Neuron', 'Name'])
-
-    for prtcl in particles:
-        l, c, w = [float(x) for x in re.sub("[^0-9|_]", "", prtcl.name).split('_')]
-        df.loc[df.shape[0]] = (prtcl.is_fixpoint, l-1, w, prtcl.name)
-        df.loc[df.shape[0]] = (prtcl.is_fixpoint, l, c, prtcl.name)
-    for layer in list(df['Layer'].unique()):
-        # Rescale
-        divisor = df.loc[(df['Layer'] == layer), 'Neuron'].max()
-        df.loc[(df['Layer'] == layer), 'Neuron'] /= divisor
-
-    tqdm.write(f'Connectivity Data gathered')
-    df = df.sort_values('Type')
-    n = 0
-    for fixtype in ft.all_types():
-        if df[df['Type'] == fixtype].shape[0] > 0:
-            plt.clf()
-            ax = sns.lineplot(y='Neuron', x='Layer', hue='Name', data=df[df['Type'] == fixtype],
-                              legend=False, estimator=None, lw=1)
-            _ = sns.lineplot(y=[0, 1], x=[-1, df['Layer'].max()], legend=False, estimator=None, lw=0)
-            ax.set_title(fixtype)
-            ax.yaxis.get_major_locator().set_params(integer=True)
-            ax.xaxis.get_major_locator().set_params(integer=True)
-            ax.set_ylabel('Normalized Neuron Position (1/n)')  # XAXIS Label
-            lines = ax.get_lines()
-            for line in lines:
-                line.set_color(PALETTE[n])
-            plt.savefig(Path(path_to_trained_model.parent / f'net_connectivity_{fixtype}.png'), dpi=300)
-            tqdm.write(f'Connectivity plottet: {fixtype} - n = {df[df["Type"] == fixtype].shape[0] // 2}')
-            n += 1
-        else:
-            # tqdm.write(f'No Connectivity {fixtype}')
-            pass
-
-
-# noinspection PyProtectedMember
-def run_particle_dropout_test(model_path, valid_loader, metric_class=torchmetrics.Accuracy):
-    diff_store_path = model_path.parent / 'diff_store.csv'
-    latest_model = torch.load(model_path, map_location=DEVICE).eval()
-    prtcl_dict = defaultdict(lambda: 0)
-    _ = test_for_fixpoints(prtcl_dict, list(latest_model.particles))
-    tqdm.write(str(dict(prtcl_dict)))
-    diff_df = pd.DataFrame(columns=['Particle Type', metric_class()._get_name(), 'Diff'])
-
-    acc_pre = validate(model_path, valid_loader, metric_class=metric_class).item()
-    diff_df.loc[diff_df.shape[0]] = ('All Organism', acc_pre, 0)
-
-    for fixpoint_type in ft.all_types():
-        new_model = torch.load(model_path, map_location=DEVICE).eval().replace_with_zero(fixpoint_type)
-        if [x for x in new_model.particles if x.is_fixpoint == fixpoint_type]:
-            new_ckpt = set_checkpoint(new_model, model_path.parent, fixpoint_type, final_model=True)
-            acc_post = validate(new_ckpt, valid_loader, metric_class=metric_class).item()
-            acc_diff = abs(acc_post - acc_pre)
-            tqdm.write(f'Zero_ident diff = {acc_diff}')
-            diff_df.loc[diff_df.shape[0]] = (fixpoint_type, acc_post, acc_diff)
-
-    diff_df.to_csv(diff_store_path, mode='w', header=True, index=False)
-    return diff_store_path
-
-
-# noinspection PyProtectedMember
-def plot_dropout_stacked_barplot(mdl_path, diff_store_path, metric_class=torchmetrics.Accuracy):
-    metric_name = metric_class()._get_name()
-    diff_df = pd.read_csv(diff_store_path).sort_values('Particle Type')
-    particle_dict = defaultdict(lambda: 0)
-    latest_model = torch.load(mdl_path, map_location=DEVICE).eval()
-    _ = test_for_fixpoints(particle_dict, list(latest_model.particles))
-    particle_dict = dict(particle_dict)
-    sorted_particle_dict = dict(sorted(particle_dict.items()))
-    tqdm.write(str(sorted_particle_dict))
-    plt.clf()
-    fig, ax = plt.subplots(ncols=2)
-    colors = PALETTE.copy()
-    colors.insert(0, colors.pop(-1))
-    palette_len = len(diff_df['Particle Type'].unique())
-    _ = sns.barplot(data=diff_df, y=metric_name, x='Particle Type', ax=ax[0], palette=colors[:palette_len], ci=None)
-
-    ax[0].set_title(f'{metric_name} after particle dropout')
-    # ax[0].set_xlabel('Particle Type')  # XAXIS Label
-    ax[0].set_xticklabels(ax[0].get_xticklabels(), rotation=30)
-
-    ax[1].pie(sorted_particle_dict.values(), labels=sorted_particle_dict.keys(),
-              colors=PALETTE[:len(sorted_particle_dict)])
-    ax[1].set_title('Particle Count')
-
-    plt.tight_layout()
-    plt.savefig(Path(diff_store_path.parent / 'dropout_stacked_barplot.png'), dpi=300)
-
-
-def run_particle_dropout_and_plot(model_path, valid_loader, metric_class=torchmetrics.Accuracy):
-    diff_store_path = run_particle_dropout_test(model_path, valid_loader=valid_loader, metric_class=metric_class)
-    plot_dropout_stacked_barplot(model_path, diff_store_path, metric_class=metric_class)
-
-
-def flat_for_store(parameters):
-    return (x.item() for y in parameters for x in y.detach().flatten())
-
-
-def train_self_replication(model, st_stps, **kwargs) -> dict:
-    self_train_loss = model.combined_self_train(st_stps, **kwargs)
-    # noinspection PyUnboundLocalVariable
-    stp_log = dict(Metric='Self Train Loss', Score=self_train_loss.item())
-    return stp_log
-
-
-def train_task(model, optimizer, loss_func, btch_x, btch_y) -> (dict, torch.Tensor):
-    # Zero your gradients for every batch!
-    optimizer.zero_grad()
-    btch_x, btch_y = btch_x.to(DEVICE), btch_y.to(DEVICE)
-    y_prd = model(btch_x)
-
-    loss = loss_func(y_prd, btch_y.to(torch.long))
-    loss.backward()
-
-    # Adjust learning weights
-    optimizer.step()
-
-    stp_log = dict(Metric='Task Loss', Score=loss.item())
-
-    return stp_log, y_prd
-
-
-def highlight_fixpoints_vs_mnist_mean(mdl_path, dataloader):
-    latest_model = torch.load(mdl_path, map_location=DEVICE).eval()
-    activation_vector = torch.as_tensor([[0, 0, 0, 0, 1]], dtype=torch.float32, device=DEVICE)
-    binary_images = []
-    real_images = []
-    with torch.no_grad():
-        # noinspection PyProtectedMember
-        for cell in latest_model._meta_layer_first.meta_cell_list:
-            cell_image_binary = torch.zeros((len(cell.meta_weight_list)), device=DEVICE)
-            cell_image_real = torch.zeros((len(cell.meta_weight_list)), device=DEVICE)
-            for idx, particle in enumerate(cell.particles):
-                if particle.is_fixpoint == ft.identity_func:
-                    cell_image_binary[idx] += 1
-                    cell_image_real[idx] = particle(activation_vector).abs().squeeze().item()
-            binary_images.append(cell_image_binary.reshape((15, 15)))
-            real_images.append(cell_image_real.reshape((15, 15)))
-
-        binary_images = torch.stack(binary_images)
-        real_images = torch.stack(real_images)
-
-        binary_image = torch.sum(binary_images, keepdim=True, dim=0)
-        real_image = torch.sum(real_images, keepdim=True, dim=0)
-
-    mnist_images = [x for x, _ in dataloader]
-    mnist_mean = torch.cat(mnist_images).reshape(10000, 15, 15).abs().sum(dim=0)
-
-    fig, axs = plt.subplots(1, 3)
-
-    for idx, (image, title) in enumerate(zip([binary_image, real_image, mnist_mean],
-                                           ["Particle Count", "Particle Value", "MNIST mean"])):
-        img = axs[idx].imshow(image.squeeze().detach().cpu())
-        img.axes.axis('off')
-        img.axes.set_title('Random Noise')
-
-    plt.tight_layout()
-    plt.savefig(mdl_path.parent / 'heatmap.png', dpi=300)
-    plt.clf()
-    plt.close('all')
-
-
-def plot_training_results_over_n_seeds(exp_path, df_train_store_name='train_store.csv', metric_name='Accuracy'):
-    combined_df_store_path = exp_path / f'comb_train_{exp_path.stem[:-1]}n.csv'
-    # noinspection PyUnboundLocalVariable
-    found_train_stores = exp_path.rglob(df_train_store_name)
-    train_dfs = []
-    for found_train_store in found_train_stores:
-        train_store_df = pd.read_csv(found_train_store, index_col=False)
-        train_store_df['Seed'] = int(found_train_store.parent.name)
-        train_dfs.append(train_store_df)
-    combined_train_df = pd.concat(train_dfs)
-    combined_train_df.to_csv(combined_df_store_path, index=False)
-    plot_training_result(combined_df_store_path, metric_name=metric_name,
-                         plot_name=f"{combined_df_store_path.stem}.png"
-                         )
-    plt.clf()
-    plt.close('all')
-
-
-def sanity_weight_swap(exp_path, dataloader, metric_class=torchmetrics.Accuracy):
-    # noinspection PyProtectedMember
-    metric_name = metric_class()._get_name()
-    found_models = exp_path.rglob(f'*{FINAL_CHECKPOINT_NAME}')
-    df = pd.DataFrame(columns=['Seed', 'Model', metric_name])
-    for model_idx, found_model in enumerate(found_models):
-        model = torch.load(found_model, map_location=DEVICE).eval()
-        weights = extract_weights_from_model(model)
-
-        results = test_weights_as_model(model, weights, dataloader, metric_class=metric_class)
-        for model_name, measurement in results.items():
-            df.loc[df.shape[0]] = (model_idx, model_name, measurement)
-        df.loc[df.shape[0]] = (model_idx, 'Difference', np.abs(np.subtract(*results.values())))
-
-    df.to_csv(exp_path / 'sanity_weight_swap.csv', index=False)
-    _ = sns.boxplot(data=df, x='Model', y=metric_name)
-    plt.tight_layout()
-    plt.savefig(exp_path / 'sanity_weight_swap.png', dpi=300)
-    plt.clf()
-    plt.close('all')
-
-
-if __name__ == '__main__':
-    raise NotImplementedError('Test this here!!!')

experiments/robustness_tester.py

@@ -1,118 +0,0 @@
-import pandas as pd
-import torch
-import random
-import copy
-
-from tqdm import tqdm
-from functionalities_test import (is_identity_function, is_zero_fixpoint, test_for_fixpoints, is_divergent,
-                                  FixTypes as FT)
-from network import Net
-from torch.nn import functional as F
-import seaborn as sns
-from matplotlib import pyplot as plt
-
-
-def prng():
-    return random.random()
-
-
-def generate_perfekt_synthetic_fixpoint_weights():
-    return torch.tensor([[1.0], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0],
-                         [1.0], [0.0], [0.0], [0.0],
-                         [1.0], [0.0]
-                         ], dtype=torch.float32)
-
-PALETTE = 10 * (
-    "#377eb8",
-    "#4daf4a",
-    "#984ea3",
-    "#e41a1c",
-    "#ff7f00",
-    "#a65628",
-    "#f781bf",
-    "#888888",
-    "#a6cee3",
-    "#b2df8a",
-    "#cab2d6",
-    "#fb9a99",
-    "#fdbf6f",
-)
-
-
-def test_robustness(model_path, noise_levels=10, seeds=10, log_step_size=10):
-    model = torch.load(model_path, map_location='cpu')
-    networks = [x for x in model.particles if x.is_fixpoint == FT.identity_func]
-    time_to_vergence = [[0 for _ in range(noise_levels)] for _ in range(len(networks))]
-    time_as_fixpoint = [[0 for _ in range(noise_levels)] for _ in range(len(networks))]
-    row_headers = []
-    df = pd.DataFrame(columns=['setting', 'Noise Level', 'Self Train Steps', 'absolute_loss',
-                               'Time to convergence', 'Time as fixpoint'])
-    with tqdm(total=(seeds * noise_levels * len(networks)), desc='Per Particle Robustness') as pbar:
-        for setting, fixpoint in enumerate(networks):  # 1 / n
-            row_headers.append(fixpoint.name)
-            for seed in range(seeds):  # n / 1
-                for noise_level in range(noise_levels):
-                    steps = 0
-                    clone = Net(fixpoint.input_size, fixpoint.hidden_size, fixpoint.out_size,
-                                f"{fixpoint.name}_clone_noise_1e-{noise_level}")
-                    clone.load_state_dict(copy.deepcopy(fixpoint.state_dict()))
-                    clone = clone.apply_noise(pow(10, -noise_level))
-
-                    while not is_zero_fixpoint(clone) and not is_divergent(clone):
-                        # -> before
-                        clone_weight_pre_application = clone.input_weight_matrix()
-                        target_data_pre_application = clone.create_target_weights(clone_weight_pre_application)
-
-                        clone.self_application(1, log_step_size)
-                        time_to_vergence[setting][noise_level] += 1
-                        # -> after
-                        clone_weight_post_application = clone.input_weight_matrix()
-                        target_data_post_application = clone.create_target_weights(clone_weight_post_application)
-
-                        absolute_loss = F.l1_loss(target_data_pre_application, target_data_post_application).item()
-
-                        if is_identity_function(clone):
-                            time_as_fixpoint[setting][noise_level] += 1
-                            # When this raises a Type Error, we found a second order fixpoint!
-                        steps += 1
-
-                        df.loc[df.shape[0]] = [f'{setting}_{seed}', fr'$\mathregular{{10^{{-{noise_level}}}}}$',
-                                               steps, absolute_loss,
-                                               time_to_vergence[setting][noise_level],
-                                               time_as_fixpoint[setting][noise_level]]
-                    pbar.update(1)
-
-    # Get the measuremts at the highest time_time_to_vergence
-    df_sorted = df.sort_values('Self Train Steps', ascending=False).drop_duplicates(['setting', 'Noise Level'])
-    df_melted = df_sorted.reset_index().melt(id_vars=['setting', 'Noise Level', 'Self Train Steps'],
-                                             value_vars=['Time to convergence', 'Time as fixpoint'],
-                                             var_name="Measurement",
-                                             value_name="Steps").sort_values('Noise Level')
-
-    df_melted.to_csv(model_path.parent / 'robustness_boxplot.csv', index=False)
-
-    # Plotting
-    # plt.rcParams.update({
-    #    "text.usetex": True,
-    #    "font.family": "sans-serif",
-    #    "font.size": 12,
-    #    "font.weight": 'bold',
-    #    "font.sans-serif": ["Helvetica"]})
-    plt.clf()
-    sns.set(style='whitegrid', font_scale=1)
-    _ = sns.boxplot(data=df_melted, y='Steps', x='Noise Level', hue='Measurement', palette=PALETTE)
-    plt.tight_layout()
-
-    # sns.set(rc={'figure.figsize': (10, 50)})
-    # bx = sns.catplot(data=df[df['absolute_loss'] < 1], y='absolute_loss', x='application_step', kind='box',
-    #                  col='noise_level', col_wrap=3, showfliers=False)
-
-    filename = f"robustness_boxplot.png"
-    filepath = model_path.parent / filename
-    plt.savefig(str(filepath))
-    plt.close('all')
-    return time_as_fixpoint, time_to_vergence
-
-
-if __name__ == "__main__":
-    raise NotImplementedError('Get out of here!')