Delete New Text Document.txt

Now at normal execution Times

.gitignore

@@ -1 +1,572 @@
-/output/
+# Created by https://www.gitignore.io/api/data,linux,macos,python,windows,pycharm,database,jupyternotebook
+# Edit at https://www.gitignore.io/?templates=data,linux,macos,python,windows,pycharm,database,jupyternotebook
+
+
+### Local Datasets ###
+/experiments
+/setups/experiments
+
+### Data ###
+*.csv
+*.dat
+*.efx
+*.gbr
+*.key
+*.pps
+*.ppt
+*.pptx
+*.sdf
+*.tax2010
+*.vcf
+*.xml
+
+### Database ###
+*.accdb
+*.db
+*.dbf
+*.mdb
+*.pdb
+*.sqlite3
+
+### JupyterNotebook ###
+.ipynb_checkpoints
+*/.ipynb_checkpoints/*
+
+# Remove previous ipynb_checkpoints
+#   git rm -r .ipynb_checkpoints/
+#
+
+### Linux ###
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### PyCharm Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+.idea/sonarlint
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+### Python Patch ###
+.venv/
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# pycharm
+.idea/
+
+#######################################
+#### Tex related
+## Core latex/pdflatex auxiliary files:
+*.aux
+*.lof
+*.log
+*.lot
+*.fls
+*.out
+*.toc
+*.fmt
+*.fot
+*.cb
+*.cb2
+.*.lb
+
+## Intermediate documents:
+*.dvi
+*.xdv
+*-converted-to.*
+# these rules might exclude image files for figures etc.
+# *.ps
+# *.eps
+# *.pdf
+
+## Generated if empty string is given at "Please type another file name for output:"
+.pdf
+
+## Bibliography auxiliary files (bibtex/biblatex/biber):
+*.bbl
+*.bcf
+*.blg
+*-blx.aux
+*-blx.bib
+*.run.xml
+
+## Build tool auxiliary files:
+*.fdb_latexmk
+*.synctex
+*.synctex(busy)
+*.synctex.gz
+*.synctex.gz(busy)
+*.pdfsync
+
+## Build tool directories for auxiliary files
+# latexrun
+latex.out/
+
+## Auxiliary and intermediate files from other packages:
+# algorithms
+*.alg
+*.loa
+
+# achemso
+acs-*.bib
+
+# amsthm
+*.thm
+
+# beamer
+*.nav
+*.pre
+*.snm
+*.vrb
+
+# changes
+*.soc
+
+# comment
+*.cut
+
+# cprotect
+*.cpt
+
+# elsarticle (documentclass of Elsevier journals)
+*.spl
+
+# endnotes
+*.ent
+
+# fixme
+*.lox
+
+# feynmf/feynmp
+*.mf
+*.mp
+*.t[1-9]
+*.t[1-9][0-9]
+*.tfm
+
+#(r)(e)ledmac/(r)(e)ledpar
+*.end
+*.?end
+*.[1-9]
+*.[1-9][0-9]
+*.[1-9][0-9][0-9]
+*.[1-9]R
+*.[1-9][0-9]R
+*.[1-9][0-9][0-9]R
+*.eledsec[1-9]
+*.eledsec[1-9]R
+*.eledsec[1-9][0-9]
+*.eledsec[1-9][0-9]R
+*.eledsec[1-9][0-9][0-9]
+*.eledsec[1-9][0-9][0-9]R
+
+# glossaries
+*.acn
+*.acr
+*.glg
+*.glo
+*.gls
+*.glsdefs
+
+# gnuplottex
+*-gnuplottex-*
+
+# gregoriotex
+*.gaux
+*.gtex
+
+# htlatex
+*.4ct
+*.4tc
+*.idv
+*.lg
+*.trc
+*.xref
+
+# hyperref
+*.brf
+
+# knitr
+*-concordance.tex
+# TODO Comment the next line if you want to keep your tikz graphics files
+*.tikz
+*-tikzDictionary
+
+# listings
+*.lol
+
+# makeidx
+*.idx
+*.ilg
+*.ind
+*.ist
+
+# minitoc
+*.maf
+*.mlf
+*.mlt
+*.mtc[0-9]*
+*.slf[0-9]*
+*.slt[0-9]*
+*.stc[0-9]*
+
+# minted
+_minted*
+*.pyg
+
+# morewrites
+*.mw
+
+# nomencl
+*.nlg
+*.nlo
+*.nls
+
+# pax
+*.pax
+
+# pdfpcnotes
+*.pdfpc
+
+# sagetex
+*.sagetex.sage
+*.sagetex.py
+*.sagetex.scmd
+
+# scrwfile
+*.wrt
+
+# sympy
+*.sout
+*.sympy
+sympy-plots-for-*.tex/
+
+# pdfcomment
+*.upa
+*.upb
+
+# pythontex
+*.pytxcode
+pythontex-files-*/
+
+# tcolorbox
+*.listing
+
+# thmtools
+*.loe
+
+# TikZ & PGF
+*.dpth
+*.md5
+*.auxlock
+
+# todonotes
+*.tdo
+
+# vhistory
+*.hst
+*.ver
+
+# easy-todo
+*.lod
+
+# xcolor
+*.xcp
+
+# xmpincl
+*.xmpi
+
+# xindy
+*.xdy
+
+# xypic precompiled matrices
+*.xyc
+
+# endfloat
+*.ttt
+*.fff
+
+# Latexian
+TSWLatexianTemp*
+
+## Editors:
+# WinEdt
+*.bak
+*.sav
+
+# Texpad
+.texpadtmp
+
+# LyX
+*.lyx~
+
+# Kile
+*.backup
+
+# KBibTeX
+*~[0-9]*
+
+# auto folder when using emacs and auctex
+./auto/*
+*.el
+
+# expex forward references with \gathertags
+*-tags.tex
+
+# standalone packages
+*.sta
+
+
+# End of https://www.gitignore.io/api/data,linux,macos,python,windows,pycharm,database,jupyternotebook
+

README.md

@@ -1,52 +1,2 @@
-# self-rep NN paper - ALIFE journal edition
-
-- [x] Plateau / Pillar sizeWhat does happen to the fixpoints after noise introduction and retraining?Options beeing: Same Fixpoint, Similar Fixpoint (Basin), 
-    - Different Fixpoint?
-      Yes, we did not found same (10-5)
-    - Do they do the clustering thingy?
-      Kind of: Small movement towards (MIM-Distance getting smaller) parent fixpoint.
-      Small movement for everyone? -> Distribution
-
-    - see `journal_basins.py` for the "train -> spawn with noise -> train again and see where they end up" functionality. Apply noise follows the `vary` function that was used in the paper robustness test with `+- prng() * eps`. Change if desired.
-
-    - there is also a distance matrix for all-to-all particle comparisons (with distance parameter one of: `MSE`, `MAE` (mean absolute error = mean manhattan) and `MIM` (mean position invariant manhattan))
-
-
-- [ ] Same Thing with Soup interaction. We would expect the same behaviour...Influence of interaction with near and far away particles.
-      - 
-      - 
-  
-- [x] Robustness test with a trained NetworkTraining for high quality fixpoints, compare with the "perfect" fixpoint. Average Loss per application step
-
-    - see `journal_robustness.py` for robustness test modeled after cristians robustness-exp (with the exeption that we put noise on the weights). Has `synthetic` bool to switch to hand-modeled perfect fixpoint instead of naturally trained ones. 
-
-    - Also added two difference between the "time-as-fixpoint" and "time-to-verge" (i.e. to divergence / zero).
-
-    - We might need to consult about the "average loss per application step", as I think application loss get gradually higher the worse the weights get. So the average might not tell us much here.
-
-- [x] Adjust Self Training so that it favors second order fixpoints-> Second order test implementation (?)
-
-- [x] Barplot over clones -> how many become a fixpoint cs how many diverge per noise level
-
-- [x] Box-Plot of Avg. Distance of clones from parent
-
-- [x] Search subspace between two fixpoints by linage(10**-5), check were they end up
-
-- [x] How are basins / "attractor areas" shaped?
-
-
-# Future Todos:
-
-- [ ] Find a statistik over weight space that provides a better init function
-- [ ] Test this init function on a mnist classifier - just for the lolz
-
----
-## Notes: 
-
-- In the spawn-experiment we now fit and transform the PCA over *ALL* trajectories, instead of each net-history by its own. This can be toggled by the `plot_pca_together` parameter in `visualisation.py/plot_3d_self_train() & plot_3d()` (default: `False` but set `True` in the spawn-experiment class).
-
-- I have also added a `start_time` property for the nets (default: `1`). This is intended to be set flexibly for e.g., clones (when they are spawned midway through the experiment), such that the PCA can start the plotting trace from this timestep. When we spawn clones we deepcopy their parent's saved weight_history too, so that the PCA transforms same lenght trajectories. With `plot_pca_together` that means that clones and their parents will literally be plotted perfectly overlayed on top, up until the spawn-time, where you can see the offset / noise we apply. By setting the start_time, you can avoid this overlap and avoid hiding the parent's trace color which gets plotted first (because the parent is always added to self.nets first). **But more importantly, you can effectively zoom into the plot, by setting the parents start-time to just shy of the end of first epoch (where they get checked on fixpoint-property and spawn clones) and the start-times of clones to the second epoch. This will make the plot begin at spawn time, cutting off the parents initial trajectory and zoom-in to the action (see. `journal_basins.py/spawn_and_continue()`).**
-
-- Now saving the whole experiment class as pickle dump (`experiment_pickle.p`, just like cristian), hope thats fine.
-
-- Added a `requirement.txt` for quick venv / pip -r installs. Append as necessary.
+# bannana-networks
+

code/experiment.py

@@ -0,0 +1,277 @@
+import os
+import time
+import dill
+from tqdm import tqdm
+from copy import copy
+
+from tensorflow.python.keras import backend as K
+
+from abc import ABC, abstractmethod
+
+
+class IllegalArgumentError(ValueError):
+    pass
+
+
+class Experiment(ABC):
+
+    @staticmethod
+    def from_dill(path):
+        with open(path, "rb") as dill_file:
+            return dill.load(dill_file)
+
+    @staticmethod
+    def reset_model():
+        K.clear_session()
+
+    def __init__(self, name=None, ident=None, **kwargs):
+        self.experiment_id = f'{ident or ""}_{time.time()}'
+        self.experiment_name = name or 'unnamed_experiment'
+        self.iteration = 0
+        self.log_messages = list()
+        self.historical_particles = dict()
+        self.params = dict(exp_iterations=100, application_steps=100, prints=True, trains_per_application=100)
+        self.with_params(**kwargs)
+
+    def __copy__(self, *args, **kwargs):
+        params = self.params
+        params.update(name=self.experiment_name)
+        params.update(**kwargs)
+        self_copy = self.__class__(*args, **params)
+        return self_copy
+
+    def __enter__(self):
+        self.dir = os.path.join('experiments', f'exp-{self.experiment_name}-{self.experiment_id}-{self.iteration}')
+        os.makedirs(self.dir)
+        print(f'** created {self.dir} **')
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.save(experiment=self.without_particles())
+        self.save_log()
+        # Clean Exit
+        self.reset_all()
+        # self.iteration += 1  Taken From here!
+
+    def with_params(self, **kwargs):
+        # Make them your own
+        self.params.update(kwargs)
+        return self
+
+    def log(self, message, **kwargs):
+        self.log_messages.append(message)
+        print(message, **kwargs)
+
+    def save_log(self, log_name="log"):
+        with open(os.path.join(self.dir, f"{log_name}.txt"), "w") as log_file:
+            for log_message in self.log_messages:
+                print(str(log_message), file=log_file)
+
+    def without_particles(self):
+        self_copy = copy(self)
+        # Check if attribute exists
+        if hasattr(self, 'historical_particles'):
+            # Check if it is empty.
+            if self.historical_particles:
+                # Do the Update
+                # self_copy.particles = [particle.states for particle in self.particles]
+                self_copy.historical_particles = {key: val.states for key, val in self.historical_particles.items()}
+        return self_copy
+
+    def save(self, **kwargs):
+        for name, value in kwargs.items():
+            with open(os.path.join(self.dir, f"{name}.dill"), "wb") as dill_file:
+                dill.dump(value, dill_file)
+
+    def reset_log(self):
+        self.log_messages = list()
+
+    @abstractmethod
+    def run_net(self, net, **kwargs):
+        raise NotImplementedError
+        pass
+
+    def run_exp(self, network_generator, reset_model=False, **kwargs):
+        # INFO Run_ID needs to be more than 0, so that exp stores the trajectories!
+        for run_id in range(self.params.get('exp_iterations')):
+            network = network_generator()
+            self.run_net(network, **kwargs)
+            self.historical_particles[run_id] = network
+            if self.params.get('prints'):
+                print("Fixpoint? " + str(network.is_fixpoint()))
+            self.iteration += 1
+            if reset_model:
+                self.reset_model()
+
+    def reset_all(self):
+        self.reset_log()
+        self.reset_model()
+
+
+class FixpointExperiment(Experiment):
+
+    def __init__(self, **kwargs):
+        kwargs['name'] = self.__class__.__name__ if 'name' not in kwargs else kwargs['name']
+        super().__init__(**kwargs)
+        self.counters = dict(divergent=0, fix_zero=0, fix_other=0, fix_sec=0, other=0)
+        self.interesting_fixpoints = []
+
+    def run_exp(self, network_generator, logging=True, reset_model=False, **kwargs):
+        kwargs.update(reset_model=False)
+        super(FixpointExperiment, self).run_exp(network_generator, **kwargs)
+        if logging:
+            self.log(self.counters)
+        if reset_model:
+            self.reset_model()
+
+    def run_net(self, net, **kwargs):
+        if len(kwargs):
+            raise IllegalArgumentError
+        for i in range(self.params.get('application_steps')):
+            if net.is_diverged() or net.is_fixpoint():
+                break
+            net.set_weights(net.apply_to_weights(net.get_weights()))
+            if self.iteration and hasattr(self, 'save_state'):
+                net.save_state(time=i)
+        self.count(net)
+
+    def count(self, net):
+        if net.is_diverged():
+            self.counters['divergent'] += 1
+        elif net.is_fixpoint():
+            if net.is_zero():
+                self.counters['fix_zero'] += 1
+            else:
+                self.counters['fix_other'] += 1
+                self.interesting_fixpoints.append(net.get_weights())
+        elif net.is_fixpoint(2):
+            self.counters['fix_sec'] += 1
+        else:
+            self.counters['other'] += 1
+
+    def reset_counters(self):
+        for key in self.counters.keys():
+            self.counters[key] = 0
+        return True
+
+    def reset_all(self):
+        super(FixpointExperiment, self).reset_all()
+        self.reset_counters()
+
+
+class MixedFixpointExperiment(FixpointExperiment):
+
+    def __init__(self, **kwargs):
+        kwargs['name'] = self.__class__.__name__ if 'name' not in kwargs else kwargs['name']
+        super(MixedFixpointExperiment, self).__init__(**kwargs)
+
+    def run_net(self, net, **kwargs):
+        assert hasattr(net, 'train'), 'This Network must be trainable, i.e. use the "TrainingNeuralNetworkDecorator"!'
+
+        for application in range(self.params.get('application_steps')):
+            epoch_num = self.params.get('trains_per_application') * application
+            net.set_weights(net.apply_to_weights(net.get_weights()))
+            if net.is_diverged() or net.is_fixpoint():
+                break
+            barformat = "Experiment Iteration: {postfix[iteration]} | "
+            barformat += "Evolution Step:{postfix[step]}| "
+            barformat += "Training Epoch:{postfix[epoch]}| "
+            barformat += "Loss: {postfix[loss]} | {bar}"
+            with tqdm(total=self.params.get('trains_per_application'),
+                      postfix={'step': 0, 'loss': 0, 'iteration': self.iteration, 'epoch': 0, None: None},
+                      bar_format=barformat) as bar:
+                # This iterates for self.trains_per_application times, the addition is just for  epoch enumeration
+                for epoch in range(epoch_num, epoch_num + self.params.get('trains_per_application')):
+                    if net.is_diverged():
+                        print('Network diverged to either inf or nan... breaking')
+                        break
+                    loss = net.train(epoch=epoch)
+                    if epoch % 10 == 0:
+                        bar.postfix.update(step=application, epoch=epoch, loss=loss, iteration=self.iteration)
+                    bar.update()
+                    epoch_num += 1
+            if self.iteration and hasattr(net, 'save_sate'):
+                net.save_state()
+        self.count(net)
+
+
+class TaskExperiment(MixedFixpointExperiment):
+
+    def __init__(self, **kwargs):
+        kwargs['name'] = self.__class__.__name__ if 'name' not in kwargs else kwargs['name']
+        super(TaskExperiment, self).__init__(**kwargs)
+
+    def run_exp(self, network_generator, reset_model=False, logging=True, **kwargs):
+        kwargs.update(reset_model=False, logging=logging)
+        super(FixpointExperiment, self).run_exp(network_generator, **kwargs)
+        if reset_model:
+            self.reset_model()
+        pass
+
+    def run_net(self, net, **kwargs):
+        assert hasattr(net, 'evaluate')
+        super(TaskExperiment, self).run_net(net, **kwargs)
+
+        # Get Performance without Training
+        task_performance = net.evaluate(*net.get_samples(task_samples=True),
+                                        batchsize=net.get_amount_of_weights())
+        self_performance = net.evaluate(*net.get_samples(self_samples=True),
+                                        batchsize=net.get_amount_of_weights())
+
+        current_performance = dict(task_performance=task_performance,
+                                   self_performance=self_performance,
+                                   counters=self.counters, id=self.iteration)
+
+        self.log(current_performance)
+        pass
+
+
+class SoupExperiment(Experiment):
+
+    def __init__(self, soup_generator, **kwargs):
+        kwargs['name'] = self.__class__.__name__ if 'name' not in kwargs else kwargs['name']
+        self.soup_generator = soup_generator
+        super(SoupExperiment, self).__init__(**kwargs)
+
+    def run_exp(self, network_generator, **kwargs):
+        for i in range(self.params.get('exp_iterations')):
+            soup = self.soup_generator()
+            soup.seed()
+            for _ in tqdm(range(self.params.get('application_steps'))):
+                soup.evolve()
+            self.log(soup.count())
+            self.save(soup=soup.without_particles())
+            K.clear_session()
+
+    def run_net(self, net, **kwargs):
+        raise NotImplementedError
+        pass
+
+
+class TaskingSoupExperiment(Experiment):
+
+    def __init__(self, soup_generator, **kwargs):
+        kwargs['name'] = self.__class__.__name__ if 'name' not in kwargs else kwargs['name']
+        super(TaskingSoupExperiment, self).__init__(**kwargs)
+        self.soup_generator = soup_generator
+
+    def __copy__(self):
+        super(TaskingSoupExperiment, self).__copy__(self.soup_generator)
+
+    def run_exp(self, **kwargs):
+        for i in range(self.params.get('exp_iterations')):
+            soup = self.soup_generator()
+            soup.seed()
+            for _ in tqdm(range(self.params.get('application_steps'))):
+                soup.evolve()
+            self.log(soup.count())
+            self.save(soup=soup.without_particles())
+            K.clear_session()
+
+    def run_net(self, net, **kwargs):
+        raise NotImplementedError()
+        pass
+
+
+if __name__ == '__main__':
+    pass

code/network.py

@@ -0,0 +1,672 @@
+# Librarys
+import numpy as np
+from abc import abstractmethod, ABC
+from typing import List, Tuple
+from types import FunctionType
+import warnings
+
+import os
+
+# Functions and Operators
+from operator import mul
+from functools import reduce
+from itertools import accumulate
+from copy import deepcopy
+
+# Deep learning Framework
+from tensorflow.python.keras.models import Sequential
+from tensorflow.python.keras.callbacks import Callback
+from tensorflow.python.keras.layers import SimpleRNN, Dense
+
+# Experiment Class
+from task import TaskAdditionOfN
+from experiment import TaskExperiment
+
+# Supress warnings and info messages
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
+
+
+class SaveStateCallback(Callback):
+    def __init__(self, network, epoch=0):
+        super(SaveStateCallback, self).__init__()
+        self.net = network
+        self.init_epoch = epoch
+
+    def on_epoch_end(self, epoch, logs=None):
+        description = dict(time=epoch+self.init_epoch)
+        description['action'] = 'train_self'
+        description['counterpart'] = None
+        self.net.save_state(**description)
+        return
+
+
+class EarlyStoppingByInfNanLoss(Callback):
+    def __init__(self, monitor='loss', verbose=0):
+        super(Callback, self).__init__()
+        self.monitor = monitor
+        self.verbose = verbose
+
+    def on_epoch_end(self, epoch, logs: dict = None):
+        logs = logs or dict()
+        current = logs.get(self.monitor)
+        if current is None:
+            warnings.warn(f'Early stopping requires {self.monitor} available!', RuntimeWarning)
+            pass
+
+        if np.isnan(current) or np.isinf(current):
+            if self.verbose > 0:
+                print(f'Epoch {epoch}: early stopping THR')
+            self.model.stop_training = True
+
+
+class NeuralNetwork(ABC):
+    """
+    This is the Base Network Class, including abstract functions that must be implemented.
+    """
+
+    @staticmethod
+    def are_weights_diverged(weights: List[np.ndarray]) -> bool:
+        return any([any((np.isnan(x).any(), np.isinf(x).any())) for x in weights])
+
+    @staticmethod
+    def are_weights_within_bounds(weights: List[np.ndarray], lower_bound: float, upper_bound: float) -> bool:
+        return any([((lower_bound < x) & (x < upper_bound)).any() for x in weights])
+
+    @staticmethod
+    def get_weight_amount(weights: List[np.ndarray]):
+        return sum([x.size for x in weights])
+
+    @staticmethod
+    def shapes(weights: List[np.ndarray]):
+        return [x.shape for x in weights]
+
+    @staticmethod
+    def num_layers(weights: List[np.ndarray]):
+        return len(weights)
+
+    def repr(self, weights: List[np.ndarray]):
+        return f'Weights({self.weights_to_flat_array(weights).tolist()})'
+
+    @staticmethod
+    def weights_to_flat_array(weights: List[np.ndarray]) -> np.ndarray:
+        return np.concatenate([d.ravel() for d in weights])
+
+    @staticmethod
+    def reshape_flat_array(array, shapes: List[Tuple[int]]) -> List[np.ndarray]:
+        # Same thing, but with an additional np call
+        # sizes: List[int] = [int(np.prod(shape)) for shape in shapes]
+
+        sizes = [reduce(mul, shape) for shape in shapes]
+        # Split the incoming array into slices for layers
+        slices = [array[x: y] for x, y in zip(accumulate([0] + sizes), accumulate(sizes))]
+        # reshape them in accordance to the given shapes
+        weights = [np.reshape(weight_slice, shape) for weight_slice, shape in zip(slices, shapes)]
+        return weights
+
+    def __init__(self, **params):
+        super().__init__()
+        self.params = dict(epsilon=0.00000000000001, early_nan_stopping=True, store_states=False)
+        self.params.update(params)
+        self.name = params.get('name', self.__class__.__name__)
+        self.keras_params = dict(activation='linear', use_bias=False)
+        self.states = []
+        self.model: Sequential
+
+    def get_params(self) -> dict:
+        return self.params
+
+    def get_keras_params(self) -> dict:
+        return self.keras_params
+
+    def with_params(self, **kwargs):
+        self.params.update(kwargs)
+        return self
+
+    def with_keras_params(self, **kwargs):
+        self.keras_params.update(kwargs)
+        return self
+
+    def print_weights(self, weights=None):
+        print(self.repr(weights or self.get_weights()))
+
+    def get_amount_of_weights(self):
+        return self.get_weight_amount(self.get_weights())
+
+    def get_model(self):
+        return self.model
+
+    def get_weights(self) -> List[np.ndarray]:
+        return self.get_model().get_weights()
+
+    def get_weights_flat(self) -> np.ndarray:
+        return self.weights_to_flat_array(self.get_weights())
+
+    def reshape_flat_array_like(self, array, weights: List[np.ndarray]) -> List[np.ndarray]:
+        return self.reshape_flat_array(array, self.shapes(weights))
+
+    def set_weights(self, new_weights: List[np.ndarray]):
+        return self.model.set_weights(new_weights)
+
+    def apply_to_network(self, other_network) -> List[np.ndarray]:
+        """
+        Take a networks weights and apply _this_ networks function.
+        :param other_network:
+        :return:
+        """
+        new_weights = self.apply_to_weights(other_network.get_weights())
+        return new_weights
+
+    def is_diverged(self):
+        return self.are_weights_diverged(self.get_weights())
+
+    def is_zero(self, epsilon=None):
+        epsilon = epsilon or self.get_params().get('epsilon')
+        return self.are_weights_within_bounds(self.get_weights(), -epsilon, epsilon)
+
+    def is_fixpoint(self, degree: int = 1, epsilon: float = None) -> bool:
+        assert degree >= 1, "degree must be >= 1"
+        epsilon = epsilon or self.get_params().get('epsilon')
+
+        new_weights = deepcopy(self.get_weights())
+
+        for _ in range(degree):
+            new_weights = self.apply_to_weights(new_weights)
+            if self.are_weights_diverged(new_weights):
+                return False
+
+        flat_new = self.weights_to_flat_array(new_weights)
+        flat_old = self.weights_to_flat_array(self.get_weights())
+        biggerEpsilon = (np.abs(flat_new - flat_old) >= epsilon).any()
+
+        # Boolean Value needs to be flipped to answer "is_fixpoint"
+        return not biggerEpsilon
+
+    def aggregate_weights_by(self, weights: List[np.ndarray], func: FunctionType, num_aggregates: int):
+        collection_sizes = self.get_weight_amount(weights) // num_aggregates
+        flat = self.weights_to_flat_array(weights)
+        array_for_aggregation = flat[:collection_sizes * num_aggregates].reshape((num_aggregates, -1))
+        left_overs = flat[collection_sizes * num_aggregates:]
+        aggregated_weights = func(array_for_aggregation, num_aggregates)
+        return aggregated_weights, left_overs
+
+    def shuffle_weights(self, weights: List[np.ndarray]):
+        flat = self.weights_to_flat_array(weights)
+        np.random.shuffle(flat)
+        return self.reshape_flat_array_like(flat, weights)
+
+    @abstractmethod
+    def get_samples(self, **kwargs):
+        # TODO: add a dogstring, telling the user what this does, e.g. what is a sample?
+        raise NotImplementedError
+
+    @abstractmethod
+    def apply_to_weights(self, old_weights) -> List[np.ndarray]:
+        """
+        Take weights as inputs; retunr the evaluation of _this_ network.
+        "Apply this function".
+
+        :param old_weights:
+        :return:
+        """
+        raise NotImplementedError
+
+
+class ParticleDecorator:
+    next_uid = 0
+
+    def __init__(self, network):
+
+        # ToDo: Add DocString, What does it do?
+
+        self.uid = self.__class__.next_uid
+        self.__class__.next_uid += 1
+        self.network = network
+        self.states = []
+        self.save_state(time=0, action='init', counterpart=None)
+
+    def __getattr__(self, name):
+        return getattr(self.network, name)
+
+    def get_uid(self):
+        return self.uid
+
+    def make_state(self, **kwargs):
+        if self.network.is_diverged():
+            return None
+        state = {'class': self.network.__class__.__name__, 'weights': self.network.get_weights_flat()}
+        state.update(kwargs)
+        return state
+
+    def save_state(self, **kwargs):
+        state = self.make_state(**kwargs)
+        if state is not None:
+            self.states += [state]
+        else:
+            pass
+        return True
+
+    def update_state(self, number, **kwargs):
+        raise NotImplementedError('Result is vague')
+        # if number < len(self.states):
+        #     self.states[number] = self.make_state(**kwargs)
+        # else:
+        #     for i in range(len(self.states), number):
+        #         self.states += [None]
+        #     self.states += self.make_state(**kwargs)
+
+    def get_states(self):
+        return self.states
+
+    def attack(self, other_network, iterations: int = 1):
+        """
+        Set a networks weights based on the output of the application of my function to its weights.
+        "Alter a networks weights based on my evaluation"
+        :param other_network:
+        :param iterations:
+        :return:
+        """
+        for _ in range(iterations):
+            other_network.set_weights(self.apply_to_network(other_network))
+        return self
+
+    def self_attack(self, iterations: int = 1):
+        """
+        Set my weights based on the output of the application of my function to its weights.
+        "Alter my network weights based on my evaluation"
+        :param iterations:
+        :return:
+        """
+        for _ in range(iterations):
+            self.attack(self)
+        return self
+
+
+class TaskDecorator(TaskAdditionOfN):
+
+    def __init__(self, network, **kwargs):
+        super(TaskDecorator, self).__init__(**kwargs)
+        self.network = network
+        self.batchsize = self.network.get_amount_of_weights()
+
+    def __getattr__(self, name):
+        return getattr(self.network, name)
+
+    def get_samples(self, task_samples=False, self_samples=False, **kwargs):
+        # XOR, cannot be true at the same time
+        assert not all([task_samples, self_samples])
+
+        if task_samples:
+            return super(TaskDecorator, self).get_samples()
+
+        elif self_samples:
+            return self.network.get_samples()
+
+        else:
+            self_x, self_y = self.network.get_samples()
+            # Super class = Task
+            task_x, task_y = super(TaskDecorator, self).get_samples()
+
+            amount_of_weights = self.network.get_amount_of_weights()
+            random_idx = np.random.choice(np.arange(amount_of_weights), amount_of_weights//2)
+
+            x = self_x[random_idx] = task_x[random_idx]
+            y = self_y[random_idx] = task_y[random_idx]
+
+            return x, y
+
+
+class WeightwiseNeuralNetwork(NeuralNetwork):
+
+    def __init__(self, width, depth, **kwargs):
+        # ToDo: Insert Docstring
+        super().__init__(**kwargs)
+        self.width: int = width
+        self.depth: int = depth
+        self.model = Sequential()
+        self.model.add(Dense(units=self.width, input_dim=4, **self.keras_params))
+        for _ in range(self.depth-1):
+            self.model.add(Dense(units=self.width, **self.keras_params))
+        self.model.add(Dense(units=1, **self.keras_params))
+
+    def apply(self, inputs):
+        # TODO: Write about it... What does it do?
+        return self.model.predict(inputs)
+
+    def get_samples(self, **kwargs: List[np.ndarray]):
+        weights = kwargs.get('weights', self.get_weights())
+        sample = np.asarray([
+            [weight, idx, *x] for idx, layer in enumerate(weights) for x, weight in np.ndenumerate(layer)
+        ])
+        # normalize [layer, cell, position]
+        for idx in range(1, sample.shape[1]):
+            sample[:, idx] = sample[:, idx] / np.max(sample[:, idx])
+        return sample, sample[:, 0]
+
+    def apply_to_weights(self, weights) -> List[np.ndarray]:
+        # ToDo: Insert DocString
+        # Transform the weight matrix in an horizontal stack as: array([[weight, layer, cell, position], ...])
+        transformed_weights, _ = self.get_samples(weights=weights)
+        new_flat_weights = self.apply(transformed_weights)
+        # use the original weight shape to transform the new tensor
+        return self.reshape_flat_array_like(new_flat_weights, weights)
+
+
+class AggregatingNeuralNetwork(NeuralNetwork):
+
+    @staticmethod
+    def aggregate_fft(array: np.ndarray, aggregates: int):
+        flat = array.flatten()
+        # noinspection PyTypeChecker
+        fft_reduction = np.fft.fftn(flat, aggregates)
+        return fft_reduction
+
+    @staticmethod
+    def aggregate_average(array, _):
+        return np.average(array, axis=1)
+
+    @staticmethod
+    def aggregate_max(array, _):
+        return np.max(array, axis=1)
+
+    @staticmethod
+    def deaggregate_identically(aggregate, amount):
+        return np.repeat(aggregate, amount, axis=0)
+
+    @staticmethod
+    def shuffle_not(weights: List[np.ndarray]):
+        """
+        Doesn't do a thing. f(x)
+
+        :param weights: A List of Weights
+        :type weights: Weights
+        :return: The same old weights.
+        :rtype: Weights
+        """
+        return weights
+
+    def shuffle_random(self, weights: List[np.ndarray]):
+        weights = self.shuffle_weights(weights)
+        return weights
+
+    def __init__(self, aggregates, width, depth, **kwargs):
+        super().__init__(**kwargs)
+        self.aggregates = aggregates
+        self.width = width
+        self.depth = depth
+        self.model = Sequential()
+        self.model.add(Dense(units=width, input_dim=self.aggregates, **self.keras_params))
+        for _ in range(depth-1):
+            self.model.add(Dense(units=width, **self.keras_params))
+        self.model.add(Dense(units=self.aggregates, **self.keras_params))
+
+    def get_aggregator(self):
+        return self.params.get('aggregator', self.aggregate_average)
+
+    def get_deaggregator(self):
+        return self.params.get('deaggregator', self.deaggregate_identically)
+
+    def get_shuffler(self):
+        return self.params.get('shuffler', self.shuffle_not)
+
+    def apply(self, inputs):
+        # You need to add an dimension here... "..." copies array values
+        return self.model.predict(inputs[None, ...])
+
+    def get_aggregated_weights(self):
+        return self.aggregate_weights_by(self.get_weights(), self.get_aggregator(), self.aggregates)
+
+    def apply_to_weights(self, old_weights) -> List[np.ndarray]:
+
+        # build aggregations of old_weights
+        old_aggregations, leftovers = self.get_aggregated_weights()
+
+        # call network
+        new_aggregations = self.apply(old_aggregations)
+        collection_sizes = self.get_amount_of_weights() // self.aggregates
+        new_aggregations = self.deaggregate_identically(new_aggregations, collection_sizes)
+        # generate new weights
+        # only include leftovers if there are some then coonvert them to Weight on base of th old shape
+        complete_weights = new_aggregations if not leftovers.shape[0] else np.hstack((new_aggregations, leftovers))
+        new_weights = self.reshape_flat_array_like(complete_weights, old_weights)
+
+        # maybe shuffle
+        new_weights = self.get_shuffler()(new_weights)
+        return new_weights
+
+    def get_samples(self, **kwargs):
+        aggregations, _ = self.get_aggregated_weights()
+        # What did that do?
+        # sample = np.transpose(np.array([[aggregations[i]] for i in range(self.aggregates)]))
+        return aggregations, aggregations
+
+    def is_fixpoint_after_aggregation(self, degree=1, epsilon=None):
+        assert degree >= 1, "degree must be >= 1"
+        epsilon = epsilon or self.get_params().get('epsilon')
+
+        old_aggregations, _ = self.get_aggregated_weights()
+        new_weights = deepcopy(self.get_weights())
+
+        for _ in range(degree):
+            new_weights = self.apply_to_weights(new_weights)
+            if self.are_weights_diverged(new_weights):
+                return False
+
+        new_aggregations, leftovers = self.get_aggregated_weights()
+
+        # ToDo: Explain This, why are you additionally checking tolerances of aggregated weights?
+        biggerEpsilon = (np.abs(np.asarray(old_aggregations) - np.asarray(new_aggregations)) >= epsilon).any()
+
+        # Boolean value has to be flipped to answer the question.
+        return True, not biggerEpsilon
+
+
+class RecurrentNeuralNetwork(NeuralNetwork):
+
+    def __init__(self, width, depth, **kwargs):
+        raise NotImplementedError
+        super(RecurrentNeuralNetwork, self).__init__()
+        self.features = 1
+        self.width = width
+        self.depth = depth
+        self.model = Sequential()
+        self.model.add(SimpleRNN(units=width, input_dim=self.features, return_sequences=True, **self.keras_params))
+        for _ in range(depth-1):
+            self.model.add(SimpleRNN(units=width, return_sequences=True, **self.keras_params))
+        self.model.add(SimpleRNN(units=self.features, return_sequences=True, **self.keras_params))
+
+    def apply(self, *inputs):
+        stuff = np.transpose(np.array([[[inputs[i]] for i in range(len(inputs))]]))
+        return self.model.predict(stuff)[0].flatten()
+
+    def apply_to_weights(self, old_weights):
+        # build list from old weights
+        new_weights = deepcopy(old_weights)
+        old_weights_list = []
+        for layer_id, layer in enumerate(old_weights):
+            for cell_id, cell in enumerate(layer):
+                for weight_id, weight in enumerate(cell):
+                    old_weights_list += [weight]
+
+        # call network
+        new_weights_list = self.apply(*old_weights_list)
+
+        # write back new weights from list of rnn returns
+        current_weight_id = 0
+        for layer_id, layer in enumerate(new_weights):
+            for cell_id, cell in enumerate(layer):
+                for weight_id, weight in enumerate(cell):
+                    new_weight = new_weights_list[current_weight_id]
+                    new_weights[layer_id][cell_id][weight_id] = new_weight
+                    current_weight_id += 1
+        return new_weights
+
+    def compute_samples(self):
+        # build list from old weights
+        old_weights_list = []
+        for layer_id, layer in enumerate(self.get_weights()):
+            for cell_id, cell in enumerate(layer):
+                for weight_id, weight in enumerate(cell):
+                    old_weights_list += [weight]
+        sample = np.asarray(old_weights_list)[None, ..., None]
+        return sample, sample
+
+
+class TrainingNeuralNetworkDecorator:
+
+    def __init__(self, network):
+        self.network = network
+        self.compile_params = dict(loss='mse', optimizer='sgd')
+        self.model_compiled = False
+
+    def __getattr__(self, name):
+        return getattr(self.network, name)
+
+    def with_params(self, **kwargs):
+        self.network.with_params(**kwargs)
+        return self
+
+    def with_keras_params(self, **kwargs):
+        self.network.with_keras_params(**kwargs)
+        return self
+
+    def get_compile_params(self):
+        return self.compile_params
+
+    def with_compile_params(self, **kwargs):
+        self.compile_params.update(kwargs)
+        return self
+
+    def compile_model(self, **kwargs):
+        compile_params = deepcopy(self.compile_params)
+        compile_params.update(kwargs)
+        return self.network.model.compile(**compile_params)
+
+    def compiled(self, **kwargs):
+        if not self.model_compiled:
+            self.compile_model(**kwargs)
+            self.model_compiled = True
+        return self
+
+    def train(self, batchsize=1, epoch=0):
+        self.compiled()
+        x, y = self.network.get_samples()
+        callbacks = []
+        if self.get_params().get('store_states'):
+            callbacks.append(SaveStateCallback(network=self, epoch=epoch))
+        if self.get_params().get('early_nan_stopping'):
+            callbacks.append(EarlyStoppingByInfNanLoss())
+
+        # 'or' does not work on empty lists
+        callbacks = callbacks if callbacks else None
+        """
+        Please Note:
+        
+        epochs: Integer. Number of epochs to train the model.
+            An epoch is an iteration over the entire `x` and `y`
+            data provided.
+            Note that in conjunction with `initial_epoch`,
+            `epochs` is to be understood as "final epoch".
+            The model is not trained for a number of iterations
+            given by `epochs`, but merely until the epoch
+            of index `epochs` is reached."""
+        history = self.network.model.fit(x=x, y=y, initial_epoch=epoch, epochs=epoch+1, verbose=0,
+                                         batch_size=batchsize, callbacks=callbacks)
+        return history.history['loss'][-1]
+
+    def learn_from(self, other_network, batchsize=1):
+        self.compiled()
+        other_network.compiled()
+        x, y = other_network.network.get_samples()
+        history = self.network.model.fit(x=x, y=y, verbose=0, batch_size=batchsize)
+        return history.history['loss'][-1]
+
+    def evaluate(self, x=None, y=None, batchsize=1):
+        self.compiled()
+        x, y = x, y if x is not None and y is not None else self.network.get_samples()
+        """
+        Please Note:
+
+        epochs: Integer. Number of epochs to train the model.
+            An epoch is an iteration over the entire `x` and `y`
+            data provided.
+            Note that in conjunction with `initial_epoch`,
+            `epochs` is to be understood as "final epoch".
+            The model is not trained for a number of iterations
+            given by `epochs`, but merely until the epoch
+            of index `epochs` is reached."""
+        loss = self.network.model.evaluate(x=x, y=y, verbose=0, batch_size=batchsize)
+        return loss
+
+
+if __name__ == '__main__':
+
+    if True:
+        # WeightWise Neural Network
+        with TaskExperiment().with_params(application_steps=10, trains_per_application=1000, exp_iterations=30) as exp:
+            net_generator = lambda: TrainingNeuralNetworkDecorator(TaskDecorator(
+                WeightwiseNeuralNetwork(width=2, depth=2))
+            ).with_keras_params(activation='linear')
+            exp.run_exp(net_generator, reset_model=True)
+
+    if False:
+        # Aggregating Neural Network
+        net_generator = lambda: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2)
+        with MixedFixpointExperiment() as exp:
+            exp.run_exp(net_generator, 10)
+            exp.reset_all()
+
+    if False:
+        # FFT Aggregation
+        net_generator = lambda: AggregatingNeuralNetwork(
+            aggregates=4, width=2, depth=2, aggregator=AggregatingNeuralNetwork.aggregate_fft)
+        with FixpointExperiment() as exp:
+            exp.run_exp(net_generator, 10)
+            exp.log(exp.counters)
+            exp.reset_model()
+            exp.reset_all()
+
+    if False:
+        # ok so this works quite realiably
+        run_count = 1000
+        net_generator = lambda: TrainingNeuralNetworkDecorator(WeightwiseNeuralNetwork(
+            width=2, depth=2).with_params(epsilon=0.0001)).with_keras_params(optimizer='sgd')
+        with MixedFixpointExperiment() as exp:
+            for run_id in tqdm(range(run_count+1)):
+                exp.run_exp(net_generator, 1)
+                if run_id % 100 == 0:
+                    exp.run_exp(net_generator, 1)
+            K.clear_session()
+
+    if False:
+        with FixpointExperiment() as exp:
+            run_count = 100
+            net = TrainingNeuralNetworkDecorator(
+                AggregatingNeuralNetwork(4, width=2, depth=2).with_params(epsilon=0.1e-6))
+            for run_id in tqdm(range(run_count+1)):
+                current_loss = net.compiled().train()
+                if run_id % 100 == 0:
+                    net.print_weights()
+                    old_aggs, _ = net.get_aggregated_weights()
+                    print("old weights agg: " + str(old_aggs))
+                    fp, new_aggs = net.is_fixpoint_after_aggregation(epsilon=0.0001)
+                    print("new weights agg: " + str(new_aggs))
+                    print("Fixpoint? " + str(net.is_fixpoint()))
+                    print("Fixpoint after Agg? " + str(fp))
+                    print("Loss " + str(current_loss))
+                    print()
+
+    if False:
+        # this explodes in our faces completely... NAN everywhere
+        # TODO: Wtf is happening here?
+        with FixpointExperiment() as exp:
+            run_count = 10000
+            net = TrainingNeuralNetworkDecorator(RecurrentNeuralNetwork(width=2, depth=2)
+                                                 ).with_keras_params(optimizer='sgd', activation='linear')
+            for run_id in tqdm(range(run_count+1)):
+                current_loss = net.compiled().train()
+                if run_id % 500 == 0:
+                    net.print_weights()
+                    # print(net.apply_to_network(net))
+                    print("Fixpoint? " + str(net.is_fixpoint()))
+                    print("Loss " + str(current_loss))
+                    print()

code/plotting/plotting_class.py

@@ -0,0 +1,66 @@
+import os
+
+from experiment import Experiment
+# noinspection PyUnresolvedReferences
+from soup import Soup
+
+from argparse import ArgumentParser
+import numpy as np
+
+import plotly as pl
+import plotly.graph_objs as go
+
+import colorlover as cl
+
+import dill
+
+from sklearn.manifold.t_sne import TSNE, PCA
+
+
+def build_args():
+    arg_parser = ArgumentParser()
+    arg_parser.add_argument('-i', '--in_file', nargs=1, type=str)
+    arg_parser.add_argument('-o', '--out_file', nargs='?', default='out', type=str)
+    return arg_parser.parse_args()
+
+
+class DataPlotter:
+
+    def __init__(self, path=None):
+        self.path = path or os.getcwd()
+        pass
+
+    def search_and_apply(self, plotting_function, files_to_look_for=None, absolut_file_or_folder=None):
+        absolut_file_or_folder, files_to_look_for = self.path or absolut_file_or_folder, list() or files_to_look_for
+        if os.path.isdir(absolut_file_or_folder):
+            for sub_file_or_folder in os.scandir(absolut_file_or_folder):
+                self.search_and_apply(plotting_function, files_to_look_for=files_to_look_for,
+                                      absolut_file_or_folder=sub_file_or_folder.path)
+        elif absolut_file_or_folder.endswith('.dill'):
+            file_or_folder = os.path.split(absolut_file_or_folder)[-1]
+            if file_or_folder in files_to_look_for and not os.path.exists(
+                    '{}.html'.format(absolut_file_or_folder[:-5])):
+                print('Apply Plotting function "{func}" on file "{file}"'.format(func=plotting_function.__name__,
+                                                                                 file=absolut_file_or_folder)
+                      )
+                with open(absolut_file_or_folder, 'rb') as in_f:
+                    exp = dill.load(in_f)
+
+                names_dill_location = os.path.join(*os.path.split(absolut_file_or_folder)[:-1], 'all_names.dill')
+                with open(names_dill_location, 'rb') as in_f:
+                    names = dill.load(in_f)
+
+                try:
+                    plotting_function((names, exp), filename='{}.html'.format(absolut_file_or_folder[:-5]))
+                except ValueError:
+                    pass
+                except AttributeError:
+                    pass
+            else:
+                # This was either another FilyType or Plot.html already exists.
+                pass
+
+
+if __name__ == '__main__':
+    plotter = DataPlotter
+    pass

code/plotting/task_learning_curves.py

@@ -0,0 +1,109 @@
+import os
+from collections import defaultdict
+
+# noinspection PyUnresolvedReferences
+from soup import Soup
+from experiment import TaskExperiment
+
+from argparse import ArgumentParser
+
+import plotly as pl
+import plotly.graph_objs as go
+
+import colorlover as cl
+
+import dill
+import numpy as np
+
+
+def build_args():
+    arg_parser = ArgumentParser()
+    arg_parser.add_argument('-i', '--in_file', nargs=1, type=str)
+    arg_parser.add_argument('-o', '--out_file', nargs='?', default='out', type=str)
+    return arg_parser.parse_args()
+
+
+def line_plot(exp: TaskExperiment, filename='lineplot'):
+    assert isinstance(exp, TaskExperiment), ' This has to be a TaskExperiment!'
+    traces, data = [], defaultdict(list)
+
+    color_scale = cl.scales['3']['div']['RdYlBu']
+
+    # Sort data per Key
+    for message in exp.log_messages:
+        for key in message.keys():
+            try:
+                data[key].append(-0.1 if np.isnan(message[key]) or np.isinf(message[key]) else message[key])
+            except:
+                data[key].append(message[key])
+
+    for line_id, key in enumerate(data.keys()):
+        if key not in ['counters', 'id']:
+            trace = go.Scatter(
+                x=[x for x in range(len(data[key]))],
+                y=data[key],
+                name=key,
+                line=dict(
+                    color=color_scale[line_id],
+                    width=5
+                ),
+            )
+
+            traces.append(trace)
+        else:
+            continue
+
+    layout = dict(xaxis=dict(title='Trains per self-application', titlefont=dict(size=20)),
+                  yaxis=dict(title='Average amount of fixpoints found',
+                             titlefont=dict(size=20),
+                             # type='log',
+                             # range=[0, 2]
+                             ),
+                  legend=dict(orientation='h', x=0.3, y=-0.3),
+                  # height=800, width=800,
+                  margin=dict(b=0)
+                  )
+
+    fig = go.Figure(data=traces, layout=layout)
+    pl.offline.plot(fig, auto_open=True, filename=filename)
+    pass
+
+
+def search_and_apply(absolut_file_or_folder, plotting_function, files_to_look_for=None, override=False):
+    # ToDo: Clean this Mess
+    assert os.path.exists(absolut_file_or_folder), f'The given path does not exist! Given: {absolut_file_or_folder}'
+    files_to_look_for = files_to_look_for or list()
+    if os.path.isdir(absolut_file_or_folder):
+        for sub_file_or_folder in os.scandir(absolut_file_or_folder):
+            search_and_apply(sub_file_or_folder.path, plotting_function,
+                             files_to_look_for=files_to_look_for, override=override)
+    elif absolut_file_or_folder.endswith('.dill'):
+        file_or_folder = os.path.split(absolut_file_or_folder)[-1]
+        if file_or_folder in files_to_look_for or not files_to_look_for:
+            if not os.path.exists('{}.html'.format(absolut_file_or_folder[:-5])) or override:
+                print('Apply Plotting function "{func}" on file "{file}"'.format(func=plotting_function.__name__,
+                                                                                 file=absolut_file_or_folder)
+                      )
+                with open(absolut_file_or_folder, 'rb') as in_f:
+                    exp = dill.load(in_f)
+
+                try:
+                    plotting_function(exp, filename='{}.html'.format(absolut_file_or_folder[:-5]))
+                except ValueError:
+                    pass
+                except AttributeError:
+                    pass
+            else:
+                # Plot.html already exists.
+                pass
+        else:
+            # This was a wrong FilyType.
+            pass
+
+
+if __name__ == '__main__':
+    args = build_args()
+    in_file = args.in_file[0]
+    out_file = args.out_file
+
+    search_and_apply(in_file, line_plot, override=True)

code/results/Soup/log.txt

@@ -0,0 +1 @@
+{'divergent': 0, 'fix_zero': 0, 'fix_other': 13, 'fix_sec': 0, 'other': 7}

code/results/Soup/weights.txt

@@ -0,0 +1,30 @@
+[-0.15321673  1.0428386  -0.7245892  -0.04343993  0.42338863  0.02538261
+ -0.40465942 -0.0242596  -1.226809   -0.8168446   0.26588777 -1.0929432
+  0.5383322  -0.73875046]
+[-0.03072096 -1.369665   -0.357126   -0.21180922  0.3853204   0.22853081
+ -0.3705557  -0.21977347 -0.6684716   0.12849599  1.0226644  -0.0922638
+ -0.7828449  -0.6572327 ]
+[-1.2444692   0.61213857  0.07965802  0.12361202  0.62641835  0.9720597
+  0.3863232   0.59948945  1.0857513   0.49231085 -0.5319295   0.29433587
+ -0.64177823  0.17603302]
+[-0.9938292  -0.4438207  -0.03172896  0.06261964 -0.3870194   0.7637992
+  0.0244509  -0.04825407  0.91551745 -0.78740424  0.29226422 -0.52767307
+ -0.41744384  0.5567152 ]
+[-0.39049304  0.8842579  -0.8447943  -0.19669186  0.7207061   0.16780053
+  0.3728221   0.08680353  0.7535456  -0.1000197   0.02029054  0.8640245
+ -0.15881588  1.1905665 ]
+[ 1.0482084   0.9248296  -0.26946014  0.57047915 -0.32660747  0.6914731
+ -0.18025818  0.3816289  -0.69358927  0.21312684 -0.39932403 -0.02991759
+ -0.83068466  0.45619962]
+[ 0.75814664  0.10328437  0.07867077 -0.0743314  -0.53440267  0.50492585
+ -0.54172474  0.51184535  0.3462249   1.0527638  -0.9503541   0.9235086
+ -0.1665241   1.1497779 ]
+[-0.77187353  1.1105504   0.24265823  0.53782856 -0.34098852 -0.75576884
+ -0.25396293 -0.56288165  0.3851537  -0.67497945  0.14336896  0.763481
+ -0.9224985   0.6374753 ]
+[-0.79123825  0.68166596 -0.30061013 -0.19360289  0.5632736   0.36276665
+  0.7470975   0.48115698  0.10046808 -0.8064349  -1.036736   -0.68296516
+ -1.156437    0.52633154]
+[ 0.1788832  -1.5321186  -0.62001514 -0.3870902   0.97524184  0.6088638
+ -0.08297889 -0.05180515 -0.29096788  0.7519439   0.8803648   0.82771575
+ -0.854887    0.1742936 ]

code/results/exp-applying_fixpoint-_1552681870.3570378-0/log.txt

@@ -0,0 +1,12 @@
+WeightwiseNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 23, 'fix_zero': 27, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+AggregatingNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 4, 'fix_zero': 46, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+RecurrentNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 46, 'fix_zero': 4, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+

code/results/exp-learn-from-soup-_1552658566.5572753-0/log.txt

@@ -0,0 +1,4 @@
+TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
+{'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], 'zs': [0.0, 1.2, 5.2, 7.4, 8.1, 9.1, 9.6, 9.8, 10.0, 9.9, 9.9]}
+
+

code/results/exp-mixed-self-fixpoints-_1552666977.5858653-0/log.txt

@@ -0,0 +1,12 @@
+WeightwiseNeuralNetwork activiation='linear' use_bias=False
+{'xs': [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500], 'ys': [0.2, 0.3, 0.15, 0.55, 0.7, 0.85, 0.8, 0.95, 0.9, 1.0, 1.0]}
+
+
+AggregatingNeuralNetwork activiation='linear' use_bias=False
+{'xs': [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500], 'ys': [1.0, 0.95, 1.0, 1.0, 0.95, 0.9, 0.8, 1.0, 0.85, 1.0, 0.9]}
+
+
+RecurrentNeuralNetwork activiation='linear' use_bias=False
+{'xs': [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500], 'ys': [0.05, 0.0, 0.05, 0.0, 0.0, 0.1, 0.1, 0.05, 0.1, 0.0, 0.0]}
+
+

code/results/exp-mixed-soup-_1552674483.9866457-0/log.txt

@@ -0,0 +1,8 @@
+TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
+{'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.0, 0.0, 0.1, 0.0, 0.0, 0.0, 0.0, 0.1, 0.0, 0.0, 0.0], 'zs': [0.0, 0.0, 0.7, 1.9, 3.6, 4.3, 6.0, 6.1, 8.3, 7.7, 8.8]}
+
+
+TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
+{'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.8, 0.4, 0.4, 0.3, 0.2, 0.2, 0.2, 0.2, 0.2, 0.4, 0.3], 'zs': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}
+
+

code/results/exp-training_fixpoint-_1552658296.0913951-0/log.txt

@@ -0,0 +1,12 @@
+WeightwiseNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 0, 'fix_zero': 0, 'fix_other': 50, 'fix_sec': 0, 'other': 0}
+
+
+AggregatingNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 50}
+
+
+RecurrentNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 38, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 12}
+
+

code/results/known_fixpoint_variation/log.txt

@@ -0,0 +1,30 @@
+variation 10e-0
+avg time to vergence 3.63
+avg time as fixpoint 0
+variation 10e-1
+avg time to vergence 5.02
+avg time as fixpoint 0
+variation 10e-2
+avg time to vergence 6.46
+avg time as fixpoint 0
+variation 10e-3
+avg time to vergence 8.04
+avg time as fixpoint 0
+variation 10e-4
+avg time to vergence 9.61
+avg time as fixpoint 0.04
+variation 10e-5
+avg time to vergence 11.23
+avg time as fixpoint 1.38
+variation 10e-6
+avg time to vergence 12.99
+avg time as fixpoint 3.23
+variation 10e-7
+avg time to vergence 14.58
+avg time as fixpoint 4.84
+variation 10e-8
+avg time to vergence 21.95
+avg time as fixpoint 11.91
+variation 10e-9
+avg time to vergence 26.45
+avg time as fixpoint 16.47

code/results/self_application_aggregation_network/log.txt

@@ -0,0 +1 @@
+{'divergent': 0, 'fix_zero': 10, 'fix_other': 0, 'fix_sec': 0, 'other': 0}

code/setups/applying-fixpoints.py

@@ -0,0 +1,70 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from experiment import *
+from network import *
+
+
+def generate_counters():
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+def count(counters, net, notable_nets: list=None):
+    notable_nets = notable_nets or list()
+    if net.is_diverged():
+        counters['divergent'] += 1
+    elif net.is_fixpoint():
+        if net.is_zero():
+            counters['fix_zero'] += 1
+        else:
+            counters['fix_other'] += 1
+            notable_nets += [net]
+    elif net.is_fixpoint(2):
+        counters['fix_sec'] += 1
+        notable_nets += [net]
+    else:
+        counters['other'] += 1
+    return counters, notable_nets
+
+
+if __name__ == '__main__':
+
+    with FixpointExperiment(name='applying_fixpoint') as exp:
+        exp.trials = 50
+        exp.run_count = 100
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:  # , 'sigmoid', 'relu']:
+            for use_bias in [False]:
+                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+        #       net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+        all_counters = []
+        all_notable_nets = []
+        all_names = []
+        for net_generator_id, net_generator in enumerate(net_generators):
+            counters = generate_counters()
+            notable_nets = []
+            for _ in tqdm(range(exp.trials)):
+                net = ParticleDecorator(net_generator())
+                net.with_params(epsilon=exp.epsilon)
+                name = str(net.name) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias=" + str(net.get_keras_params().get('use_bias'))
+                for run_id in range(exp.run_count):
+                    loss = net.self_attack()
+                count(counters, net, notable_nets)
+            all_counters += [counters]
+            all_notable_nets += [notable_nets]
+            all_names += [name]
+            exp.reset_model()
+        exp.save(all_counters=all_counters)
+        exp.save(trajectorys=exp.without_particles())
+        # net types reached in the end
+        # exp.save(all_notable_nets=all_notable_nets)
+        exp.save(all_names=all_names) #experiment setups
+        for exp_id, counter in enumerate(all_counters):
+            exp.log(all_names[exp_id])
+            exp.log(all_counters[exp_id])
+            exp.log('\n')

code/setups/experiments/exp-training_fixpoint-_1552658296.0913951-0/log.txt

@@ -0,0 +1,12 @@
+WeightwiseNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 0, 'fix_zero': 0, 'fix_other': 50, 'fix_sec': 0, 'other': 0}
+
+
+AggregatingNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 50}
+
+
+RecurrentNeuralNetwork activiation='linear' use_bias=False
+{'divergent': 38, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 12}
+
+

code/setups/experiments/exp-weightwise_self_application-_1552664922.4501734-0/log.txt

@@ -0,0 +1 @@
+{'divergent': 11, 'fix_zero': 9, 'fix_other': 0, 'fix_sec': 0, 'other': 0}

code/setups/fixpoint-density.py

@@ -0,0 +1,69 @@
+import sys
+import os
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from experiment import *
+from network import *
+
+import tensorflow.python.keras.backend as K
+
+
+def generate_counters():
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+def count(counters, net, notable_nets=None):
+    notable_nets = notable_nets or []
+    if net.is_diverged():
+        counters['divergent'] += 1
+    elif net.is_fixpoint():
+        if net.is_zero():
+            counters['fix_zero'] += 1
+        else:
+            counters['fix_other'] += 1
+            notable_nets += [net]
+    elif net.is_fixpoint(2):
+        counters['fix_sec'] += 1
+        notable_nets += [net]
+    else:
+        counters['other'] += 1
+    return counters, notable_nets
+
+
+if __name__ == '__main__':
+    with Experiment('fixpoint-density') as exp:
+        #NOTE: settings could/should stay this way
+        #FFT doesn't work though
+        exp.trials = 100000
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:
+            net_generators += [lambda activation=activation: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
+            net_generators += [lambda activation=activation: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
+            # net_generators += [lambda activation=activation: FFTNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
+            # net_generators += [lambda activation=activation: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
+        all_counters = []
+        all_notable_nets = []
+        all_names = []
+        for net_generator_id, net_generator in enumerate(net_generators):
+            counters = generate_counters()
+            notable_nets = []
+            for _ in tqdm(range(exp.trials)):
+                net = net_generator().with_params(epsilon=exp.epsilon)
+                net = ParticleDecorator(net)
+                name = str(net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias='" + str(net.get_keras_params().get('use_bias')) + "'"
+                count(counters, net, notable_nets)
+                K.clear_session()
+            all_counters += [counters]
+            # all_notable_nets += [notable_nets]
+            all_names += [name]
+        exp.save(all_counters=all_counters)
+        exp.save(all_notable_nets=all_notable_nets)
+        exp.save(all_names=all_names)
+        for exp_id, counter in enumerate(all_counters):
+            exp.log(all_names[exp_id])
+            exp.log(all_counters[exp_id])
+            exp.log('\n')
+
+    print('Done')

code/setups/known-fixpoint-variation.py

@@ -0,0 +1,92 @@
+import sys
+
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from experiment import *
+from network import *
+from soup import prng
+
+import tensorflow.python.keras.backend as K
+
+
+from statistics import mean
+avg = mean
+
+
+def generate_fixpoint_weights():
+    return [
+        np.array([[1.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0]], dtype=np.float32),
+        np.array([[1.0, 0.0], [0.0, 0.0]], dtype=np.float32),
+        np.array([[1.0], [0.0]], dtype=np.float32)
+    ]
+
+
+def generate_fixpoint_net():
+    #NOTE: Weightwise only is all we can do right now IMO
+    net = WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation='sigmoid')
+    # I don't know if this work for aggregaeting. We don't actually need it, though.
+    # net = AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation='sigmoid')
+    net.set_weights(generate_fixpoint_weights())
+    return net
+
+
+def vary(old_weights, e=1.0):
+    new_weights = copy.deepcopy(old_weights)
+    for layer_id, layer in enumerate(new_weights):
+        for cell_id, cell in enumerate(layer):
+            for weight_id, weight in enumerate(cell):
+                if prng() < 0.5:
+                    new_weights[layer_id][cell_id][weight_id] = weight + prng() * e
+                else:
+                    new_weights[layer_id][cell_id][weight_id] = weight - prng() * e
+    return new_weights
+
+
+if __name__ == '__main__':
+    with Experiment('known-fixpoint-variation') as exp:
+        exp.depth = 10
+        exp.trials = 100
+        exp.max_steps = 100
+        exp.epsilon = 1e-4
+        exp.xs = []
+        exp.ys = []
+        exp.zs = []
+        exp.notable_nets = []
+        current_scale = 1.0
+        for _ in range(exp.depth):
+            print('variation scale ' + str(current_scale))
+            for _ in tqdm(range(exp.trials)):
+                net = generate_fixpoint_net().with_params(epsilon=exp.epsilon)
+                net = ParticleDecorator(net)
+                net.set_weights(vary(net.get_weights(), current_scale))
+                time_to_something = 0
+                time_as_fixpoint = 0
+                still_fixpoint = True
+                for _ in range(exp.max_steps):
+                    net.self_attack()
+                    if net.is_zero() or net.is_diverged():
+                        break
+                    if net.is_fixpoint():
+                        if still_fixpoint:
+                            time_as_fixpoint += 1
+                        else:
+                            print('remarkable')
+                            exp.notable_nets += [net.get_weights()]
+                            still_fixpoint = True
+                    else:
+                        still_fixpoint = False
+                    time_to_something += 1
+                exp.xs += [current_scale]
+                # time steps taken to reach divergence or zero (reaching another fix-point is basically never happening)
+                exp.ys += [time_to_something]
+                # time steps still regarded as sthe initial fix-point
+                exp.zs += [time_as_fixpoint]
+                K.backend.clear_session()
+            current_scale /= 10.0
+        for d in range(exp.depth):
+            exp.log('variation 10e-' + str(d))
+            exp.log('avg time to vergence ' + str(avg(exp.ys[d*exp.trials:(d+1) * exp.trials])))
+            exp.log('avg time as fixpoint ' + str(avg(exp.zs[d*exp.trials:(d+1) * exp.trials])))

code/setups/learn_from_soup.py

@@ -0,0 +1,110 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from typing import Tuple
+
+from experiment import *
+from network import *
+from soup import *
+
+
+from tensorflow.python.keras import backend as K
+
+from statistics import mean
+avg = mean
+
+
+def generate_counters():
+    """
+    Initial build of the counter dict, to store counts.
+
+    :rtype: dict
+    :return: dictionary holding counter for: 'divergent', 'fix_zero', 'fix_sec', 'other'
+    """
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+def count(counters, soup, notable_nets=None):
+    """
+    Count the occurences ot the types of weight trajectories.
+
+    :param counters:      A counter dictionary.
+    :param soup:          A Soup
+    :param notable_nets:  A list to store and save intersting candidates
+
+    :rtype      Tuple[dict, list]
+    :return:    Both the counter dictionary and the list of interessting nets.
+    """
+
+    notable_nets = notable_nets or list()
+    for net in soup.particles:
+        if net.is_diverged():
+            counters['divergent'] += 1
+        elif net.is_fixpoint():
+            if net.is_zero():
+                counters['fix_zero'] += 1
+            else:
+                counters['fix_other'] += 1
+                # notable_nets += [net]
+        # elif net.is_fixpoint(2):
+        #     counters['fix_sec'] += 1
+        #     notable_nets += [net]
+        else:
+            counters['other'] += 1
+    return counters, notable_nets
+
+
+if __name__ == '__main__':
+
+    with SoupExperiment(name='learn-from-soup') as exp:
+        exp.soup_size = 10
+        exp.soup_life = 100
+        exp.trials = 10
+        exp.learn_from_severity_values = [10 * i for i in range(11)]
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:  # ['sigmoid', 'linear', 'relu']:
+            for use_bias in [False]:
+                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                # net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                # net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+
+        all_names = []
+        all_data = []
+        for net_generator_id, net_generator in enumerate(net_generators):
+            xs = []
+            ys = []
+            zs = []
+            notable_nets = []
+            for learn_from_severity in exp.learn_from_severity_values:
+                counters = generate_counters()
+                results = []
+                for _ in tqdm(range(exp.trials)):
+                    soup = Soup(exp.soup_size, lambda net_generator=net_generator, exp=exp: TrainingNeuralNetworkDecorator(net_generator()).with_params(epsilon=exp.epsilon))
+                    soup.with_params(attacking_rate=-1, learn_from_rate=0.1, train=0, learn_from_severity=learn_from_severity)
+                    soup.seed()
+                    name = str(soup.particles[0].name) + " activiation='" + str(soup.particles[0].get_keras_params().get('activation')) + "' use_bias=" + str(soup.particles[0].get_keras_params().get('use_bias'))
+                    for time in range(exp.soup_life):
+                        soup.evolve()
+                    count(counters, soup, notable_nets)
+                    K.clear_session()
+
+                xs += [learn_from_severity]
+                ys += [float(counters['fix_zero']) / float(exp.trials)]
+                zs += [float(counters['fix_other']) / float(exp.trials)]
+            all_names += [name]
+            # xs: learn_from_intensity according to exp.learn_from_intensity_values
+            # ys: zero-fixpoints after life time
+            # zs: non-zero-fixpoints after life time
+            all_data += [{'xs':xs, 'ys':ys, 'zs':zs}]
+
+        exp.save(all_names=all_names)
+        exp.save(all_data=all_data)
+        exp.save(soup=soup.without_particles())
+        for exp_id, name in enumerate(all_names):
+            exp.log(all_names[exp_id])
+            exp.log(all_data[exp_id])
+            exp.log('\n')

code/setups/mixed-self-fixpoints.py

@@ -0,0 +1,98 @@
+import sys
+import os
+
+from typing import Tuple
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from experiment import *
+from network import *
+
+
+def generate_counters():
+    """
+    Initial build of the counter dict, to store counts.
+
+    :rtype: dict
+    :return: dictionary holding counter for: 'divergent', 'fix_zero', 'fix_sec', 'other'
+    """
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+def count(counters, net, notable_nets=None):
+    """
+    Count the occurences ot the types of weight trajectories.
+
+    :param counters:      A counter dictionary.
+    :param net:           A Neural Network
+    :param notable_nets:  A list to store and save intersting candidates
+
+    :rtype      Tuple[dict, list]
+    :return:    Both the counter dictionary and the list of interessting nets.
+    """
+    notable_nets = notable_nets or list()
+    if net.is_diverged():
+        counters['divergent'] += 1
+    elif net.is_fixpoint():
+        if net.is_zero():
+            counters['fix_zero'] += 1
+        else:
+            counters['fix_other'] += 1
+            notable_nets += [net]
+    elif net.is_fixpoint(2):
+        counters['fix_sec'] += 1
+        notable_nets += [net]
+    else:
+        counters['other'] += 1
+    return counters, notable_nets
+
+if __name__ == '__main__':
+
+    with Experiment('mixed-self-fixpoints') as exp:
+        exp.trials = 20
+        exp.selfattacks = 4
+        exp.trains_per_selfattack_values = [50 * i for i in range(11)]
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:  # , 'sigmoid', 'relu']:
+            for use_bias in [False]:
+                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                # net_generators += [lambda activation=activation, use_bias=use_bias: FFTNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+
+        all_names = []
+        all_data = []
+
+        for net_generator_id, net_generator in enumerate(net_generators):
+            xs = []
+            ys = []
+            for trains_per_selfattack in exp.trains_per_selfattack_values:
+                counters = generate_counters()
+                notable_nets = []
+                for _ in tqdm(range(exp.trials)):
+                    net = ParticleDecorator(net_generator())
+                    net = TrainingNeuralNetworkDecorator(net).with_params(epsilon=exp.epsilon)
+                    name = str(net.net.net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias=" + str(net.get_keras_params().get('use_bias'))
+                    for selfattack_id in range(exp.selfattacks):
+                        net.self_attack()
+                        for train_id in range(trains_per_selfattack):
+                            loss = net.compiled().train(epoch=selfattack_id*trains_per_selfattack+train_id)
+                        if net.is_diverged() or net.is_fixpoint():
+                            break
+                    count(counters, net, notable_nets)
+                    exp.reset_model()
+                xs += [trains_per_selfattack]
+                ys += [float(counters['fix_zero'] + counters['fix_other']) / float(exp.trials)]
+            all_names += [name]
+            # xs: how many trains per self-attack from exp.trains_per_selfattack_values
+            # ys: average amount of fixpoints found
+            all_data += [{'xs': xs, 'ys': ys}]
+
+        exp.save(all_names=all_names)
+        exp.save(all_data=all_data)
+        for exp_id, name in enumerate(all_names):
+            exp.log(all_names[exp_id])
+            exp.log(all_data[exp_id])
+            exp.log('\n')

code/setups/mixed-soup.py

@@ -0,0 +1,108 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from typing import Tuple
+
+from experiment import *
+from network import *
+from soup import *
+
+import tensorflow.python.keras.backend as K
+
+
+def generate_counters():
+    """
+    Initial build of the counter dict, to store counts.
+
+    :rtype: dict
+    :return: dictionary holding counter for: 'divergent', 'fix_zero', 'fix_sec', 'other'
+    """
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+
+def count(counters, soup, notable_nets=None):
+    """
+    Count the occurences ot the types of weight trajectories.
+
+    :param counters:      A counter dictionary.
+    :param soup:          A Soup
+    :param notable_nets:  A list to store and save intersting candidates
+
+    :rtype      Tuple[dict, list]
+    :return:    Both the counter dictionary and the list of interessting nets.
+    """
+
+    notable_nets = notable_nets or list()
+    for net in soup.particles:
+        if net.is_diverged():
+            counters['divergent'] += 1
+        elif net.is_fixpoint():
+            if net.is_zero():
+                counters['fix_zero'] += 1
+            else:
+                counters['fix_other'] += 1
+                # notable_nets += [net]
+        # elif net.is_fixpoint(2):
+        #     counters['fix_sec'] += 1
+        #     notable_nets += [net]
+        else:
+            counters['other'] += 1
+    return counters, notable_nets
+
+if __name__ == '__main__':
+
+    with Experiment('mixed-soup') as exp:
+        exp.trials = 10
+        exp.soup_size = 10
+        exp.soup_life = 5
+        exp.trains_per_selfattack_values = [10 * i for i in range(11)]
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:  # ['linear', 'sigmoid', 'relu']:
+            for use_bias in [False]:
+                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                # net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+
+        all_names = []
+        all_data = []
+        for net_generator_id, net_generator in enumerate(net_generators):
+            xs = []
+            ys = []
+            zs = []
+            for trains_per_selfattack in exp.trains_per_selfattack_values:
+                counters = generate_counters()
+                notable_nets = []
+                for soup_idx in tqdm(range(exp.trials)):
+                    soup = Soup(exp.soup_size,
+                                lambda net_generator=net_generator, exp=exp: TrainingNeuralNetworkDecorator(
+                                    net_generator()).with_params(epsilon=exp.epsilon))
+                    soup.with_params(attacking_rate=0.1, learn_from_rate=-1, train=trains_per_selfattack,
+                                     learn_from_severity=-1)
+                    soup.seed()
+                    name = str(soup.particles[0].net.__class__.__name__) + " activiation='" + str(
+                        soup.particles[0].get_keras_params().get('activation')) + "' use_bias=" + str(
+                        soup.particles[0].get_keras_params().get('use_bias'))
+                    for _ in range(exp.soup_life):
+                        soup.evolve()
+                    count(counters, soup, notable_nets)
+                    K.clear_session()
+
+                xs += [trains_per_selfattack]
+                ys += [float(counters['fix_zero']) / float(exp.trials)]
+                zs += [float(counters['fix_other']) / float(exp.trials)]
+            all_names += [name]
+            # xs: how many trains per self-attack from exp.trains_per_selfattack_values
+            # ys: average amount of zero-fixpoints found
+            # zs: average amount of non-zero fixpoints
+            all_data += [{'xs': xs, 'ys': ys, 'zs': zs}]
+
+        exp.save(all_names=all_names)
+        exp.save(all_data=all_data)
+        for exp_id, name in enumerate(all_names):
+            exp.log(all_names[exp_id])
+            exp.log(all_data[exp_id])
+            exp.log('\n')

code/setups/network_trajectorys.py

@@ -0,0 +1,112 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from soup import *
+from experiment import *
+
+
+if __name__ == '__main__':
+    def run_exp(net, prints=False):
+        # INFO Run_ID needs to be more than 0, so that exp stores the trajectories!
+        exp.run_net(net, 100, run_id=run_id + 1)
+        exp.historical_particles[run_id] = net
+        if prints:
+            print("Fixpoint? " + str(net.is_fixpoint()))
+            print("Loss " + str(loss))
+
+    if True:
+        # WeightWise Neural Network
+        with FixpointExperiment(name="weightwise_self_application") as exp:
+            for run_id in tqdm(range(20)):
+                net = ParticleDecorator(WeightwiseNeuralNetwork(width=2, depth=2)
+                                        .with_keras_params(activation='linear'))
+                run_exp(net)
+                K.clear_session()
+            exp.log(exp.counters)
+            exp.save(trajectorys=exp.without_particles())
+
+    if False:
+        # Aggregating Neural Network
+        with FixpointExperiment(name="aggregating_self_application") as exp:
+            for run_id in tqdm(range(10)):
+                net = ParticleDecorator(AggregatingNeuralNetwork(aggregates=4, width=2, depth=2)
+                                        .with_keras_params(activation='linear'))
+                run_exp(net)
+                K.clear_session()
+            exp.log(exp.counters)
+            exp.save(trajectorys=exp.without_particles())
+
+    if False:
+        #FFT Neural Network
+        with FixpointExperiment() as exp:
+            for run_id in tqdm(range(10)):
+                net = ParticleDecorator(FFTNeuralNetwork(aggregates=4, width=2, depth=2)
+                                        .with_keras_params(activation='linear'))
+                run_exp(net)
+                K.clear_session()
+            exp.log(exp.counters)
+            exp.save(trajectorys=exp.without_particles())
+
+    if False:
+        # ok so this works quite realiably
+        with FixpointExperiment(name="weightwise_learning") as exp:
+            for i in range(10):
+                run_count = 100
+                net = TrainingNeuralNetworkDecorator(ParticleDecorator(WeightwiseNeuralNetwork(width=2, depth=2)))
+                net.with_params(epsilon=0.0001).with_keras_params(activation='linear')
+                exp.historical_particles[net.get_uid()] = net
+                for run_id in tqdm(range(run_count+1)):
+                    net.compiled()
+                    loss = net.train(epoch=run_id)
+                    # run_exp(net)
+                    # net.save_state(time=run_id)
+                K.clear_session()
+            exp.save(trajectorys=exp.without_particles())
+
+    if False:
+        # ok so this works quite realiably
+        with FixpointExperiment(name="aggregating_learning") as exp:
+            for i in range(10):
+                run_count = 100
+                net = TrainingNeuralNetworkDecorator(ParticleDecorator(AggregatingNeuralNetwork(4, width=2, depth=2)))
+                net.with_params(epsilon=0.0001).with_keras_params(activation='linear')
+                exp.historical_particles[net.get_uid()] = net
+                for run_id in tqdm(range(run_count+1)):
+                    net.compiled()
+                    loss = net.train(epoch=run_id)
+                    # run_exp(net)
+                    # net.save_state(time=run_id)
+                K.clear_session()
+            exp.save(trajectorys=exp.without_particles())
+
+    if False:
+        # this explodes in our faces completely... NAN everywhere
+        # TODO: Wtf is happening here?
+        with FixpointExperiment() as exp:
+            run_count = 10000
+            net = TrainingNeuralNetworkDecorator(RecurrentNeuralNetwork(width=2, depth=2))\
+                .with_params(epsilon=0.1e-2).with_keras_params(optimizer='sgd', activation='linear')
+            for run_id in tqdm(range(run_count+1)):
+                loss = net.compiled().train()
+                if run_id % 500 == 0:
+                    net.print_weights()
+                    # print(net.apply_to_network(net))
+                    print("Fixpoint? " + str(net.is_fixpoint()))
+                    print("Loss " + str(loss))
+                    print()
+    if False:
+        # and this gets somewhat interesting... we can still achieve non-trivial fixpoints
+        # over multiple applications when training enough in-between
+        with MixedFixpointExperiment() as exp:
+            for run_id in range(10):
+                net = TrainingNeuralNetworkDecorator(FFTNeuralNetwork(2, width=2, depth=2))\
+                    .with_params(epsilon=0.0001, activation='sigmoid')
+                exp.run_net(net)
+
+                net.print_weights()
+
+                print("Fixpoint? " + str(net.is_fixpoint()))
+            exp.log(exp.counters)

code/setups/soup_trajectorys.py

@@ -0,0 +1,32 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from soup import *
+from experiment import *
+
+
+if __name__ == '__main__':
+    if True:
+        with SoupExperiment(namne="soup") as exp:
+            net_generator = lambda: TrainingNeuralNetworkDecorator(WeightwiseNeuralNetwork(2, 2)) \
+                 .with_keras_params(activation='linear').with_params(epsilon=0.0001)
+            # net_generator = lambda: TrainingNeuralNetworkDecorator(AggregatingNeuralNetwork(4, 2, 2))\
+            #     .with_keras_params(activation='linear')
+            # net_generator = lambda: TrainingNeuralNetworkDecorator(FFTNeuralNetwork(4, 2, 2))\
+            #    .with_keras_params(activation='linear')
+            # net_generator = lambda: RecurrentNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
+            soup = Soup(20, net_generator).with_params(remove_divergent=True, remove_zero=True,
+                                                       train=30,
+                                                       learn_from_rate=-1)
+            soup.seed()
+            for _ in tqdm(range(100)):
+                soup.evolve()
+            exp.log(soup.count())
+            # you can access soup.historical_particles[particle_uid].states[time_step]['loss']
+            #             or soup.historical_particles[particle_uid].states[time_step]['weights']
+            # from soup.dill
+            exp.save(soup=soup.without_particles())
+            K.clear_session()

code/setups/training-fixpoints.py

@@ -0,0 +1,70 @@
+import sys
+import os
+
+# Concat top Level dir to system environmental variables
+sys.path += os.path.join('..', '.')
+
+from experiment import *
+from network import *
+
+import tensorflow.python.keras.backend as K
+
+def generate_counters():
+    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
+
+def count(counters, net, notable_nets=None):
+    notable_nets = notable_nets or list()
+    if net.is_diverged():
+        counters['divergent'] += 1
+    elif net.is_fixpoint():
+        if net.is_zero():
+            counters['fix_zero'] += 1
+        else:
+            counters['fix_other'] += 1
+            notable_nets += [net]
+    elif net.is_fixpoint(2):
+        counters['fix_sec'] += 1
+        notable_nets += [net]
+    else:
+        counters['other'] += 1
+    return counters, notable_nets
+
+
+if __name__ == '__main__':
+
+    with Experiment('training_fixpoint') as exp:
+        exp.trials = 50
+        exp.run_count = 1000
+        exp.epsilon = 1e-4
+        net_generators = []
+        for activation in ['linear']:  # , 'sigmoid', 'relu']:
+            for use_bias in [False]:
+                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+                net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
+        all_counters = []
+        all_notable_nets = []
+        all_names = []
+        for net_generator_id, net_generator in enumerate(net_generators):
+            counters = generate_counters()
+            notable_nets = []
+            for _ in tqdm(range(exp.trials)):
+                net = ParticleDecorator(net_generator())
+                net = TrainingNeuralNetworkDecorator(net).with_params(epsilon=exp.epsilon)
+                name = str(net.net.net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias=" + str(net.get_keras_params().get('use_bias'))
+                for run_id in range(exp.run_count):
+                    loss = net.compiled().train(epoch=run_id+1)
+                count(counters, net, notable_nets)
+            all_counters += [counters]
+            all_notable_nets += [notable_nets]
+            all_names += [name]
+            K.clear_session()
+        exp.save(all_counters=all_counters)
+        exp.save(trajectorys=exp.without_particles())
+        # net types reached in the end
+        # exp.save(all_notable_nets=all_notable_nets)
+        exp.save(all_names=all_names) #experiment setups
+        for exp_id, counter in enumerate(all_counters):
+            exp.log(all_names[exp_id])
+            exp.log(all_counters[exp_id])
+            exp.log('\n')

code/soup.py

@@ -0,0 +1,376 @@
+import random
+from tensorflow.python.keras.layers import Dense, Dropout, BatchNormalization
+from tensorflow.python.keras.layers import Input, Layer, Concatenate, RepeatVector, Reshape
+from tensorflow.python.keras.models import Sequential, Model
+from tensorflow.python.keras import backend as K
+
+from typing import List, Tuple
+
+# Functions and Operators
+from operator import mul
+from functools import reduce
+from itertools import accumulate
+
+import numpy as np
+
+from task import Task, TaskAdditionOfN
+
+from copy import copy, deepcopy
+
+from network import ParticleDecorator, WeightwiseNeuralNetwork, TrainingNeuralNetworkDecorator, \
+    EarlyStoppingByInfNanLoss
+
+from experiment import TaskingSoupExperiment
+
+from math import sqrt
+
+
+def prng():
+    return random.random()
+
+
+class SlicingLayer(Layer):
+
+    def __init__(self):
+        self.kernel: None
+        self.inputs: int
+        super(SlicingLayer, self).__init__()
+
+    def build(self, input_shape):
+        # Create a trainable weight variable for this layer.
+        self.kernel = None
+        self.inputs = input_shape[-1]
+        super(SlicingLayer, self).build(input_shape)  # Be sure to call this at the end
+
+    def call(self, x, **kwargs):
+        concats = [Concatenate()([x[:, i][..., None]] * self.inputs) for i in range(x.shape[-1].value)]
+        return concats
+
+    def compute_output_shape(self, input_shape):
+        return [Concatenate()([(None, 1)] * 4) for _ in range(input_shape[-1])]
+
+
+class Soup(object):
+
+    def __init__(self, size, generator, **kwargs):
+        self.size = size
+        self.generator = generator
+        self.particles = []
+        self.historical_particles = {}
+        self.soup_params = dict(attacking_rate=0.1, learn_from_rate=0.1, train=0, learn_from_severity=1)
+        self.soup_params.update(kwargs)
+        self.time = 0
+        self.is_seeded = False
+        self.is_compiled = False
+
+    def __len__(self):
+        return len(self.particles)
+
+    def __copy__(self):
+        copy_ = Soup(self.size, self.generator, **self.soup_params)
+        copy_.__dict__ = {attr: self.__dict__[attr] for attr in self.__dict__ if
+                          attr not in ['particles', 'historical_particles']}
+        return copy_
+
+    def without_particles(self):
+        self_copy = copy(self)
+        # self_copy.particles = [particle.states for particle in self.particles]
+        self_copy.historical_particles = {key: val.states for key, val in self.historical_particles.items()}
+        return self_copy
+
+    def with_soup_params(self, **kwargs):
+        self.soup_params.update(kwargs)
+        return self
+
+    def generate_particle(self):
+        new_particle = ParticleDecorator(self.generator())
+        self.historical_particles[new_particle.get_uid()] = new_particle
+        return new_particle
+
+    def get_particle(self, uid, otherwise=None):
+        return self.historical_particles.get(uid, otherwise)
+
+    def seed(self):
+        if not self.is_seeded:
+            self.particles = []
+            for _ in range(self.size):
+                self.particles += [self.generate_particle()]
+        else:
+            print('already seeded!')
+        self.is_seeded = True
+        return self
+
+    def evolve(self, iterations=1):
+        for _ in range(iterations):
+            self.time += 1
+            for particle_id, particle in enumerate(self.particles):
+                description = {'time': self.time}
+                if prng() < self.soup_params.get('attacking_rate'):
+                    other_particle_id = int(prng() * len(self.particles))
+                    other_particle = self.particles[other_particle_id]
+                    particle.attack(other_particle)
+                    description['action'] = 'attacking'
+                    description['counterpart'] = other_particle.get_uid()
+
+                if prng() < self.soup_params.get('learn_from_rate'):
+                    other_particle_id = int(prng() * len(self.particles))
+                    other_particle = self.particles[other_particle_id]
+                    for _ in range(self.soup_params.get('learn_from_severity', 1)):
+                        particle.learn_from(other_particle)
+                    description['action'] = 'learn_from'
+                    description['counterpart'] = other_particle.get_uid()
+
+                for _ in range(self.soup_params.get('train', 0)):
+                    # callbacks on save_state are broken for TrainingNeuralNetwork
+                    loss = particle.train(store_states=False)
+                    description['fitted'] = self.soup_params.get('train', 0)
+                    description['loss'] = loss
+                    description['action'] = 'train_self'
+                    description['counterpart'] = None
+
+                if self.soup_params.get('remove_divergent') and particle.is_diverged():
+                    new_particle = self.generate_particle()
+                    self.particles[particle_id] = new_particle
+                    description['action'] = 'divergent_dead'
+                    description['counterpart'] = new_particle.get_uid()
+
+                if self.soup_params.get('remove_zero') and particle.is_zero():
+                    new_particle = self.generate_particle()
+                    self.particles[particle_id] = new_particle
+                    description['action'] = 'zweo_dead'
+                    description['counterpart'] = new_particle.get_uid()
+                particle.save_state(**description)
+
+    def count(self):
+        counters = dict(divergent=0, fix_zero=0, fix_other=0, fix_sec=0, other=0)
+        for particle in self.particles:
+            if particle.is_diverged():
+                counters['divergent'] += 1
+            elif particle.is_fixpoint():
+                if particle.is_zero():
+                    counters['fix_zero'] += 1
+                else:
+                    counters['fix_other'] += 1
+            elif particle.is_fixpoint(2):
+                counters['fix_sec'] += 1
+            else:
+                counters['other'] += 1
+        return counters
+
+    def print_all(self):
+        for particle in self.particles:
+            particle.print_weights()
+            print(particle.is_fixpoint())
+
+
+class TaskingSoup(Soup):
+
+    @staticmethod
+    def weights_to_flat_array(weights: List[np.ndarray]) -> np.ndarray:
+        return np.concatenate([d.ravel() for d in weights])
+
+    @staticmethod
+    def reshape_flat_array(array, shapes: List[Tuple[int]]) -> List[np.ndarray]:
+        # Same thing, but with an additional np call
+        # sizes: List[int] = [int(np.prod(shape)) for shape in shapes]
+
+        sizes = [reduce(mul, shape) for shape in shapes]
+        # Split the incoming array into slices for layers
+        slices = [array[x: y] for x, y in zip(accumulate([0] + sizes), accumulate(sizes))]
+        # reshape them in accordance to the given shapes
+        weights = [np.reshape(weight_slice, shape) for weight_slice, shape in zip(slices, shapes)]
+        return weights
+
+    def __init__(self, population_size: int, task: Task, particle_generator, sparsity_rate=0.1, use_bias=False,
+                 safe=True, **kwargs):
+
+        if safe:
+            input_shape_error_message = f'The population size must be devideable by {task.input_shape[-1]}'
+            assert population_size % task.input_shape[-1] == 0, input_shape_error_message
+            assert population_size % 2 == 0, 'The population size needs to be of even value'
+
+        super(TaskingSoup, self).__init__(population_size, particle_generator, **kwargs)
+        self.task = task
+        self.model: Sequential
+
+        self.network_params = dict(sparsity_rate=sparsity_rate, early_nan_stopping=True, use_bias=use_bias,
+                                   depth=population_size // task.input_shape[-1])
+        self.network_params.update(kwargs.get('network_params', {}))
+        self.compile_params = dict(loss='mse', optimizer='sgd')
+        self.compile_params.update(kwargs.get('compile_params', {}))
+
+    def with_network_params(self, **params):
+        self.network_params.update(params)
+
+    def _generate_model(self):
+        particle_idx_list = list(range(len(self)))
+        particles_per_layer = len(self) // self.network_params.get('depth')
+        task_input = Input(self.task.input_shape, name='Task_Input')
+        # First layer, which is conected to the input layer and independently trainable / not trainable at all.
+        input_neurons = particles_per_layer * self.task.output_shape
+        x = Dense(input_neurons, use_bias=self.network_params.get('use_bias'))(task_input)
+        x = SlicingLayer()(x)
+
+        for layer_num in range(self.network_params.get('depth')):
+            # This needs to be tensors, because particles come as keras models that applicable
+            x = [self.particles[layer_num*particles_per_layer + i].get_model()(x[i]) for
+                 i in range(particles_per_layer)]
+            x = [RepeatVector(particles_per_layer)(x[i]) for i in range(particles_per_layer)]
+            x = [Reshape((particles_per_layer,))(x[i]) for i in range(particles_per_layer)]
+        x = Concatenate()(x)
+        x = Dense(self.task.output_shape, use_bias=self.network_params.get('use_bias'), activation='linear')(x)
+
+        model = Model(inputs=task_input, outputs=x)
+        return model
+
+    def get_weights(self):
+        return self.model.get_weights()
+
+    def set_weights(self, weights: List[np.ndarray]):
+        self.model.set_weights(weights)
+
+    def set_intermediate_weights(self, weights: List[np.ndarray]):
+        all_weights = self.get_weights()
+        all_weights[1:-1] = weights
+        self.set_weights(all_weights)
+
+    def get_intermediate_weights(self):
+        return self.get_weights()[1:-1]
+
+    def seed(self):
+        K.clear_session()
+        self.is_compiled = False
+        super(TaskingSoup, self).seed()
+        self.model = self._generate_model()
+        pass
+
+    def compile_model(self, **kwargs):
+        if not self.is_compiled:
+            compile_params = deepcopy(self.compile_params)
+            compile_params.update(kwargs)
+            return self.model.compile(**compile_params)
+        else:
+            raise BrokenPipeError('This Model is not compiled yet! Something went wrong in the Pipeline!')
+
+    def get_total_weight_amount(self):
+        if self.is_seeded:
+            return sum([x.get_amount_of_weights() for x in self.particles])
+        else:
+            return 0
+
+    def get_shapes(self):
+        return [x.shape for x in self.get_weights()]
+
+    def get_intermediate_shapes(self):
+        weights = [x.shape for x in self.get_weights()]
+        return weights[2:-2] if self.network_params.get('use_bias') else weights[1:-1]
+
+    def predict(self, x):
+        return self.model.predict(x)
+
+    def evolve(self, iterations=1):
+        for iteration in range(iterations):
+            super(TaskingSoup, self).evolve(iterations=1)
+            self.train_particles()
+
+    def get_particle_weights(self):
+        return np.concatenate([x.get_weights_flat() for x in self.particles])
+
+    def get_particle_input_shape(self):
+        if self.is_seeded:
+            return tuple([x if x else -1 for x in self.particles[0].get_model().input_shape])
+        else:
+            return -1
+
+    def set_particle_weights(self, weights):
+        particle_weight_shape = self.particles[0].shapes(self.particles[0].get_weights())
+        sizes = [x.get_amount_of_weights() for x in self.particles]
+        flat_weights = self.weights_to_flat_array(weights)
+        slices = [flat_weights[x: y] for x, y in zip(accumulate([0] + sizes), accumulate(sizes))]
+        for particle, slice in zip(self.particles, slices):
+            new_weights = self.reshape_flat_array(slice, particle_weight_shape)
+            particle.set_weights(new_weights)
+        return True
+
+    def compiled(self, **kwargs):
+        if not self.is_compiled:
+            self.compile_model(**kwargs)
+            self.is_compiled = True
+        return self
+
+    def train(self, batchsize=1, epoch=0):
+        self.compiled()
+        x, y = self.task.get_samples()
+        callbacks = []
+        if self.network_params.get('early_nan_stopping'):
+            callbacks.append(EarlyStoppingByInfNanLoss())
+
+        # 'or' does not work on empty lists
+        callbacks = callbacks if callbacks else None
+        """
+        Please Note:
+
+        epochs: Integer. Number of epochs to train the model.
+            An epoch is an iteration over the entire `x` and `y`
+            data provided.
+            Note that in conjunction with `initial_epoch`,
+            `epochs` is to be understood as "final epoch".
+            The model is not trained for a number of iterations
+            given by `epochs`, but merely until the epoch
+            of index `epochs` is reached."""
+        history = self.model.fit(x=x, y=y, initial_epoch=epoch, epochs=epoch + 1, verbose=0,
+                                 batch_size=batchsize, callbacks=callbacks)
+        return history.history['loss'][-1]
+
+    def train_particles(self, **kwargs):
+        self.compiled()
+        weights = self.get_particle_weights()
+        shaped_weights = self.reshape_flat_array(weights, self.get_intermediate_shapes())
+        self.set_intermediate_weights(shaped_weights)
+        _ = self.train(**kwargs)  # This returns the loss values
+        new_weights = self.get_intermediate_weights()
+        self.set_particle_weights(new_weights)
+        return
+
+
+if __name__ == '__main__':
+    if True:
+        from task import TaskAdditionOfN
+
+        net_generator = lambda: TrainingNeuralNetworkDecorator(
+            WeightwiseNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
+        )
+        soup_generator = lambda: TaskingSoup(20, TaskAdditionOfN(4), net_generator)
+        with TaskingSoupExperiment(soup_generator, name='solving_soup') as exp:
+            exp.run_exp(reset_model=False)
+
+    if False:
+        soup_generator = lambda: Soup(10, net_generator).with_soup_params(remove_divergent=True, remove_zero=True)
+        with SoupExperiment(soup_generator, name='soup') as exp:
+            net_generator = lambda: TrainingNeuralNetworkDecorator(
+                WeightwiseNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
+            )
+
+            exp.run_exp(net_generator)
+
+            # net_generator = lambda: FFTNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
+            # net_generator = lambda: AggregatingNeuralNetwork(4, 2, 2).with_keras_params(activation='sigmoid')\
+            # .with_params(shuffler=AggregatingNeuralNetwork.shuffle_random)
+            # net_generator = lambda: RecurrentNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
+
+    if False:
+        soup_generator = lambda: Soup(10, net_generator).with_soup_params(remove_divergent=True, remove_zero=True)
+        with SoupExperiment(soup_generator, name='soup') as exp:
+            net_generator = lambda: TrainingNeuralNetworkDecorator(WeightwiseNeuralNetwork(2, 2)) \
+                .with_keras_params(activation='linear').with_params(epsilon=0.0001)
+
+            exp.run_exp(net_generator)
+
+            # net_generator = lambda: TrainingNeuralNetworkDecorator(AggregatingNeuralNetwork(4, 2, 2))
+            # .with_keras_params(activation='linear')\
+            # .with_params(shuffler=AggregatingNeuralNetwork.shuffle_random)
+            # net_generator = lambda: TrainingNeuralNetworkDecorator(FFTNeuralNetwork(4, 2, 2))\
+            #     .with_keras_params(activation='linear')\
+            # .with_params(shuffler=AggregatingNeuralNetwork.shuffle_random)
+            # net_generator = lambda: RecurrentNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
+

code/task.py

@@ -0,0 +1,32 @@
+from abc import ABC, abstractmethod
+import numpy as np
+
+from typing import Tuple
+
+
+class Task(ABC):
+
+    def __init__(self, input_shape, output_shape, **kwargs):
+        assert any([x not in kwargs.keys() for x in ["input_shape", "output_shape"]]), 'Dublicated arguments were given'
+        self.input_shape = input_shape
+        self.output_shape = output_shape
+        self.batchsize = kwargs.get('batchsize', 100)
+
+    def get_samples(self) -> Tuple[np.ndarray, np.ndarray]:
+        raise NotImplementedError
+
+
+class TaskAdditionOfN(Task):
+
+    def __init__(self, n: int, input_shape=(4,), output_shape=1, **kwargs):
+        assert any([x not in kwargs.keys() for x in ["input_shape", "output_shape"]]), 'Dublicated arguments were given'
+        assert n <= input_shape[0], f'You cannot Add more values (n={n}) than your input is long (in={input_shape}).'
+        kwargs.update(input_shape=input_shape, output_shape=output_shape)
+        super(TaskAdditionOfN, self).__init__(**kwargs)
+        self.n = n
+
+    def get_samples(self) -> Tuple[np.ndarray, np.ndarray]:
+        x = np.zeros((self.batchsize, *self.input_shape))
+        x[:, :self.n] = np.random.standard_normal((self.batchsize, self.n)) * 0.5
+        y = np.sum(x, axis=1)
+        return x, y

experiments/__init__.py

@@ -1,6 +0,0 @@
-from .mixed_setting_exp import run_mixed_experiment
-from .robustness_exp import run_robustness_experiment
-from .self_application_exp import run_SA_experiment
-from .self_train_exp import run_ST_experiment
-from .soup_exp import run_soup_experiment
-import functionalities_test

experiments/helpers.py

@@ -1,59 +0,0 @@
-""" -------------------------------- Methods for summarizing the experiments --------------------------------- """
-from pathlib import Path
-
-from visualization import line_chart_fixpoints, bar_chart_fixpoints
-
-
-def summary_fixpoint_experiment(runs, population_size, epochs, experiments, net_learning_rate, directory,
-                                summary_pre_title):
-    avg_fixpoint_counters = {
-        "avg_identity_func": 0,
-        "avg_divergent": 0,
-        "avg_fix_zero": 0,
-        "avg_fix_weak": 0,
-        "avg_fix_sec": 0,
-        "avg_other_func": 0
-    }
-
-    for i in range(len(experiments)):
-        fixpoint_counters = experiments[i].fixpoint_counters
-
-        avg_fixpoint_counters["avg_identity_func"] += fixpoint_counters["identity_func"]
-        avg_fixpoint_counters["avg_divergent"] += fixpoint_counters["divergent"]
-        avg_fixpoint_counters["avg_fix_zero"] += fixpoint_counters["fix_zero"]
-        avg_fixpoint_counters["avg_fix_weak"] += fixpoint_counters["fix_weak"]
-        avg_fixpoint_counters["avg_fix_sec"] += fixpoint_counters["fix_sec"]
-        avg_fixpoint_counters["avg_other_func"] += fixpoint_counters["other_func"]
-
-    # Calculating the average for each fixpoint
-    avg_fixpoint_counters.update((x, y / len(experiments)) for x, y in avg_fixpoint_counters.items())
-
-    # Checking where the data is coming from to have a relevant title in the plot.
-    if summary_pre_title not in ["ST", "SA", "soup", "mixed", "robustness"]:
-        summary_pre_title = ""
-
-    # Plotting the summary
-    source_checker = "summary"
-    exp_details = f"{summary_pre_title}: {runs} runs & {epochs} epochs each."
-    bar_chart_fixpoints(avg_fixpoint_counters, population_size, directory, net_learning_rate, exp_details,
-                        source_checker)
-
-
-def summary_fixpoint_percentage(runs, epochs, fixpoints_percentages, ST_steps, SA_steps, directory_name,
-                                population_size):
-    fixpoints_percentages = [round(fixpoints_percentages[i] / runs, 1) for i in range(len(fixpoints_percentages))]
-
-    # Plotting summary
-    if "soup" in directory_name:
-        line_chart_fixpoints(fixpoints_percentages, epochs / ST_steps, ST_steps, SA_steps, directory_name,
-                             population_size)
-    else:
-        line_chart_fixpoints(fixpoints_percentages, epochs, ST_steps, SA_steps, directory_name, population_size)
-
-
-""" -------------------------------------------- Miscellaneous --------------------------------------------------- """
-
-
-def check_folder(experiment_folder: str):
-    exp_path = Path('experiments') / experiment_folder
-    exp_path.mkdir(parents=True, exist_ok=True)

experiments/meta_task_exp.py

@@ -1,271 +0,0 @@
-import pickle
-from collections import defaultdict
-from pathlib import Path
-import sys
-import platform
-
-import pandas as pd
-import torchmetrics
-import numpy as np
-import torch
-from matplotlib import pyplot as plt
-import seaborn as sns
-from torch import nn
-from torch.nn import Flatten
-from torch.utils.data import Dataset, DataLoader
-from torchvision.datasets import MNIST
-from torchvision.transforms import ToTensor, Compose, Resize
-from tqdm import tqdm
-
-if platform.node() == 'CarbonX':
-    debug = True
-    print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
-    print("@ Warning, Debugging Config@!!!!!! @")
-    print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
-else:
-    debug = False
-    try:
-        # noinspection PyUnboundLocalVariable
-        if __package__ is None:
-            DIR = Path(__file__).resolve().parent
-            sys.path.insert(0, str(DIR.parent))
-            __package__ = DIR.name
-        else:
-            DIR = None
-    except NameError:
-        DIR = None
-        pass
-
-from network import MetaNet
-from functionalities_test import test_for_fixpoints
-
-WORKER = 10 if not debug else 2
-BATCHSIZE = 500 if not debug else 50
-EPOCH = 100 if not debug else 3
-VALIDATION_FRQ = 5 if not debug else 1
-SELF_TRAIN_FRQ = 1 if not debug else 1
-DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-
-if debug:
-    torch.autograd.set_detect_anomaly(True)
-
-
-class ToFloat:
-
-    def __call__(self, x):
-        return x.to(torch.float32)
-
-
-class AddTaskDataset(Dataset):
-    def __init__(self, length=int(5e5)):
-        super().__init__()
-        self.length = length
-        self.prng = np.random.default_rng()
-
-    def __len__(self):
-        return self.length
-
-    def __getitem__(self, _):
-        ab = self.prng.normal(size=(2,)).astype(np.float32)
-        return ab, ab.sum(axis=-1, keepdims=True)
-
-
-def set_checkpoint(model, out_path, epoch_n, final_model=False):
-    epoch_n = str(epoch_n)
-    if not final_model:
-        ckpt_path = Path(out_path) / 'ckpt' / f'{epoch_n.zfill(4)}_model_ckpt.tp'
-    else:
-        ckpt_path = Path(out_path) / f'trained_model_ckpt_e{epoch_n}.tp'
-    ckpt_path.parent.mkdir(exist_ok=True, parents=True)
-
-    torch.save(model, ckpt_path, pickle_protocol=pickle.HIGHEST_PROTOCOL)
-    return ckpt_path
-
-
-def validate(checkpoint_path, ratio=0.1):
-    checkpoint_path = Path(checkpoint_path)
-    import torchmetrics
-
-    # initialize metric
-    validmetric = torchmetrics.Accuracy()
-    ut = Compose([ToTensor(), ToFloat(), Resize((15, 15)), Flatten(start_dim=0)])
-
-    try:
-        datas = MNIST(str(data_path), transform=ut, train=False)
-    except RuntimeError:
-        datas = MNIST(str(data_path), transform=ut, train=False, download=True)
-    valid_d = DataLoader(datas, batch_size=BATCHSIZE, shuffle=True, drop_last=True, num_workers=WORKER)
-
-    model = torch.load(checkpoint_path, map_location=DEVICE).eval()
-    n_samples = int(len(valid_d) * ratio)
-
-    with tqdm(total=n_samples, desc='Validation Run: ') as pbar:
-        for idx, (valid_batch_x, valid_batch_y) in enumerate(valid_d):
-            valid_batch_x, valid_batch_y = valid_batch_x.to(DEVICE), valid_batch_y.to(DEVICE)
-            y_valid = model(valid_batch_x)
-
-            # metric on current batch
-            acc = validmetric(y_valid.cpu(), valid_batch_y.cpu())
-            pbar.set_postfix_str(f'Acc: {acc}')
-            pbar.update()
-            if idx == n_samples:
-                break
-
-    # metric on all batches using custom accumulation
-    acc = validmetric.compute()
-    tqdm.write(f"Avg. accuracy on all data: {acc}")
-    return acc
-
-
-def new_train_storage_df():
-    return pd.DataFrame(columns=['Epoch', 'Batch', 'Metric', 'Score'])
-
-
-def checkpoint_and_validate(model, out_path, epoch_n, final_model=False):
-    out_path = Path(out_path)
-    ckpt_path = set_checkpoint(model, out_path, epoch_n, final_model=final_model)
-    result = validate(ckpt_path)
-    return result
-
-
-def plot_training_result(path_to_dataframe):
-    # load from Drive
-    df = pd.read_csv(path_to_dataframe, index_col=0)
-
-    # Set up figure
-    fig, ax1 = plt.subplots()  # initializes figure and plots
-    ax2 = ax1.twinx()  # applies twinx to ax2, which is the second y-axis.
-
-    # plots the first set of data
-    data = df[(df['Metric'] == 'Task Loss') | (df['Metric'] == 'Self Train Loss')].groupby(['Epoch', 'Metric']).mean()
-    palette = sns.color_palette()[0:data.reset_index()['Metric'].unique().shape[0]]
-    sns.lineplot(data=data.groupby(['Epoch', 'Metric']).mean(), x='Epoch', y='Score', hue='Metric',
-                 palette=palette, ax=ax1)
-
-    # plots the second set of data
-    data = df[(df['Metric'] == 'Test Accuracy') | (df['Metric'] == 'Train Accuracy')]
-    palette = sns.color_palette()[len(palette):data.reset_index()['Metric'].unique().shape[0] + len(palette)]
-    sns.lineplot(data=data, x='Epoch', y='Score', marker='o', hue='Metric', palette=palette)
-
-    ax1.set(yscale='log', ylabel='Losses')
-    ax1.set_title('Training Lineplot')
-    ax2.set(ylabel='Accuracy')
-
-    fig.legend(loc="center right", title='Metric', bbox_to_anchor=(0.85, 0.5))
-    ax1.get_legend().remove()
-    ax2.get_legend().remove()
-    plt.tight_layout()
-    if debug:
-        plt.show()
-    else:
-        plt.savefig(Path(path_to_dataframe.parent / 'training_lineplot.png'), dpi=300)
-
-
-if __name__ == '__main__':
-
-    self_train = False
-    training = False
-    plotting = False
-    particle_analysis = True
-    as_sparse_network_test = True
-
-    data_path = Path('data')
-    data_path.mkdir(exist_ok=True, parents=True)
-
-    run_path = Path('output') / 'mnist_self_train_100_NEW_STYLE'
-    model_path = run_path / '0000_trained_model.zip'
-    df_store_path = run_path / 'train_store.csv'
-
-    if training:
-        utility_transforms = Compose([ToTensor(), ToFloat(), Resize((15, 15)), Flatten(start_dim=0)])
-        try:
-            dataset = MNIST(str(data_path), transform=utility_transforms)
-        except RuntimeError:
-            dataset = MNIST(str(data_path), transform=utility_transforms, download=True)
-        d = DataLoader(dataset, batch_size=BATCHSIZE, shuffle=True, drop_last=True, num_workers=WORKER)
-
-        interface = np.prod(dataset[0][0].shape)
-        metanet = MetaNet(interface, depth=4, width=6, out=10).to(DEVICE).train()
-
-        loss_fn = nn.CrossEntropyLoss()
-        optimizer = torch.optim.SGD(metanet.parameters(), lr=0.004, momentum=0.9)
-
-        train_store = new_train_storage_df()
-        for epoch in tqdm(range(EPOCH), desc='MetaNet Train - Epochs'):
-            is_validation_epoch = epoch % VALIDATION_FRQ == 0 if not debug else True
-            is_self_train_epoch = epoch % SELF_TRAIN_FRQ == 0 if not debug else True
-            if is_validation_epoch:
-                metric = torchmetrics.Accuracy()
-            else:
-                metric = None
-            for batch, (batch_x, batch_y) in tqdm(enumerate(d), total=len(d), desc='MetaNet Train - Batch'):
-                if self_train and is_self_train_epoch:
-                    self_train_loss = metanet.combined_self_train(optimizer)
-                    step_log = dict(Epoch=epoch, Batch=batch, Metric='Self Train Loss', Score=self_train_loss.item())
-                    train_store.loc[train_store.shape[0]] = step_log
-
-                # Zero your gradients for every batch!
-                optimizer.zero_grad()
-                batch_x, batch_y = batch_x.to(DEVICE), batch_y.to(DEVICE)
-                y = metanet(batch_x)
-                # loss = loss_fn(y, batch_y.unsqueeze(-1).to(torch.float32))
-                loss = loss_fn(y, batch_y.to(torch.long))
-                loss.backward()
-
-                # Adjust learning weights
-                optimizer.step()
-
-                step_log = dict(Epoch=epoch, Batch=batch,
-                                Metric='Task Loss', Score=loss.item())
-                train_store.loc[train_store.shape[0]] = step_log
-                if is_validation_epoch:
-                    metric(y.cpu(), batch_y.cpu())
-
-                if batch >= 3 and debug:
-                    break
-
-            if is_validation_epoch:
-                validation_log = dict(Epoch=int(epoch), Batch=BATCHSIZE,
-                                      Metric='Train Accuracy', Score=metric.compute().item())
-                train_store.loc[train_store.shape[0]] = validation_log
-
-                accuracy = checkpoint_and_validate(metanet, run_path, epoch)
-                validation_log = dict(Epoch=int(epoch), Batch=BATCHSIZE,
-                                      Metric='Test Accuracy', Score=accuracy.item())
-                train_store.loc[train_store.shape[0]] = validation_log
-                if particle_analysis:
-                    counter_dict = defaultdict(lambda: 0)
-                    # This returns ID-functions
-                    _ = test_for_fixpoints(counter_dict, list(metanet.particles))
-                    for key, value in dict(counter_dict).items():
-                        step_log = dict(Epoch=int(epoch), Batch=BATCHSIZE, Metric=key, Score=value)
-                        train_store.loc[train_store.shape[0]] = step_log
-                train_store.to_csv(df_store_path, mode='a', header=not df_store_path.exists())
-                train_store = new_train_storage_df()
-
-        accuracy = checkpoint_and_validate(metanet, run_path, EPOCH, final_model=True)
-        validation_log = dict(Epoch=EPOCH, Batch=BATCHSIZE,
-                              Metric='Test Accuracy', Score=accuracy.item())
-
-        train_store.loc[train_store.shape[0]] = validation_log
-        train_store.to_csv(df_store_path)
-
-    if plotting:
-        plot_training_result(df_store_path)
-
-    if particle_analysis:
-        model_path = next(run_path.glob('*ckpt.tp'))
-        latest_model = torch.load(model_path, map_location=DEVICE).eval()
-        counter_dict = defaultdict(lambda: 0)
-        _ = test_for_fixpoints(counter_dict, list(latest_model.particles))
-        tqdm.write(str(dict(counter_dict)))
-        zero_ident = torch.load(model_path, map_location=DEVICE).eval().replace_with_zero('identity_func')
-        zero_other = torch.load(model_path, map_location=DEVICE).eval().replace_with_zero('other_func')
-        if as_sparse_network_test:
-            acc_pre = validate(model_path, ratio=1)
-            ident_ckpt = set_checkpoint(zero_ident, model_path.parent, -1, final_model=True)
-            ident_acc_post = validate(ident_ckpt, ratio=1)
-            tqdm.write(f'Zero_ident diff = {abs(ident_acc_post-acc_pre)}')
-            other_ckpt = set_checkpoint(zero_other, model_path.parent, -2, final_model=True)
-            other_acc_post = validate(other_ckpt, ratio=1)
-            tqdm.write(f'Zero_other diff = {abs(other_acc_post - acc_pre)}')

experiments/mixed_setting_exp.py

@@ -1,177 +0,0 @@
-import os.path
-import pickle
-
-from tqdm import tqdm
-
-from experiments.helpers import check_folder, summary_fixpoint_experiment, summary_fixpoint_percentage
-from functionalities_test import test_for_fixpoints
-from network import Net
-from visualization import plot_loss, bar_chart_fixpoints, line_chart_fixpoints
-from visualization import plot_3d_self_train
-
-
-class MixedSettingExperiment:
-    def __init__(self, population_size, net_i_size, net_h_size, net_o_size, learning_rate, train_nets,
-                 epochs, SA_steps, ST_steps_between_SA, log_step_size, directory_name):
-        super().__init__()
-        self.population_size = population_size
-
-        self.net_input_size = net_i_size
-        self.net_hidden_size = net_h_size
-        self.net_out_size = net_o_size
-        self.net_learning_rate = learning_rate
-        self.train_nets = train_nets
-        self.epochs = epochs
-        self.SA_steps = SA_steps
-        self.ST_steps_between_SA = ST_steps_between_SA
-        self.log_step_size = log_step_size
-
-        self.fixpoint_counters = {
-            "identity_func": 0,
-            "divergent": 0,
-            "fix_zero": 0,
-            "fix_weak": 0,
-            "fix_sec": 0,
-            "other_func": 0
-        }
-
-        self.loss_history = []
-
-        self.fixpoint_counters_history = []
-
-        self.directory_name = directory_name
-        os.mkdir(self.directory_name)
-
-        self.nets = []
-        self.populate_environment()
-
-        self.fixpoint_percentage()
-        self.weights_evolution_3d_experiment()
-        self.count_fixpoints()
-        self.visualize_loss()
-
-    def populate_environment(self):
-        loop_population_size = tqdm(range(self.population_size))
-        for i in loop_population_size:
-            loop_population_size.set_description("Populating mixed experiment %s" % i)
-
-            net_name = f"mixed_net_{str(i)}"
-            net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name)
-            self.nets.append(net)
-
-        loop_epochs = tqdm(range(self.epochs))
-        for j in loop_epochs:
-            loop_epochs.set_description("Running mixed experiment %s" % j)
-
-            for i in loop_population_size:
-                net = self.nets[i]
-
-                if self.train_nets == "before_SA":
-                    for _ in range(self.ST_steps_between_SA):
-                        net.self_train(1, self.log_step_size, self.net_learning_rate)
-                    net.self_application(self.SA_steps, self.log_step_size)
-
-                elif self.train_nets == "after_SA":
-                    net.self_application(self.SA_steps, self.log_step_size)
-                    for _ in range(self.ST_steps_between_SA):
-                        net.self_train(1, self.log_step_size, self.net_learning_rate)
-
-                print(
-                    f"\nLast weight matrix (epoch: {j}):\n{net.input_weight_matrix()}\nLossHistory: {net.loss_history[-10:]}")
-            test_for_fixpoints(self.fixpoint_counters, self.nets)
-            # Rounding the result not to run into other problems later regarding the exact representation of floating number
-            fixpoints_percentage = round((self.fixpoint_counters["fix_zero"] + self.fixpoint_counters[
-                "fix_sec"]) / self.population_size, 1)
-            self.fixpoint_counters_history.append(fixpoints_percentage)
-
-            # Resetting the fixpoint counter. Last iteration not to be reset - it is important for the bar_chart_fixpoints().
-            if j < self.epochs:
-                self.reset_fixpoint_counters()
-
-    def weights_evolution_3d_experiment(self):
-        exp_name = f"Mixed {str(len(self.nets))}"
-
-        # This batch size is not relevant for mixed settings because during an epoch there are more steps of SA & ST happening
-        # and only they need the batch size. To not affect the number of epochs shown in the 3D plot, will send
-        # forward the number "1" for batch size with the variable <irrelevant_batch_size>
-        irrelevant_batch_size = 1
-        plot_3d_self_train(self.nets, exp_name, self.directory_name, irrelevant_batch_size, True)
-
-    def count_fixpoints(self):
-        exp_details = f"SA steps: {self.SA_steps}; ST steps: {self.ST_steps_between_SA}"
-
-        test_for_fixpoints(self.fixpoint_counters, self.nets)
-        bar_chart_fixpoints(self.fixpoint_counters, self.population_size, self.directory_name, self.net_learning_rate,
-                            exp_details)
-
-    def fixpoint_percentage(self):
-        line_chart_fixpoints(self.fixpoint_counters_history, self.epochs, self.ST_steps_between_SA,
-                             self.SA_steps, self.directory_name, self.population_size)
-
-    def visualize_loss(self):
-        for i in range(len(self.nets)):
-            net_loss_history = self.nets[i].loss_history
-            self.loss_history.append(net_loss_history)
-
-        plot_loss(self.loss_history, self.directory_name)
-
-    def reset_fixpoint_counters(self):
-        self.fixpoint_counters = {
-            "identity_func": 0,
-            "divergent": 0,
-            "fix_zero": 0,
-            "fix_weak": 0,
-            "fix_sec": 0,
-            "other_func": 0
-        }
-
-
-def run_mixed_experiment(population_size, net_input_size, net_hidden_size, net_out_size, net_learning_rate, train_nets,
-                         epochs, SA_steps, ST_steps_between_SA, batch_size, name_hash, runs, run_name):
-    experiments = {}
-    fixpoints_percentages = []
-
-    check_folder("mixed")
-
-    # Running the experiments
-    for i in range(runs):
-        directory_name = f"experiments/mixed/{run_name}_run_{i}_{str(population_size)}_nets_{SA_steps}_SA_{ST_steps_between_SA}_ST_{str(name_hash)}"
-
-        mixed_experiment = MixedSettingExperiment(
-            population_size,
-            net_input_size,
-            net_hidden_size,
-            net_out_size,
-            net_learning_rate,
-            train_nets,
-            epochs,
-            SA_steps,
-            ST_steps_between_SA,
-            batch_size,
-            directory_name
-        )
-        pickle.dump(mixed_experiment, open(f"{directory_name}/full_experiment_pickle.p", "wb"))
-        experiments[i] = mixed_experiment
-
-        # Building history of fixpoint percentages for summary
-        fixpoint_counters_history = mixed_experiment.fixpoint_counters_history
-        if not fixpoints_percentages:
-            fixpoints_percentages = mixed_experiment.fixpoint_counters_history
-        else:
-            # Using list comprehension to make the sum of all the percentages
-            fixpoints_percentages = [fixpoints_percentages[i] + fixpoint_counters_history[i] for i in
-                                     range(len(fixpoints_percentages))]
-
-    # Building a summary of all the runs
-    directory_name = f"experiments/mixed/summary_{run_name}_{runs}_runs_{str(population_size)}_nets_{str(name_hash)}"
-    os.mkdir(directory_name)
-
-    summary_pre_title = "mixed"
-    summary_fixpoint_experiment(runs, population_size, epochs, experiments, net_learning_rate, directory_name,
-                                summary_pre_title)
-    summary_fixpoint_percentage(runs, epochs, fixpoints_percentages, ST_steps_between_SA, SA_steps, directory_name,
-                                population_size)
-
-
-if __name__ == '__main__':
-    raise NotImplementedError('Test this here!!!')

experiments/robustness_exp.py

@@ -1,151 +0,0 @@
-import copy
-import os.path
-import pickle
-import random
-
-from tqdm import tqdm
-
-from experiments.helpers import check_folder, summary_fixpoint_experiment
-from functionalities_test import test_for_fixpoints, is_identity_function
-from network import Net
-from visualization import bar_chart_fixpoints, box_plot, write_file
-
-
-def add_noise(input_data, epsilon=pow(10, -5)):
-
-    output = copy.deepcopy(input_data)
-    for k in range(len(input_data)):
-        output[k][0] += random.random() * epsilon
-
-    return output
-
-
-class RobustnessExperiment:
-    def __init__(self, population_size, log_step_size, net_input_size, net_hidden_size, net_out_size, net_learning_rate,
-                 ST_steps, directory_name) -> None:
-        self.population_size = population_size
-        self.log_step_size = log_step_size
-        self.net_input_size = net_input_size
-        self.net_hidden_size = net_hidden_size
-        self.net_out_size = net_out_size
-
-        self.net_learning_rate = net_learning_rate
-
-        self.ST_steps = ST_steps
-        self.fixpoint_counters = {
-            "identity_func": 0,
-            "divergent": 0,
-            "fix_zero": 0,
-            "fix_weak": 0,
-            "fix_sec": 0,
-            "other_func": 0
-        }
-        self.id_functions = []
-
-        self.directory_name = directory_name
-        os.mkdir(self.directory_name)
-
-        self.nets = []
-        # Create population:
-        self.populate_environment()
-        print("Nets:\n", self.nets)
-
-        self.count_fixpoints()
-        [print(net.is_fixpoint) for net in self.nets]
-        self.test_robustness()
-
-    def populate_environment(self):
-        loop_population_size = tqdm(range(self.population_size))
-        for i in loop_population_size:
-            loop_population_size.set_description("Populating robustness experiment %s" % i)
-
-            net_name = f"net_{str(i)}"
-            net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name)
-
-            for _ in range(self.ST_steps):
-                net.self_train(1, self.log_step_size, self.net_learning_rate)
-
-            self.nets.append(net)
-
-    def test_robustness(self):
-        # test_for_fixpoints(self.fixpoint_counters, self.nets, self.id_functions)
-
-        zero_epsilon = pow(10, -5)
-        data = [[0 for _ in range(10)] for _ in range(len(self.id_functions))]
-
-        for i in range(len(self.id_functions)):
-            for j in range(10):
-                original_net = self.id_functions[i]
-
-                # Creating a clone of the network. Not by copying it, but by creating a completely new network
-                # and changing its weights to the original ones.
-                original_net_clone = Net(original_net.input_size, original_net.hidden_size, original_net.out_size,
-                                         original_net.name)
-                # Extra safety for the value of the weights
-                original_net_clone.load_state_dict(copy.deepcopy(original_net.state_dict()))
-
-                noisy_weights = add_noise(original_net_clone.input_weight_matrix(), epsilon=pow(10, -j))
-                original_net_clone.apply_weights(noisy_weights)
-
-                # Testing if the new net is still an identity function after applying noise
-                still_id_func = is_identity_function(original_net_clone, zero_epsilon)
-
-                # If the net is still an id. func. after applying the first run of noise, continue to apply it until otherwise
-                while still_id_func and data[i][j] <= 1000:
-                    data[i][j] += 1
-
-                    original_net_clone = original_net_clone.self_application(1, self.log_step_size)
-
-                    still_id_func = is_identity_function(original_net_clone, zero_epsilon)
-
-        print(f"Data {data}")
-
-        if data.count(0) == 10:
-            print(f"There is no network resisting the robustness test.")
-            text = f"For this population of \n {self.population_size} networks \n there is no" \
-                   f" network resisting the robustness test."
-            write_file(text, self.directory_name)
-        else:
-            box_plot(data, self.directory_name, self.population_size)
-
-    def count_fixpoints(self):
-        exp_details = f"ST steps: {self.ST_steps}"
-
-        self.id_functions = test_for_fixpoints(self.fixpoint_counters, self.nets)
-        bar_chart_fixpoints(self.fixpoint_counters, self.population_size, self.directory_name, self.net_learning_rate,
-                            exp_details)
-
-
-def run_robustness_experiment(population_size, batch_size, net_input_size, net_hidden_size, net_out_size,
-                              net_learning_rate, epochs, runs, run_name, name_hash):
-    experiments = {}
-
-    check_folder("robustness")
-
-    # Running the experiments
-    for i in range(runs):
-        ST_directory_name = f"experiments/robustness/{run_name}_run_{i}_{str(population_size)}_nets_{epochs}_epochs_{str(name_hash)}"
-
-        robustness_experiment = RobustnessExperiment(
-            population_size,
-            batch_size,
-            net_input_size,
-            net_hidden_size,
-            net_out_size,
-            net_learning_rate,
-            epochs,
-            ST_directory_name
-        )
-        pickle.dump(robustness_experiment, open(f"{ST_directory_name}/full_experiment_pickle.p", "wb"))
-        experiments[i] = robustness_experiment
-
-    # Building a summary of all the runs
-    directory_name = f"experiments/robustness/summary_{run_name}_{runs}_runs_{str(population_size)}_nets_{str(name_hash)}"
-    os.mkdir(directory_name)
-
-    summary_pre_title = "robustness"
-    summary_fixpoint_experiment(runs, population_size, epochs, experiments, net_learning_rate, directory_name,
-                                summary_pre_title)
-
-if __name__ == '__main__':
-    raise NotImplementedError('Test this here!!!')

experiments/self_application_exp.py

@@ -1,120 +0,0 @@
-import os.path
-import pickle
-
-from tqdm import tqdm
-
-from experiments.helpers import check_folder, summary_fixpoint_experiment
-from functionalities_test import test_for_fixpoints
-from network import Net
-from visualization import bar_chart_fixpoints
-from visualization import plot_3d_self_application
-
-
-class SelfApplicationExperiment:
-    def __init__(self, population_size, log_step_size, net_input_size, net_hidden_size, net_out_size,
-                 net_learning_rate, application_steps, train_nets, directory_name, training_steps
-                 ) -> None:
-        self.population_size = population_size
-        self.log_step_size = log_step_size
-        self.net_input_size = net_input_size
-        self.net_hidden_size = net_hidden_size
-        self.net_out_size = net_out_size
-
-        self.net_learning_rate = net_learning_rate
-        self.SA_steps = application_steps  #
-
-        self.train_nets = train_nets
-        self.ST_steps = training_steps
-
-        self.directory_name = directory_name
-        os.mkdir(self.directory_name)
-
-        """ Creating the nets & making the SA steps & (maybe) also training the networks. """
-        self.nets = []
-        # Create population:
-        self.populate_environment()
-
-        self.fixpoint_counters = {
-            "identity_func": 0,
-            "divergent": 0,
-            "fix_zero": 0,
-            "fix_weak": 0,
-            "fix_sec": 0,
-            "other_func": 0
-        }
-
-        self.weights_evolution_3d_experiment()
-        self.count_fixpoints()
-
-    def populate_environment(self):
-        loop_population_size = tqdm(range(self.population_size))
-        for i in loop_population_size:
-            loop_population_size.set_description("Populating SA experiment %s" % i)
-
-            net_name = f"SA_net_{str(i)}"
-
-            net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name
-                      )
-            for _ in range(self.SA_steps):
-                input_data = net.input_weight_matrix()
-                target_data = net.create_target_weights(input_data)
-
-                if self.train_nets == "before_SA":
-                    net.self_train(1, self.log_step_size, self.net_learning_rate)
-                    net.self_application(self.SA_steps, self.log_step_size)
-                elif self.train_nets == "after_SA":
-                    net.self_application(self.SA_steps, self.log_step_size)
-                    net.self_train(1, self.log_step_size, self.net_learning_rate)
-                else:
-                    net.self_application(self.SA_steps, self.log_step_size)
-
-            self.nets.append(net)
-
-    def weights_evolution_3d_experiment(self):
-        exp_name = f"SA_{str(len(self.nets))}_nets_3d_weights_PCA"
-        plot_3d_self_application(self.nets, exp_name, self.directory_name, self.log_step_size)
-
-    def count_fixpoints(self):
-        test_for_fixpoints(self.fixpoint_counters, self.nets)
-        exp_details = f"{self.SA_steps} SA steps"
-        bar_chart_fixpoints(self.fixpoint_counters, self.population_size, self.directory_name, self.net_learning_rate,
-                            exp_details)
-
-
-def run_SA_experiment(population_size, batch_size, net_input_size, net_hidden_size, net_out_size,
-                      net_learning_rate, runs, run_name, name_hash, application_steps, train_nets, training_steps):
-    experiments = {}
-
-    check_folder("self_application")
-
-    # Running the experiments
-    for i in range(runs):
-        directory_name = f"experiments/self_application/{run_name}_run_{i}_{str(population_size)}_nets_{application_steps}_SA_{str(name_hash)}"
-
-        SA_experiment = SelfApplicationExperiment(
-            population_size,
-            batch_size,
-            net_input_size,
-            net_hidden_size,
-            net_out_size,
-            net_learning_rate,
-            application_steps,
-            train_nets,
-            directory_name,
-            training_steps
-        )
-        pickle.dump(SA_experiment, open(f"{directory_name}/full_experiment_pickle.p", "wb"))
-        experiments[i] = SA_experiment
-
-    # Building a summary of all the runs
-    directory_name = f"experiments/self_application/summary_{run_name}_{runs}_runs_{str(population_size)}_nets_{application_steps}_SA_{str(name_hash)}"
-    os.mkdir(directory_name)
-
-    summary_pre_title = "SA"
-    summary_fixpoint_experiment(runs, population_size, application_steps, experiments, net_learning_rate,
-                                directory_name,
-                                summary_pre_title)
-
-
-if __name__ == '__main__':
-    raise NotImplementedError('Test this here!!!')

experiments/self_train_exp.py

@@ -1,116 +0,0 @@
-import os.path
-import pickle
-from pathlib import Path
-
-from tqdm import tqdm
-
-from experiments.helpers import check_folder, summary_fixpoint_experiment
-from functionalities_test import test_for_fixpoints
-from network import Net
-from visualization import plot_loss, bar_chart_fixpoints
-from visualization import plot_3d_self_train
-
-
-
-class SelfTrainExperiment:
-    def __init__(self, population_size, log_step_size, net_input_size, net_hidden_size, net_out_size, net_learning_rate,
-                 epochs, directory_name) -> None:
-        self.population_size = population_size
-        self.log_step_size = log_step_size
-        self.net_input_size = net_input_size
-        self.net_hidden_size = net_hidden_size
-        self.net_out_size = net_out_size
-
-        self.net_learning_rate = net_learning_rate
-        self.epochs = epochs
-
-        self.loss_history = []
-
-        self.fixpoint_counters = {
-            "identity_func": 0,
-            "divergent": 0,
-            "fix_zero": 0,
-            "fix_weak": 0,
-            "fix_sec": 0,
-            "other_func": 0
-        }
-
-        self.directory_name = directory_name
-        os.mkdir(self.directory_name)
-
-        self.nets = []
-        # Create population:
-        self.populate_environment()
-
-        self.weights_evolution_3d_experiment()
-        self.count_fixpoints()
-        self.visualize_loss()
-
-    def populate_environment(self):
-        loop_population_size = tqdm(range(self.population_size))
-        for i in loop_population_size:
-            loop_population_size.set_description("Populating ST experiment %s" % i)
-
-            net_name = f"ST_net_{str(i)}"
-            net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name)
-
-            for _ in range(self.epochs):
-              net.self_train(1, self.log_step_size, self.net_learning_rate)
-
-            print(f"\nLast weight matrix (epoch: {self.epochs}):\n{net.input_weight_matrix()}\nLossHistory: {net.loss_history[-10:]}")
-            self.nets.append(net)
-
-    def weights_evolution_3d_experiment(self):
-        exp_name = f"ST_{str(len(self.nets))}_nets_3d_weights_PCA"
-        return plot_3d_self_train(self.nets, exp_name, self.directory_name, self.log_step_size)
-
-    def count_fixpoints(self):
-        test_for_fixpoints(self.fixpoint_counters, self.nets)
-        exp_details = f"Self-train for {self.epochs} epochs"
-        bar_chart_fixpoints(self.fixpoint_counters, self.population_size, self.directory_name, self.net_learning_rate,
-                            exp_details)
-
-    def visualize_loss(self):
-        for i in range(len(self.nets)):
-            net_loss_history = self.nets[i].loss_history
-            self.loss_history.append(net_loss_history)
-
-        plot_loss(self.loss_history, self.directory_name)
-
-
-def run_ST_experiment(population_size, batch_size, net_input_size, net_hidden_size, net_out_size, net_learning_rate,
-                      epochs, runs, run_name, name_hash):
-    experiments = {}
-    logging_directory = Path('output') / 'self_training'
-    logging_directory.mkdir(parents=True, exist_ok=True)
-
-    # Running the experiments
-    for i in range(runs):
-        experiment_name = f"{run_name}_run_{i}_{str(population_size)}_nets_{epochs}_epochs_{str(name_hash)}"
-        this_exp_directory = logging_directory / experiment_name
-        ST_experiment = SelfTrainExperiment(
-            population_size,
-            batch_size,
-            net_input_size,
-            net_hidden_size,
-            net_out_size,
-            net_learning_rate,
-            epochs,
-            this_exp_directory
-        )
-        with (this_exp_directory / 'full_experiment_pickle.p').open('wb') as f:
-            pickle.dump(ST_experiment, f)
-        experiments[i] = ST_experiment
-
-    # Building a summary of all the runs
-    summary_name = f"/summary_{run_name}_{runs}_runs_{str(population_size)}_nets_{epochs}_epochs_{str(name_hash)}"
-    summary_directory_name = logging_directory / summary_name
-    summary_directory_name.mkdir(parents=True, exist_ok=True)
-
-    summary_pre_title = "ST"
-    summary_fixpoint_experiment(runs, population_size, epochs, experiments, net_learning_rate, summary_directory_name,
-                                summary_pre_title)
-
-
-if __name__ == '__main__':
-    raise NotImplementedError('Test this here!!!')

experiments/self_train_secondary_exp.py

@@ -1,114 +0,0 @@
-import pickle
-from pathlib import Path
-
-from tqdm import tqdm
-
-from experiments.helpers import check_folder, summary_fixpoint_experiment
-from functionalities_test import test_for_fixpoints
-from network import SecondaryNet
-from visualization import plot_loss, bar_chart_fixpoints
-from visualization import plot_3d_self_train
-
-
-class SelfTrainExperimentSecondary:
-    def __init__(self, population_size, log_step_size, net_input_size, net_hidden_size, net_out_size, net_learning_rate,
-                 epochs, directory: Path) -> None:
-        self.population_size = population_size
-        self.log_step_size = log_step_size
-        self.net_input_size = net_input_size
-        self.net_hidden_size = net_hidden_size
-        self.net_out_size = net_out_size
-
-        self.net_learning_rate = net_learning_rate
-        self.epochs = epochs
-
-        self.loss_history = []
-
-        self.fixpoint_counters = {
-            "identity_func": 0,
-            "divergent": 0,
-            "fix_zero": 0,
-            "fix_weak": 0,
-            "fix_sec": 0,
-            "other_func": 0
-        }
-
-        self.directory_name = Path(directory)
-        self.directory_name.mkdir(parents=True, exist_ok=True)
-
-        self.nets = []
-        # Create population:
-        self.populate_environment()
-
-        self.weights_evolution_3d_experiment()
-        self.count_fixpoints()
-        self.visualize_loss()
-
-    def populate_environment(self):
-        loop_population_size = tqdm(range(self.population_size))
-        for i in loop_population_size:
-            loop_population_size.set_description("Populating ST experiment %s" % i)
-
-            net_name = f"ST_net_{str(i)}"
-            net = SecondaryNet(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name)
-
-            for _ in range(self.epochs):
-                net.self_train(1, self.log_step_size, self.net_learning_rate)
-
-            print(f"\nLast weight matrix (epoch: {self.epochs}):\n{net.input_weight_matrix()}\nLossHistory: {net.loss_history[-10:]}")
-            self.nets.append(net)
-
-    def weights_evolution_3d_experiment(self):
-        exp_name = f"ST_{str(len(self.nets))}_nets_3d_weights_PCA"
-        return plot_3d_self_train(self.nets, exp_name, self.directory_name, self.log_step_size)
-
-    def count_fixpoints(self):
-        test_for_fixpoints(self.fixpoint_counters, self.nets)
-        exp_details = f"Self-train for {self.epochs} epochs"
-        bar_chart_fixpoints(self.fixpoint_counters, self.population_size, self.directory_name, self.net_learning_rate,
-                            exp_details)
-
-    def visualize_loss(self):
-        for i in range(len(self.nets)):
-            net_loss_history = self.nets[i].loss_history
-            self.loss_history.append(net_loss_history)
-
-        plot_loss(self.loss_history, self.directory_name)
-
-
-def run_ST_experiment(population_size, batch_size, net_input_size, net_hidden_size, net_out_size, net_learning_rate,
-                      epochs, runs, run_name, name_hash):
-    experiments = {}
-    logging_directory = Path('output') / 'self_training'
-    logging_directory.mkdir(parents=True, exist_ok=True)
-
-    # Running the experiments
-    for i in range(runs):
-        experiment_name = f"{run_name}_run_{i}_{str(population_size)}_nets_{epochs}_epochs_{str(name_hash)}"
-        this_exp_directory = logging_directory / experiment_name
-        ST_experiment = SelfTrainExperimentSecondary(
-            population_size,
-            batch_size,
-            net_input_size,
-            net_hidden_size,
-            net_out_size,
-            net_learning_rate,
-            epochs,
-            this_exp_directory
-        )
-        with (this_exp_directory / 'full_experiment_pickle.p').open('wb') as f:
-            pickle.dump(ST_experiment, f)
-        experiments[i] = ST_experiment
-
-    # Building a summary of all the runs
-    summary_name = f"/summary_{run_name}_{runs}_runs_{str(population_size)}_nets_{epochs}_epochs_{str(name_hash)}"
-    summary_directory_name = logging_directory / summary_name
-    summary_directory_name.mkdir(parents=True, exist_ok=True)
-
-    summary_pre_title = "ST"
-    summary_fixpoint_experiment(runs, population_size, epochs, experiments, net_learning_rate, summary_directory_name,
-                                summary_pre_title)
-
-
-if __name__ == '__main__':
-    raise NotImplementedError('Test this here!!!')

experiments/soup_exp.py

@@ -1,190 +0,0 @@
-import random
-import os.path
-import pickle
-from pathlib import Path
-from typing import Union
-
-from tqdm import tqdm
-
-from experiments.helpers import check_folder, summary_fixpoint_percentage, summary_fixpoint_experiment
-from functionalities_test import test_for_fixpoints
-from network import Net
-from visualization import plot_loss, bar_chart_fixpoints, plot_3d_soup, line_chart_fixpoints
-
-
-class SoupExperiment:
-    def __init__(self, population_size, net_i_size, net_h_size, net_o_size, learning_rate, attack_chance,
-                 train_nets, ST_steps, epochs, log_step_size, directory: Union[str, Path]):
-        super().__init__()
-        self.population_size = population_size
-
-        self.net_input_size = net_i_size
-        self.net_hidden_size = net_h_size
-        self.net_out_size = net_o_size
-        self.net_learning_rate = learning_rate
-        self.attack_chance = attack_chance
-        self.train_nets = train_nets
-        # self.SA_steps = SA_steps
-        self.ST_steps = ST_steps
-        self.epochs = epochs
-        self.log_step_size = log_step_size
-
-        self.loss_history = []
-
-        self.fixpoint_counters = {
-            "identity_func": 0,
-            "divergent": 0,
-            "fix_zero": 0,
-            "fix_weak": 0,
-            "fix_sec": 0,
-            "other_func": 0
-        }
-        # <self.fixpoint_counters_history> is used for keeping track of the amount of fixpoints in %
-        self.fixpoint_counters_history = []
-
-        self.directory = Path(directory)
-        self.directory.mkdir(parents=True, exist_ok=True)
-
-        self.population = []
-        self.populate_environment()
-
-        self.evolve()
-        self.fixpoint_percentage()
-        self.weights_evolution_3d_experiment()
-        self.count_fixpoints()
-        self.visualize_loss()
-
-    def populate_environment(self):
-        loop_population_size = tqdm(range(self.population_size))
-        for i in tqdm(range(self.population_size)):
-            loop_population_size.set_description("Populating soup experiment %s" % i)
-
-            net_name = f"soup_network_{i}"
-            net = Net(self.net_input_size, self.net_hidden_size, self.net_out_size, net_name)
-            self.population.append(net)
-
-    def population_self_train(self):
-        #  Self-training each network in the population
-        for j in range(self.population_size):
-            net = self.population[j]
-
-            for _ in range(self.ST_steps):
-                net.self_train(1, self.log_step_size, self.net_learning_rate)
-
-    def population_attack(self):
-        # A network attacking another network with a given percentage
-        if random.randint(1, 100) <= self.attack_chance:
-            random_net1, random_net2 = random.sample(range(self.population_size), 2)
-            random_net1 = self.population[random_net1]
-            random_net2 = self.population[random_net2]
-            print(f"\n Attack: {random_net1.name} -> {random_net2.name}")
-            random_net1.attack(random_net2)
-
-    def evolve(self):
-        """ Evolving consists of attacking & self-training. """
-
-        loop_epochs = tqdm(range(self.epochs))
-        for i in loop_epochs:
-            loop_epochs.set_description("Evolving soup %s" % i)
-
-            # A network attacking another network with a given percentage
-            self.population_attack()
-
-            #  Self-training each network in the population
-            self.population_self_train()
-
-            # Testing for fixpoints after each batch of ST steps to see relevant data
-            if i % self.ST_steps == 0:
-                test_for_fixpoints(self.fixpoint_counters, self.population)
-                fixpoints_percentage = round(self.fixpoint_counters["identity_func"] / self.population_size, 1)
-                self.fixpoint_counters_history.append(fixpoints_percentage)
-
-            # Resetting the fixpoint counter. Last iteration not to be reset -
-            #  it is important for the bar_chart_fixpoints().
-            if i < self.epochs:
-                self.reset_fixpoint_counters()
-
-    def weights_evolution_3d_experiment(self):
-        exp_name = f"soup_{self.population_size}_nets_{self.ST_steps}_training_{self.epochs}_epochs"
-        return plot_3d_soup(self.population, exp_name, self.directory)
-
-    def count_fixpoints(self):
-        test_for_fixpoints(self.fixpoint_counters, self.population)
-        exp_details = f"Evolution steps: {self.epochs} epochs"
-        bar_chart_fixpoints(self.fixpoint_counters, self.population_size, self.directory, self.net_learning_rate,
-                            exp_details)
-
-    def fixpoint_percentage(self):
-        runs = self.epochs / self.ST_steps
-        SA_steps = None
-        line_chart_fixpoints(self.fixpoint_counters_history, runs, self.ST_steps, SA_steps, self.directory,
-                             self.population_size)
-
-    def visualize_loss(self):
-        for i in range(len(self.population)):
-            net_loss_history = self.population[i].loss_history
-            self.loss_history.append(net_loss_history)
-
-        plot_loss(self.loss_history, self.directory)
-
-    def reset_fixpoint_counters(self):
-        self.fixpoint_counters = {
-            "identity_func": 0,
-            "divergent": 0,
-            "fix_zero": 0,
-            "fix_weak": 0,
-            "fix_sec": 0,
-            "other_func": 0
-        }
-
-
-def run_soup_experiment(population_size, attack_chance, net_input_size, net_hidden_size, net_out_size,
-                        net_learning_rate, epochs, batch_size, runs, run_name, name_hash, ST_steps, train_nets):
-    experiments = {}
-    fixpoints_percentages = []
-
-    check_folder("soup")
-
-    # Running the experiments
-    for i in range(runs):
-        # FIXME: Make this a pathlib.Path() Operation
-        directory_name = f"experiments/soup/{run_name}_run_{i}_{str(population_size)}_nets_{epochs}_epochs_{str(name_hash)}"
-
-        soup_experiment = SoupExperiment(
-            population_size,
-            net_input_size,
-            net_hidden_size,
-            net_out_size,
-            net_learning_rate,
-            attack_chance,
-            train_nets,
-            ST_steps,
-            epochs,
-            batch_size,
-            directory_name
-        )
-        pickle.dump(soup_experiment, open(f"{directory_name}/full_experiment_pickle.p", "wb"))
-        experiments[i] = soup_experiment
-
-        # Building history of fixpoint percentages for summary
-        fixpoint_counters_history = soup_experiment.fixpoint_counters_history
-        if not fixpoints_percentages:
-            fixpoints_percentages = soup_experiment.fixpoint_counters_history
-        else:
-            # Using list comprehension to make the sum of all the percentages
-            fixpoints_percentages = [fixpoints_percentages[i] + fixpoint_counters_history[i] for i in
-                                     range(len(fixpoints_percentages))]
-
-    # Creating a folder for the summary of the current runs
-    # FIXME: Make this a pathlib.Path() Operation
-    directory_name = f"experiments/soup/summary_{run_name}_{runs}_runs_{str(population_size)}_nets_{epochs}_epochs_{str(name_hash)}"
-    os.mkdir(directory_name)
-
-    # Building a summary of all the runs
-    summary_pre_title = "soup"
-    summary_fixpoint_experiment(runs, population_size, epochs, experiments, net_learning_rate, directory_name,
-                                summary_pre_title)
-    SA_steps = None
-    summary_fixpoint_percentage(runs, epochs, fixpoints_percentages, ST_steps, SA_steps, directory_name,
-                                population_size)
-

experiments/soup_melt_exp.py

@@ -1,50 +0,0 @@
-import random
-
-from tqdm import tqdm
-
-from experiments.soup_exp import SoupExperiment
-from functionalities_test import test_for_fixpoints
-
-
-class MeltingSoupExperiment(SoupExperiment):
-
-    def __init__(self, melt_chance, *args, keep_population_size=True, **kwargs):
-        super(MeltingSoupExperiment, self).__init__(*args, **kwargs)
-        self.keep_population_size = keep_population_size
-        self.melt_chance = melt_chance
-
-    def population_melt(self):
-        # A network melting with another network by a given percentage
-        if random.randint(1, 100) <= self.melt_chance:
-            random_net1_idx, random_net2_idx, destroy_idx = random.sample(range(self.population_size), 3)
-            random_net1 = self.population[random_net1_idx]
-            random_net2 = self.population[random_net2_idx]
-            print(f"\n Melt: {random_net1.name} -> {random_net2.name}")
-            melted_network = random_net1.melt(random_net2)
-            if self.keep_population_size:
-                del self.population[destroy_idx]
-            self.population.append(melted_network)
-
-    def evolve(self):
-        """ Evolving consists of attacking, melting & self-training. """
-
-        loop_epochs = tqdm(range(self.epochs))
-        for i in loop_epochs:
-            loop_epochs.set_description("Evolving soup %s" % i)
-
-            self.population_attack()
-
-            self.population_melt()
-
-            self.population_self_train()
-
-            # Testing for fixpoints after each batch of ST steps to see relevant data
-            if i % self.ST_steps == 0:
-                test_for_fixpoints(self.fixpoint_counters, self.population)
-                fixpoints_percentage = round(self.fixpoint_counters["identity_func"] / self.population_size, 1)
-                self.fixpoint_counters_history.append(fixpoints_percentage)
-
-            # Resetting the fixpoint counter. Last iteration not to be reset -
-            #  it is important for the bar_chart_fixpoints().
-            if i < self.epochs:
-                self.reset_fixpoint_counters()

functionalities_test.py

@@ -1,95 +0,0 @@
-import copy
-from typing import Dict, List
-import torch
-from tqdm import tqdm
-
-from network import Net
-
-
-def is_divergent(network: Net) -> bool:
-    return network.input_weight_matrix().isinf().any().item() or network.input_weight_matrix().isnan().any().item()
-
-
-def is_identity_function(network: Net, epsilon=pow(10, -5)) -> bool:
-
-    input_data = network.input_weight_matrix()
-    target_data = network.create_target_weights(input_data)
-    predicted_values = network(input_data)
-
-
-    return torch.allclose(target_data.detach(), predicted_values.detach(),
-                          rtol=0, atol=epsilon)
-
-
-def is_zero_fixpoint(network: Net, epsilon=pow(10, -5)) -> bool:
-    target_data = network.create_target_weights(network.input_weight_matrix().detach())
-    result = torch.allclose(target_data, torch.zeros_like(target_data), rtol=0, atol=epsilon)
-    # result = bool(len(np.nonzero(network.create_target_weights(network.input_weight_matrix()))))
-    return result
-
-
-def is_secondary_fixpoint(network: Net, epsilon: float = pow(10, -5)) -> bool:
-    """ Secondary fixpoint check is done like this: compare first INPUT with second OUTPUT.
-    If they are within the boundaries, then is secondary fixpoint. """
-
-    input_data = network.input_weight_matrix()
-    target_data = network.create_target_weights(input_data)
-
-    # Calculating first output
-    first_output = network(input_data)
-
-    # Getting the second output by initializing a new net with the weights of the original net.
-    net_copy = copy.deepcopy(network)
-    net_copy.apply_weights(first_output)
-    input_data_2 = net_copy.input_weight_matrix()
-
-    # Calculating second output
-    second_output = network(input_data_2)
-
-    # Perform the Check: all(epsilon > abs(input_data - second_output))
-    check_abs_within_epsilon = torch.allclose(target_data.detach(), second_output.detach(),
-                                              rtol=0, atol=epsilon)
-    return check_abs_within_epsilon
-
-
-def test_for_fixpoints(fixpoint_counter: Dict, nets: List, id_functions=None):
-    id_functions = id_functions or list()
-
-    for net in tqdm(nets, desc='Fixpoint Tester', total=len(nets)):
-        if is_divergent(net):
-            fixpoint_counter["divergent"] += 1
-            net.is_fixpoint = "divergent"
-        elif is_identity_function(net):  # is default value
-            fixpoint_counter["identity_func"] += 1
-            net.is_fixpoint = "identity_func"
-            id_functions.append(net)
-        elif is_zero_fixpoint(net):
-            fixpoint_counter["fix_zero"] += 1
-            net.is_fixpoint = "fix_zero"
-        elif is_secondary_fixpoint(net):
-            fixpoint_counter["fix_sec"] += 1
-            net.is_fixpoint = "fix_sec"
-        else:
-            fixpoint_counter["other_func"] += 1
-            net.is_fixpoint = "other_func"
-    return id_functions
-
-
-def changing_rate(x_new, x_old):
-    return x_new - x_old
-
-
-def test_status(net: Net) -> Net:
-
-    if is_divergent(net):
-        net.is_fixpoint = "divergent"
-    elif is_identity_function(net):  # is default value
-        net.is_fixpoint = "identity_func"
-    elif is_zero_fixpoint(net):
-        net.is_fixpoint = "fix_zero"
-    elif is_secondary_fixpoint(net):
-        net.is_fixpoint = "fix_sec"
-    else:
-        net.is_fixpoint = "other_func"
-
-    return net

journal_basin_linspace_clones.py

@@ -1,203 +0,0 @@
-import copy
-import itertools
-from pathlib import Path
-import random
-import pickle
-import pandas as pd
-import numpy as np
-import torch
-
-from functionalities_test import is_identity_function, test_status
-from journal_basins import SpawnExperiment, mean_invariate_manhattan_distance
-from network import Net
-
-from sklearn.metrics import mean_absolute_error as MAE
-from sklearn.metrics import mean_squared_error as MSE
-
-
-class SpawnLinspaceExperiment(SpawnExperiment):
-
-    def spawn_and_continue(self, number_clones: int = None):
-        number_clones = number_clones or self.nr_clones
-
-        df = pd.DataFrame(
-            columns=['clone', 'parent', 'parent2',
-                     'MAE_pre', 'MAE_post',
-                     'MSE_pre', 'MSE_post',
-                     'MIM_pre', 'MIM_post',
-                     'noise', 'status_pst'])
-
-        # For every initial net {i} after populating (that is fixpoint after first epoch);
-        # parent = self.parents[0]
-        # parent_clone = clone = Net(parent.input_size, parent.hidden_size, parent.out_size,
-        #                         name=f"{parent.name}_clone_{0}", start_time=self.ST_steps)
-        # parent_clone.apply_weights(torch.as_tensor(parent.create_target_weights(parent.input_weight_matrix())))
-        # parent_clone = parent_clone.apply_noise(self.noise)
-        # self.parents.append(parent_clone)
-        pairwise_net_list = list(itertools.combinations(self.parents, 2))
-        for net1, net2 in pairwise_net_list:
-            # We set parent start_time to just before this epoch ended, so plotting is zoomed in. Comment out to
-            # to see full trajectory (but the clones will be very hard to see).
-            # Make one target to compare distances to clones later when they have trained.
-            net1.start_time = self.ST_steps - 150
-            net1_input_data = net1.input_weight_matrix().detach()
-            net1_target_data = net1.create_target_weights(net1_input_data).detach()
-
-            net2.start_time = self.ST_steps - 150
-            net2_input_data = net2.input_weight_matrix().detach()
-            net2_target_data = net2.create_target_weights(net2_input_data).detach()
-
-            if is_identity_function(net1) and is_identity_function(net2):
-                # if True:
-                # Clone the fixpoint x times and add (+-)self.noise to weight-sets randomly;
-                # To plot clones starting after first epoch (z=ST_steps), set that as start_time!
-                # To make sure PCA will plot the same trajectory up until this point, we clone the
-                # parent-net's weight history as well.
-
-                in_between_weights = np.linspace(net1_target_data, net2_target_data, number_clones, endpoint=False)
-                # in_between_weights = np.logspace(net1_target_data, net2_target_data, number_clones, endpoint=False)
-
-                for j, in_between_weight in enumerate(in_between_weights):
-                    clone = Net(net1.input_size, net1.hidden_size, net1.out_size,
-                                name=f"{net1.name}_{net2.name}_clone_{str(j)}", start_time=self.ST_steps + 100)
-                    clone.apply_weights(torch.as_tensor(in_between_weight))
-
-                    clone.s_train_weights_history = copy.deepcopy(net1.s_train_weights_history)
-                    clone.number_trained = copy.deepcopy(net1.number_trained)
-
-                    # Pre Training distances (after noise application of course)
-                    clone_pre_weights = clone.create_target_weights(clone.input_weight_matrix()).detach()
-                    MAE_pre = MAE(net1_target_data, clone_pre_weights)
-                    MSE_pre = MSE(net1_target_data, clone_pre_weights)
-                    MIM_pre = mean_invariate_manhattan_distance(net1_target_data, clone_pre_weights)
-
-                    try:
-                        # Then finish training each clone {j} (for remaining epoch-1 * ST_steps) ..
-                        for _ in range(self.epochs - 1):
-                            for _ in range(self.ST_steps):
-                                clone.self_train(1, self.log_step_size, self.net_learning_rate)
-                                if any([torch.isnan(x).any() for x in clone.parameters()]):
-                                    raise ValueError
-                    except ValueError:
-                        print("Ran into nan in 'in beetween weights' array.")
-                        df.loc[len(df)] = [j, net1.name, net2.name,
-                                           MAE_pre, 0,
-                                           MSE_pre, 0,
-                                           MIM_pre, 0,
-                                           self.noise, clone.is_fixpoint]
-                        continue
-
-                    # Post Training distances for comparison
-                    clone_post_weights = clone.create_target_weights(clone.input_weight_matrix()).detach()
-                    MAE_post = MAE(net1_target_data, clone_post_weights)
-                    MSE_post = MSE(net1_target_data, clone_post_weights)
-                    MIM_post = mean_invariate_manhattan_distance(net1_target_data, clone_post_weights)
-
-                    # .. log to data-frame and add to nets for 3d plotting if they are fixpoints themselves.
-                    test_status(clone)
-                    if is_identity_function(clone):
-                        print(f"Clone {j} (between {net1.name} and {net2.name}) is fixpoint."
-                              f"\nMSE({net1.name},{j}): {MSE_post}"
-                              f"\nMAE({net1.name},{j}): {MAE_post}"
-                              f"\nMIM({net1.name},{j}): {MIM_post}\n")
-                        self.nets.append(clone)
-
-                    df.loc[len(df)] = [j, net1.name, net2.name,
-                                       MAE_pre, MAE_post,
-                                       MSE_pre, MSE_post,
-                                       MIM_pre, MIM_post,
-                                       self.noise, clone.is_fixpoint]
-
-        for net1, net2 in pairwise_net_list:
-            try:
-                value = 'MAE'
-                c_selector = [f'{value}_pre', f'{value}_post']
-                values = df.loc[(df['parent'] == net1.name) & (df['parent2'] == net2.name)][c_selector]
-                this_min, this_max = values.values.min(), values.values.max()
-                df.loc[(df['parent'] == net1.name) &
-                       (df['parent2'] == net2.name), c_selector] = (values - this_min) / (this_max - this_min)
-            except ValueError:
-                pass
-
-        for parent in self.parents:
-            for _ in range(self.epochs - 1):
-                for _ in range(self.ST_steps):
-                    parent.self_train(1, self.log_step_size, self.net_learning_rate)
-
-        self.df = df
-
-
-if __name__ == '__main__':
-    NET_INPUT_SIZE = 4
-    NET_OUT_SIZE = 1
-
-    # Define number of runs & name:
-    ST_runs = 1
-    ST_runs_name = "test-27"
-    ST_steps = 2000
-    ST_epochs = 2
-    ST_log_step_size = 10
-
-    # Define number of networks & their architecture
-    nr_clones = 25
-    ST_population_size = 10
-    ST_net_hidden_size = 2
-    ST_net_learning_rate = 0.04
-    ST_name_hash = random.getrandbits(32)
-
-    print(f"Running the Spawn experiment:")
-    exp = SpawnLinspaceExperiment(
-        population_size=ST_population_size,
-        log_step_size=ST_log_step_size,
-        net_input_size=NET_INPUT_SIZE,
-        net_hidden_size=ST_net_hidden_size,
-        net_out_size=NET_OUT_SIZE,
-        net_learning_rate=ST_net_learning_rate,
-        epochs=ST_epochs,
-        st_steps=ST_steps,
-        nr_clones=nr_clones,
-        noise=1e-8,
-        directory=Path('output') / 'spawn_basin' / f'{ST_name_hash}' / f'linage'
-    )
-    df = exp.df
-
-    directory = Path('output') / 'spawn_basin' / f'{ST_name_hash}' / 'linage'
-    with (directory / f"experiment_pickle_{ST_name_hash}.p").open('wb') as f:
-        pickle.dump(exp, f)
-    print(f"\nSaved experiment to {directory}.")
-
-    # Boxplot with counts of nr_fixpoints, nr_other, nr_etc. on y-axis
-    # sns.countplot(data=df, x="noise", hue="status_post")
-    # plt.savefig(f"output/spawn_basin/{ST_name_hash}/fixpoint_status_countplot.png")
-
-    # Catplot (either kind="point" or "box") that shows before-after training distances to parent
-    # mlt = df[["MIM_pre", "MIM_post", "noise"]].melt("noise", var_name="time", value_name='Average Distance')
-    # sns.catplot(data=mlt, x="time", y="Average Distance", col="noise", kind="point", col_wrap=5, sharey=False)
-    # plt.savefig(f"output/spawn_basin/{ST_name_hash}/clone_distance_catplot.png")
-
-    # Pointplot with pre and after parent Distances
-    import seaborn as sns
-    from matplotlib import pyplot as plt, ticker
-
-    # ptplt = sns.pointplot(data=exp.df, x='MAE_pre', y='MAE_post', join=False)
-    ptplt = sns.scatterplot(x=exp.df['MAE_pre'], y=exp.df['MAE_post'])
-    # ptplt.set(xscale='log', yscale='log')
-    x0, x1 = ptplt.axes.get_xlim()
-    y0, y1 = ptplt.axes.get_ylim()
-    lims = [max(x0, y0), min(x1, y1)]
-    # This is the x=y line using transforms
-    ptplt.plot(lims, lims, 'w', linestyle='dashdot', transform=ptplt.axes.transData)
-    ptplt.plot([0, 1], [0, 1], ':k', transform=ptplt.axes.transAxes)
-    ptplt.set(xlabel='Mean Absolute Distance before Self-Training',
-              ylabel='Mean Absolute Distance after Self-Training')
-    # ptplt.axes.xaxis.set_major_formatter(ticker.FuncFormatter(lambda x, pos: round(float(x), 2)))
-    # ptplt.xticks(rotation=45)
-    #for ind, label in enumerate(ptplt.get_xticklabels()):
-    #    if ind % 10 == 0:  # every 10th label is kept
-    #        label.set_visible(True)
-    #    else:
-    #        label.set_visible(False)
-
-    filepath = exp.directory / 'mim_dist_plot.pdf'
-    plt.tight_layout()
-    plt.savefig(filepath, dpi=600, format='pdf', bbox_inches='tight')